Standalone SlapOS and Slapos instance TestCase

This is about replacing the `utils.py` we have duplicated in each software release tests by a more "official API".

This introduce a  "sandalone slapos API" which are helpers around slapos API to create a recursive SlapOS. This can eventually replace implementation of erp5testnode and slaprunner (but there are no plans for that)

/reviewed-on nexedi/slapos.core!64

slapos/proxy/views.py

@@ -44,6 +44,7 @@ from slapos.util import loads, dumps
 
 import six
 from six.moves import range
+from six.moves.urllib.parse import urlparse
 
 app = Flask(__name__)
 
@@ -316,11 +317,14 @@ def loadComputerConfigurationFromXML():
   computer_dict = loads(xml.encode('utf-8'))
   execute_db('computer', 'INSERT OR REPLACE INTO %s values(:reference, :address, :netmask)',
              computer_dict)
+
+  # remove references to old partitions.
+  execute_db('partition', 'DELETE FROM %s WHERE computer_reference = :reference', computer_dict)
+  execute_db('partition_network', 'DELETE FROM %s WHERE computer_reference = :reference', computer_dict)
+
   for partition in computer_dict['partition_list']:
     partition['computer_reference'] = computer_dict['reference']
     execute_db('partition', 'INSERT OR IGNORE INTO %s (reference, computer_reference) values(:reference, :computer_reference)', partition)
-    execute_db('partition_network', 'DELETE FROM %s WHERE partition_reference = ? AND computer_reference = ?',
-               [partition['reference'], partition['computer_reference']])
     for address in partition['address_list']:
       address['reference'] = partition['tap']['name']
       address['partition_reference'] = partition['reference']
@@ -358,12 +362,61 @@ def supplySupply():
 @app.route('/requestComputerPartition', methods=['POST'])
 def requestComputerPartition():
   parsed_request_dict = parseRequestComputerPartitionForm(request.form)
-
   # Is it a slave instance?
   slave = loads(request.form.get('shared_xml', EMPTY_DICT_XML).encode('utf-8'))
 
   # Check first if instance is already allocated
   if slave:
+    # slapproxy cannot request frontends, but we can workaround common cases,
+    # so that during tests promises are succesful.
+    if not isRequestToBeForwardedToExternalMaster(parsed_request_dict):
+      # if client request a "simple" frontend for an URL, we can tell this
+      # client to use the URL directly.
+      apache_frontend_sr_url_list = (
+          'http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg',
+      )
+      if parsed_request_dict['software_release'] in apache_frontend_sr_url_list \
+        and parsed_request_dict.get('software_type', '') in ('', 'RootSoftwareInstance', 'default'):
+        url = parsed_request_dict['partition_parameter_kw'].get('url')
+        if url:
+          app.logger.warning("Bypassing frontend for %s => %s", parsed_request_dict, url)
+          partition = ComputerPartition('', 'Fake frontend for {}'.format(url))
+          partition.slap_computer_id = ''
+          partition.slap_computer_partition_id = ''
+          partition._parameter_dict = {}
+          partition._connection_dict = {
+            'secure_access': url,
+            'domain': urlparse(url).netloc,
+          }
+          return dumps(partition)
+      # another similar case is for KVM frontends. This is used in
+      # request-slave-frontend from software/kvm/instance-kvm.cfg.jinja2
+      # requested values by 'return' recipe are: url resource port domainname
+      kvm_frontend_sr_url_list = (
+          'http://git.erp5.org/gitweb/slapos.git/blob_plain/refs/tags/slapos-0.92:/software/kvm/software.cfg',
+      )
+      if parsed_request_dict['software_release'] in kvm_frontend_sr_url_list \
+          and parsed_request_dict.get('software_type') in ('frontend', ):
+        host = parsed_request_dict['partition_parameter_kw'].get('host')
+        port = parsed_request_dict['partition_parameter_kw'].get('port')
+        if host and port:
+          # host is supposed to be ipv6 without brackets.
+          if ':' in host and host[0] != '[':
+            host = '[%s]' % host
+          url = 'https://%s:%s/' % (host, port)
+          app.logger.warning("Bypassing KVM VNC frontend for %s => %s", parsed_request_dict, url)
+          partition = ComputerPartition('', 'Fake KVM VNC frontend for {}'.format(url))
+          partition.slap_computer_id = ''
+          partition.slap_computer_partition_id = ''
+          partition._parameter_dict = {}
+          partition._connection_dict = {
+            'url': url,
+            'domainname': host,
+            'port': port,
+            'path': '/'
+          }
+          return dumps(partition)
+
     # XXX: change schema to include a simple "partition_reference" which
     # is name of the instance. Then, no need to do complex search here.
     slave_reference = parsed_request_dict['partition_id'] + '_' + parsed_request_dict['partition_reference']

slapos/testing/__init__.py

+# -*- coding: utf-8 -*-
+##############################################################################
+#
+# Copyright (c) 2018 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################

slapos/testing/utils.py

+# -*- coding: utf-8 -*-
+##############################################################################
+#
+# Copyright (c) 2018 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import socket
+import hashlib
+from contextlib import closing
+
+
+# Utility functions
+def findFreeTCPPort(ip=''):
+  # type: (str) -> int
+  """Find a free TCP port to listen to.
+  """
+  s = socket.socket(
+      socket.AF_INET6 if ':' in ip else socket.AF_INET, socket.SOCK_STREAM)
+  with closing(s):
+    s.bind((ip, 0))
+    return s.getsockname()[1]
+
+
+def getPortFromPath(path):
+  # type: (str) -> int
+  """A stable port using a hash from path.
+  """
+  return 1024 + int(
+      hashlib.md5(path.encode('utf-8', 'backslashreplace')).hexdigest(),
+      16) % (65535 - 1024)

slapos/tests/test_slapproxy.py

@@ -129,8 +129,8 @@ database_uri = %(tempdir)s/lib/proxy.db
       computer_id = self.computer_id
     computer_dict = {
         'reference': computer_id,
-        'address': '123.456.789',
-        'netmask': 'fffffffff',
+        'address': '12.34.56.78',
+        'netmask': '255.255.255.255',
         'partition_list': [],
     }
     for i in range(partition_amount):
@@ -160,6 +160,47 @@ database_uri = %(tempdir)s/lib/proxy.db
     views.is_schema_already_executed = False
 
 
+class TestLoadComputerConfiguration(BasicMixin, unittest.TestCase):
+  """tests /loadComputerConfigurationFromXML the endpoint for format
+  """
+  def test_loadComputerConfigurationFromXML_remove_partitions(self):
+    computer_dict = {
+        'reference': self.computer_id,
+        'address': '12.34.56.78',
+        'netmask': '255.255.255.255',
+        'partition_list': [
+            {
+                'reference': 'slappart1',
+                'address_list': [
+                    {
+                        'addr': '1.2.3.4',
+                        'netmask': '255.255.255.255'
+                    },
+                ],
+                'tap': {'name': 'tap0'},
+            }
+        ],
+    }
+    rv = self.app.post('/loadComputerConfigurationFromXML', data={
+        'computer_id': self.computer_id,
+        'xml': dumps(computer_dict),
+    })
+    self.assertEqual(rv._status_code, 200)
+    # call again with different partition reference, old partition will be removed
+    # and a new partition will be used.
+    computer_dict['partition_list'][0]['reference'] = 'something else'
+    rv = self.app.post('/loadComputerConfigurationFromXML', data={
+        'computer_id': self.computer_id,
+        'xml': dumps(computer_dict),
+    })
+    self.assertEqual(rv._status_code, 200)
+    computer = loads(
+        self.app.get('/getFullComputerInformation', query_string={'computer_id': self.computer_id}).data)
+    self.assertEqual(
+        ['something else'],
+        [p.getId() for p in computer._computer_partition_list])
+
+
 class TestInformation(BasicMixin, unittest.TestCase):
   """
   Test Basic response of slapproxy
@@ -682,6 +723,44 @@ class TestRequest(MasterMixin):
     partition_new = self.request('http://sr//', None, 'myinstance', 'slappart0')
     self.assertEqual(partition_new.getConnectionParameter('foo'), '1')
 
+  def test_request_frontend(self):
+    # slapproxy tells client to bypass "simple" frontends by just using the URL.
+    request = self.request(
+        'http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg',
+        None,
+        self.id(),
+        'slappart0',
+        shared=True,
+        partition_parameter_kw={'url': 'https://[::1]:123/', })
+    self.assertEqual(
+        'https://[::1]:123/',
+        request.getConnectionParameterDict()['secure_access'])
+    self.assertEqual(
+        '[::1]:123',
+        request.getConnectionParameterDict()['domain'])
+
+  def test_request_kvm_frontend(self):
+    # slapproxy tells client to bypass kvm vnc frontends by building an URL using the backend.
+    request = self.request(
+        'http://git.erp5.org/gitweb/slapos.git/blob_plain/refs/tags/slapos-0.92:/software/kvm/software.cfg',
+        'frontend',
+        self.id(),
+        'slappart0',
+        shared=True,
+        partition_parameter_kw={'host': '::1', 'port': '123'})
+    self.assertEqual(
+        'https://[::1]:123/',
+        request.getConnectionParameterDict()['url'])
+    self.assertEqual(
+        '[::1]',
+        request.getConnectionParameterDict()['domainname'])
+    self.assertEqual(
+        '123',
+        request.getConnectionParameterDict()['port'])
+    self.assertEqual(
+        '/',
+        request.getConnectionParameterDict()['path'])
+
 
 class TestSlaveRequest(MasterMixin):
   """
@@ -1221,8 +1300,8 @@ database_uri = %(tempdir)s/lib/external_proxy.db
       computer_id = self.external_computer_id
     computer_dict = {
         'reference': computer_id,
-        'address': '123.456.789',
-        'netmask': 'fffffffff',
+        'address': '12.34.56.78',
+        'netmask': '255.255.255.255',
         'partition_list': [],
     }
     for i in range(partition_amount):