Proxy: Forward requests as partition and use it in standalone

The idea is to make it possible while embedding SlapOS in an existing partition to configure that some requests are forwarded to the master of the partition and to make these requests as the partition - so that when this partition is destroyed all other partitions from the embedded partitions are destroyed.

A bit more integration in slapos is still needed, but is intended to be used in theia (and why not webrunner):

At the beginning theia's instance  looks like this:

![screenshot of theia hosting subscription on slapos master, with the default 3 partitions](/uploads/9a297dbbae49996fb7ee69e89ea75da1/image.png)

after an ERP5 instance is requested in theia's embedded slapos, the partition requests are forwarded as this partition and appear here, as requested by theia:

![screenshot of theia hosting subscription on slapos master, frontend partitions requested by ERP5 appears as requested by theia](/uploads/37b8b30ca117a1251fc2bd099a5dac82/image.png)

This can also allow transparent integration, because the partition credentials are known by the partition, unlike the user credentials.

See merge request !218

slapos/proxy/views.py

@@ -563,21 +563,47 @@ def forwardRequestToExternalMaster(master_url, request_form):
     slap.initializeConnection(master_url)
 
   partition_reference = unicode2str(request_form['partition_reference'])
+  filter_kw = loads(request_form['filter_xml'].encode('utf-8'))
+  partition_parameter_kw = loads(request_form['partition_parameter_xml'].encode('utf-8'))
+
+  app.logger.info("Forwarding request of %s to %s", partition_reference, master_url)
+  app.logger.debug("request_form: %s", request_form)
+
   # Store in database
   execute_db('forwarded_partition_request', 'INSERT OR REPLACE INTO %s values(:partition_reference, :master_url)',
              {'partition_reference':partition_reference, 'master_url': master_url})
 
-  new_request_form = request_form.copy()
-  filter_kw = loads(new_request_form['filter_xml'].encode('utf-8'))
+  if master_entry.get('computer') and master_entry.get('partition'):
+    app.logger.debug("requesting from partition %s", master_entry)
+    # XXX ComputerPartition.request and OpenOrder.request have different signatures
+    partition = slap.registerComputerPartition(
+        master_entry['computer'],
+        master_entry['partition'],
+    ).request(
+        software_release=request_form['software_release'],
+        software_type=request_form.get('software_type', ''),
+        partition_reference=partition_reference,
+        shared=loads(request_form['shared_xml'].encode('utf-8')),
+        partition_parameter_kw=partition_parameter_kw,
+        filter_kw=filter_kw,
+        state=loads(request_form['state'].encode('utf-8')),
+    )
+  else:
     filter_kw['source_instance_id'] = partition_reference
-  new_request_form['filter_xml'] = dumps(filter_kw)
-
-  xml = slap._connection_helper.POST('/requestComputerPartition', data=new_request_form)
-  partition = loads(xml)
+    partition = slap.registerOpenOrder().request(
+        software_release=request_form['software_release'],
+        partition_reference=partition_reference,
+        partition_parameter_kw=partition_parameter_kw,
+        software_type=request_form.get('software_type', ''),
+        filter_kw=filter_kw,
+        state=loads(request_form['state'].encode('utf-8')),
+        shared=loads(request_form['shared_xml'].encode('utf-8')),
+    )
 
   # XXX move to other end
-  partition._master_url = master_url
-
+  partition._master_url = master_url # type: ignore
+  partition._connection_helper = None
+  partition._software_release_document = request_form['software_release'] # type: ignore
   return dumps(partition)
 
 def getAllocatedInstance(partition_reference):

slapos/slap/standalone.py

@@ -34,6 +34,7 @@ import time
 import errno
 import socket
 import shutil
+import collections
 
 from six.moves import urllib
 from six.moves import http_client
@@ -43,6 +44,13 @@ try:
 except ImportError:
   import subprocess
 
+try:
+  from typing import TYPE_CHECKING, Optional, Iterable, Dict, Union
+  if TYPE_CHECKING:
+    import subprocess
+except ImportError: # XXX to be removed once we depend on typing
+  pass
+
 import xml_marshaller
 import zope.interface
 import psutil
@@ -151,10 +159,27 @@ class SupervisorConfigWriter(ConfigWriter):
 class SlapOSConfigWriter(ConfigWriter):
   """Write slapos configuration at etc/slapos.cfg
   """
+  def _getPartitionForwardConfiguration(self):
+    # type: () -> Iterable[str]
+    for pfc in self._standalone_slapos._partition_forward_configuration:
+      software_release_list = '\n  '.join(pfc.software_release_list)
+      config = '[multimaster/{pfc.master_url}]\n'.format(pfc=pfc)
+      if pfc.cert:
+        config += 'cert = {pfc.cert}\n'.format(pfc=pfc)
+      if pfc.key:
+        config += 'key = {pfc.key}\n'.format(pfc=pfc)
+      config += 'software_release_list =\n  {}\n'.format('\n  '.join(pfc.software_release_list))
+      if isinstance(pfc, PartitionForwardAsPartitionConfiguration):
+        config += "computer = {pfc.computer}\n".format(pfc=pfc)
+        config += "partition = {pfc.partition}\n".format(pfc=pfc)
+      yield config
+
   def writeConfig(self, path):
-    standalone_slapos = self._standalone_slapos  # type: StandaloneSlapOS
+    # type: (str) -> None
+    standalone_slapos = self._standalone_slapos
     read_only_shared_part_list = '\n  '.join( #  pylint: disable=unused-variable; used in format()
         standalone_slapos._shared_part_list)
+    partition_forward_configuration = '\n'.join(self._getPartitionForwardConfiguration())
     with open(path, 'w') as f:
       f.write(
           textwrap.dedent(
@@ -176,6 +201,8 @@ class SlapOSConfigWriter(ConfigWriter):
               host = {standalone_slapos._server_ip}
               port = {standalone_slapos._server_port}
               database_uri = {standalone_slapos._proxy_database}
+
+              {partition_forward_configuration}
               """).format(**locals()))
 
 
@@ -195,6 +222,46 @@ class SlapOSCommandWriter(ConfigWriter):
     os.chmod(path, 0o755)
 
 
+class PartitionForwardConfiguration(object):
+  """Specification of request forwarding to another master, requested as user.
+  """
+  def __init__(
+      self,
+      master_url,
+      cert=None,
+      key=None,
+      software_release_list=(),
+  ):
+    # type: (str, Optional[str], Optional[str], Iterable[str]) -> None
+    self.master_url = master_url
+    self.cert = cert
+    self.key = key
+    self.software_release_list = list(software_release_list)
+
+
+class PartitionForwardAsPartitionConfiguration(PartitionForwardConfiguration):
+  """Specification of request forwarding to another master, requested as partition.
+  """
+  def __init__(
+      self,
+      master_url,
+      computer,
+      partition,
+      cert=None,
+      key=None,
+      software_release_list=(),
+  ):
+    # type: (str, str, str, Optional[str], Optional[str], Iterable[str]) -> None
+    super(PartitionForwardAsPartitionConfiguration, self).__init__(
+        master_url,
+        cert,
+        key,
+        software_release_list,
+    )
+    self.computer = computer
+    self.partition = partition
+
+
 @zope.interface.implementer(ISupply, IRequester)
 class StandaloneSlapOS(object):
   """A SlapOS that can be embedded in other applications, also useful for testing.
@@ -215,7 +282,10 @@ class StandaloneSlapOS(object):
       shared_part_list=(),
       software_root=None,
       instance_root=None,
-      shared_part_root=None):
+      shared_part_root=None,
+      partition_forward_configuration=(),
+    ):
+    # type: (str, str, int, str, Iterable[str], Optional[str], Optional[str], Optional[str], Iterable[Union[PartitionForwardConfiguration, PartitionForwardAsPartitionConfiguration]]) -> None
     """Constructor, creates a standalone slapos in `base_directory`.
 
     Arguments:
@@ -226,6 +296,7 @@ class StandaloneSlapOS(object):
       * `software_root` -- directory to install software, default to "soft" in `base_directory`
       * `instance_root` -- directory to create instances, default to "inst" in `base_directory`
       * `shared_part_root` -- directory to hold shared parts software, default to "shared" in `base_directory`.
+      * `partition_forward_configuration` -- configuration of partition request forwarding to external SlapOS master.
 
     Error cases:
       * `PathTooDeepError` when `base_directory` is too deep. Because of limitation
@@ -241,6 +312,7 @@ class StandaloneSlapOS(object):
 
     self._base_directory = base_directory
     self._shared_part_list = list(shared_part_list)
+    self._partition_forward_configuration = list(partition_forward_configuration)
 
     self._slapos_commands = {
         'slapos-node-software': {

slapos/tests/test_slapproxy.py

@@ -78,6 +78,7 @@ class BasicMixin(object):
     logging.basicConfig(level=logging.DEBUG)
     self.setFiles()
     self.startProxy()
+    os.environ.pop('SLAPGRID_INSTANCE_ROOT', None)
 
   def createSlapOSConfigurationFile(self):
     with open(self.slapos_cfg, 'w') as f:
@@ -1455,6 +1456,25 @@ database_uri = %(tempdir)s/lib/external_proxy.db
     self.external_proxy_slap._connection_helper.POST('/loadComputerConfigurationFromXML',
                                                      data=request_dict)
 
+  def external_proxy_create_requested_partition(self):
+    # type: () -> None
+    """Create an already requested partition as slappart0, so that we can
+    request from this partition.
+    """
+    external_slap = slapos.slap.slap()
+    external_slap.initializeConnection(self.external_master_url)
+    external_slap.registerSupply().supply(
+        'https://example.com/dummy/software.cfg',
+         computer_guid=self.external_computer_id,
+    )
+    partition = external_slap.registerOpenOrder().request(
+        'https://example.com/dummy/software.cfg',
+        'instance',
+    )
+    # XXX this has to match what is set in slapos_multimaster.cfg.in
+    self.assertEqual('external_computer', partition.slap_computer_id)
+    self.assertEqual('slappart0', partition.slap_computer_partition_id)
+
   def _checkInstanceIsFowarded(self, name, partition_parameter_kw, software_release):
     """
     Test there is no instance on local proxy.
@@ -1574,6 +1594,51 @@ database_uri = %(tempdir)s/lib/external_proxy.db
 
     self._checkInstanceIsFowarded(instance_reference, dummy_parameter_dict, self.external_software_release)
 
+    instance_parameter_dict = partition.getInstanceParameterDict()
+    instance_parameter_dict.pop('timestamp')
+    self.assertEqual(dummy_parameter_dict, instance_parameter_dict)
+    self.assertEqual(self.external_software_release, partition.getSoftwareRelease())
+    self.assertEqual({}, partition.getConnectionParameterDict())
+
+  def testForwardRequestFromPartition(self):
+    """
+    Test that instance request is forwarded and requested from computer partition.
+    """
+    dummy_parameter_dict = {'foo': 'bar'}
+    instance_reference = 'MyFirstInstance'
+    self.format_for_number_of_partitions(1)
+    self.external_proxy_format_for_number_of_partitions(2)
+    self.external_proxy_create_requested_partition()
+
+    partition = self.request(
+        'https://example.com/request/from/partition/software.cfg',
+        None,
+        instance_reference,
+        'slappart0',
+        partition_parameter_kw=dummy_parameter_dict,
+    )
+
+    instance_parameter_dict = partition.getInstanceParameterDict()
+    instance_parameter_dict.pop('timestamp')
+    self.assertEqual(dummy_parameter_dict, instance_parameter_dict)
+    self.assertEqual('https://example.com/request/from/partition/software.cfg', partition.getSoftwareRelease())
+    self.assertEqual({}, partition.getConnectionParameterDict())
+
+    with sqlite3.connect(os.path.join(
+        self._tempdir,
+        'lib',
+        'external_proxy.db',
+    )) as db:
+      requested_by = slapos.proxy.views.execute_db(
+          "partition", "select reference, requested_by from %s", db=db)
+    self.assertEqual([{
+        'reference': 'slappart0',
+        'requested_by': None
+    }, {
+        'reference': 'slappart1',
+        'requested_by': 'slappart0'
+    }], requested_by)
+
   def testRequestToCurrentMaster(self):
     """
     Explicitely ask deployment of an instance to current master

slapos/tests/test_slapproxy/slapos_multimaster.cfg.in

@@ -27,3 +27,19 @@ software_release_list =
 software_release_list =
   http://mywebsite.me/exteral_software_release.cfg
 
+# Request as a computer partition, so that requested partitions are linked
+# to the partition requesting them.
+[multimaster/https://slap.example.com]
+key = /path/to/cert.key
+cert = /path/to/cert.cert
+computer = COMP-12345
+partition = slappart1
+software_release_list =
+  https://example.com/software.cfg
+
+[multimaster/http://%(external_proxy_host)s:%(external_proxy_port)s/]
+# No certificate here: it is http.
+computer = external_computer
+partition = slappart0
+software_release_list =
+  https://example.com/request/from/partition/software.cfg

slapos/tests/test_standalone.py

@@ -37,11 +37,14 @@ import errno
 import time
 import multiprocessing
 from contextlib import closing
+from six.moves.configparser import ConfigParser
 
 import psutil
 
 from slapos.slap.standalone import StandaloneSlapOS
 from slapos.slap.standalone import SlapOSNodeCommandError
+from slapos.slap.standalone import PartitionForwardConfiguration
+from slapos.slap.standalone import PartitionForwardAsPartitionConfiguration
 
 SLAPOS_TEST_IPV4 = os.environ['SLAPOS_TEST_IPV4']
 SLAPOS_TEST_IPV6 = os.environ['SLAPOS_TEST_IPV6']
@@ -165,6 +168,89 @@ class TestSlapOSStandaloneSetup(unittest.TestCase):
     with self.assertRaises(BaseException):
       standalone1.stop()
 
+  def test_partition_forward(self):
+    # type: () -> None
+    working_dir = tempfile.mkdtemp(prefix=__name__)
+    self.addCleanup(shutil.rmtree, working_dir)
+    partition_forward_config = [
+        PartitionForwardConfiguration(
+            'https://slapos1.example.com',
+            'path/to/cert',
+            'path/to/key',
+            software_release_list=('https://example.com/software-1.cfg', ),
+        ),
+        PartitionForwardConfiguration(
+            'https://slapos2.example.com',
+            software_release_list=('https://example.com/software-2.cfg', ),
+        ),
+        PartitionForwardAsPartitionConfiguration(
+            'https://slapos3.example.com',
+            'computer',
+            'partition',
+            'path/to/cert',
+            'path/to/key',
+            software_release_list=('https://example.com/software-3.cfg', ),
+        ),
+        PartitionForwardAsPartitionConfiguration(
+            'https://slapos4.example.com',
+            'computer',
+            'partition',
+            software_release_list=('https://example.com/software-4.cfg', ),
+        ),
+    ]
+    standalone = StandaloneSlapOS(
+        working_dir,
+        SLAPOS_TEST_IPV4,
+        SLAPOS_TEST_PORT,
+        partition_forward_configuration=partition_forward_config,
+    )
+    self.addCleanup(standalone.stop)
+
+    config_parser = ConfigParser()
+    config_parser.read([os.path.join(working_dir, 'etc', 'slapos.cfg')])
+    self.assertTrue(
+        config_parser.has_section('multimaster/https://slapos1.example.com'))
+    self.assertEqual(
+        'path/to/cert',
+        config_parser.get('multimaster/https://slapos1.example.com', 'cert'))
+    self.assertEqual(
+        'path/to/key',
+        config_parser.get('multimaster/https://slapos1.example.com', 'key'))
+    self.assertEqual(
+        'https://example.com/software-1.cfg',
+        config_parser.get(
+            'multimaster/https://slapos1.example.com',
+            'software_release_list').strip())
+    self.assertFalse(
+        config_parser.has_option(
+            'multimaster/https://slapos2.example.com', 'computer'))
+    self.assertFalse(
+        config_parser.has_option(
+            'multimaster/https://slapos2.example.com', 'partition'))
+
+    self.assertTrue(
+        config_parser.has_section('multimaster/https://slapos2.example.com'))
+    self.assertFalse(
+        config_parser.has_option(
+            'multimaster/https://slapos2.example.com', 'cert'))
+    self.assertFalse(
+        config_parser.has_option(
+            'multimaster/https://slapos2.example.com', 'key'))
+
+    self.assertTrue(
+        config_parser.has_section('multimaster/https://slapos3.example.com'))
+    self.assertEqual(
+        'computer',
+        config_parser.get(
+            'multimaster/https://slapos3.example.com', 'computer'))
+    self.assertEqual(
+        'partition',
+        config_parser.get(
+            'multimaster/https://slapos3.example.com', 'partition'))
+
+    self.assertTrue(
+        config_parser.has_section('multimaster/https://slapos4.example.com'))
+
 
 class SlapOSStandaloneTestCase(unittest.TestCase):
   # This test case takes care of stopping the standalone instance