Commit e0b4339e authored by Jérome Perrin's avatar Jérome Perrin

Proxy: Forward requests as partition and use it in standalone

The idea is to make it possible while embedding SlapOS in an existing partition to configure that some requests are forwarded to the master of the partition and to make these requests as the partition - so that when this partition is destroyed all other partitions from the embedded partitions are destroyed.

A bit more integration in slapos is still needed, but is intended to be used in theia (and why not webrunner):

At the beginning theia's instance  looks like this:

![screenshot of theia hosting subscription on slapos master, with the default 3 partitions](/uploads/9a297dbbae49996fb7ee69e89ea75da1/image.png)

after an ERP5 instance is requested in theia's embedded slapos, the partition requests are forwarded as this partition and appear here, as requested by theia:

![screenshot of theia hosting subscription on slapos master, frontend partitions requested by ERP5 appears as requested by theia](/uploads/37b8b30ca117a1251fc2bd099a5dac82/image.png)

This can also allow transparent integration, because the partition credentials are known by the partition, unlike the user credentials.

See merge request !218
parents bbde89e3 4825a8c4
Pipeline #9541 passed with stage
...@@ -563,21 +563,47 @@ def forwardRequestToExternalMaster(master_url, request_form): ...@@ -563,21 +563,47 @@ def forwardRequestToExternalMaster(master_url, request_form):
slap.initializeConnection(master_url) slap.initializeConnection(master_url)
partition_reference = unicode2str(request_form['partition_reference']) partition_reference = unicode2str(request_form['partition_reference'])
filter_kw = loads(request_form['filter_xml'].encode('utf-8'))
partition_parameter_kw = loads(request_form['partition_parameter_xml'].encode('utf-8'))
app.logger.info("Forwarding request of %s to %s", partition_reference, master_url)
app.logger.debug("request_form: %s", request_form)
# Store in database # Store in database
execute_db('forwarded_partition_request', 'INSERT OR REPLACE INTO %s values(:partition_reference, :master_url)', execute_db('forwarded_partition_request', 'INSERT OR REPLACE INTO %s values(:partition_reference, :master_url)',
{'partition_reference':partition_reference, 'master_url': master_url}) {'partition_reference':partition_reference, 'master_url': master_url})
new_request_form = request_form.copy() if master_entry.get('computer') and master_entry.get('partition'):
filter_kw = loads(new_request_form['filter_xml'].encode('utf-8')) app.logger.debug("requesting from partition %s", master_entry)
filter_kw['source_instance_id'] = partition_reference # XXX ComputerPartition.request and OpenOrder.request have different signatures
new_request_form['filter_xml'] = dumps(filter_kw) partition = slap.registerComputerPartition(
master_entry['computer'],
xml = slap._connection_helper.POST('/requestComputerPartition', data=new_request_form) master_entry['partition'],
partition = loads(xml) ).request(
software_release=request_form['software_release'],
software_type=request_form.get('software_type', ''),
partition_reference=partition_reference,
shared=loads(request_form['shared_xml'].encode('utf-8')),
partition_parameter_kw=partition_parameter_kw,
filter_kw=filter_kw,
state=loads(request_form['state'].encode('utf-8')),
)
else:
filter_kw['source_instance_id'] = partition_reference
partition = slap.registerOpenOrder().request(
software_release=request_form['software_release'],
partition_reference=partition_reference,
partition_parameter_kw=partition_parameter_kw,
software_type=request_form.get('software_type', ''),
filter_kw=filter_kw,
state=loads(request_form['state'].encode('utf-8')),
shared=loads(request_form['shared_xml'].encode('utf-8')),
)
# XXX move to other end # XXX move to other end
partition._master_url = master_url partition._master_url = master_url # type: ignore
partition._connection_helper = None
partition._software_release_document = request_form['software_release'] # type: ignore
return dumps(partition) return dumps(partition)
def getAllocatedInstance(partition_reference): def getAllocatedInstance(partition_reference):
......
...@@ -34,6 +34,7 @@ import time ...@@ -34,6 +34,7 @@ import time
import errno import errno
import socket import socket
import shutil import shutil
import collections
from six.moves import urllib from six.moves import urllib
from six.moves import http_client from six.moves import http_client
...@@ -43,6 +44,13 @@ try: ...@@ -43,6 +44,13 @@ try:
except ImportError: except ImportError:
import subprocess import subprocess
try:
from typing import TYPE_CHECKING, Optional, Iterable, Dict, Union
if TYPE_CHECKING:
import subprocess
except ImportError: # XXX to be removed once we depend on typing
pass
import xml_marshaller import xml_marshaller
import zope.interface import zope.interface
import psutil import psutil
...@@ -151,10 +159,27 @@ class SupervisorConfigWriter(ConfigWriter): ...@@ -151,10 +159,27 @@ class SupervisorConfigWriter(ConfigWriter):
class SlapOSConfigWriter(ConfigWriter): class SlapOSConfigWriter(ConfigWriter):
"""Write slapos configuration at etc/slapos.cfg """Write slapos configuration at etc/slapos.cfg
""" """
def _getPartitionForwardConfiguration(self):
# type: () -> Iterable[str]
for pfc in self._standalone_slapos._partition_forward_configuration:
software_release_list = '\n '.join(pfc.software_release_list)
config = '[multimaster/{pfc.master_url}]\n'.format(pfc=pfc)
if pfc.cert:
config += 'cert = {pfc.cert}\n'.format(pfc=pfc)
if pfc.key:
config += 'key = {pfc.key}\n'.format(pfc=pfc)
config += 'software_release_list =\n {}\n'.format('\n '.join(pfc.software_release_list))
if isinstance(pfc, PartitionForwardAsPartitionConfiguration):
config += "computer = {pfc.computer}\n".format(pfc=pfc)
config += "partition = {pfc.partition}\n".format(pfc=pfc)
yield config
def writeConfig(self, path): def writeConfig(self, path):
standalone_slapos = self._standalone_slapos # type: StandaloneSlapOS # type: (str) -> None
standalone_slapos = self._standalone_slapos
read_only_shared_part_list = '\n '.join( # pylint: disable=unused-variable; used in format() read_only_shared_part_list = '\n '.join( # pylint: disable=unused-variable; used in format()
standalone_slapos._shared_part_list) standalone_slapos._shared_part_list)
partition_forward_configuration = '\n'.join(self._getPartitionForwardConfiguration())
with open(path, 'w') as f: with open(path, 'w') as f:
f.write( f.write(
textwrap.dedent( textwrap.dedent(
...@@ -176,6 +201,8 @@ class SlapOSConfigWriter(ConfigWriter): ...@@ -176,6 +201,8 @@ class SlapOSConfigWriter(ConfigWriter):
host = {standalone_slapos._server_ip} host = {standalone_slapos._server_ip}
port = {standalone_slapos._server_port} port = {standalone_slapos._server_port}
database_uri = {standalone_slapos._proxy_database} database_uri = {standalone_slapos._proxy_database}
{partition_forward_configuration}
""").format(**locals())) """).format(**locals()))
...@@ -195,6 +222,46 @@ class SlapOSCommandWriter(ConfigWriter): ...@@ -195,6 +222,46 @@ class SlapOSCommandWriter(ConfigWriter):
os.chmod(path, 0o755) os.chmod(path, 0o755)
class PartitionForwardConfiguration(object):
"""Specification of request forwarding to another master, requested as user.
"""
def __init__(
self,
master_url,
cert=None,
key=None,
software_release_list=(),
):
# type: (str, Optional[str], Optional[str], Iterable[str]) -> None
self.master_url = master_url
self.cert = cert
self.key = key
self.software_release_list = list(software_release_list)
class PartitionForwardAsPartitionConfiguration(PartitionForwardConfiguration):
"""Specification of request forwarding to another master, requested as partition.
"""
def __init__(
self,
master_url,
computer,
partition,
cert=None,
key=None,
software_release_list=(),
):
# type: (str, str, str, Optional[str], Optional[str], Iterable[str]) -> None
super(PartitionForwardAsPartitionConfiguration, self).__init__(
master_url,
cert,
key,
software_release_list,
)
self.computer = computer
self.partition = partition
@zope.interface.implementer(ISupply, IRequester) @zope.interface.implementer(ISupply, IRequester)
class StandaloneSlapOS(object): class StandaloneSlapOS(object):
"""A SlapOS that can be embedded in other applications, also useful for testing. """A SlapOS that can be embedded in other applications, also useful for testing.
...@@ -215,7 +282,10 @@ class StandaloneSlapOS(object): ...@@ -215,7 +282,10 @@ class StandaloneSlapOS(object):
shared_part_list=(), shared_part_list=(),
software_root=None, software_root=None,
instance_root=None, instance_root=None,
shared_part_root=None): shared_part_root=None,
partition_forward_configuration=(),
):
# type: (str, str, int, str, Iterable[str], Optional[str], Optional[str], Optional[str], Iterable[Union[PartitionForwardConfiguration, PartitionForwardAsPartitionConfiguration]]) -> None
"""Constructor, creates a standalone slapos in `base_directory`. """Constructor, creates a standalone slapos in `base_directory`.
Arguments: Arguments:
...@@ -226,6 +296,7 @@ class StandaloneSlapOS(object): ...@@ -226,6 +296,7 @@ class StandaloneSlapOS(object):
* `software_root` -- directory to install software, default to "soft" in `base_directory` * `software_root` -- directory to install software, default to "soft" in `base_directory`
* `instance_root` -- directory to create instances, default to "inst" in `base_directory` * `instance_root` -- directory to create instances, default to "inst" in `base_directory`
* `shared_part_root` -- directory to hold shared parts software, default to "shared" in `base_directory`. * `shared_part_root` -- directory to hold shared parts software, default to "shared" in `base_directory`.
* `partition_forward_configuration` -- configuration of partition request forwarding to external SlapOS master.
Error cases: Error cases:
* `PathTooDeepError` when `base_directory` is too deep. Because of limitation * `PathTooDeepError` when `base_directory` is too deep. Because of limitation
...@@ -241,6 +312,7 @@ class StandaloneSlapOS(object): ...@@ -241,6 +312,7 @@ class StandaloneSlapOS(object):
self._base_directory = base_directory self._base_directory = base_directory
self._shared_part_list = list(shared_part_list) self._shared_part_list = list(shared_part_list)
self._partition_forward_configuration = list(partition_forward_configuration)
self._slapos_commands = { self._slapos_commands = {
'slapos-node-software': { 'slapos-node-software': {
......
...@@ -78,6 +78,7 @@ class BasicMixin(object): ...@@ -78,6 +78,7 @@ class BasicMixin(object):
logging.basicConfig(level=logging.DEBUG) logging.basicConfig(level=logging.DEBUG)
self.setFiles() self.setFiles()
self.startProxy() self.startProxy()
os.environ.pop('SLAPGRID_INSTANCE_ROOT', None)
def createSlapOSConfigurationFile(self): def createSlapOSConfigurationFile(self):
with open(self.slapos_cfg, 'w') as f: with open(self.slapos_cfg, 'w') as f:
...@@ -1455,6 +1456,25 @@ database_uri = %(tempdir)s/lib/external_proxy.db ...@@ -1455,6 +1456,25 @@ database_uri = %(tempdir)s/lib/external_proxy.db
self.external_proxy_slap._connection_helper.POST('/loadComputerConfigurationFromXML', self.external_proxy_slap._connection_helper.POST('/loadComputerConfigurationFromXML',
data=request_dict) data=request_dict)
def external_proxy_create_requested_partition(self):
# type: () -> None
"""Create an already requested partition as slappart0, so that we can
request from this partition.
"""
external_slap = slapos.slap.slap()
external_slap.initializeConnection(self.external_master_url)
external_slap.registerSupply().supply(
'https://example.com/dummy/software.cfg',
computer_guid=self.external_computer_id,
)
partition = external_slap.registerOpenOrder().request(
'https://example.com/dummy/software.cfg',
'instance',
)
# XXX this has to match what is set in slapos_multimaster.cfg.in
self.assertEqual('external_computer', partition.slap_computer_id)
self.assertEqual('slappart0', partition.slap_computer_partition_id)
def _checkInstanceIsFowarded(self, name, partition_parameter_kw, software_release): def _checkInstanceIsFowarded(self, name, partition_parameter_kw, software_release):
""" """
Test there is no instance on local proxy. Test there is no instance on local proxy.
...@@ -1574,6 +1594,51 @@ database_uri = %(tempdir)s/lib/external_proxy.db ...@@ -1574,6 +1594,51 @@ database_uri = %(tempdir)s/lib/external_proxy.db
self._checkInstanceIsFowarded(instance_reference, dummy_parameter_dict, self.external_software_release) self._checkInstanceIsFowarded(instance_reference, dummy_parameter_dict, self.external_software_release)
instance_parameter_dict = partition.getInstanceParameterDict()
instance_parameter_dict.pop('timestamp')
self.assertEqual(dummy_parameter_dict, instance_parameter_dict)
self.assertEqual(self.external_software_release, partition.getSoftwareRelease())
self.assertEqual({}, partition.getConnectionParameterDict())
def testForwardRequestFromPartition(self):
"""
Test that instance request is forwarded and requested from computer partition.
"""
dummy_parameter_dict = {'foo': 'bar'}
instance_reference = 'MyFirstInstance'
self.format_for_number_of_partitions(1)
self.external_proxy_format_for_number_of_partitions(2)
self.external_proxy_create_requested_partition()
partition = self.request(
'https://example.com/request/from/partition/software.cfg',
None,
instance_reference,
'slappart0',
partition_parameter_kw=dummy_parameter_dict,
)
instance_parameter_dict = partition.getInstanceParameterDict()
instance_parameter_dict.pop('timestamp')
self.assertEqual(dummy_parameter_dict, instance_parameter_dict)
self.assertEqual('https://example.com/request/from/partition/software.cfg', partition.getSoftwareRelease())
self.assertEqual({}, partition.getConnectionParameterDict())
with sqlite3.connect(os.path.join(
self._tempdir,
'lib',
'external_proxy.db',
)) as db:
requested_by = slapos.proxy.views.execute_db(
"partition", "select reference, requested_by from %s", db=db)
self.assertEqual([{
'reference': 'slappart0',
'requested_by': None
}, {
'reference': 'slappart1',
'requested_by': 'slappart0'
}], requested_by)
def testRequestToCurrentMaster(self): def testRequestToCurrentMaster(self):
""" """
Explicitely ask deployment of an instance to current master Explicitely ask deployment of an instance to current master
......
...@@ -27,3 +27,19 @@ software_release_list = ...@@ -27,3 +27,19 @@ software_release_list =
software_release_list = software_release_list =
http://mywebsite.me/exteral_software_release.cfg http://mywebsite.me/exteral_software_release.cfg
# Request as a computer partition, so that requested partitions are linked
# to the partition requesting them.
[multimaster/https://slap.example.com]
key = /path/to/cert.key
cert = /path/to/cert.cert
computer = COMP-12345
partition = slappart1
software_release_list =
https://example.com/software.cfg
[multimaster/http://%(external_proxy_host)s:%(external_proxy_port)s/]
# No certificate here: it is http.
computer = external_computer
partition = slappart0
software_release_list =
https://example.com/request/from/partition/software.cfg
...@@ -37,11 +37,14 @@ import errno ...@@ -37,11 +37,14 @@ import errno
import time import time
import multiprocessing import multiprocessing
from contextlib import closing from contextlib import closing
from six.moves.configparser import ConfigParser
import psutil import psutil
from slapos.slap.standalone import StandaloneSlapOS from slapos.slap.standalone import StandaloneSlapOS
from slapos.slap.standalone import SlapOSNodeCommandError from slapos.slap.standalone import SlapOSNodeCommandError
from slapos.slap.standalone import PartitionForwardConfiguration
from slapos.slap.standalone import PartitionForwardAsPartitionConfiguration
SLAPOS_TEST_IPV4 = os.environ['SLAPOS_TEST_IPV4'] SLAPOS_TEST_IPV4 = os.environ['SLAPOS_TEST_IPV4']
SLAPOS_TEST_IPV6 = os.environ['SLAPOS_TEST_IPV6'] SLAPOS_TEST_IPV6 = os.environ['SLAPOS_TEST_IPV6']
...@@ -165,6 +168,89 @@ class TestSlapOSStandaloneSetup(unittest.TestCase): ...@@ -165,6 +168,89 @@ class TestSlapOSStandaloneSetup(unittest.TestCase):
with self.assertRaises(BaseException): with self.assertRaises(BaseException):
standalone1.stop() standalone1.stop()
def test_partition_forward(self):
# type: () -> None
working_dir = tempfile.mkdtemp(prefix=__name__)
self.addCleanup(shutil.rmtree, working_dir)
partition_forward_config = [
PartitionForwardConfiguration(
'https://slapos1.example.com',
'path/to/cert',
'path/to/key',
software_release_list=('https://example.com/software-1.cfg', ),
),
PartitionForwardConfiguration(
'https://slapos2.example.com',
software_release_list=('https://example.com/software-2.cfg', ),
),
PartitionForwardAsPartitionConfiguration(
'https://slapos3.example.com',
'computer',
'partition',
'path/to/cert',
'path/to/key',
software_release_list=('https://example.com/software-3.cfg', ),
),
PartitionForwardAsPartitionConfiguration(
'https://slapos4.example.com',
'computer',
'partition',
software_release_list=('https://example.com/software-4.cfg', ),
),
]
standalone = StandaloneSlapOS(
working_dir,
SLAPOS_TEST_IPV4,
SLAPOS_TEST_PORT,
partition_forward_configuration=partition_forward_config,
)
self.addCleanup(standalone.stop)
config_parser = ConfigParser()
config_parser.read([os.path.join(working_dir, 'etc', 'slapos.cfg')])
self.assertTrue(
config_parser.has_section('multimaster/https://slapos1.example.com'))
self.assertEqual(
'path/to/cert',
config_parser.get('multimaster/https://slapos1.example.com', 'cert'))
self.assertEqual(
'path/to/key',
config_parser.get('multimaster/https://slapos1.example.com', 'key'))
self.assertEqual(
'https://example.com/software-1.cfg',
config_parser.get(
'multimaster/https://slapos1.example.com',
'software_release_list').strip())
self.assertFalse(
config_parser.has_option(
'multimaster/https://slapos2.example.com', 'computer'))
self.assertFalse(
config_parser.has_option(
'multimaster/https://slapos2.example.com', 'partition'))
self.assertTrue(
config_parser.has_section('multimaster/https://slapos2.example.com'))
self.assertFalse(
config_parser.has_option(
'multimaster/https://slapos2.example.com', 'cert'))
self.assertFalse(
config_parser.has_option(
'multimaster/https://slapos2.example.com', 'key'))
self.assertTrue(
config_parser.has_section('multimaster/https://slapos3.example.com'))
self.assertEqual(
'computer',
config_parser.get(
'multimaster/https://slapos3.example.com', 'computer'))
self.assertEqual(
'partition',
config_parser.get(
'multimaster/https://slapos3.example.com', 'partition'))
self.assertTrue(
config_parser.has_section('multimaster/https://slapos4.example.com'))
class SlapOSStandaloneTestCase(unittest.TestCase): class SlapOSStandaloneTestCase(unittest.TestCase):
# This test case takes care of stopping the standalone instance # This test case takes care of stopping the standalone instance
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment