grid: new shared_part_list config file option

This new option for slapos.cfg allow to define a list of paths to use
for shared parts feature of `slapos.recipe.cmmi`.
This will be injected in the buildout profile used to install software,
as `${buildout:shared-part-list}` which is the option `slapos.recipe.cmmi`
use since 0.11.

Note that the option changed in `slapos.recipe.cmmi` because this was
incompatible. In 0.10, the option was named `${buildout:shared-parts}` and
it was supposed to be a single path. `slapos.recipe.cmmi` 0.10 was trying
to create this path. If we continued using this `${buildout:shared-parts}`
option and changed to passe a newline-separated list of paths, then
`slapos.recipe.cmmi` would just create one directory with \n in the name
and will probably fail, which would make all software from 1.0.73 to 1.0.121
not installable on next `slapos.core`, so we used another buildout option
to prevent this problem.

Because generally, profiles using slapos.recipe.cmmi 0.10 were not
"safely" shared, we don't even provide compatibility with old option.

/reviewed-on nexedi/slapos.core!46

slapos.cfg.example

@@ -8,6 +8,12 @@ cert_file =  /etc/opt/slapos/ssl/computer.crt
 certificate_repository_path = /etc/opt/slapos/ssl/partition_pki
 software_root = /opt/slapgrid
 instance_root = /srv/slapgrid
+# A list of directories containing shared parts for slapos.recipe.cmmi.
+# Only the last one is used to install new parts, other ones can have
+# already installed parts.
+shared_part_list =
+  /opt/slapgrid/read_only_shared
+  /opt/slapgrid/shared
 
 [slapformat]
 # Replace by your network interface providing IPv6 if you don't use re6st

slapos/grid/SlapObject.py

@@ -39,6 +39,7 @@ import tarfile
 import tempfile
 import time
 from six.moves import xmlrpc_client as xmlrpclib, range
+from six.moves.configparser import ConfigParser
 
 from supervisor import xmlrpc
 
@@ -112,7 +113,8 @@ class Software(object):
                download_from_binary_cache_url_blacklist=None,
                upload_to_binary_cache_url_blacklist=None,
                software_min_free_space=None,
-               buildout_debug=False,):
+               buildout_debug=False,
+               shared_part_list=''):
     """Initialisation of class parameters
     """
 
@@ -127,6 +129,7 @@ class Software(object):
     self.software_url_hash = md5digest(self.url)
     self.software_path = os.path.join(self.software_root,
                                       self.software_url_hash)
+    self.shared_part_list = shared_part_list
     self.buildout = buildout
     self.buildout_debug = buildout_debug
     self.logger = logger
@@ -260,7 +263,7 @@ class Software(object):
     try:
       buildout_cfg = os.path.join(self.software_path, 'buildout.cfg')
       if not os.path.exists(buildout_cfg):
-        self._create_buildout_profile(buildout_cfg, self.url)
+        self._create_buildout_profile(buildout_cfg, self.url, self.shared_part_list)
 
       additional_parameters = list(self._additional_buildout_parameters(extends_cache))
       additional_parameters.extend(['-c', buildout_cfg])
@@ -292,9 +295,16 @@ class Software(object):
       if f is not None:
         f.close()
 
-  def _create_buildout_profile(self, buildout_cfg, url):
+  def _create_buildout_profile(self, buildout_cfg, url, shared_part_list):
+    parser = ConfigParser()
+    parser.add_section('buildout')
+    parser.set('buildout', 'extends', url)
+    # Set shared-part-list for slapos.recipe.cmmi >= 0.10.
+    # slapos.recipe.cmmi was using shared-parts and expecting a single path.
+    # This is not supported.
+    parser.set('buildout', 'shared-part-list', shared_part_list)
     with open(buildout_cfg, 'w') as fout:
-      fout.write('[buildout]\nextends = ' + url + '\n')
+      parser.write(fout)
     self._set_ownership(buildout_cfg)
 
   def uploadSoftwareRelease(self, tarpath):

slapos/grid/slapgrid.py

@@ -241,6 +241,7 @@ def create_slapgrid_object(options, logger):
 
   return Slapgrid(software_root=op['software_root'],
                   instance_root=op['instance_root'],
+                  shared_part_list=op.get('shared_part_list', ''),
                   master_url=op['master_url'],
                   computer_id=op['computer_id'],
                   buildout=op.get('buildout'),
@@ -340,6 +341,7 @@ class Slapgrid(object):
                firewall_conf={},
                config=None,
                buildout_debug=False,
+               shared_part_list=''
                ):
     """Makes easy initialisation of class parameters"""
     # Parses arguments
@@ -405,6 +407,7 @@ class Slapgrid(object):
     self.firewall_conf = firewall_conf
     self.config = config
     self._manager_list = slapmanager.from_config(config)
+    self.shared_part_list = shared_part_list
 
   def _getWatchdogLine(self):
     invocation_list = [WATCHDOG_PATH]
@@ -562,7 +565,8 @@ stderr_logfile_backups=1
             shadir_ca_file=self.shadir_ca_file,
             shadir_cert_file=self.shadir_cert_file,
             shadir_key_file=self.shadir_key_file,
-            software_min_free_space=self.software_min_free_space)
+            software_min_free_space=self.software_min_free_space,
+            shared_part_list=self.shared_part_list)
 
         # call manager for every software release
         for manager in self._manager_list:

slapos/tests/test_slapgrid.py

@@ -113,6 +113,7 @@ class BasicMixin(object):
     self._tempdir = tempfile.mkdtemp()
     self.manager_list = []
     self.software_root = os.path.join(self._tempdir, 'software')
+    self.shared_parts_root = os.path.join(self._tempdir, 'shared')
     self.instance_root = os.path.join(self._tempdir, 'instance')
     if 'SLAPGRID_INSTANCE_ROOT' in os.environ:
       del os.environ['SLAPGRID_INSTANCE_ROOT']
@@ -134,7 +135,8 @@ class BasicMixin(object):
                                   self.computer_id,
                                   self.buildout,
                                   develop=develop,
-                                  logger=logging.getLogger())
+                                  logger=logging.getLogger(),
+                                  shared_part_list=self.shared_parts_root)
     self.grid._manager_list = self.manager_list
     # monkey patch buildout bootstrap
 
@@ -1867,6 +1869,16 @@ echo %s; echo %s; exit 42""" % (line1, line2))
       self.assertIn(line2, software.error_log)
       self.assertIn('Failed to run buildout', software.error_log)
 
+  def test_software_install_generate_buildout_cfg_with_shared_part_list(self):
+    computer = ComputerForTest(self.software_root, self.instance_root, 1, 1)
+    with httmock.HTTMock(computer.request_handler):
+      software = computer.software_list[0]
+      # examine the genrated buildout
+      software.setBuildout("""#!/bin/sh
+        cat buildout.cfg; exit 1""")
+      self.launchSlapgridSoftware()
+    self.assertIn('shared-part-list = %s' % self.shared_parts_root, software.error_log)
+
 
 class SlapgridInitialization(unittest.TestCase):
   """