Support multiple paths for shared parts

Use a new option `buildout:shared-path-list` that is now a new line separated list of path, only the last part is used to install new parts, other are read-only.

Also some cleanups in the test suite.

/reviewed-on nexedi/slapos.recipe.cmmi!9

setup.py

@@ -18,7 +18,7 @@ setup(
         + '\n' +
         read('CHANGELOG.rst')
         + '\n' +
-        read('slapos', 'recipe', 'cmmi', 'README.txt')
+        read('slapos', 'recipe', 'cmmi', 'README.rst')
         + '\n' +
         'Download\n'
         '========\n'

slapos/recipe/cmmi/__init__.py

@@ -41,13 +41,8 @@ class Recipe(object):
             options.update(platform_options)
 
         shared = ((options.get('shared', '').lower() == 'true') and
-                  buildout['buildout'].get('shared-parts', None))
+                  buildout['buildout'].get('shared-part-list', None))
         if shared:
-           shared_part = buildout['buildout'].get('shared-parts', None)
-           shared = os.path.join(shared_part.strip().rstrip('/'), self.name)
-           if not os.path.exists(shared):
-              os.makedirs(shared)
-
             self._signature = slapos.recipe.downloadunpacked.Signature('.slapos.recipe.cmmi.signature')
             buildout_directory = buildout['buildout']['directory']
             profile_base_location = options.get('_profile_base_location_', '')
@@ -57,7 +52,15 @@ class Recipe(object):
                     v = v.replace(profile_base_location, '${:_profile_base_location_}')
                 self._signature.update(k, v)
 
-           shared = os.path.join(shared, self._signature.hexdigest())
+            shared_parts = [part.strip().rstrip('/')
+                    for part in buildout['buildout']['shared-part-list'].splitlines()
+                    if part.strip()]
+            for shared in shared_parts:
+                shared = os.path.join(os.path.join(shared, self.name), self._signature.hexdigest())
+                if os.path.exists(shared):
+                    break
+            else:
+                shared = os.path.join(os.path.join(shared_parts[-1], self.name), self._signature.hexdigest())
             log.info('shared directory %s set for %s', shared, self.name)
         else:
             shared = ''
@@ -303,7 +306,7 @@ class Recipe(object):
                 # leftovers from a previous failed attempt, removing it.
                 log.warning('Removing already existing directory %s' % compile_dir)
                 shutil.rmtree(compile_dir)
-            os.mkdir(compile_dir)
+            os.makedirs(compile_dir)
             try:
                 self.options.get('md5sum') # so that buildout does not complain "unused option md5sum"
                 opt = self.options.copy()

slapos/recipe/cmmi/tests.py

+from slapos.recipe.downloadunpacked import Signature
 from zope.testing import renormalizing
 import doctest
 import errno
@@ -319,17 +320,47 @@ class NonInformativeTests(unittest.TestCase):
         self.assertTrue(os.path.exists(build_directory))
 
 def test_suite():
+    # Hash used in the test depend on the file:// url of the package, so
+    # we use re-normalizer to replace SHARED_PACKAGE_HASH with the actual
+    # hash for this path.
+    package_url = 'file://%s/package-0.0.0.tar.gz' % os.path.join(
+        os.path.dirname(__file__), 'testdata')
+
+    signature_FIRST_SHARED_PACKAGE_HASH = Signature('.slapos.recipe.cmmi.signature')
+    signature_FIRST_SHARED_PACKAGE_HASH.update('environment', 'FOO=bar')
+    signature_FIRST_SHARED_PACKAGE_HASH.update('recipe', 'slapos.recipe.cmmi')
+    signature_FIRST_SHARED_PACKAGE_HASH.update('shared', 'True')
+    signature_FIRST_SHARED_PACKAGE_HASH.update('url', package_url)
+
+    signature_ANOTHER_SHARED_PACKAGE_HASH = Signature('.slapos.recipe.cmmi.signature')
+    signature_ANOTHER_SHARED_PACKAGE_HASH.update('change', 'True')
+    signature_ANOTHER_SHARED_PACKAGE_HASH.update('recipe', 'slapos.recipe.cmmi')
+    signature_ANOTHER_SHARED_PACKAGE_HASH.update('shared', 'True')
+    signature_ANOTHER_SHARED_PACKAGE_HASH.update('url', package_url)
+
     suite = unittest.TestSuite((
         doctest.DocFileSuite(
-            'README.txt',
+            'README.rst',
             setUp=setUp,
             tearDown=zc.buildout.testing.buildoutTearDown,
             optionflags=optionflags,
             checker=renormalizing.RENormalizing([
-                (re.compile('--prefix=\S+sample-buildout'),
+                (re.compile(r'--prefix=\S+sample-buildout'),
                  '--prefix=/sample_buildout'),
-                (re.compile('\s/\S+sample-buildout'),
+                (re.compile(r'\s/\S+sample-buildout'),
                  ' /sample_buildout'),
+                (re.compile(r'--prefix=\S+\/shared\/'),
+                 '--prefix=/shared/'),
+               (re.compile(r'\s/\S+\/shared\/'),
+                 ' /shared/'),
+               (re.compile('FIRST_SHARED_PACKAGE_HASH'),
+                signature_FIRST_SHARED_PACKAGE_HASH.hexdigest()),
+               (re.compile('ANOTHER_SHARED_PACKAGE_HASH'),
+                signature_ANOTHER_SHARED_PACKAGE_HASH.hexdigest()),
+                # Normalize subprocess.CalledProcessError message, on python >= 3.6
+                # there's an extra . at the end.
+               (re.compile(r'Command (.*) returned non-zero exit status (\d+)[\.]{0,1}'),
+                  r'Command \1 returned non-zero exit status \2.'),
                 zc.buildout.testing.normalize_path,
                 zc.buildout.testing.not_found,
             ]),