Add part dependencies to buildout signature

Add referred parts' hash strings in __buildout_signature__,
that invokes rebuild of a part when one of its (recursive)
dependencies are modified.

Also remove duplicates and sort entries in __buildout_signature__.

Squashed with fixup!:

The purpose is to make the part signature available sooner,
during recipe's __init__.

Original commit already broke detection of unused options in sections.

src/zc/buildout/buildout.py

@@ -468,6 +468,8 @@ class Buildout(DictMixin):
         self._raw = _unannotate(data)
         self._data = {}
         self._parts = []
+        self._initializing = []
+        self._signature_cache = {}
 
         # provide some defaults before options are parsed
         # because while parsing options those attributes might be
@@ -771,9 +773,7 @@ class Buildout(DictMixin):
                 _save_options(section, self[section], sys.stdout)
             print_()
 
-
-        # compute new part recipe signatures
-        self._compute_part_signatures(install_parts)
+        del self._signature_cache
 
         # uninstall parts that are no-longer used or who's configs
         # have changed
@@ -1011,16 +1011,7 @@ class Buildout(DictMixin):
                 self._logger.warning(
                     "Unexpected entry, %r, in develop-eggs directory.", f)
 
-    def _compute_part_signatures(self, parts):
-        # Compute recipe signature and add to options
-        for part in parts:
-            options = self.get(part)
-            if options is None:
-                options = self[part] = {}
-            recipe, entry = _recipe(options)
-            req = pkg_resources.Requirement.parse(recipe)
-            sig = _dists_sig(pkg_resources.working_set.resolve([req]))
-            options['__buildout_signature__'] = ' '.join(sig)
+    _compute_part_signatures = None # BBB: monkey-patched by slapos.rebootstrap
 
     def _read_installed_part_options(self):
         old = self['buildout']['installed']
@@ -1377,13 +1368,27 @@ class Buildout(DictMixin):
                     v = v.replace(os.getcwd(), base_path)
                 print_("%s =%s" % (k, v))
 
+    def initialize(self, options, reqs, entry):
+        recipe_class = _install_and_load(reqs, 'zc.buildout', entry, self)
+        try:
+            sig = self._signature_cache[reqs]
+        except KeyError:
+            req = pkg_resources.Requirement.parse(reqs)
+            sig = self._signature_cache[reqs] = sorted(set(
+                _dists_sig(pkg_resources.working_set.resolve([req]))))
+        self._initializing.append((options, sig))
+        try:
+            recipe = recipe_class(self, options.name, options)
+            options['__buildout_signature__']
+        finally:
+            del self._initializing[-1]
+        return recipe
+
     def __getitem__(self, section):
         __doing__ = 'Getting section %s.', section
         try:
-            return self._data[section]
+            options = self._data[section]
         except KeyError:
-            pass
-
             try:
                 data = self._raw[section]
             except KeyError:
@@ -1392,6 +1397,15 @@ class Buildout(DictMixin):
             options = self.Options(self, section, data)
             self._data[section] = options
             options._initialize()
+
+        if self._initializing:
+            caller = self._initializing[-1][0]
+            if 'buildout' != section and not (
+                  section in caller.depends or
+                  # Do not only check the caller,
+                  # because of circular dependencies during substitutions.
+                  section in (x[0].name for x in self._initializing)):
+                caller.depends.add(section)
         return options
 
     def __setitem__(self, name, data):
@@ -1478,6 +1492,7 @@ class Options(DictMixin):
         self._raw = data
         self._cooked = {}
         self._data = {}
+        self.depends = set()
 
     def _initialize(self):
         name = self.name
@@ -1499,16 +1514,13 @@ class Options(DictMixin):
             self.buildout[dname]
 
         if self.get('recipe'):
-            self.initialize()
+            self.recipe = self.buildout.initialize(self, *_recipe(self._data))
             self.buildout._parts.append(name)
 
-    def initialize(self):
-        reqs, entry = _recipe(self._data)
-        buildout = self.buildout
-        recipe_class = _install_and_load(reqs, 'zc.buildout', entry, buildout)
-
-        name = self.name
-        self.recipe = recipe_class(buildout, name, self)
+        m = md5()
+        for item in sorted(self.items()):
+            m.update(('%r\0%r\0' % item).encode())
+        self.items_signature = '%s:%s' % (name, m.hexdigest())
 
     def _do_extend_raw(self, name, data, doing):
         if name == 'buildout':
@@ -1557,6 +1569,16 @@ class Options(DictMixin):
         if v is None:
             v = self._raw.get(option)
             if v is None:
+                if option == '__buildout_signature__':
+                    buildout = self.buildout
+                    options, sig = buildout._initializing[-1]
+                    if options is self:
+                        self.depends = frozenset(self.depends)
+                        v = self._data[option] = ' '.join(sig + [
+                            buildout[dependency].items_signature
+                            for dependency in sorted(self.depends)])
+                        return v
+                    raise zc.buildout.UserError("premature access to " + option)
                 return default
 
         __doing__ = 'Getting option %s:%s.', self.name, option
@@ -1608,7 +1630,14 @@ class Options(DictMixin):
             section, option = s
             if not section:
                 section = self.name
-            v = self.buildout[section].get(option, None, seen)
+                options = self.buildout[section]
+            else:
+                self.buildout._initializing.append((self,))
+                try:
+                    options = self.buildout[section]
+                finally:
+                    del self.buildout._initializing[-1]
+            v = options.get(option, None, seen)
             if v is None:
                 if option == '_buildout_section_name_':
                     v = self.name

src/zc/buildout/testing.py

@@ -202,15 +202,6 @@ def wait_until(label, func, *args, **kw):
         time.sleep(0.01)
     raise ValueError('Timed out waiting for: '+label)
 
-class TestOptions(zc.buildout.buildout.Options):
-
-    def __init__(self, *args):
-        zc.buildout.buildout.Options.__init__(self, *args)
-        self._created = []
-
-    def initialize(self):
-        pass
-
 class Buildout(zc.buildout.buildout.Buildout):
 
     def __init__(self):
@@ -220,7 +211,8 @@ class Buildout(zc.buildout.buildout.Buildout):
         zc.buildout.buildout.Buildout.__init__(
             self, '', [('buildout', 'directory', os.getcwd())], False)
 
-    Options = TestOptions
+    def initialize(self, *args):
+        pass
 
 def buildoutSetUp(test):
 

src/zc/buildout/tests/buildout.txt

@@ -2796,7 +2796,7 @@ were created.
 The ``.installed.cfg`` is only updated for the recipes that ran::
 
     >>> cat(sample_buildout, '.installed.cfg')
-    ... # doctest: +NORMALIZE_WHITESPACE
+    ... # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
     [buildout]
     installed_develop_eggs = /sample-buildout/develop-eggs/recipes.egg-link
     parts = debug d1 d2 d3 d4
@@ -2826,7 +2826,7 @@ The ``.installed.cfg`` is only updated for the recipes that ran::
     <BLANKLINE>
     [d4]
     __buildout_installed__ = /sample-buildout/data2-extra
-    __buildout_signature__ = recipes-PiIFiO8ny5yNZ1S3JfT0xg==
+    __buildout_signature__ = recipes-PiIFiO8ny5yNZ1S3JfT0xg== d2:...
     path = /sample-buildout/data2-extra
     recipe = recipes:mkdir
 
@@ -2839,6 +2839,7 @@ Now, if we run the buildout without the install command::
 
     >>> print_(system(buildout), end='')
     Develop: '/sample-buildout/recipes'
+    Uninstalling d4.
     Uninstalling d2.
     Uninstalling d1.
     Uninstalling debug.
@@ -2848,7 +2849,8 @@ Now, if we run the buildout without the install command::
     Installing d2.
     d2: Creating directory data2
     Updating d3.
-    Updating d4.
+    Installing d4.
+    d4: Creating directory data2-extra
 
 We see the output of the debug recipe, and that ``data2`` was created.  We
 also see that ``d1`` and ``d2`` have gone away::

src/zc/buildout/tests/test_all.py

@@ -2259,6 +2259,80 @@ def dealing_with_extremely_insane_dependencies():
     Error: Couldn't find a distribution for 'pack5'.
     """
 
+def test_part_pulled_by_recipe():
+    """
+    >>> mkdir(sample_buildout, 'recipes')
+    >>> write(sample_buildout, 'recipes', 'test.py',
+    ... '''
+    ... class Recipe:
+    ...
+    ...     def __init__(self, buildout, name, options):
+    ...         self.x = buildout[options['x']][name]
+    ...
+    ...     def install(self):
+    ...         print self.x
+    ...         return ()
+    ...
+    ...     update = install
+    ... ''')
+
+
+    >>> write(sample_buildout, 'recipes', 'setup.py',
+    ... '''
+    ... from setuptools import setup
+    ... setup(
+    ...     name = "recipes",
+    ...     entry_points = {'zc.buildout': ['test = test:Recipe']},
+    ...     )
+    ... ''')
+
+    >>> write(sample_buildout, 'buildout.cfg',
+    ... '''
+    ... [buildout]
+    ... develop = recipes
+    ... parts = a
+    ... [a]
+    ... recipe = recipes:test
+    ... x = b
+    ... [b]
+    ... <= a
+    ... a = A
+    ... b = B
+    ... c = ${c:x}
+    ... [c]
+    ... x = c
+    ... ''')
+
+    >>> os.chdir(sample_buildout)
+    >>> buildout = os.path.join(sample_buildout, 'bin', 'buildout')
+
+    >>> print_(system(buildout), end='')
+    Develop: '/sample-buildout/recipes'
+    Installing b.
+    B
+    Installing a.
+    A
+
+    >>> print_(system(buildout), end='')
+    Develop: '/sample-buildout/recipes'
+    Updating b.
+    B
+    Updating a.
+    A
+
+    >>> cat('.installed.cfg') # doctest: +ELLIPSIS
+    [buildout]
+    ...
+    [b]
+    __buildout_installed__ = 
+    __buildout_signature__ = recipes-... c:...
+    ...
+    [a]
+    __buildout_installed__ = 
+    __buildout_signature__ = recipes-... b:...
+    ...
+    """
+
 def read_find_links_to_load_extensions():
     r"""
 We'll create a wacky buildout extension that just announces itself when used:

zc.recipe.egg_/src/zc/recipe/egg/api.rst

@@ -97,14 +97,14 @@ of extra requirements to be included in the working set.
 We can see that the options were augmented with additional data
 computed by the egg recipe by looking at .installed.cfg:
 
-    >>> cat(sample_buildout, '.installed.cfg')
+    >>> cat(sample_buildout, '.installed.cfg') # doctest: +ELLIPSIS
     [buildout]
     installed_develop_eggs = /sample-buildout/develop-eggs/sample.egg-link
     parts = sample-part
     <BLANKLINE>
     [sample-part]
     __buildout_installed__ = 
-    __buildout_signature__ = ...
+    __buildout_signature__ = sample-... setuptools-...egg zc.buildout-... zc.recipe.egg-...
     _b = /sample-buildout/bin
     _d = /sample-buildout/develop-eggs
     _e = /sample-buildout/eggs