Commit 1d7fa371 authored by Kazuhiko Shiozaki's avatar Kazuhiko Shiozaki Committed by Xavier Thompson

Add part dependencies to buildout signature

Add referred parts' hash strings in __buildout_signature__,
that invokes rebuild of a part when one of its (recursive)
dependencies are modified.

Also remove duplicates and sort entries in __buildout_signature__.

Squashed with fixup!:

The purpose is to make the part signature available sooner,
during recipe's __init__.

Original commit already broke detection of unused options in sections.
parent 44f7950a
......@@ -468,6 +468,8 @@ class Buildout(DictMixin):
self._raw = _unannotate(data)
self._data = {}
self._parts = []
self._initializing = []
self._signature_cache = {}
# provide some defaults before options are parsed
# because while parsing options those attributes might be
......@@ -771,9 +773,7 @@ class Buildout(DictMixin):
_save_options(section, self[section], sys.stdout)
print_()
# compute new part recipe signatures
self._compute_part_signatures(install_parts)
del self._signature_cache
# uninstall parts that are no-longer used or who's configs
# have changed
......@@ -1011,16 +1011,7 @@ class Buildout(DictMixin):
self._logger.warning(
"Unexpected entry, %r, in develop-eggs directory.", f)
def _compute_part_signatures(self, parts):
# Compute recipe signature and add to options
for part in parts:
options = self.get(part)
if options is None:
options = self[part] = {}
recipe, entry = _recipe(options)
req = pkg_resources.Requirement.parse(recipe)
sig = _dists_sig(pkg_resources.working_set.resolve([req]))
options['__buildout_signature__'] = ' '.join(sig)
_compute_part_signatures = None # BBB: monkey-patched by slapos.rebootstrap
def _read_installed_part_options(self):
old = self['buildout']['installed']
......@@ -1377,21 +1368,44 @@ class Buildout(DictMixin):
v = v.replace(os.getcwd(), base_path)
print_("%s =%s" % (k, v))
def __getitem__(self, section):
__doing__ = 'Getting section %s.', section
def initialize(self, options, reqs, entry):
recipe_class = _install_and_load(reqs, 'zc.buildout', entry, self)
try:
return self._data[section]
sig = self._signature_cache[reqs]
except KeyError:
pass
req = pkg_resources.Requirement.parse(reqs)
sig = self._signature_cache[reqs] = sorted(set(
_dists_sig(pkg_resources.working_set.resolve([req]))))
self._initializing.append((options, sig))
try:
recipe = recipe_class(self, options.name, options)
options['__buildout_signature__']
finally:
del self._initializing[-1]
return recipe
def __getitem__(self, section):
__doing__ = 'Getting section %s.', section
try:
data = self._raw[section]
options = self._data[section]
except KeyError:
raise MissingSection(section)
options = self.Options(self, section, data)
self._data[section] = options
options._initialize()
try:
data = self._raw[section]
except KeyError:
raise MissingSection(section)
options = self.Options(self, section, data)
self._data[section] = options
options._initialize()
if self._initializing:
caller = self._initializing[-1][0]
if 'buildout' != section and not (
section in caller.depends or
# Do not only check the caller,
# because of circular dependencies during substitutions.
section in (x[0].name for x in self._initializing)):
caller.depends.add(section)
return options
def __setitem__(self, name, data):
......@@ -1478,6 +1492,7 @@ class Options(DictMixin):
self._raw = data
self._cooked = {}
self._data = {}
self.depends = set()
def _initialize(self):
name = self.name
......@@ -1499,16 +1514,13 @@ class Options(DictMixin):
self.buildout[dname]
if self.get('recipe'):
self.initialize()
self.recipe = self.buildout.initialize(self, *_recipe(self._data))
self.buildout._parts.append(name)
def initialize(self):
reqs, entry = _recipe(self._data)
buildout = self.buildout
recipe_class = _install_and_load(reqs, 'zc.buildout', entry, buildout)
name = self.name
self.recipe = recipe_class(buildout, name, self)
m = md5()
for item in sorted(self.items()):
m.update(('%r\0%r\0' % item).encode())
self.items_signature = '%s:%s' % (name, m.hexdigest())
def _do_extend_raw(self, name, data, doing):
if name == 'buildout':
......@@ -1557,6 +1569,16 @@ class Options(DictMixin):
if v is None:
v = self._raw.get(option)
if v is None:
if option == '__buildout_signature__':
buildout = self.buildout
options, sig = buildout._initializing[-1]
if options is self:
self.depends = frozenset(self.depends)
v = self._data[option] = ' '.join(sig + [
buildout[dependency].items_signature
for dependency in sorted(self.depends)])
return v
raise zc.buildout.UserError("premature access to " + option)
return default
__doing__ = 'Getting option %s:%s.', self.name, option
......@@ -1608,7 +1630,14 @@ class Options(DictMixin):
section, option = s
if not section:
section = self.name
v = self.buildout[section].get(option, None, seen)
options = self.buildout[section]
else:
self.buildout._initializing.append((self,))
try:
options = self.buildout[section]
finally:
del self.buildout._initializing[-1]
v = options.get(option, None, seen)
if v is None:
if option == '_buildout_section_name_':
v = self.name
......
......@@ -202,15 +202,6 @@ def wait_until(label, func, *args, **kw):
time.sleep(0.01)
raise ValueError('Timed out waiting for: '+label)
class TestOptions(zc.buildout.buildout.Options):
def __init__(self, *args):
zc.buildout.buildout.Options.__init__(self, *args)
self._created = []
def initialize(self):
pass
class Buildout(zc.buildout.buildout.Buildout):
def __init__(self):
......@@ -220,7 +211,8 @@ class Buildout(zc.buildout.buildout.Buildout):
zc.buildout.buildout.Buildout.__init__(
self, '', [('buildout', 'directory', os.getcwd())], False)
Options = TestOptions
def initialize(self, *args):
pass
def buildoutSetUp(test):
......
......@@ -2796,7 +2796,7 @@ were created.
The ``.installed.cfg`` is only updated for the recipes that ran::
>>> cat(sample_buildout, '.installed.cfg')
... # doctest: +NORMALIZE_WHITESPACE
... # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
[buildout]
installed_develop_eggs = /sample-buildout/develop-eggs/recipes.egg-link
parts = debug d1 d2 d3 d4
......@@ -2826,7 +2826,7 @@ The ``.installed.cfg`` is only updated for the recipes that ran::
<BLANKLINE>
[d4]
__buildout_installed__ = /sample-buildout/data2-extra
__buildout_signature__ = recipes-PiIFiO8ny5yNZ1S3JfT0xg==
__buildout_signature__ = recipes-PiIFiO8ny5yNZ1S3JfT0xg== d2:...
path = /sample-buildout/data2-extra
recipe = recipes:mkdir
......@@ -2839,6 +2839,7 @@ Now, if we run the buildout without the install command::
>>> print_(system(buildout), end='')
Develop: '/sample-buildout/recipes'
Uninstalling d4.
Uninstalling d2.
Uninstalling d1.
Uninstalling debug.
......@@ -2848,7 +2849,8 @@ Now, if we run the buildout without the install command::
Installing d2.
d2: Creating directory data2
Updating d3.
Updating d4.
Installing d4.
d4: Creating directory data2-extra
We see the output of the debug recipe, and that ``data2`` was created. We
also see that ``d1`` and ``d2`` have gone away::
......
......@@ -2259,6 +2259,80 @@ def dealing_with_extremely_insane_dependencies():
Error: Couldn't find a distribution for 'pack5'.
"""
def test_part_pulled_by_recipe():
"""
>>> mkdir(sample_buildout, 'recipes')
>>> write(sample_buildout, 'recipes', 'test.py',
... '''
... class Recipe:
...
... def __init__(self, buildout, name, options):
... self.x = buildout[options['x']][name]
...
... def install(self):
... print self.x
... return ()
...
... update = install
... ''')
>>> write(sample_buildout, 'recipes', 'setup.py',
... '''
... from setuptools import setup
... setup(
... name = "recipes",
... entry_points = {'zc.buildout': ['test = test:Recipe']},
... )
... ''')
>>> write(sample_buildout, 'buildout.cfg',
... '''
... [buildout]
... develop = recipes
... parts = a
... [a]
... recipe = recipes:test
... x = b
... [b]
... <= a
... a = A
... b = B
... c = ${c:x}
... [c]
... x = c
... ''')
>>> os.chdir(sample_buildout)
>>> buildout = os.path.join(sample_buildout, 'bin', 'buildout')
>>> print_(system(buildout), end='')
Develop: '/sample-buildout/recipes'
Installing b.
B
Installing a.
A
>>> print_(system(buildout), end='')
Develop: '/sample-buildout/recipes'
Updating b.
B
Updating a.
A
>>> cat('.installed.cfg') # doctest: +ELLIPSIS
[buildout]
...
[b]
__buildout_installed__ =
__buildout_signature__ = recipes-... c:...
...
[a]
__buildout_installed__ =
__buildout_signature__ = recipes-... b:...
...
"""
def read_find_links_to_load_extensions():
r"""
We'll create a wacky buildout extension that just announces itself when used:
......
......@@ -97,14 +97,14 @@ of extra requirements to be included in the working set.
We can see that the options were augmented with additional data
computed by the egg recipe by looking at .installed.cfg:
>>> cat(sample_buildout, '.installed.cfg')
>>> cat(sample_buildout, '.installed.cfg') # doctest: +ELLIPSIS
[buildout]
installed_develop_eggs = /sample-buildout/develop-eggs/sample.egg-link
parts = sample-part
<BLANKLINE>
[sample-part]
__buildout_installed__ =
__buildout_signature__ = ...
__buildout_signature__ = sample-... setuptools-...egg zc.buildout-... zc.recipe.egg-...
_b = /sample-buildout/bin
_d = /sample-buildout/develop-eggs
_e = /sample-buildout/eggs
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment