Skip to content

S
slapos.buildout
Commit 517e6352 authored by Jim Fulton's avatar Jim Fulton
Browse files
Options

- Changed the way the installed database (.installed.cfg) is handled 

  to avoid database corruption when a user breaks out of a buildout
  with control-c.

- Don't save an installed database if there are no installed parts or
  develop egg links.
parent 4f330ba0
Hide whitespace changes
Inline Side-by-side
Showing with 415 additions and 129 deletions
CHANGES.txt
View file @ 517e6352
...@@ -17,18 +17,25 @@ Change History ...@@ -17,18 +17,25 @@ Change History
Feature Changes Feature Changes
--------------- ---------------
Improved error reporting and debugging support: - Improved error reporting and debugging support:
- Added "logical tracebacks" that show functionally what the buildout - Added "logical tracebacks" that show functionally what the buildout
was doing when an error occurs. Don't show a Python traceback was doing when an error occurs. Don't show a Python traceback
unless the -D option is used. unless the -D option is used.
- Added a -D option that causes the buildout to print a traceback and - Added a -D option that causes the buildout to print a traceback and
start the pdb post-mortem debugger when an error occurs. start the pdb post-mortem debugger when an error occurs.
- Warnings are printed for unused options in the buildout section and - Warnings are printed for unused options in the buildout section and
installed-part sections. This should make it easier to catch option installed-part sections. This should make it easier to catch option
misspellings. misspellings.
- Changed the way the installed database (.installed.cfg) is handled
to avoid database corruption when a user breaks out of a buildout
with control-c.
- Don't save an installed database if there are no installed parts or
develop egg links.
1.0.0b21 (2007-03-06) 1.0.0b21 (2007-03-06)
===================== =====================
......
src/zc/buildout/buildout.py
View file @ 517e6352
...@@ -202,7 +202,8 @@ class Buildout(UserDict.DictMixin): ...@@ -202,7 +202,8 @@ class Buildout(UserDict.DictMixin):
self._maybe_upgrade() self._maybe_upgrade()
# load installed data # load installed data
installed_part_options = self._read_installed_part_options() (installed_part_options, installed_exists
)= self._read_installed_part_options()
# Remove old develop eggs # Remove old develop eggs
self._uninstall( self._uninstall(
...@@ -212,6 +213,12 @@ class Buildout(UserDict.DictMixin): ...@@ -212,6 +213,12 @@ class Buildout(UserDict.DictMixin):
# Build develop eggs # Build develop eggs
installed_develop_eggs = self._develop() installed_develop_eggs = self._develop()
installed_part_options['buildout']['installed_develop_eggs'
] = installed_develop_eggs
if installed_exists:
self._update_installed(
installed_develop_eggs=installed_develop_eggs)
# get configured and installed part lists # get configured and installed part lists
conf_parts = self['buildout']['parts'] conf_parts = self['buildout']['parts']
...@@ -244,118 +251,147 @@ class Buildout(UserDict.DictMixin): ...@@ -244,118 +251,147 @@ class Buildout(UserDict.DictMixin):
# compute new part recipe signatures # compute new part recipe signatures
self._compute_part_signatures(install_parts) self._compute_part_signatures(install_parts)
try: # uninstall parts that are no-longer used or who's configs
# uninstall parts that are no-longer used or who's configs # have changed
# have changed for part in reversed(installed_parts):
for part in reversed(installed_parts): if part in install_parts:
if part in install_parts: old_options = installed_part_options[part].copy()
old_options = installed_part_options[part].copy() installed_files = old_options.pop('__buildout_installed__')
installed_files = old_options.pop('__buildout_installed__') new_options = self.get(part)
new_options = self.get(part) if old_options == new_options:
if old_options == new_options: # The options are the same, but are all of the
# The options are the same, but are all of the # installed files still there? If not, we should
# installed files still there? If not, we should # reinstall.
# reinstall. if not installed_files:
if not installed_files: continue
continue for f in installed_files.split('\n'):
for f in installed_files.split('\n'): if not os.path.exists(self._buildout_path(f)):
if not os.path.exists(self._buildout_path(f)): break
break
else:
continue
# output debugging info
for k in old_options:
if k not in new_options:
self._logger.debug("Part: %s, dropped option %s",
part, k)
elif old_options[k] != new_options[k]:
self._logger.debug(
"Part: %s, option %s, %r != %r",
part, k, new_options[k], old_options[k],
)
for k in new_options:
if k not in old_options:
self._logger.debug("Part: %s, new option %s",
part, k)
elif not uninstall_missing:
continue
self._uninstall_part(part, installed_part_options)
installed_parts = [p for p in installed_parts if p != part]
# Check for unused buildout options:
_check_for_unused_options_in_section(self, 'buildout')
# install new parts
for part in install_parts:
signature = self[part].pop('__buildout_signature__')
saved_options = self[part].copy()
recipe = self[part].recipe
if part in installed_parts:
__doing__ = 'Updating %s', part
self._logger.info(*__doing__)
old_options = installed_part_options[part]
old_installed_files = old_options['__buildout_installed__']
try:
update = recipe.update
except AttributeError:
update = recipe.install
self._logger.warning(
"The recipe for %s doesn't define an update "
"method. Using its install method.",
part)
try:
installed_files = update()
except:
installed_parts.remove(part)
self._uninstall(old_installed_files)
raise
if installed_files is None:
installed_files = old_installed_files.split('\n')
else: else:
if isinstance(installed_files, str): continue
installed_files = [installed_files]
else: # output debugging info
installed_files = list(installed_files) for k in old_options:
if k not in new_options:
installed_files += [ self._logger.debug("Part: %s, dropped option %s",
p for p in old_installed_files.split('\n') part, k)
if p and p not in installed_files] elif old_options[k] != new_options[k]:
self._logger.debug(
"Part: %s, option %s, %r != %r",
part, k, new_options[k], old_options[k],
)
for k in new_options:
if k not in old_options:
self._logger.debug("Part: %s, new option %s",
part, k)
elif not uninstall_missing:
continue
self._uninstall_part(part, installed_part_options)
installed_parts = [p for p in installed_parts if p != part]
if installed_exists:
self._update_installed(parts=' '.join(installed_parts))
# Check for unused buildout options:
_check_for_unused_options_in_section(self, 'buildout')
# install new parts
for part in install_parts:
signature = self[part].pop('__buildout_signature__')
saved_options = self[part].copy()
recipe = self[part].recipe
if part in installed_parts: # update
need_to_save_installed = False
__doing__ = 'Updating %s', part
self._logger.info(*__doing__)
old_options = installed_part_options[part]
old_installed_files = old_options['__buildout_installed__']
try:
update = recipe.update
except AttributeError:
update = recipe.install
self._logger.warning(
"The recipe for %s doesn't define an update "
"method. Using its install method.",
part)
try:
installed_files = update()
except:
installed_parts.remove(part)
self._uninstall(old_installed_files)
if installed_exists:
self._update_installed(
parts=' '.join(installed_parts))
raise
old_installed_files = old_installed_files.split('\n')
if installed_files is None:
installed_files = old_installed_files
else: else:
__doing__ = 'Installing %s', part if isinstance(installed_files, str):
self._logger.info(*__doing__) installed_files = [installed_files]
installed_files = recipe.install() else:
if installed_files is None: installed_files = list(installed_files)
self._logger.warning(
"The %s install returned None. A path or " need_to_save_installed = [
"iterable os paths should be returned.", p for p in installed_files
part) if p not in old_installed_files]
installed_files = ()
if need_to_save_installed:
if isinstance(installed_files, str): installed_files = (old_installed_files
+ need_to_save_installed)
else: # install
need_to_save_installed = True
__doing__ = 'Installing %s', part
self._logger.info(*__doing__)
installed_files = recipe.install()
if installed_files is None:
self._logger.warning(
"The %s install returned None. A path or "
"iterable os paths should be returned.",
part)
installed_files = ()
elif isinstance(installed_files, str):
installed_files = [installed_files] installed_files = [installed_files]
else:
installed_files = list(installed_files)
installed_part_options[part] = saved_options
saved_options['__buildout_installed__'
] = '\n'.join(installed_files)
saved_options['__buildout_signature__'] = signature
installed_parts = [p for p in installed_parts if p != part]
installed_parts.append(part)
_check_for_unused_options_in_section(self, part)
if need_to_save_installed:
installed_part_options['buildout']['parts'] = (
' '.join(installed_parts))
self._save_installed_options(installed_part_options)
installed_exists = True
else:
assert installed_exists
self._update_installed(parts=' '.join(installed_parts))
installed_part_options[part] = saved_options if installed_develop_eggs:
saved_options['__buildout_installed__' if not installed_exists:
] = '\n'.join(installed_files) self._save_installed_options(installed_part_options)
saved_options['__buildout_signature__'] = signature elif (not installed_parts) and installed_exists:
os.remove(self['buildout']['installed'])
installed_parts = [p for p in installed_parts if p != part]
installed_parts.append(part)
_check_for_unused_options_in_section(self, part)
finally: def _update_installed(self, **buildout_options):
installed_part_options['buildout']['parts'] = ( installed = self['buildout']['installed']
' '.join(installed_parts)) f = open(installed, 'a')
installed_part_options['buildout']['installed_develop_eggs' f.write('\n[buildout]\n')
] = installed_develop_eggs for option, value in buildout_options.items():
_save_option(option, value, f)
self._save_installed_options(installed_part_options) f.close()
def _uninstall_part(self, part, installed_part_options): def _uninstall_part(self, part, installed_part_options):
# ununstall part # ununstall part
...@@ -466,9 +502,11 @@ class Buildout(UserDict.DictMixin): ...@@ -466,9 +502,11 @@ class Buildout(UserDict.DictMixin):
options[option] = value options[option] = value
result[section] = Options(self, section, options) result[section] = Options(self, section, options)
return result return result, True
else: else:
return {'buildout': Options(self, 'buildout', {'parts': ''})} return ({'buildout': Options(self, 'buildout', {'parts': ''})},
False,
)
def _uninstall(self, installed): def _uninstall(self, installed):
for f in installed.split('\n'): for f in installed.split('\n'):
...@@ -891,17 +929,20 @@ def _quote_spacey_nl(match): ...@@ -891,17 +929,20 @@ def _quote_spacey_nl(match):
) )
return result return result
def _save_option(option, value, f):
value = _spacey_nl.sub(_quote_spacey_nl, value)
if value.startswith('\n\t'):
value = '%(__buildout_space_n__)s' + value[2:]
if value.endswith('\n\t'):
value = value[:-2] + '%(__buildout_space_n__)s'
print >>f, option, '=', value
def _save_options(section, options, f): def _save_options(section, options, f):
print >>f, '[%s]' % section print >>f, '[%s]' % section
items = options.items() items = options.items()
items.sort() items.sort()
for option, value in items: for option, value in items:
value = _spacey_nl.sub(_quote_spacey_nl, value) _save_option(option, value, f)
if value.startswith('\n\t'):
value = '%(__buildout_space_n__)s' + value[2:]
if value.endswith('\n\t'):
value = value[:-2] + '%(__buildout_space_n__)s'
print >>f, option, '=', value
def _open(base, filename, seen): def _open(base, filename, seen):
"""Open a configuration file and return the result as a dictionary, """Open a configuration file and return the result as a dictionary,
......
src/zc/buildout/buildout.txt
View file @ 517e6352
...@@ -1756,8 +1756,20 @@ information on installed parts. This option is initialized to ...@@ -1756,8 +1756,20 @@ information on installed parts. This option is initialized to
".installed.cfg", but it can be overridded in the configuration file ".installed.cfg", but it can be overridded in the configuration file
or on the command line: or on the command line:
>>> os.remove('.installed.cfg') >>> write('buildout.cfg',
... """
... [buildout]
... develop = recipes
... parts = debug
...
... [debug]
... recipe = recipes:debug
... """)
>>> print system(buildout+' buildout:installed=inst.cfg'), >>> print system(buildout+' buildout:installed=inst.cfg'),
buildout: Develop: /sample-buildout/recipes
buildout: Installing debug
recipe recipes:debug
>>> ls(sample_buildout) >>> ls(sample_buildout)
- b1.cfg - b1.cfg
...@@ -1771,12 +1783,14 @@ or on the command line: ...@@ -1771,12 +1783,14 @@ or on the command line:
d parts d parts
d recipes d recipes
The installation database can be disabled by supplying an empty The installation database can be disabled by supplying an empty
buildout installed opttion: buildout installed opttion:
>>> os.remove('inst.cfg') >>> os.remove('inst.cfg')
>>> print system(buildout+' buildout:installed='), >>> print system(buildout+' buildout:installed='),
buildout: Develop: /sample-buildout/recipes
buildout: Installing debug
recipe recipes:debug
>>> ls(sample_buildout) >>> ls(sample_buildout)
- b1.cfg - b1.cfg
...@@ -1790,6 +1804,28 @@ buildout installed opttion: ...@@ -1790,6 +1804,28 @@ buildout installed opttion:
d recipes d recipes
Note that there will be no installation database if there are no
parts:
>>> write('buildout.cfg',
... """
... [buildout]
... parts =
... """)
>>> print system(buildout+' buildout:installed=inst.cfg'),
>>> ls(sample_buildout)
- b1.cfg
- b2.cfg
- base.cfg
d bin
- buildout.cfg
d develop-eggs
d eggs
d parts
d recipes
Extensions Extensions
---------- ----------
......
src/zc/buildout/tests.py
View file @ 517e6352
...@@ -1240,11 +1240,11 @@ uninstall ...@@ -1240,11 +1240,11 @@ uninstall
[buildout] [buildout]
... ...
[foo] [foo]
__buildout_installed__ = c __buildout_installed__ = a
b
c
d d
e e
a
b
__buildout_signature__ = ... __buildout_signature__ = ...
""" """
...@@ -1386,6 +1386,208 @@ def whine_about_unused_options(): ...@@ -1386,6 +1386,208 @@ def whine_about_unused_options():
buildout: Unused options for foo: 'z' buildout: Unused options for foo: 'z'
''' '''
def abnormal_exit():
"""
People sometimes hit control-c while running a builout. We need to make
sure that the installed database Isn't corrupted. To test this, we'll create
some evil recipes that exit uncleanly:
>>> mkdir('recipes')
>>> write('recipes', 'recipes.py',
... '''
... import os
...
... class Clean:
... def __init__(*_): pass
... def install(_): return ()
... def update(_): pass
...
... class EvilInstall(Clean):
... def install(_): os._exit(1)
...
... class EvilUpdate(Clean):
... def update(_): os._exit(1)
... ''')
>>> write('recipes', 'setup.py',
... '''
... import setuptools
... setuptools.setup(name='recipes',
... entry_points = {
... 'zc.buildout': [
... 'clean = recipes:Clean',
... 'evil_install = recipes:EvilInstall',
... 'evil_update = recipes:EvilUpdate',
... 'evil_uninstall = recipes:Clean',
... ],
... },
... )
... ''')
Now let's look at 3 cases:
1. We exit during installation after installing some other parts:
>>> write('buildout.cfg',
... '''
... [buildout]
... develop = recipes
... parts = p1 p2 p3 p4
...
... [p1]
... recipe = recipes:clean
...
... [p2]
... recipe = recipes:clean
...
... [p3]
... recipe = recipes:evil_install
...
... [p4]
... recipe = recipes:clean
... ''')
>>> print system(buildout),
buildout: Develop: /sample-buildout/recipes
buildout: Installing p1
buildout: Installing p2
buildout: Installing p3
>>> print system(buildout),
buildout: Develop: /sample-buildout/recipes
buildout: Updating p1
buildout: Updating p2
buildout: Installing p3
>>> print system(buildout+' buildout:parts='),
buildout: Develop: /sample-buildout/recipes
buildout: Uninstalling p2
buildout: Uninstalling p1
2. We exit while updating:
>>> write('buildout.cfg',
... '''
... [buildout]
... develop = recipes
... parts = p1 p2 p3 p4
...
... [p1]
... recipe = recipes:clean
...
... [p2]
... recipe = recipes:clean
...
... [p3]
... recipe = recipes:evil_update
...
... [p4]
... recipe = recipes:clean
... ''')
>>> print system(buildout),
buildout: Develop: /sample-buildout/recipes
buildout: Installing p1
buildout: Installing p2
buildout: Installing p3
buildout: Installing p4
>>> print system(buildout),
buildout: Develop: /sample-buildout/recipes
buildout: Updating p1
buildout: Updating p2
buildout: Updating p3
>>> print system(buildout+' buildout:parts='),
buildout: Develop: /sample-buildout/recipes
buildout: Uninstalling p2
buildout: Uninstalling p1
buildout: Uninstalling p4
buildout: Uninstalling p3
3. We exit while installing or updating after uninstalling:
>>> write('buildout.cfg',
... '''
... [buildout]
... develop = recipes
... parts = p1 p2 p3 p4
...
... [p1]
... recipe = recipes:evil_update
...
... [p2]
... recipe = recipes:clean
...
... [p3]
... recipe = recipes:clean
...
... [p4]
... recipe = recipes:clean
... ''')
>>> print system(buildout),
buildout: Develop: /sample-buildout/recipes
buildout: Installing p1
buildout: Installing p2
buildout: Installing p3
buildout: Installing p4
>>> write('buildout.cfg',
... '''
... [buildout]
... develop = recipes
... parts = p1 p2 p3 p4
...
... [p1]
... recipe = recipes:evil_update
...
... [p2]
... recipe = recipes:clean
...
... [p3]
... recipe = recipes:clean
...
... [p4]
... recipe = recipes:clean
... x = 1
... ''')
>>> print system(buildout),
buildout: Develop: /sample-buildout/recipes
buildout: Uninstalling p4
buildout: Updating p1
>>> write('buildout.cfg',
... '''
... [buildout]
... develop = recipes
... parts = p1 p2 p3 p4
...
... [p1]
... recipe = recipes:clean
...
... [p2]
... recipe = recipes:clean
...
... [p3]
... recipe = recipes:clean
...
... [p4]
... recipe = recipes:clean
... ''')
>>> print system(buildout),
buildout: Develop: /sample-buildout/recipes
buildout: Uninstalling p1
buildout: Installing p1
buildout: Updating p2
buildout: Updating p3
buildout: Installing p4
"""
###################################################################### ######################################################################
def create_sample_eggs(test, executable=sys.executable): def create_sample_eggs(test, executable=sys.executable):
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7