Skip to content

S
slapos.buildout
Commit 3dcc234b authored by Jim Fulton's avatar Jim Fulton
Browse files
Options

Improved error reporting/handling: 

- Added "logical tracebacks" that show functionally what the buildout
  was doing when an error occurs.  Don't show a Python traceback
  unless the -D option is used.

- Added a -D option that causes the buildout to print a traceback and
  start the pdb post-mortem debugger when an error occurs.
parent 33bd492b
Hide whitespace changes
Inline Side-by-side
Showing with 320 additions and 44 deletions
src/zc/buildout/buildout.py
View file @ 3dcc234b
#############################################################################
############################################################################
#
# Copyright (c) 2005 Zope Corporation and Contributors.
# All Rights Reserved.
......@@ -61,6 +61,8 @@ class Buildout(UserDict.DictMixin):
def __init__(self, config_file, cloptions,
user_defaults=True, windows_restart=False):
__doing__ = 'Initializing'
self.__windows_restart = windows_restart
# default options
......@@ -150,6 +152,8 @@ class Buildout(UserDict.DictMixin):
return os.path.join(self._buildout_dir, *names)
def bootstrap(self, args):
__doing__ = 'Bootstraping'
self._setup_directories()
# Now copy buildout and setuptools eggs, amd record destination eggs:
......@@ -180,6 +184,8 @@ class Buildout(UserDict.DictMixin):
self['buildout']['bin-directory'])
def install(self, install_args):
__doing__ = 'Installing'
self._load_extensions()
self._setup_directories()
......@@ -271,22 +277,7 @@ class Buildout(UserDict.DictMixin):
elif not uninstall_missing:
continue
# ununstall part
self._logger.info('Uninstalling %s', part)
# run uinstall recipe
recipe, entry = _recipe(installed_part_options[part])
try:
uninstaller = _install_and_load(
recipe, 'zc.buildout.uninstall', entry, self)
self._logger.info('Running uninstall recipe')
uninstaller(part, installed_part_options[part])
except (ImportError, pkg_resources.DistributionNotFound), v:
pass
# remove created files and directories
self._uninstall(
installed_part_options[part]['__buildout_installed__'])
self._uninstall_part(part, installed_part_options)
installed_parts = [p for p in installed_parts if p != part]
# install new parts
......@@ -295,7 +286,8 @@ class Buildout(UserDict.DictMixin):
saved_options = self[part].copy()
recipe = self[part].recipe
if part in installed_parts:
self._logger.info('Updating %s', part)
__doing__ = 'Updating %s', part
self._logger.info(*__doing__)
old_options = installed_part_options[part]
old_installed_files = old_options['__buildout_installed__']
try:
......@@ -321,14 +313,14 @@ class Buildout(UserDict.DictMixin):
installed_files = [installed_files]
else:
installed_files = list(installed_files)
installed_files += [
p for p in old_installed_files.split('\n')
if p and p not in installed_files]
else:
self._logger.info('Installing %s', part)
__doing__ = 'Installing %s', part
self._logger.info(*__doing__)
installed_files = recipe.install()
if installed_files is None:
self._logger.warning(
......@@ -356,7 +348,28 @@ class Buildout(UserDict.DictMixin):
self._save_installed_options(installed_part_options)
def _uninstall_part(self, part, installed_part_options):
# ununstall part
__doing__ = 'Uninstalling %s', part
self._logger.info(*__doing__)
# run uinstall recipe
recipe, entry = _recipe(installed_part_options[part])
try:
uninstaller = _install_and_load(
recipe, 'zc.buildout.uninstall', entry, self)
self._logger.info('Running uninstall recipe')
uninstaller(part, installed_part_options[part])
except (ImportError, pkg_resources.DistributionNotFound), v:
pass
# remove created files and directories
self._uninstall(
installed_part_options[part]['__buildout_installed__'])
def _setup_directories(self):
__doing__ = 'Setting up buildout directories'
# Create buildout directories
for name in ('bin', 'parts', 'eggs', 'develop-eggs'):
......@@ -368,6 +381,8 @@ class Buildout(UserDict.DictMixin):
def _develop(self):
"""Install sources by running setup.py develop on them
"""
__doing__ = 'Processing directories listed in the develop option'
develop = self['buildout'].get('develop')
if not develop:
return ''
......@@ -382,6 +397,7 @@ class Buildout(UserDict.DictMixin):
for setup in develop.split():
setup = self._buildout_path(setup)
self._logger.info("Develop: %s", setup)
__doing__ = 'Processing develop directory %s', setup
zc.buildout.easy_install.develop(setup, dest)
except:
# if we had an error, we need to roll back changes, by
......@@ -482,9 +498,8 @@ class Buildout(UserDict.DictMixin):
_save_options(part, installed_options[part], f)
f.close()
def _error(self, message, *args, **kw):
self._logger.error(message, *args, **kw)
sys.exit(1)
def _error(self, message, *args):
raise zc.buildout.UserError(message % args)
def _setup_logging(self):
root_logger = logging.getLogger()
......@@ -513,6 +528,7 @@ class Buildout(UserDict.DictMixin):
def _maybe_upgrade(self):
# See if buildout or setuptools need to be upgraded.
# If they do, do the upgrade and restart the buildout process.
__doing__ = 'Checking for upgrades'
if not self.newest:
return
......@@ -537,6 +553,8 @@ class Buildout(UserDict.DictMixin):
if not upgraded:
return
__doing__ = 'Upgrading'
should_run = realpath(
os.path.join(os.path.abspath(self['buildout']['bin-directory']),
'buildout')
......@@ -583,6 +601,7 @@ class Buildout(UserDict.DictMixin):
sys.exit(os.spawnv(os.P_WAIT, sys.executable, args))
def _load_extensions(self):
__doing__ = 'Loading extensions'
specs = self['buildout'].get('extensions', '').split()
if specs:
path = [self['buildout']['develop-eggs-directory']]
......@@ -628,6 +647,7 @@ class Buildout(UserDict.DictMixin):
runsetup = setup # backward compat.
def __getitem__(self, section):
__doing__ = 'Getting section %s', section
try:
return self._data[section]
except KeyError:
......@@ -657,12 +677,13 @@ class Buildout(UserDict.DictMixin):
def _install_and_load(spec, group, entry, buildout):
__doing__ = 'Loading recipe %s', spec
try:
req = pkg_resources.Requirement.parse(spec)
buildout_options = buildout['buildout']
if pkg_resources.working_set.find(req) is None:
__doing__ = 'Installing recipe %s', spec
if buildout.offline:
dest = None
path = [buildout_options['develop-eggs-directory'],
......@@ -681,6 +702,7 @@ def _install_and_load(spec, group, entry, buildout):
newest=buildout.newest,
)
__doing__ = 'Loading %s recipe entry %s:%s', group, spec, entry
return pkg_resources.load_entry_point(
req.project_name, group, entry)
......@@ -700,6 +722,8 @@ class Options(UserDict.DictMixin):
self._data = {}
def _initialize(self):
name = self.name
__doing__ = 'Initializing section %s', name
# force substitutions
for k in self._raw:
self.get(k)
......@@ -712,8 +736,9 @@ class Options(UserDict.DictMixin):
buildout = self.buildout
recipe_class = _install_and_load(reqs, 'zc.buildout', entry, buildout)
self.recipe = recipe_class(buildout, self.name, self)
buildout._parts.append(self.name)
__doing__ = 'Initializing part %s', name
self.recipe = recipe_class(buildout, name, self)
buildout._parts.append(name)
def get(self, option, default=None, seen=None):
try:
......@@ -725,6 +750,8 @@ class Options(UserDict.DictMixin):
if v is None:
return default
__doing__ = 'Getting option %s:%s', self.name, option
if '${' in v:
key = self.name, option
if seen is None:
......@@ -732,10 +759,6 @@ class Options(UserDict.DictMixin):
elif key in seen:
raise zc.buildout.UserError(
"Circular reference in substitutions.\n"
"We're evaluating %s\nand are referencing: %s.\n"
% (", ".join([":".join(k) for k in seen]),
":".join(key)
)
)
else:
seen.append(key)
......@@ -789,8 +812,7 @@ class Options(UserDict.DictMixin):
v = self.get(key)
if v is None:
raise MissingOption("Missing option: %s:%s"
% (self.name, key))
raise MissingOption("Missing option: %s:%s" % (self.name, key))
return v
def __setitem__(self, option, value):
......@@ -954,10 +976,39 @@ def _recipe(options):
return recipe, entry
def _doing():
_, v, tb = sys.exc_info()
message = str(v)
doing = []
while tb is not None:
d = tb.tb_frame.f_locals.get('__doing__')
if d:
doing.append(d)
tb = tb.tb_next
if doing:
sys.stderr.write('While:\n')
for d in doing:
if not isinstance(d, str):
d = d[0] % d[1:]
sys.stderr.write(' %s\n' % d)
def _error(*message):
sys.stderr.write('Error: ' + ' '.join(message) +'\n')
sys.exit(1)
_internal_error_template = """
An internal error occured due to a bug in either zc.buildout or in a
recipe being used:
%s:
%s
"""
def _internal_error(v):
sys.stderr.write(_internal_error_template % (v.__class__.__name__, v))
_usage = """\
Usage: buildout [options] [assignments] [command [command arguments]]
......@@ -1011,6 +1062,12 @@ Options:
new distributions if installed distributions satisfy it's
requirements.
-D
Debug errors. If an error occurs, then the post-mortem debugger
will be started. This is especially useful for debuging recipe
problems.
Assignments are of the form: section:option=value and are used to
provide configuration options that override those given in the
configuration file. For example, to run the buildout in offline mode,
......@@ -1046,11 +1103,12 @@ def main(args=None):
options = []
windows_restart = False
user_defaults = True
debug = False
while args:
if args[0][0] == '-':
op = orig_op = args.pop(0)
op = op[1:]
while op and op[0] in 'vqhWUoOnN':
while op and op[0] in 'vqhWUoOnND':
if op[0] == 'v':
verbosity += 10
elif op[0] == 'q':
......@@ -1067,6 +1125,8 @@ def main(args=None):
options.append(('buildout', 'newest', 'true'))
elif op[0] == 'N':
options.append(('buildout', 'newest', 'false'))
elif op[0] == 'D':
debug = True
else:
_help()
op = op[1:]
......@@ -1110,8 +1170,21 @@ def main(args=None):
buildout = Buildout(config_file, options,
user_defaults, windows_restart)
getattr(buildout, command)(args)
except zc.buildout.UserError, v:
_error(str(v))
except SystemExit:
pass
except Exception, v:
_doing()
if debug:
exc_info = sys.exc_info()
import pdb, traceback
traceback.print_exception(*exc_info)
sys.stderr.write('\nStarting pdb:\n')
pdb.post_mortem(exc_info[2])
else:
if isinstance(v, zc.buildout.UserError):
_error(str(v))
else:
_internal_error(v)
finally:
logging.shutdown()
......
src/zc/buildout/buildout.txt
View file @ 3dcc234b
......@@ -120,9 +120,7 @@ and then we'll create a source file for our mkdir recipe:
... class Mkdir:
...
... def __init__(self, buildout, name, options):
... self.buildout = buildout
... self.name = name
... self.options = options
... self.name, self.options = name, options
... options['path'] = os.path.join(
... buildout['buildout']['directory'],
... options['path'],
......@@ -367,10 +365,10 @@ Error reporting
If a user makes an error, an error needs to be printed and work needs
to stop. This is accomplished by logging a detailed error message and
then raising a (or an instance of a subclass of a)
zc.buildout.UserError exception. Raising UserError causes the
buildout to print the error and exit without printing a traceback. In
the sample above, of someone gives a non-existant directory to create
the directory in:
zc.buildout.UserError exception. Raising an error other than a
UserError still displays the error, but labels it as a bug in the
buildout software or recipe. In the sample above, of someone gives a
non-existant directory to create the directory in:
>>> write(sample_buildout, 'buildout.cfg',
......@@ -389,6 +387,10 @@ We'll get a user error, not a traceback.
>>> print system(buildout),
buildout: Develop: /sample-buildout/recipes
data-dir: Cannot create /xxx/mydata. /xxx is not a directory.
While:
Installing
Getting section data-dir
Initializing part data-dir
Error: Invalid Path
......@@ -813,6 +815,8 @@ buildout.
... """)
>>> print system(buildout + ' -c ' + server_url + '/remote.cfg'),
While:
Initializing
Error: Missing option: buildout:directory
Normally, the buildout directory defaults to directory
......
src/zc/buildout/debugging.txt 0 → 100644
View file @ 3dcc234b
Debugging buildouts
===================
Buildouts can be pretty complex. When things go wrong, it isn't
always obvious why. Errors can occur due to problems in user input or
due to bugs in zc.buildout or recipes. When an error occurs, Python's
post-mortem debugger can be used to inspect the state of the buildout
or recipe code were there error occured. To enable this, use the -D
option to the buildout. Let's create a recipe that has a bug:
>>> mkdir(sample_buildout, 'recipes')
>>> write(sample_buildout, 'recipes', 'mkdir.py',
... """
... import os, zc.buildout
...
... class Mkdir:
...
... def __init__(self, buildout, name, options):
... self.name, self.options = name, options
... options['path'] = os.path.join(
... buildout['buildout']['directory'],
... options['path'],
... )
...
... def install(self):
... directory = self.options['directory']
... os.mkdir(directory)
... return directory
...
... def update(self):
... pass
... """)
>>> write(sample_buildout, 'recipes', 'setup.py',
... """
... from setuptools import setup
...
... setup(name = "recipes",
... entry_points = {'zc.buildout': ['mkdir = mkdir:Mkdir']},
... )
... """)
And create a buildout that uses it:
>>> write(sample_buildout, 'buildout.cfg',
... """
... [buildout]
... develop = recipes
... parts = data-dir
...
... [data-dir]
... recipe = recipes:mkdir
... path = mystuff
... """)
If we run the buildout, we'll get an error:
>>> print system(buildout),
buildout: Develop: /sample-buildout/recipes
buildout: Installing data-dir
While:
Installing data-dir
Error: Missing option: data-dir:directory
If we want to debug the error, we can add the -D option. Here's we'll
supply some input:
>>> print system(buildout+" -D", """\
... up
... p self.options.keys()
... q
... """),
buildout: Develop: /tmp/tmpozk_tH/_TEST_/sample-buildout/recipes
buildout: Installing data-dir
While:
Installing data-dir
Traceback (most recent call last):
File "/zc/buildout/buildout.py", line 1173, in main
getattr(buildout, command)(args)
File "/zc/buildout/buildout.py", line 324, in install
installed_files = recipe.install()
File "/sample-buildout/recipes/mkdir.py", line 14, in install
directory = self.options['directory']
File "/zc/buildout/buildout.py", line 815, in __getitem__
raise MissingOption("Missing option: %s:%s" % (self.name, key))
MissingOption: Missing option: data-dir:directory
<BLANKLINE>
Starting pdb:
> /zc/buildout/buildout.py(815)__getitem__()
-> raise MissingOption("Missing option: %s:%s" % (self.name, key))
(Pdb) > /sample-buildout/recipes/mkdir.py(14)install()
-> directory = self.options['directory']
(Pdb) ['path', 'recipe']
(Pdb)
src/zc/buildout/tests.py
View file @ 3dcc234b
......@@ -116,9 +116,15 @@ It is an error to create a variable-reference cycle:
>>> print system(os.path.join(sample_buildout, 'bin', 'buildout')),
... # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
While:
Initializing
Getting section buildout
Initializing section buildout
Getting option buildout:y
Getting option buildout:z
Getting option buildout:x
Getting option buildout:y
Error: Circular reference in substitutions.
We're evaluating buildout:y, buildout:z, buildout:x
and are referencing: buildout:y.
It is an error to use funny characters in variable refereces:
......@@ -131,6 +137,11 @@ It is an error to use funny characters in variable refereces:
... ''')
>>> print system(os.path.join(sample_buildout, 'bin', 'buildout')),
While:
Initializing
Getting section buildout
Initializing section buildout
Getting option buildout:x
Error: The section name in substitution, ${bui$ldout:y},
has invalid characters.
......@@ -143,6 +154,11 @@ It is an error to use funny characters in variable refereces:
... ''')
>>> print system(os.path.join(sample_buildout, 'bin', 'buildout')),
While:
Initializing
Getting section buildout
Initializing section buildout
Getting option buildout:x
Error: The option name in substitution, ${buildout:y{z},
has invalid characters.
......@@ -157,6 +173,11 @@ and too have too many or too few colons:
... ''')
>>> print system(os.path.join(sample_buildout, 'bin', 'buildout')),
While:
Initializing
Getting section buildout
Initializing section buildout
Getting option buildout:x
Error: The substitution, ${parts},
doesn't contain a colon.
......@@ -169,6 +190,11 @@ and too have too many or too few colons:
... ''')
>>> print system(os.path.join(sample_buildout, 'bin', 'buildout')),
While:
Initializing
Getting section buildout
Initializing section buildout
Getting option buildout:x
Error: The substitution, ${buildout:y:z},
has too many colons.
......@@ -181,6 +207,9 @@ Al parts have to have a section:
... ''')
>>> print system(os.path.join(sample_buildout, 'bin', 'buildout')),
While:
Installing
Getting section x
Error: The referenced section, 'x', was not defined.
and all parts have to have a specified recipe:
......@@ -196,6 +225,8 @@ and all parts have to have a specified recipe:
... ''')
>>> print system(os.path.join(sample_buildout, 'bin', 'buildout')),
While:
Installing
Error: Missing option: x:recipe
"""
......@@ -477,6 +508,12 @@ Options:
buildout:newest=false. With this setting, buildout will not seek
new distributions if installed distributions satisfy it's
requirements.
<BLANKLINE>
-D
<BLANKLINE>
Debug errors. If an error occurs, then the post-mortem debugger
will be started. This is especially useful for debuging recipe
problems.
<BLANKLINE>
Assignments are of the form: section:option=value and are used to
provide configuration options that override those given in the
......@@ -554,6 +591,12 @@ Options:
buildout:newest=false. With this setting, buildout will not seek
new distributions if installed distributions satisfy it's
requirements.
<BLANKLINE>
-D
<BLANKLINE>
Debug errors. If an error occurs, then the post-mortem debugger
will be started. This is especially useful for debuging recipe
problems.
<BLANKLINE>
Assignments are of the form: section:option=value and are used to
provide configuration options that override those given in the
......@@ -1233,6 +1276,55 @@ def log_when_there_are_not_local_distros():
"""
def internal_errors():
"""Internal errors are clearly marked and don't generate tracebacks:
>>> mkdir(sample_buildout, 'recipes')
>>> write(sample_buildout, 'recipes', 'mkdir.py',
... '''
... class Mkdir:
... def __init__(self, buildout, name, options):
... self.name, self.options = name, options
... options['path'] = os.path.join(
... buildout['buildout']['directory'],
... options['path'],
... )
... ''')
>>> write(sample_buildout, 'recipes', 'setup.py',
... '''
... from setuptools import setup
... setup(name = "recipes",
... entry_points = {'zc.buildout': ['mkdir = mkdir:Mkdir']},
... )
... ''')
>>> write(sample_buildout, 'buildout.cfg',
... '''
... [buildout]
... develop = recipes
... parts = data-dir
...
... [data-dir]
... recipe = recipes:mkdir
... ''')
>>> print system(buildout),
buildout: Develop: /sample-buildout/recipes
While:
Installing
Getting section data-dir
Initializing part data-dir
<BLANKLINE>
An internal error occured due to a bug in either zc.buildout or in a
recipe being used:
<BLANKLINE>
NameError:
global name 'os' is not defined
"""
######################################################################
def create_sample_eggs(test, executable=sys.executable):
......@@ -1440,6 +1532,17 @@ def test_suite():
'picked \\1 = V.V'),
])
),
doctest.DocFileSuite(
'debugging.txt',
setUp=zc.buildout.testing.buildoutSetUp,
tearDown=zc.buildout.testing.buildoutTearDown,
checker=renormalizing.RENormalizing([
zc.buildout.testing.normalize_path,
(re.compile(r'\S+buildout.py'), 'buildout.py'),
(re.compile(r'line \d+'), 'line NNN'),
(re.compile(r'py\(\d+\)'), 'py(NNN)'),
])
),
doctest.DocFileSuite(
'update.txt',
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7