From 8a087b51b5d5263df46a4d7d46c842f86783808e Mon Sep 17 00:00:00 2001 From: Jim Fulton <jim@zope.com> Date: Sat, 12 May 2012 15:26:28 -0400 Subject: [PATCH] normalize the way default options are handled and documented Cherry-picked from trunk. Thanks Gary. --- src/zc/buildout/buildout.py | 76 +++--- src/zc/buildout/buildout.txt | 325 +++++++++++++++--------- src/zc/buildout/tests.py | 5 + zc.recipe.egg_/src/zc/recipe/egg/egg.py | 14 +- 4 files changed, 259 insertions(+), 161 deletions(-) diff --git a/src/zc/buildout/buildout.py b/src/zc/buildout/buildout.py index 4b84fe2..76686f5 100644 --- a/src/zc/buildout/buildout.py +++ b/src/zc/buildout/buildout.py @@ -108,13 +108,24 @@ def _unannotate(data): return data _buildout_default_options = _annotate_section({ - 'eggs-directory': 'eggs', - 'develop-eggs-directory': 'develop-eggs', + 'allow-hosts': '*', + 'allow-picked-versions': 'true', 'bin-directory': 'bin', - 'parts-directory': 'parts', + 'develop-eggs-directory': 'develop-eggs', + 'eggs-directory': 'eggs', + 'executable': sys.executable, + 'find-links': '', + 'install-from-cache': 'false', 'installed': '.installed.cfg', - 'log-level': 'INFO', 'log-format': '', + 'log-level': 'INFO', + 'newest': 'true', + 'offline': 'false', + 'parts-directory': 'parts', + 'prefer-final': 'false', + 'python': 'buildout', + 'unzip': 'false', + 'use-dependency-links': 'true', }, 'DEFAULT_VALUE') # _buildout_version and _buildout_1_4_default_versions are part of a @@ -201,7 +212,7 @@ class Buildout(UserDict.DictMixin): # provide some defaults before options are parsed # because while parsing options those attributes might be # used already (Gottfried Ganssauge) - buildout_section = data.get('buildout') + buildout_section = data['buildout'] # Try to make sure we have absolute paths for standard # directories. We do this before doing substitutions, in case @@ -214,22 +225,28 @@ class Buildout(UserDict.DictMixin): d = self._buildout_path(buildout_section[name+'-directory']) buildout_section[name+'-directory'] = d - links = buildout_section and buildout_section.get('find-links', '') + # Attributes on this buildout object shouldn't be used by + # recipes in their __init__. It can cause bugs, because the + # recipes will be instantiated below (``options = self['buildout']``) + # before this has completed initializing. These attributes are + # left behind for legacy support but recipe authors should + # beware of using them. A better practice is for a recipe to + # use the buildout['buildout'] options. + links = buildout_section['find-links'] self._links = links and links.split() or () - - allow_hosts = buildout_section and buildout_section.get( - 'allow-hosts', '*').split('\n') + allow_hosts = buildout_section['allow-hosts'].split('\n') self._allow_hosts = tuple([host.strip() for host in allow_hosts if host.strip() != '']) - self._logger = logging.getLogger('zc.buildout') - self.offline = False - self.newest = True + self.offline = (buildout_section['offline'] == 'true') + self.newest = (buildout_section['newest'] == 'true') ################################################################## ## WARNING!!! ## ALL ATTRIBUTES MUST HAVE REASONABLE DEFAULTS AT THIS POINT - ## OTHERWISE ATTRIBUTEERRORS MIGHT HAPPEN ANY TIME + ## OTHERWISE ATTRIBUTEERRORS MIGHT HAPPEN ANY TIME FROM RECIPES. + ## RECIPES SHOULD GENERALLY USE buildout['buildout'] OPTIONS, NOT + ## BUILDOUT ATTRIBUTES. ################################################################## # initialize some attrs and buildout directories. options = self['buildout'] @@ -238,7 +255,7 @@ class Buildout(UserDict.DictMixin): links = options.get('find-links', '') self._links = links and links.split() or () - allow_hosts = options.get('allow-hosts', '*').split('\n') + allow_hosts = options['allow-hosts'].split('\n') self._allow_hosts = tuple([host.strip() for host in allow_hosts if host.strip() != '']) @@ -256,20 +273,18 @@ class Buildout(UserDict.DictMixin): self._setup_logging() - offline = options.get('offline', 'false') + offline = options['offline'] if offline not in ('true', 'false'): self._error('Invalid value for offline option: %s', offline) - options['offline'] = offline - self.offline = offline == 'true' + self.offline = (offline == 'true') if self.offline: newest = options['newest'] = 'false' else: - newest = options.get('newest', 'true') + newest = options['newest'] if newest not in ('true', 'false'): self._error('Invalid value for newest option: %s', newest) - options['newest'] = newest - self.newest = newest == 'true' + self.newest = (newest == 'true') # This is a hacked version of zc.buildout for 1.4.4. # This means that buildout uses the defaults set up above. The point @@ -282,25 +297,25 @@ class Buildout(UserDict.DictMixin): versions.update(dict(self[versions_section])) zc.buildout.easy_install.default_versions(versions) - prefer_final = options.get('prefer-final', 'false') + prefer_final = options['prefer-final'] if prefer_final not in ('true', 'false'): self._error('Invalid value for prefer-final option: %s', prefer_final) zc.buildout.easy_install.prefer_final(prefer_final=='true') - use_dependency_links = options.get('use-dependency-links', 'true') + use_dependency_links = options['use-dependency-links'] if use_dependency_links not in ('true', 'false'): self._error('Invalid value for use-dependency-links option: %s', use_dependency_links) zc.buildout.easy_install.use_dependency_links( use_dependency_links == 'true') - allow_picked_versions = options.get('allow-picked-versions', 'true') + allow_picked_versions = options['allow-picked-versions'] if allow_picked_versions not in ('true', 'false'): self._error('Invalid value for allow-picked-versions option: %s', allow_picked_versions) zc.buildout.easy_install.allow_picked_versions( - allow_picked_versions=='true') + allow_picked_versions == 'true') download_cache = options.get('download-cache') if download_cache: @@ -317,13 +332,12 @@ class Buildout(UserDict.DictMixin): zc.buildout.easy_install.download_cache(download_cache) - install_from_cache = options.get('install-from-cache') - if install_from_cache: - if install_from_cache not in ('true', 'false'): - self._error('Invalid value for install-from-cache option: %s', - install_from_cache) - if install_from_cache == 'true': - zc.buildout.easy_install.install_from_cache(True) + install_from_cache = options['install-from-cache'] + if install_from_cache not in ('true', 'false'): + self._error('Invalid value for install-from-cache option: %s', + install_from_cache) + zc.buildout.easy_install.install_from_cache( + install_from_cache=='true') # "Use" each of the defaults so they aren't reported as unused options. for name in _buildout_default_options: diff --git a/src/zc/buildout/buildout.txt b/src/zc/buildout/buildout.txt index 0bc80c1..a66e75b 100644 --- a/src/zc/buildout/buildout.txt +++ b/src/zc/buildout/buildout.txt @@ -504,7 +504,7 @@ Let's fix the recipe: >>> write(sample_buildout, 'recipes', 'mkdir.py', ... """ - ... import logging, os, zc.buildout + ... import logging, os, zc.buildout, sys ... ... class Mkdir: ... @@ -533,13 +533,15 @@ Let's fix the recipe: ... 'Creating directory %s', os.path.basename(path)) ... os.mkdir(path) ... created.append(path) - ... except: + ... except Exception: ... for d in created: ... os.rmdir(d) ... assert not os.path.exists(d) ... logging.getLogger(self.name).info( ... 'Removed %s due to error', ... os.path.basename(d)) + ... sys.stderr.flush() + ... sys.stdout.flush() ... raise ... ... return paths @@ -581,7 +583,7 @@ When we rerun the buildout: .. Wait for the file to really disappear. My linux is weird. >>> wait_until("foo goes away", lambda : not os.path.exists('foo'), - ... timeout=100) + ... timeout=200) we get the same error, but we don't get the directory left behind: @@ -729,6 +731,10 @@ COMMAND_LINE_VALUE). ================== <BLANKLINE> [buildout] + allow-hosts= * + DEFAULT_VALUE + allow-picked-versions= true + DEFAULT_VALUE bin-directory= bin DEFAULT_VALUE develop= recipes @@ -739,16 +745,34 @@ COMMAND_LINE_VALUE). COMPUTED_VALUE eggs-directory= eggs DEFAULT_VALUE + executable= ... + DEFAULT_VALUE + find-links= + DEFAULT_VALUE + install-from-cache= false + DEFAULT_VALUE installed= .installed.cfg DEFAULT_VALUE log-format= DEFAULT_VALUE log-level= INFO DEFAULT_VALUE + newest= true + DEFAULT_VALUE + offline= false + DEFAULT_VALUE parts= data-dir /sample-buildout/buildout.cfg parts-directory= parts DEFAULT_VALUE + prefer-final= false + DEFAULT_VALUE + python= buildout + DEFAULT_VALUE + unzip= false + DEFAULT_VALUE + use-dependency-links= true + DEFAULT_VALUE <BLANKLINE> [data-dir] path= foo bins @@ -2201,10 +2225,15 @@ database is shown. <BLANKLINE> Configuration data: [buildout] + allow-hosts = * + allow-picked-versions = true bin-directory = /sample-buildout/bin develop-eggs-directory = /sample-buildout/develop-eggs directory = /sample-buildout eggs-directory = /sample-buildout/eggs + executable = python + find-links = + install-from-cache = false installed = /sample-buildout/.installed.cfg log-format = log-level = INFO @@ -2212,6 +2241,10 @@ database is shown. offline = false parts = parts-directory = /sample-buildout/parts + prefer-final = false + python = buildout + unzip = false + use-dependency-links = true verbosity = 20 <BLANKLINE> @@ -2219,6 +2252,37 @@ All of these options can be overridden by configuration files or by command-line assignments. We've discussed most of these options already, but let's review them and touch on some we haven't discussed: +allow-hosts + On some environments the links visited by `zc.buildout` can be forbidden by + paranoid firewalls. These URLs might be in the chain of links visited by + `zc.buildout` as defined by buildout's `find-links` option, or as defined + by various eggs in their `url`, `download_url`, `dependency_links` metadata. + + The fact that package_index works like a spider and might visit links and + go to other locations makes this even harder. + + The `allow-hosts` option provides a way to prevent this, and + works exactly like the one provided in `easy_install`. + + You can provide a list of allowed host, together with wildcards:: + + [buildout] + ... + + allow-hosts = + *.python.org + example.com + + All URLs that does not match these hosts will not be visited. + +allow-picked-versions + By default, the buildout will choose the best match for a given requirement + if the requirement is not specified precisely (for instance, using the + "versions" option. This behavior corresponds to the + "allow-picked-versions" being set to its default value, "true". If + "allow-picked-versions" is "false," instead of picking the best match, + buildout will raise an error. This helps enforce repeatability. + bin-directory The directory path where scripts are written. This can be a relative path, which is interpreted relative to the directory @@ -2239,6 +2303,51 @@ eggs-directory *never* be modified. This can be a relative path, which is interpreted relative to the directory option. +executable + The Python executable used to run the buildout. See the python + option below. + +find-links + You can specify more locations to search for distributions using the + `find-links` option. All locations specified will be searched for + distributions along with the package index as described before. + + Locations can be urls:: + + [buildout] + ... + find-links = http://download.zope.org/distribution/ + + They can also be directories on disk:: + + [buildout] + ... + find-links = /some/path + + Finally, they can also be direct paths to distributions:: + + [buildout] + ... + find-links = /some/path/someegg-1.0.0-py2.3.egg + + Any number of locations can be specified in the `find-links` option:: + + [buildout] + ... + find-links = + http://download.zope.org/distribution/ + /some/otherpath + /some/path/someegg-1.0.0-py2.3.egg + +install-from-cache + A download cache can be used as the basis of application source releases. + In an application source release, we want to distribute an application that + can be built without making any network accesses. In this case, we + distribute a buildout with download cache and tell the buildout to install + from the download cache only, without making network accesses. The + buildout install-from-cache option can be used to signal that packages + should be installed only from the download cache. + installed The file path where information about the results of the previous buildout run is written. This can be a relative path, which is @@ -2252,12 +2361,101 @@ log-format log-level The log level before verbosity adjustment +newest + By default buildout and recipes will try to find the newest versions of + distributions needed to satisfy requirements. This can be very time + consuming, especially when incrementally working on setting up a buildout + or working on a recipe. The buildout "newest" option can be used to to + suppress this. If the "newest" option is set to false, then new + distributions won't be sought if an installed distribution meets + requirements. The "newest" option can also be set to false using the -N + command-line option. See also the "offline" option. + +offline + The "offline" option goes a bit further than the "newest" option. If the + buildout "offline" option is given a value of "true", the buildout and + recipes that are aware of the option will avoid doing network access. This + is handy when running the buildout when not connected to the internet. It + also makes buildouts run much faster. This option is typically set using + the buildout -o option. + parts A white space separated list of parts to be installed. parts-directory A working directory that parts can used to store data. +prefer-final + Currently, when searching for new releases, the newest available + release is used. This isn't usually ideal, as you may get a + development release or alpha releases not ready to be widely used. + You can request that final releases be preferred using the prefer + final option in the buildout section:: + + [buildout] + ... + prefer-final = true + + When the prefer-final option is set to true, then when searching for + new releases, final releases are preferred. If there are final + releases that satisfy distribution requirements, then those releases + are used even if newer non-final releases are available. The buildout + prefer-final option can be used to override this behavior. + + In buildout version 2, final releases will be preferred by default. + You will then need to use a false value for prefer-final to get the + newest releases. + +python + The name of a section containing information about the default + Python interpreter. Recipes that need a installation + typically have options to tell them which Python installation to + use. By convention, if a section-specific option isn't used, the + option is looked for in the buildout section. The option must + point to a section with an executable option giving the path to a + Python executable. By default, the buildout section defines the + default Python as the Python used to run the buildout. + +unzip + By default, zc.buildout doesn't unzip zip-safe eggs ("unzip = false"). + This follows the policy followed by setuptools itself. Experience shows + this policy to to be inconvenient. Zipped eggs make debugging more + difficult and often import more slowly. You can include an unzip option in + the buildout section to change the default unzipping policy ("unzip = + true"). + +use-dependency-links + By default buildout will obey the setuptools dependency_links metadata + when it looks for dependencies. This behavior can be controlled with + the use-dependency-links buildout option:: + + [buildout] + ... + use-dependency-links = false + + The option defaults to true. If you set it to false, then dependency + links are only looked for in the locations specified by find-links. + +unzip + By default, zc.buildout doesn't unzip zip-safe eggs ("unzip = false"). + This follows the policy followed by setuptools itself. Experience shows + this policy to to be inconvenient. Zipped eggs make debugging more + difficult and often import more slowly. You can include an unzip option in + the buildout section to change the default unzipping policy ("unzip = + true"). + +use-dependency-links + By default buildout will obey the setuptools dependency_links metadata + when it looks for dependencies. This behavior can be controlled with + the use-dependency-links buildout option:: + + [buildout] + ... + use-dependency-links = false + + The option defaults to true. If you set it to false, then dependency + links are only looked for in the locations specified by find-links. + verbosity A log-level adjustment. Typically, this is set via the -q and -v command-line options. @@ -2336,48 +2534,6 @@ if there isn't a configuration file: Generated script '/sample-bootstrapped2/bin/buildout'. -Newest and Offline Modes ------------------------- - -By default buildout and recipes will try to find the newest versions -of distributions needed to satisfy requirements. This can be very -time consuming, especially when incrementally working on setting up a -buildout or working on a recipe. The buildout newest option can be -used to to suppress this. If the newest option is set to false, then -new distributions won't be sought if an installed distribution meets -requirements. The newest option can be set to false using the -N -command-line option. - -The offline option goes a bit further. If the buildout offline option -is given a value of "true", the buildout and recipes that are aware of -the option will avoid doing network access. This is handy when -running the buildout when not connected to the internet. It also -makes buildouts run much faster. This option is typically set using -the buildout -o option. - -Preferring Final Releases -------------------------- - -Currently, when searching for new releases, the newest available -release is used. This isn't usually ideal, as you may get a -development release or alpha releases not ready to be widely used. -You can request that final releases be preferred using the prefer -final option in the buildout section:: - - [buildout] - ... - prefer-final = true - -When the prefer-final option is set to true, then when searching for -new releases, final releases are preferred. If there are final -releases that satisfy distribution requirements, then those releases -are used even if newer non-final releases are available. The buildout -prefer-final option can be used to override this behavior. - -In buildout version 2, final releases will be preferred by default. -You will then need to use a false value for prefer-final to get the -newest releases. - Finding distributions --------------------- @@ -2396,49 +2552,7 @@ distributions. The latest version of the distribution that meets the requirements of the buildout will always be used. You can also specify more locations to search for distributions using -the `find-links` option. All locations specified will be searched for -distributions along with the package index as described before. - -Locations can be urls:: - - [buildout] - ... - find-links = http://download.zope.org/distribution/ - -They can also be directories on disk:: - - [buildout] - ... - find-links = /some/path - -Finally, they can also be direct paths to distributions:: - - [buildout] - ... - find-links = /some/path/someegg-1.0.0-py2.3.egg - -Any number of locations can be specified in the `find-links` option:: - - [buildout] - ... - find-links = - http://download.zope.org/distribution/ - /some/otherpath - /some/path/someegg-1.0.0-py2.3.egg - -Dependency links ----------------- - -By default buildout will obey the setuptools dependency_links metadata -when it looks for dependencies. This behavior can be controlled with -the use-dependency-links buildout option:: - - [buildout] - ... - use-dependency-links = false - -The option defaults to true. If you set it to false, then dependency -links are only looked for in the locations specified by find-links. +the `find-links` option. See its description above. Controlling the installation database ------------------------------------- @@ -2599,38 +2713,3 @@ We see that our extension is loaded and executed: ext ['buildout'] Develop: '/sample-bootstrapped/demo' unload ['buildout'] - -Allow hosts ------------ - -On some environments the links visited by `zc.buildout` can be forbidden -by paranoiac firewalls. These URL might be on the chain of links -visited by `zc.buildout` wheter they are defined in the `find-links` option, -wheter they are defined by various eggs in their `url`, `download_url`, -`dependency_links` metadata. - -It is even harder to track that package_index works like a spider and -might visit links and go to other location. - -The `allow-hosts` option provides a way to prevent this, and -works exactly like the one provided in `easy_install`. - -You can provide a list of allowed host, together with wildcards:: - - [buildout] - ... - - allow-hosts = - *.python.org - example.com - -All urls that does not match these hosts will not be visited. - -.. [#future_recipe_methods] In the future, additional methods may be - added. Older recipes with fewer methods will still be - supported. - -.. [#packaging_info] If we wanted to create a distribution from this - package, we would need specify much more information. See the - `setuptools documentation - <http://peak.telecommunity.com/DevCenter/setuptools>`_. diff --git a/src/zc/buildout/tests.py b/src/zc/buildout/tests.py index 8c49a6d..11d8294 100644 --- a/src/zc/buildout/tests.py +++ b/src/zc/buildout/tests.py @@ -2851,6 +2851,8 @@ def test_suite(): r'when that file already exists: '), '[Errno 17] File exists: ' ), + (re.compile('executable = %s' % re.escape(sys.executable)), + 'executable = python'), ]) ), doctest.DocFileSuite( @@ -2947,6 +2949,9 @@ def test_suite(): '-q develop -mxN -d /sample-buildout/develop-eggs' ), (re.compile(r'^[*]...'), '...'), + # for bug_92891_bootstrap_crashes_with_egg_recipe_in_buildout_section + (re.compile(r"Unused options for buildout: 'eggs' 'scripts'\."), + "Unused options for buildout: 'scripts' 'eggs'."), ]), ), zc.buildout.rmtree.test_suite(), diff --git a/zc.recipe.egg_/src/zc/recipe/egg/egg.py b/zc.recipe.egg_/src/zc/recipe/egg/egg.py index 393e7a2..34cc4c8 100644 --- a/zc.recipe.egg_/src/zc/recipe/egg/egg.py +++ b/zc.recipe.egg_/src/zc/recipe/egg/egg.py @@ -27,8 +27,8 @@ class Eggs(object): self.buildout = buildout self.name = name self.options = options - links = options.get('find-links', - buildout['buildout'].get('find-links')) + b_options = buildout['buildout'] + links = options.get('find-links', b_options['find-links']) if links: links = links.split() options['find-links'] = '\n'.join(links) @@ -36,20 +36,19 @@ class Eggs(object): links = () self.links = links - index = options.get('index', buildout['buildout'].get('index')) + index = options.get('index', b_options.get('index')) if index is not None: options['index'] = index self.index = index - allow_hosts = buildout['buildout'].get('allow-hosts', '*') + allow_hosts = b_options['allow-hosts'] allow_hosts = tuple([host.strip() for host in allow_hosts.split('\n') if host.strip()!='']) self.allow_hosts = allow_hosts - options['eggs-directory'] = buildout['buildout']['eggs-directory'] + options['eggs-directory'] = b_options['eggs-directory'] options['_e'] = options['eggs-directory'] # backward compat. - options['develop-eggs-directory' - ] = buildout['buildout']['develop-eggs-directory'] + options['develop-eggs-directory'] = b_options['develop-eggs-directory'] options['_d'] = options['develop-eggs-directory'] # backward compat. def working_set(self, extra=()): @@ -58,6 +57,7 @@ class Eggs(object): This is intended for reuse by similar recipes. """ options = self.options + b_options = self.buildout['buildout'] # Backward compat. :( options['executable'] = sys.executable -- 2.30.9