normalize the way default options are handled and documented

Cherry-picked from trunk. Thanks Gary.

src/zc/buildout/buildout.py

@@ -108,13 +108,24 @@ def _unannotate(data):
     return data
 
 _buildout_default_options = _annotate_section({
-    'eggs-directory': 'eggs',
-    'develop-eggs-directory': 'develop-eggs',
+    'allow-hosts': '*',
+    'allow-picked-versions': 'true',
     'bin-directory': 'bin',
-    'parts-directory': 'parts',
+    'develop-eggs-directory': 'develop-eggs',
+    'eggs-directory': 'eggs',
+    'executable': sys.executable,
+    'find-links': '',
+    'install-from-cache': 'false',
     'installed': '.installed.cfg',
-    'log-level': 'INFO',
     'log-format': '',
+    'log-level': 'INFO',
+    'newest': 'true',
+    'offline': 'false',
+    'parts-directory': 'parts',
+    'prefer-final': 'false',
+    'python': 'buildout',
+    'unzip': 'false',
+    'use-dependency-links': 'true',
     }, 'DEFAULT_VALUE')
 
 # _buildout_version and _buildout_1_4_default_versions are part of a
@@ -201,7 +212,7 @@ class Buildout(UserDict.DictMixin):
         # provide some defaults before options are parsed
         # because while parsing options those attributes might be
         # used already (Gottfried Ganssauge)
-        buildout_section = data.get('buildout')
+        buildout_section = data['buildout']
 
         # Try to make sure we have absolute paths for standard
         # directories. We do this before doing substitutions, in case
@@ -214,22 +225,28 @@ class Buildout(UserDict.DictMixin):
                 d = self._buildout_path(buildout_section[name+'-directory'])
                 buildout_section[name+'-directory'] = d
 
-        links = buildout_section and buildout_section.get('find-links', '')
+        # Attributes on this buildout object shouldn't be used by
+        # recipes in their __init__.  It can cause bugs, because the
+        # recipes will be instantiated below (``options = self['buildout']``)
+        # before this has completed initializing.  These attributes are
+        # left behind for legacy support but recipe authors should
+        # beware of using them.  A better practice is for a recipe to
+        # use the buildout['buildout'] options.
+        links = buildout_section['find-links']
         self._links = links and links.split() or ()
-
-        allow_hosts = buildout_section and buildout_section.get(
-             'allow-hosts', '*').split('\n')
+        allow_hosts = buildout_section['allow-hosts'].split('\n')
         self._allow_hosts = tuple([host.strip() for host in allow_hosts
                                    if host.strip() != ''])
-
         self._logger = logging.getLogger('zc.buildout')
-        self.offline = False
-        self.newest = True
+        self.offline = (buildout_section['offline'] == 'true')
+        self.newest = (buildout_section['newest'] == 'true')
 
         ##################################################################
         ## WARNING!!!
         ## ALL ATTRIBUTES MUST HAVE REASONABLE DEFAULTS AT THIS POINT
-        ## OTHERWISE ATTRIBUTEERRORS MIGHT HAPPEN ANY TIME
+        ## OTHERWISE ATTRIBUTEERRORS MIGHT HAPPEN ANY TIME FROM RECIPES.
+        ## RECIPES SHOULD GENERALLY USE buildout['buildout'] OPTIONS, NOT
+        ## BUILDOUT ATTRIBUTES.
         ##################################################################
         # initialize some attrs and buildout directories.
         options = self['buildout']
@@ -238,7 +255,7 @@ class Buildout(UserDict.DictMixin):
         links = options.get('find-links', '')
         self._links = links and links.split() or ()
 
-        allow_hosts = options.get('allow-hosts', '*').split('\n')
+        allow_hosts = options['allow-hosts'].split('\n')
         self._allow_hosts = tuple([host.strip() for host in allow_hosts
                                    if host.strip() != ''])
 
@@ -256,20 +273,18 @@ class Buildout(UserDict.DictMixin):
 
         self._setup_logging()
 
-        offline = options.get('offline', 'false')
+        offline = options['offline']
         if offline not in ('true', 'false'):
             self._error('Invalid value for offline option: %s', offline)
-        options['offline'] = offline
-        self.offline = offline == 'true'
+        self.offline = (offline == 'true')
 
         if self.offline:
             newest = options['newest'] = 'false'
         else:
-            newest = options.get('newest', 'true')
+            newest = options['newest']
             if newest not in ('true', 'false'):
                 self._error('Invalid value for newest option: %s', newest)
-            options['newest'] = newest
-        self.newest = newest == 'true'
+        self.newest = (newest == 'true')
 
         # This is a hacked version of zc.buildout for 1.4.4.
         # This means that buildout uses the defaults set up above.  The point
@@ -282,25 +297,25 @@ class Buildout(UserDict.DictMixin):
             versions.update(dict(self[versions_section]))
         zc.buildout.easy_install.default_versions(versions)
 
-        prefer_final = options.get('prefer-final', 'false')
+        prefer_final = options['prefer-final']
         if prefer_final not in ('true', 'false'):
             self._error('Invalid value for prefer-final option: %s',
                         prefer_final)
         zc.buildout.easy_install.prefer_final(prefer_final=='true')
 
-        use_dependency_links = options.get('use-dependency-links', 'true')
+        use_dependency_links = options['use-dependency-links']
         if use_dependency_links not in ('true', 'false'):
             self._error('Invalid value for use-dependency-links option: %s',
                         use_dependency_links)
         zc.buildout.easy_install.use_dependency_links(
             use_dependency_links == 'true')
 
-        allow_picked_versions = options.get('allow-picked-versions', 'true')
+        allow_picked_versions = options['allow-picked-versions']
         if allow_picked_versions not in ('true', 'false'):
             self._error('Invalid value for allow-picked-versions option: %s',
                         allow_picked_versions)
         zc.buildout.easy_install.allow_picked_versions(
-            allow_picked_versions=='true')
+            allow_picked_versions == 'true')
 
         download_cache = options.get('download-cache')
         if download_cache:
@@ -317,13 +332,12 @@ class Buildout(UserDict.DictMixin):
 
             zc.buildout.easy_install.download_cache(download_cache)
 
-        install_from_cache = options.get('install-from-cache')
-        if install_from_cache:
+        install_from_cache = options['install-from-cache']
         if install_from_cache not in ('true', 'false'):
             self._error('Invalid value for install-from-cache option: %s',
                         install_from_cache)
-            if install_from_cache == 'true':
-                zc.buildout.easy_install.install_from_cache(True)
+        zc.buildout.easy_install.install_from_cache(
+            install_from_cache=='true')
 
         # "Use" each of the defaults so they aren't reported as unused options.
         for name in _buildout_default_options:

src/zc/buildout/buildout.txt

@@ -504,7 +504,7 @@ Let's fix the recipe:
 
     >>> write(sample_buildout, 'recipes', 'mkdir.py',
     ... """
-    ... import logging, os, zc.buildout
+    ... import logging, os, zc.buildout, sys
     ...
     ... class Mkdir:
     ...
@@ -533,13 +533,15 @@ Let's fix the recipe:
     ...                     'Creating directory %s', os.path.basename(path))
     ...                 os.mkdir(path)
     ...                 created.append(path)
-    ...         except:
+    ...         except Exception:
     ...             for d in created:
     ...                 os.rmdir(d)
     ...                 assert not os.path.exists(d)
     ...                 logging.getLogger(self.name).info(
     ...                     'Removed %s due to error',
     ...                      os.path.basename(d))
+    ...             sys.stderr.flush()
+    ...             sys.stdout.flush()
     ...             raise
     ...
     ...         return paths
@@ -581,7 +583,7 @@ When we rerun the buildout:
 .. Wait for the file to really disappear. My linux is weird.
 
     >>> wait_until("foo goes away", lambda : not os.path.exists('foo'),
-    ...            timeout=100)
+    ...            timeout=200)
 
 we get the same error, but we don't get the directory left behind:
 
@@ -729,6 +731,10 @@ COMMAND_LINE_VALUE).
     ==================
     <BLANKLINE>
     [buildout]
+    allow-hosts= *
+        DEFAULT_VALUE
+    allow-picked-versions= true
+        DEFAULT_VALUE
     bin-directory= bin
         DEFAULT_VALUE
     develop= recipes
@@ -739,16 +745,34 @@ COMMAND_LINE_VALUE).
         COMPUTED_VALUE
     eggs-directory= eggs
         DEFAULT_VALUE
+    executable= ...
+        DEFAULT_VALUE
+    find-links=
+        DEFAULT_VALUE
+    install-from-cache= false
+        DEFAULT_VALUE
     installed= .installed.cfg
         DEFAULT_VALUE
     log-format=
         DEFAULT_VALUE
     log-level= INFO
         DEFAULT_VALUE
+    newest= true
+        DEFAULT_VALUE
+    offline= false
+        DEFAULT_VALUE
     parts= data-dir
         /sample-buildout/buildout.cfg
     parts-directory= parts
         DEFAULT_VALUE
+    prefer-final= false
+        DEFAULT_VALUE
+    python= buildout
+        DEFAULT_VALUE
+    unzip= false
+        DEFAULT_VALUE
+    use-dependency-links= true
+        DEFAULT_VALUE
     <BLANKLINE>
     [data-dir]
     path= foo bins
@@ -2201,10 +2225,15 @@ database is shown.
     <BLANKLINE>
     Configuration data:
     [buildout]
+    allow-hosts = *
+    allow-picked-versions = true
     bin-directory = /sample-buildout/bin
     develop-eggs-directory = /sample-buildout/develop-eggs
     directory = /sample-buildout
     eggs-directory = /sample-buildout/eggs
+    executable = python
+    find-links =
+    install-from-cache = false
     installed = /sample-buildout/.installed.cfg
     log-format =
     log-level = INFO
@@ -2212,6 +2241,10 @@ database is shown.
     offline = false
     parts =
     parts-directory = /sample-buildout/parts
+    prefer-final = false
+    python = buildout
+    unzip = false
+    use-dependency-links = true
     verbosity = 20
     <BLANKLINE>
 
@@ -2219,6 +2252,37 @@ All of these options can be overridden by configuration files or by
 command-line assignments.  We've discussed most of these options
 already, but let's review them and touch on some we haven't discussed:
 
+allow-hosts
+    On some environments the links visited by `zc.buildout` can be forbidden by
+    paranoid firewalls. These URLs might be in the chain of links visited by
+    `zc.buildout` as defined by buildout's `find-links` option, or as defined
+    by various eggs in their `url`, `download_url`, `dependency_links` metadata.
+
+    The fact that package_index works like a spider and might visit links and
+    go to other locations makes this even harder.
+
+    The `allow-hosts` option provides a way to prevent this, and
+    works exactly like the one provided in `easy_install`.
+
+    You can provide a list of allowed host, together with wildcards::
+
+        [buildout]
+        ...
+
+        allow-hosts =
+            *.python.org
+            example.com
+
+    All URLs that does not match these hosts will not be visited.
+
+allow-picked-versions
+    By default, the buildout will choose the best match for a given requirement
+    if the requirement is not specified precisely (for instance, using the
+    "versions" option.  This behavior corresponds to the
+    "allow-picked-versions" being set to its default value, "true".  If
+    "allow-picked-versions" is "false," instead of picking the best match,
+    buildout will raise an error.  This helps enforce repeatability.
+
 bin-directory
    The directory path where scripts are written.  This can be a
    relative path, which is interpreted relative to the directory
@@ -2239,6 +2303,51 @@ eggs-directory
    *never* be modified.  This can be a relative path, which is
    interpreted relative to the directory option.
 
+executable
+   The Python executable used to run the buildout.  See the python
+   option below.
+
+find-links
+    You can specify more locations to search for distributions using the
+    `find-links` option. All locations specified will be searched for
+    distributions along with the package index as described before.
+
+    Locations can be urls::
+
+      [buildout]
+      ...
+      find-links = http://download.zope.org/distribution/
+
+    They can also be directories on disk::
+
+      [buildout]
+      ...
+      find-links = /some/path
+
+    Finally, they can also be direct paths to distributions::
+
+      [buildout]
+      ...
+      find-links = /some/path/someegg-1.0.0-py2.3.egg
+
+    Any number of locations can be specified in the `find-links` option::
+
+      [buildout]
+      ...
+      find-links =
+          http://download.zope.org/distribution/
+          /some/otherpath
+          /some/path/someegg-1.0.0-py2.3.egg
+
+install-from-cache
+    A download cache can be used as the basis of application source releases.
+    In an application source release, we want to distribute an application that
+    can be built without making any network accesses.  In this case, we
+    distribute a buildout with download cache and tell the buildout to install
+    from the download cache only, without making network accesses.  The
+    buildout install-from-cache option can be used to signal that packages
+    should be installed only from the download cache.
+
 installed
    The file path where information about the results of the previous
    buildout run is written.  This can be a relative path, which is
@@ -2252,12 +2361,101 @@ log-format
 log-level
    The log level before verbosity adjustment
 
+newest
+    By default buildout and recipes will try to find the newest versions of
+    distributions needed to satisfy requirements.  This can be very time
+    consuming, especially when incrementally working on setting up a buildout
+    or working on a recipe.  The buildout "newest" option can be used to to
+    suppress this.  If the "newest" option is set to false, then new
+    distributions won't be sought if an installed distribution meets
+    requirements.  The "newest" option can also be set to false using the -N
+    command-line option.  See also the "offline" option.
+
+offline
+    The "offline" option goes a bit further than the "newest" option.  If the
+    buildout "offline" option is given a value of "true", the buildout and
+    recipes that are aware of the option will avoid doing network access.  This
+    is handy when running the buildout when not connected to the internet.  It
+    also makes buildouts run much faster. This option is typically set using
+    the buildout -o option.
+
 parts
    A white space separated list of parts to be installed.
 
 parts-directory
    A working directory that parts can used to store data.
 
+prefer-final
+    Currently, when searching for new releases, the newest available
+    release is used.  This isn't usually ideal, as you may get a
+    development release or alpha releases not ready to be widely used.
+    You can request that final releases be preferred using the prefer
+    final option in the buildout section::
+
+      [buildout]
+      ...
+      prefer-final = true
+
+    When the prefer-final option is set to true, then when searching for
+    new releases, final releases are preferred.  If there are final
+    releases that satisfy distribution requirements, then those releases
+    are used even if newer non-final releases are available.  The buildout
+    prefer-final option can be used to override this behavior.
+
+    In buildout version 2, final releases will be preferred by default.
+    You will then need to use a false value for prefer-final to get the
+    newest releases.
+
+python
+   The name of a section containing information about the default
+   Python interpreter.  Recipes that need a installation
+   typically have options to tell them which Python installation to
+   use.  By convention, if a section-specific option isn't used, the
+   option is looked for in the buildout section.  The option must
+   point to a section with an executable option giving the path to a
+   Python executable.  By default, the buildout section defines the
+   default Python as the Python used to run the buildout.
+
+unzip
+    By default, zc.buildout doesn't unzip zip-safe eggs ("unzip = false").
+    This follows the policy followed by setuptools itself.  Experience shows
+    this policy to to be inconvenient.  Zipped eggs make debugging more
+    difficult and often import more slowly.  You can include an unzip option in
+    the buildout section to change the default unzipping policy ("unzip =
+    true").
+
+use-dependency-links
+    By default buildout will obey the setuptools dependency_links metadata
+    when it looks for dependencies. This behavior can be controlled with
+    the use-dependency-links buildout option::
+
+      [buildout]
+      ...
+      use-dependency-links = false
+
+    The option defaults to true. If you set it to false, then dependency
+    links are only looked for in the locations specified by find-links.
+
+unzip
+    By default, zc.buildout doesn't unzip zip-safe eggs ("unzip = false").
+    This follows the policy followed by setuptools itself.  Experience shows
+    this policy to to be inconvenient.  Zipped eggs make debugging more
+    difficult and often import more slowly.  You can include an unzip option in
+    the buildout section to change the default unzipping policy ("unzip =
+    true").
+
+use-dependency-links
+    By default buildout will obey the setuptools dependency_links metadata
+    when it looks for dependencies. This behavior can be controlled with
+    the use-dependency-links buildout option::
+
+      [buildout]
+      ...
+      use-dependency-links = false
+
+    The option defaults to true. If you set it to false, then dependency
+    links are only looked for in the locations specified by find-links.
+
 verbosity
    A log-level adjustment.  Typically, this is set via the -q and -v
    command-line options.
@@ -2336,48 +2534,6 @@ if there isn't a configuration file:
     Generated script '/sample-bootstrapped2/bin/buildout'.
 
 
-Newest and Offline Modes
-------------------------
-
-By default buildout and recipes will try to find the newest versions
-of distributions needed to satisfy requirements.  This can be very
-time consuming, especially when incrementally working on setting up a
-buildout or working on a recipe.  The buildout newest option can be
-used to to suppress this.  If the newest option is set to false, then
-new distributions won't be sought if an installed distribution meets
-requirements.  The newest option can be set to false using the -N
-command-line option.
-
-The offline option goes a bit further.  If the buildout offline option
-is given a value of "true", the buildout and recipes that are aware of
-the option will avoid doing network access.  This is handy when
-running the buildout when not connected to the internet.  It also
-makes buildouts run much faster. This option is typically set using
-the buildout -o option.
-
-Preferring Final Releases
--------------------------
-
-Currently, when searching for new releases, the newest available
-release is used.  This isn't usually ideal, as you may get a
-development release or alpha releases not ready to be widely used.
-You can request that final releases be preferred using the prefer
-final option in the buildout section::
-
-  [buildout]
-  ...
-  prefer-final = true
-
-When the prefer-final option is set to true, then when searching for
-new releases, final releases are preferred.  If there are final
-releases that satisfy distribution requirements, then those releases
-are used even if newer non-final releases are available.  The buildout
-prefer-final option can be used to override this behavior.
-
-In buildout version 2, final releases will be preferred by default.
-You will then need to use a false value for prefer-final to get the
-newest releases.
-
 Finding distributions
 ---------------------
 
@@ -2396,49 +2552,7 @@ distributions. The latest version of the distribution that meets the
 requirements of the buildout will always be used.
 
 You can also specify more locations to search for distributions using
-the `find-links` option. All locations specified will be searched for
-distributions along with the package index as described before.
-
-Locations can be urls::
-
-  [buildout]
-  ...
-  find-links = http://download.zope.org/distribution/
-
-They can also be directories on disk::
-
-  [buildout]
-  ...
-  find-links = /some/path
-
-Finally, they can also be direct paths to distributions::
-
-  [buildout]
-  ...
-  find-links = /some/path/someegg-1.0.0-py2.3.egg
-
-Any number of locations can be specified in the `find-links` option::
-
-  [buildout]
-  ...
-  find-links =
-      http://download.zope.org/distribution/
-      /some/otherpath
-      /some/path/someegg-1.0.0-py2.3.egg
-
-Dependency links
-----------------
-
-By default buildout will obey the setuptools dependency_links metadata
-when it looks for dependencies. This behavior can be controlled with
-the use-dependency-links buildout option::
-
-  [buildout]
-  ...
-  use-dependency-links = false
-
-The option defaults to true. If you set it to false, then dependency
-links are only looked for in the locations specified by find-links.
+the `find-links` option. See its description above.
 
 Controlling the installation database
 -------------------------------------
@@ -2599,38 +2713,3 @@ We see that our extension is loaded and executed:
     ext ['buildout']
     Develop: '/sample-bootstrapped/demo'
     unload ['buildout']
-
-Allow hosts
------------
-
-On some environments the links visited by `zc.buildout` can be forbidden
-by paranoiac firewalls. These URL might be on the chain of links
-visited by `zc.buildout` wheter they are defined in the `find-links` option,
-wheter they are defined by various eggs in their `url`, `download_url`,
-`dependency_links` metadata.
-
-It is even harder to track that package_index works like a spider and
-might visit links and go to other location.
-
-The `allow-hosts` option provides a way to prevent this, and
-works exactly like the one provided in `easy_install`.
-
-You can provide a list of allowed host, together with wildcards::
-
-    [buildout]
-    ...
-
-    allow-hosts =
-        *.python.org
-        example.com
-
-All urls that does not match these hosts will not be visited.
-
-.. [#future_recipe_methods] In the future, additional methods may be
-       added. Older recipes with fewer methods will still be
-       supported.
-
-.. [#packaging_info] If we wanted to create a distribution from this
-       package, we would need specify much more information.  See the
-       `setuptools documentation
-       <http://peak.telecommunity.com/DevCenter/setuptools>`_.

src/zc/buildout/tests.py

@@ -2851,6 +2851,8 @@ def test_suite():
                            r'when that file already exists: '),
                 '[Errno 17] File exists: '
                 ),
+               (re.compile('executable = %s' % re.escape(sys.executable)),
+                'executable = python'),
                ])
             ),
         doctest.DocFileSuite(
@@ -2947,6 +2949,9 @@ def test_suite():
                    '-q develop -mxN -d /sample-buildout/develop-eggs'
                 ),
                (re.compile(r'^[*]...'), '...'),
+               # for bug_92891_bootstrap_crashes_with_egg_recipe_in_buildout_section
+               (re.compile(r"Unused options for buildout: 'eggs' 'scripts'\."),
+                "Unused options for buildout: 'scripts' 'eggs'."),
                ]),
             ),
         zc.buildout.rmtree.test_suite(),

zc.recipe.egg_/src/zc/recipe/egg/egg.py

@@ -27,8 +27,8 @@ class Eggs(object):
         self.buildout = buildout
         self.name = name
         self.options = options
-        links = options.get('find-links',
-                            buildout['buildout'].get('find-links'))
+        b_options = buildout['buildout']
+        links = options.get('find-links', b_options['find-links'])
         if links:
             links = links.split()
             options['find-links'] = '\n'.join(links)
@@ -36,20 +36,19 @@ class Eggs(object):
             links = ()
         self.links = links
 
-        index = options.get('index', buildout['buildout'].get('index'))
+        index = options.get('index', b_options.get('index'))
         if index is not None:
             options['index'] = index
         self.index = index
 
-        allow_hosts = buildout['buildout'].get('allow-hosts', '*')
+        allow_hosts = b_options['allow-hosts']
         allow_hosts = tuple([host.strip() for host in allow_hosts.split('\n')
                                if host.strip()!=''])
         self.allow_hosts = allow_hosts
 
-        options['eggs-directory'] = buildout['buildout']['eggs-directory']
+        options['eggs-directory'] = b_options['eggs-directory']
         options['_e'] = options['eggs-directory'] # backward compat.
-        options['develop-eggs-directory'
-                ] = buildout['buildout']['develop-eggs-directory']
+        options['develop-eggs-directory'] = b_options['develop-eggs-directory']
         options['_d'] = options['develop-eggs-directory'] # backward compat.
 
     def working_set(self, extra=()):
@@ -58,6 +57,7 @@ class Eggs(object):
         This is intended for reuse by similar recipes.
         """
         options = self.options
+        b_options = self.buildout['buildout']
 
         # Backward compat. :(
         options['executable'] = sys.executable