merge of gary-2-bootstrap-changes into trunk, resolving lots of conflicts with...

merge of gary-2-bootstrap-changes into trunk, resolving lots of conflicts with the distribute changes.

CHANGES.txt

 Change History
 **************
 
-1.4.4 (?)
-=========
+1.?.? (201?-??-??)
+==================
+
+New Features:
+
+- Improve bootstrap.
+
+  * New options let you specify where to find ez_setup.py and where to find
+    a download cache.  These options can keep bootstrap from going over the
+    network.
+
+  * Another new option lets you specify where to put generated eggs.
 
+  * The buildout script generated by bootstrap honors more of the settings
+    in the designated configuration file (e.g., buildout.cfg).
 
 1.4.3 (2009-12-10)
 ==================

bootstrap/bootstrap.py

@@ -20,21 +20,62 @@ use the -c option to specify an alternate configuration file.
 $Id$
 """
 
-import os, shutil, sys, tempfile, urllib2
+import os, shutil, sys, tempfile, textwrap, urllib, urllib2
 from optparse import OptionParser
 
-tmpeggs = tempfile.mkdtemp()
-
 is_jython = sys.platform.startswith('java')
 
+setuptools_source = 'http://peak.telecommunity.com/dist/ez_setup.py'
+distribute_source = 'http://python-distribute.org/distribute_setup.py'
+
 # parsing arguments
-parser = OptionParser()
+def normalize_to_url(option, opt_str, value, parser):
+    if value:
+        if '://' not in value: # It doesn't smell like a URL.
+            value = 'file://%s' % (
+                urllib.pathname2url(
+                    os.path.abspath(os.path.expanduser(value))),)
+        if opt_str == '--download-base' and not value.endswith('/'):
+            # Download base needs a trailing slash to make the world happy.
+            value += '/'
+    else:
+        value = None
+    name = opt_str[2:].replace('-', '_')
+    setattr(parser.values, name, value)
+
+usage = '''\
+[DESIRED PYTHON FOR BUILDOUT] bootstrap.py [options]
+
+Bootstraps a buildout-based project.
+
+Simply run this script in a directory containing a buildout.cfg, using the
+Python that you want bin/buildout to use.
+
+Note that by using --setup-source and --download-base to point to
+local resources, you can keep this script from going over the network.
+'''
+
+parser = OptionParser(usage=usage)
 parser.add_option("-v", "--version", dest="version",
                           help="use a specific zc.buildout version")
 parser.add_option("-d", "--distribute",
-                   action="store_true", dest="distribute", default=False,
-                   help="Use Disribute rather than Setuptools.")
-
+                   action="store_true", dest="use_distribute", default=False,
+                   help="Use Distribute rather than Setuptools.")
+parser.add_option("--setup-source", action="callback", dest="setup_source",
+                  callback=normalize_to_url, nargs=1, type="string",
+                  help=("Specify a URL or file location for the setup file. "
+                        "If you use Setuptools, this will default to " +
+                        setuptools_source + "; if you use Distribute, this "
+                        "will default to " + distribute_source +"."))
+parser.add_option("--download-base", action="callback", dest="download_base",
+                  callback=normalize_to_url, nargs=1, type="string",
+                  help=("Specify a URL or directory for downloading "
+                        "zc.buildout and either Setuptools or Distribute. "
+                        "Defaults to PyPI."))
+parser.add_option("--eggs",
+                  help=("Specify a directory for storing eggs.  Defaults to "
+                        "a temporary directory that is deleted when the "
+                        "bootstrap script completes."))
 parser.add_option("-c", None, action="store", dest="config_file",
                    help=("Specify the path to the buildout configuration "
                          "file to be used."))
@@ -45,35 +86,46 @@ options, args = parser.parse_args()
 if options.config_file is not None:
     args += ['-c', options.config_file]
 
-if options.version is not None:
-    VERSION = '==%s' % options.version
+if options.eggs:
+    eggs_dir = os.path.abspath(os.path.expanduser(options.eggs))
 else:
-    VERSION = ''
+    eggs_dir = tempfile.mkdtemp()
+
+if options.setup_source is None:
+    if options.use_distribute:
+        options.setup_source = distribute_source
+    else:
+        options.setup_source = setuptools_source
 
-USE_DISTRIBUTE = options.distribute
 args = args + ['bootstrap']
 
 to_reload = False
+
 try:
+    import setuptools # A flag.  Sometimes pkg_resources is installed alone.
     import pkg_resources
     if not hasattr(pkg_resources, '_distribute'):
         to_reload = True
         raise ImportError
 except ImportError:
     ez = {}
-    if USE_DISTRIBUTE:
-        exec urllib2.urlopen('http://python-distribute.org/distribute_setup.py'
-                         ).read() in ez
-        ez['use_setuptools'](to_dir=tmpeggs, download_delay=0, no_fake=True)
-    else:
-        exec urllib2.urlopen('http://peak.telecommunity.com/dist/ez_setup.py'
-                             ).read() in ez
-        ez['use_setuptools'](to_dir=tmpeggs, download_delay=0)
+    exec urllib2.urlopen(options.setup_source).read() in ez
+    setup_args = dict(to_dir=eggs_dir, download_delay=0)
+    if options.download_base:
+        setup_args['download_base'] = options.download_base
+    if options.use_distribute:
+        setup_args['no_fake'] = True
+    ez['use_setuptools'](**setup_args)
 
     if to_reload:
         reload(pkg_resources)
     else:
         import pkg_resources
+    # This does not (always?) update the default working set.  We will
+    # do it.
+    for path in sys.path:
+        if path not in pkg_resources.working_set.entries:
+            pkg_resources.working_set.add_entry(path)
 
 if sys.platform == 'win32':
     def quote(c):
@@ -85,37 +137,46 @@ else:
     def quote (c):
         return c
 
-cmd = 'from setuptools.command.easy_install import main; main()'
-ws  = pkg_resources.working_set
+cmd = [quote(sys.executable),
+       '-c',
+       quote('from setuptools.command.easy_install import main; main()'),
+       '-mqNxd',
+       quote(eggs_dir)]
+
+if options.download_base:
+    cmd.extend(['-f', quote(options.download_base)])
 
-if USE_DISTRIBUTE:
-    requirement = 'distribute'
+requirement = 'zc.buildout'
+if options.version:
+    requirement = '=='.join((requirement, options.version))
+cmd.append(requirement)
+
+if options.use_distribute:
+    setup_requirement = 'distribute'
 else:
-    requirement = 'setuptools'
+    setup_requirement = 'setuptools'
+ws = pkg_resources.working_set
+env = dict(
+    os.environ,
+    PYTHONPATH=ws.find(
+        pkg_resources.Requirement.parse(setup_requirement)).location)
 
 if is_jython:
     import subprocess
-
-    assert subprocess.Popen([sys.executable] + ['-c', quote(cmd), '-mqNxd',
-           quote(tmpeggs), 'zc.buildout' + VERSION],
-           env=dict(os.environ,
-               PYTHONPATH=
-               ws.find(pkg_resources.Requirement.parse(requirement)).location
-               ),
-           ).wait() == 0
-
-else:
-    assert os.spawnle(
-        os.P_WAIT, sys.executable, quote (sys.executable),
-        '-c', quote (cmd), '-mqNxd', quote (tmpeggs), 'zc.buildout' + VERSION,
-        dict(os.environ,
-            PYTHONPATH=
-            ws.find(pkg_resources.Requirement.parse(requirement)).location
-            ),
-        ) == 0
-
-ws.add_entry(tmpeggs)
-ws.require('zc.buildout' + VERSION)
+    exitcode = subprocess.Popen(cmd, env=env).wait()
+else: # Windows prefers this, apparently; otherwise we would prefer subprocess
+    exitcode = os.spawnle(*([os.P_WAIT, sys.executable] + cmd + [env]))
+if exitcode != 0:
+    sys.stdout.flush()
+    sys.stderr.flush()
+    print ("An error occured when trying to install zc.buildout. "
+           "Look above this message for any errors that "
+           "were output by easy_install.")
+    sys.exit(exitcode)
+
+ws.add_entry(eggs_dir)
+ws.require(requirement)
 import zc.buildout.buildout
 zc.buildout.buildout.main(args)
-shutil.rmtree(tmpeggs)
+if not options.eggs: # clean up temporary egg directory
+    shutil.rmtree(eggs_dir)

dev.py

@@ -13,7 +13,7 @@
 ##############################################################################
 """Bootstrap the buildout project itself.
 
-This is different from a normal boostrapping process because the
+This is different from a normal bootstrapping process because the
 buildout egg itself is installed as a develop egg.
 
 $Id$

src/zc/buildout/bootstrap.txt

@@ -57,7 +57,7 @@ Let's try with an unknown version::
     ...     'bootstrap.py --version UNKNOWN'); print 'X' # doctest: +ELLIPSIS
     ...
     X
-    No local packages or download links found for zc.buildout==UNKNOWN
+    No local packages or download links found for zc.buildout==UNKNOWN...
     ...
 
 Now let's try with `1.1.2`, which happens to exist::
@@ -119,8 +119,8 @@ Let's make sure the generated `buildout` script uses it::
         zc.buildout.buildout.main()
     <BLANKLINE>
 
-`zc.buildout` now can also run with `Distribute` with the `--distribute` option::
-
+`zc.buildout` now can also run with `Distribute` with the `--distribute`
+option::
 
     >>> print 'X'; print system(
     ...     zc.buildout.easy_install._safe_arg(sys.executable)+' '+
@@ -153,7 +153,8 @@ Make sure both options can be used together::
 
     >>> print 'X'; print system(
     ...     zc.buildout.easy_install._safe_arg(sys.executable)+' '+
-    ...     'bootstrap.py --distribute --version 1.2.1'); print 'X' # doctest: +ELLIPSIS
+    ...     'bootstrap.py --distribute --version 1.2.1'); print 'X'
+    ... # doctest: +ELLIPSIS
     ...
     X
     ...
@@ -161,7 +162,8 @@ Make sure both options can be used together::
     <BLANKLINE>
     X
 
-Let's make sure the generated `buildout` script uses ``Distribute`` *and* ``zc.buildout-1.2.1``::
+Let's make sure the generated `buildout` script uses ``Distribute`` *and*
+``zc.buildout-1.2.1``::
 
     >>> print open(buildout_script).read() # doctest: +ELLIPSIS
     #...
@@ -194,4 +196,70 @@ Last, the -c option needs to work on bootstrap.py::
     <BLANKLINE>
     X
 
+You can specify a location of ez_setup.py or distribute_setup, so you
+can rely on a local or remote location.  We'll write our own ez_setup.py
+that we will also use to test some other bootstrap options.
+
+    >>> write('ez_setup.py', '''\
+    ... def use_setuptools(**kwargs):
+    ...     import sys, pprint
+    ...     pprint.pprint(kwargs, width=40)
+    ...     sys.exit()
+    ... ''')
+    >>> print system(
+    ...     zc.buildout.easy_install._safe_arg(sys.executable)+' '+
+    ...     'bootstrap.py --setup-source=./ez_setup.py')
+    ... # doctest: +ELLIPSIS
+    {'download_delay': 0,
+     'to_dir': '...'}
+    <BLANKLINE>
+
+You can also pass a download-cache, and a place in which eggs should be stored
+(they are normally stored in a temporary directory).
+
+    >>> print system(
+    ...     zc.buildout.easy_install._safe_arg(sys.executable)+' '+
+    ...     'bootstrap.py --setup-source=./ez_setup.py '+
+    ...     '--download-base=./download-cache --eggs=eggs')
+    ... # doctest: +ELLIPSIS
+    {'download_base': '/sample/download-cache/',
+     'download_delay': 0,
+     'to_dir': '/sample/eggs'}
+    <BLANKLINE>
+
+Here's the entire help text.
 
+    >>> print system(
+    ...     zc.buildout.easy_install._safe_arg(sys.executable)+' '+
+    ...     'bootstrap.py --help'),
+    ... # doctest: +ELLIPSIS
+    usage: [DESIRED PYTHON FOR BUILDOUT] bootstrap.py [options]
+    <BLANKLINE>
+    Bootstraps a buildout-based project.
+    <BLANKLINE>
+    Simply run this script in a directory containing a buildout.cfg, using the
+    Python that you want bin/buildout to use.
+    <BLANKLINE>
+    Note that by using --setup-source and --download-base to point to
+    local resources, you can keep this script from going over the network.
+    <BLANKLINE>
+    <BLANKLINE>
+    options:
+      -h, --help            show this help message and exit
+      -v VERSION, --version=VERSION
+                            use a specific zc.buildout version
+      -d, --distribute      Use Distribute rather than Setuptools.
+      --setup-source=SETUP_SOURCE
+                            Specify a URL or file location for the setup file. If
+                            you use Setuptools, this will default to
+                            http://peak.telecommunity.com/dist/ez_setup.py; if you
+                            use Distribute, this will default to http://python-
+                            distribute.org/distribute_setup.py.
+      --download-base=DOWNLOAD_BASE
+                            Specify a URL or directory for downloading zc.buildout
+                            and either Setuptools or Distribute. Defaults to PyPI.
+      --eggs=EGGS           Specify a directory for storing eggs.  Defaults to a
+                            temporary directory that is deleted when the bootstrap
+                            script completes.
+      -c CONFIG_FILE        Specify the path to the buildout configuration file to
+                            be used.

src/zc/buildout/buildout.py

@@ -338,11 +338,32 @@ class Buildout(UserDict.DictMixin):
 
         self._setup_directories()
 
+        options = self['buildout']
+
+        # Get a base working set for our distributions that corresponds to the
+        # stated desires in the configuration.
+        distributions = ['setuptools', 'zc.buildout']
+        if options.get('offline') == 'true':
+            ws = zc.buildout.easy_install.working_set(
+                distributions, options['executable'],
+                [options['develop-eggs-directory'], options['eggs-directory']]
+                )
+        else:
+            ws = zc.buildout.easy_install.install(
+                distributions, options['eggs-directory'],
+                links=self._links,
+                index=options.get('index'),
+                executable=options['executable'],
+                path=[options['develop-eggs-directory']],
+                newest=self.newest,
+                allow_hosts=self._allow_hosts
+                )
+
         # Now copy buildout and setuptools eggs, and record destination eggs:
         entries = []
         for name in 'setuptools', 'zc.buildout':
             r = pkg_resources.Requirement.parse(name)
-            dist = pkg_resources.working_set.find(r)
+            dist = ws.find(r)
             if dist.precedence == pkg_resources.DEVELOP_DIST:
                 dest = os.path.join(self['buildout']['develop-eggs-directory'],
                                     name+'.egg-link')
@@ -362,8 +383,8 @@ class Buildout(UserDict.DictMixin):
         ws = pkg_resources.WorkingSet(entries)
         ws.require('zc.buildout')
         zc.buildout.easy_install.scripts(
-            ['zc.buildout'], ws, sys.executable,
-            self['buildout']['bin-directory'])
+            ['zc.buildout'], ws, options['executable'],
+            options['bin-directory'])
 
     init = bootstrap