merge lp:~gary/zc.buildout/python-support-10: add significantly improved Distribute support.

CHANGES.txt

@@ -43,22 +43,34 @@ New Features:
   * The buildout script generated by bootstrap honors more of the settings
     in the designated configuration file (e.g., buildout.cfg).
 
+- You can develop zc.buildout using Distribute instead of Setuptools.  Use
+  the --distribute option on the dev.py script.  (Releases should be tested
+  with both Distribute and Setuptools.)
+
+- The ``distribute-version`` now works in the [buildout] section, mirroring
+  the ``setuptools-version`` option (this is for consistency; using the
+  general-purpose ``versions`` option is preferred).
+
 Bugs fixed:
 
+- Using Distribute with the ``allow-picked-versions = false`` buildout
+  option no longer causes an error.
+
 - The handling and documenting of default buildout options was normalized.
   This means, among other things, that ``bin/buildout -vv`` and
   ``bin/buildout annotate`` correctly list more of the options.
 
 - Installing a namespace package using a Python that already has a package
   in the same namespace (e.g., in the Python's site-packages) failed in
-  some cases.
+  some cases.  It is now handled correctly.
 
 - Another variation of this error showed itself when at least two
   dependencies were in a shared location like site-packages, and the
   first one met the "versions" setting.  The first dependency would be
   added, but subsequent dependencies from the same location (e.g.,
   site-packages) would use the version of the package found in the
-  shared location, ignoring the version setting.
+  shared location, ignoring the version setting.  This is also now
+  handled correctly.
 
 1.4.3 (2009-12-10)
 ==================

bootstrap/bootstrap.py

@@ -115,7 +115,7 @@ parser.add_option("-c", None, action="store", dest="config_file",
 
 options, args = parser.parse_args()
 
-# if -c was provided, we push it back into args for buildout' main function
+# if -c was provided, we push it back into args for buildout's main function
 if options.config_file is not None:
     args += ['-c', options.config_file]
 

dev.py

@@ -20,6 +20,7 @@ $Id$
 """
 
 import os, shutil, sys, subprocess, urllib2
+from optparse import OptionParser
 
 if sys.platform == 'win32':
     def quote(c):
@@ -56,10 +57,36 @@ for k, v in sys.modules.items():
 
 is_jython = sys.platform.startswith('java')
 
+setuptools_source = 'http://peak.telecommunity.com/dist/ez_setup.py'
+distribute_source = 'http://python-distribute.org/distribute_setup.py'
+
+usage = '''\
+[DESIRED PYTHON FOR DEVELOPING BUILDOUT] dev.py [options]
+
+Bootstraps buildout itself for development.
+
+This is different from a normal bootstrapping process because the
+buildout egg itself is installed as a develop egg.
+'''
+
+parser = OptionParser(usage=usage)
+parser.add_option("-d", "--distribute",
+                   action="store_true", dest="use_distribute", default=False,
+                   help="Use Distribute rather than Setuptools.")
+
+options, args = parser.parse_args()
+
+if args:
+    parser.error('This script accepts no arguments other than its options.')
+
+if options.use_distribute:
+    setup_source = distribute_source
+else:
+    setup_source = setuptools_source
+
 for d in 'eggs', 'develop-eggs', 'bin':
     if not os.path.exists(d):
         os.mkdir(d)
-
 if os.path.isdir('build'):
     shutil.rmtree('build')
 
@@ -67,22 +94,32 @@ try:
     to_reload = False
     import pkg_resources
     to_reload = True
+    if not hasattr(pkg_resources, '_distribute'):
+        raise ImportError
     import setuptools # A flag.  Sometimes pkg_resources is installed alone.
 except ImportError:
+    ez_code = urllib2.urlopen(setup_source).read().replace('\r\n', '\n')
     ez = {}
-    exec urllib2.urlopen('http://peak.telecommunity.com/dist/ez_setup.py'
-                         ).read() in ez
-    ez['use_setuptools'](to_dir='eggs', download_delay=0)
-
-    import pkg_resources
+    exec ez_code in ez
+    setup_args = dict(to_dir='eggs', download_delay=0)
+    if options.use_distribute:
+        setup_args['no_fake'] = True
+    ez['use_setuptools'](**setup_args)
     if to_reload:
         reload(pkg_resources)
+    else:
+        import pkg_resources
+    # This does not (always?) update the default working set.  We will
+    # do it.
+    for path in sys.path:
+        if path not in pkg_resources.working_set.entries:
+            pkg_resources.working_set.add_entry(path)
 
 env = os.environ.copy() # Windows needs yet-to-be-determined values from this.
 env['PYTHONPATH'] = os.path.dirname(pkg_resources.__file__)
 subprocess.Popen(
     [sys.executable] +
-    ['-S', 'setup.py', '-q', 'develop', '-m', '-x', '-d', 'develop-eggs'],
+    ['setup.py', '-q', 'develop', '-m', '-x', '-d', 'develop-eggs'],
     env=env).wait()
 
 pkg_resources.working_set.add_entry('src')

src/zc/buildout/buildout.py

@@ -858,10 +858,15 @@ class Buildout(UserDict.DictMixin):
 
         options = self['buildout']
 
+        specs = ['zc.buildout']
+        if zc.buildout.easy_install.is_distribute:
+            specs.append('distribute')
+        else:
+            specs.append('setuptools')
         ws = zc.buildout.easy_install.install(
             [
             (spec + ' ' + options.get(spec+'-version', '')).strip()
-            for spec in ('zc.buildout', 'setuptools')
+            for spec in specs
             ],
             options['eggs-directory'],
             links = options.get('find-links', '').split(),

src/zc/buildout/buildout.txt

@@ -770,6 +770,7 @@ COMMAND_LINE_VALUE).
     relative-paths= false
         DEFAULT_VALUE
     socket-timeout=
+        DEFAULT_VALUE
     unzip= false
         DEFAULT_VALUE
     use-dependency-links= true

src/zc/buildout/easy_install.py

@@ -51,6 +51,8 @@ url_match = re.compile('[a-z0-9+.-]+://').match
 
 is_win32 = sys.platform == 'win32'
 is_jython = sys.platform.startswith('java')
+is_distribute = (
+    pkg_resources.Requirement.parse('setuptools').key=='distribute')
 
 if is_jython:
     import java.lang.System
@@ -802,6 +804,8 @@ class Installer:
 
 
     def _constrain(self, requirement):
+        if is_distribute and requirement.key == 'setuptools':
+            requirement = pkg_resources.Requirement.parse('distribute')
         version = self._versions.get(requirement.project_name)
         if version:
             if version not in requirement:
@@ -1484,7 +1488,7 @@ def _get_module_file(executable, name):
     - executable is a path to the desired Python executable.
     - name is the name of the (pure, not C) Python module.
     """
-    cmd = [executable, "-c",
+    cmd = [executable, "-Sc",
            "import imp; "
            "fp, path, desc = imp.find_module(%r); "
            "fp.close; "
@@ -1607,6 +1611,7 @@ addsitedir_namespace_originalpackages_snippet = '''
             pkg_resources.working_set.add_entry(sitedir)'''
 
 original_path_snippet = '''
+    sys.__egginsert = len(buildout_paths) # Support distribute.
     original_paths = [
         %s
         ]

src/zc/buildout/easy_install.txt

@@ -222,7 +222,9 @@ dependencies.  We might do this to specify a specific version.
     d  other-1.0-py2.4.egg
 
 We can request that eggs be unzipped even if they are zip safe.  This
-can be useful when debugging.
+can be useful when debugging.  (Note that Distribute will unzip eggs by
+default, so if you are using Distribute, most or all eggs will already be
+unzipped without this flag.)
 
     >>> rmdir(dest)
     >>> dest = tmpdir('sample-install')
@@ -1284,6 +1286,7 @@ this is very straightforward.
             if not sitedircase in known_paths and os.path.exists(sitedir):
                 sys.path.append(sitedir)
                 known_paths.add(sitedircase)
+        sys.__egginsert = len(buildout_paths) # Support distribute.
         original_paths = [
             ...
             ]
@@ -1360,6 +1363,7 @@ call to another text fixture to create.
                 sys.path.append(sitedir)
                 known_paths.add(sitedircase)
                 pkg_resources.working_set.add_entry(sitedir)
+        sys.__egginsert = len(buildout_paths) # Support distribute.
         original_paths = [
             ...
             ]
@@ -1423,6 +1427,7 @@ at that result.
                 sys.path.append(sitedir)
                 known_paths.add(sitedircase)
                 pkg_resources.working_set.add_entry(sitedir)
+        sys.__egginsert = len(buildout_paths) # Support distribute.
         original_paths = [
             ...
             ]

src/zc/buildout/tests.py

@@ -864,7 +864,8 @@ On the other hand, if we have a regular egg, rather than a develop egg:
     -  z3c.recipe.scripts.egg-link
     -  zc.recipe.egg.egg-link
 
-    >>> ls('eggs') # doctest: +ELLIPSIS
+    >>> print 'START ->'; ls('eggs') # doctest: +ELLIPSIS
+    START...
     -  foox-0.0.0-py2.4.egg
     ...
 
@@ -3103,6 +3104,13 @@ honoring our version specification.
     >>> write('foo.py', '')
     >>> _ = system(buildout+' setup . sdist')
 
+    >>> if zc.buildout.easy_install.is_distribute:
+    ...     distribute_version = 'distribute = %s' % (
+    ...         pkg_resources.working_set.find(
+    ...             pkg_resources.Requirement.parse('distribute')).version,)
+    ... else:
+    ...     distribute_version = ''
+    ...
     >>> write('buildout.cfg',
     ... '''
     ... [buildout]
@@ -3114,12 +3122,14 @@ honoring our version specification.
     ... [versions]
     ... setuptools = %s
     ... foo = 1
+    ... %s
     ...
     ... [foo]
     ... recipe = zc.recipe.egg
     ... eggs = foo
-    ... ''' % pkg_resources.working_set.find(
-    ...    pkg_resources.Requirement.parse('setuptools')).version)
+    ... ''' % (pkg_resources.working_set.find(
+    ...         pkg_resources.Requirement.parse('setuptools')).version,
+    ...        distribute_version))
 
     >>> print system(buildout),
     Installing foo.
@@ -3623,11 +3633,13 @@ def updateSetup(test):
 
     # now let's make the new releases
     makeNewRelease('zc.buildout', ws, new_releases)
-    makeNewRelease('setuptools', ws, new_releases)
-
     os.mkdir(os.path.join(new_releases, 'zc.buildout'))
-    os.mkdir(os.path.join(new_releases, 'setuptools'))
-
+    if zc.buildout.easy_install.is_distribute:
+        makeNewRelease('distribute', ws, new_releases)
+        os.mkdir(os.path.join(new_releases, 'distribute'))
+    else:
+        makeNewRelease('setuptools', ws, new_releases)
+        os.mkdir(os.path.join(new_releases, 'setuptools'))
 
 
 normalize_bang = (
@@ -3651,7 +3663,8 @@ def test_suite():
                 '__buildout_signature__ = recipes-SSSSSSSSSSS'),
                (re.compile('executable = [\S ]+python\S*', re.I),
                 'executable = python'),
-               (re.compile('[-d]  setuptools-\S+[.]egg'), 'setuptools.egg'),
+               (re.compile('[-d]  (setuptools|distribute)-\S+[.]egg'),
+                'setuptools.egg'),
                (re.compile('zc.buildout(-\S+)?[.]egg(-link)?'),
                 'zc.buildout.egg'),
                (re.compile('creating \S*setup.cfg'), 'creating setup.cfg'),
@@ -3666,6 +3679,7 @@ def test_suite():
                            r'when that file already exists: '),
                 '[Errno 17] File exists: '
                 ),
+               (re.compile('distribute'), 'setuptools'),
                ])
             ),
         doctest.DocFileSuite(
@@ -3695,9 +3709,18 @@ def test_suite():
                (re.compile('(zc.buildout|setuptools)-\d+[.]\d+\S*'
                            '-py\d.\d.egg'),
                 '\\1.egg'),
+               (re.compile('distribute-\d+[.]\d+\S*'
+                           '-py\d.\d.egg'),
+                'setuptools.egg'),
                (re.compile('(zc.buildout|setuptools)( version)? \d+[.]\d+\S*'),
                 '\\1 V.V'),
-               (re.compile('[-d]  setuptools'), '-  setuptools'),
+               (re.compile('distribute( version)? \d+[.]\d+\S*'),
+                'setuptools V.V'),
+               (re.compile('[-d]  (setuptools|distribute)'), '-  setuptools'),
+               (re.compile('distribute'), 'setuptools'),
+               (re.compile("\nUnused options for buildout: "
+                           "'(distribute|setuptools)\-version'\."),
+                '')
                ])
             ),
 
@@ -3713,13 +3736,17 @@ def test_suite():
                zc.buildout.testing.normalize_egg_py,
                normalize_bang,
                (re.compile('extdemo[.]pyd'), 'extdemo.so'),
-               (re.compile('[-d]  setuptools-\S+[.]egg'), 'setuptools.egg'),
+               (re.compile('[-d]  (setuptools|distribute)-\S+[.]egg'),
+                'setuptools.egg'),
                (re.compile(r'\\[\\]?'), '/'),
                (re.compile(r'\#!\S+\bpython\S*'), '#!/usr/bin/python'),
                # Normalize generate_script's Windows interpreter to UNIX:
                (re.compile(r'\nimport subprocess\n'), '\n'),
                (re.compile('subprocess\\.call\\(argv, env=environ\\)'),
                 'os.execve(sys.executable, argv, environ)'),
+               (re.compile('distribute'), 'setuptools'),
+               # Distribute unzips eggs by default.
+               (re.compile('\-  demoneeded'), 'd  demoneeded'),
                ]+(sys.version_info < (2, 5) and [
                   (re.compile('.*No module named runpy.*', re.S), ''),
                   (re.compile('.*usage: pdb.py scriptfile .*', re.S), ''),
@@ -3751,7 +3778,7 @@ def test_suite():
                zc.buildout.testing.normalize_egg_py,
                (re.compile("buildout: Running \S*setup.py"),
                 'buildout: Running setup.py'),
-               (re.compile('setuptools-\S+-'),
+               (re.compile('(setuptools|distribute)-\S+-'),
                 'setuptools.egg'),
                (re.compile('zc.buildout-\S+-'),
                 'zc.buildout.egg'),
@@ -3759,7 +3786,7 @@ def test_suite():
                 'File "one.py"'),
                (re.compile(r'We have a develop egg: (\S+) (\S+)'),
                 r'We have a develop egg: \1 V'),
-               (re.compile('Picked: setuptools = \S+'),
+               (re.compile('Picked: (setuptools|distribute) = \S+'),
                 'Picked: setuptools = V'),
                (re.compile(r'\\[\\]?'), '/'),
                (re.compile(
@@ -3770,6 +3797,9 @@ def test_suite():
                # for bug_92891_bootstrap_crashes_with_egg_recipe_in_buildout_section
                (re.compile(r"Unused options for buildout: 'eggs' 'scripts'\."),
                 "Unused options for buildout: 'scripts' 'eggs'."),
+               (re.compile('distribute'), 'setuptools'),
+               # Distribute unzips eggs by default.
+               (re.compile('\-  demoneeded'), 'd  demoneeded'),
                ]),
             ),
         zc.buildout.testselectingpython.test_suite(),

src/zc/buildout/update.txt

@@ -100,8 +100,8 @@ Our buildout script's site.py has been updated to use the new eggs:
     ...
 
 Now, let's recreate the sample buildout. If we specify constraints on
-the versions of zc.buildout and setuptools to use, running the
-buildout will install earlier versions of these packages:
+the versions of zc.buildout and setuptools (or distribute) to use,
+running the buildout will install earlier versions of these packages:
 
     >>> write(sample_buildout, 'buildout.cfg',
     ... """
@@ -112,6 +112,7 @@ buildout will install earlier versions of these packages:
     ... develop = showversions
     ... zc.buildout-version = < 99
     ... setuptools-version = < 99
+    ... distribute-version = < 99
     ...
     ... [show-versions]
     ... recipe = showversions

z3c.recipe.scripts_/src/z3c/recipe/scripts/tests.py

@@ -423,8 +423,8 @@ def test_suite():
                zc.buildout.tests.normalize_bang,
                (re.compile(r'zc.buildout(-\S+)?[.]egg(-link)?'),
                 'zc.buildout.egg'),
-               (re.compile('[-d]  setuptools-[^-]+-'), 'setuptools-X-'),
-               (re.compile(r'setuptools-[\w.]+-py'), 'setuptools-X-py'),
+               (re.compile('[-d]  (setuptools|distribute)-[^-]+-'), 'setuptools-X-'),
+               (re.compile(r'(setuptools|distribute)-[\w.]+-py'), 'setuptools-X-py'),
                (re.compile(r'eggs\\\\demo'), 'eggs/demo'),
                (re.compile(r'[a-zA-Z]:\\\\foo\\\\bar'), '/foo/bar'),
                (re.compile(r'\#!\S+\bpython\S*'), '#!/usr/bin/python'),
@@ -432,6 +432,7 @@ def test_suite():
                (re.compile(r'\nimport subprocess\n'), '\n'),
                (re.compile('subprocess\\.call\\(argv, env=environ\\)'),
                 'os.execve(sys.executable, argv, environ)'),
+               (re.compile('distribute'), 'setuptools'),
                ])
             ),
         doctest.DocTestSuite(

zc.recipe.egg_/src/zc/recipe/egg/tests.py

@@ -50,9 +50,12 @@ def test_suite():
                zc.buildout.tests.normalize_bang,
                (re.compile('zc.buildout(-\S+)?[.]egg(-link)?'),
                 'zc.buildout.egg'),
-               (re.compile('[-d]  setuptools-[^-]+-'), 'setuptools-X-'),
+               (re.compile('[-d]  (setuptools|distribute)-[^-]+-'),
+                'setuptools-X-'),
                (re.compile(r'eggs\\\\demo'), 'eggs/demo'),
                (re.compile(r'[a-zA-Z]:\\\\foo\\\\bar'), '/foo/bar'),
+               # Distribute unzips eggs by default.
+               (re.compile('\-  demoneeded'), 'd  demoneeded'),
                ])
             ),
         doctest.DocFileSuite(
@@ -64,7 +67,7 @@ def test_suite():
                (re.compile('__buildout_signature__ = '
                            'sample-\S+\s+'
                            'zc.recipe.egg-\S+\s+'
-                           'setuptools-\S+\s+'
+                           '(setuptools|distribute)-\S+\s+'
                            'zc.buildout-\S+\s*'
                            ),
                 '__buildout_signature__ = sample- zc.recipe.egg-\n'),
@@ -104,14 +107,17 @@ def test_suite():
                    zc.buildout.testing.normalize_path,
                    zc.buildout.testing.normalize_endings,
                    zc.buildout.testing.normalize_script,
-                   (re.compile('Got setuptools \S+'), 'Got setuptools V'),
-                   (re.compile('([d-]  )?setuptools-\S+-py'),
+                   (re.compile('Got (setuptools|distribute) \S+'),
+                    'Got setuptools V'),
+                   (re.compile('([d-]  )?(setuptools|distribute)-\S+-py'),
                     'setuptools-V-py'),
                    (re.compile('-py2[.][0-35-9][.]'), 'py2.5.'),
                    (re.compile('zc.buildout-\S+[.]egg'),
                     'zc.buildout.egg'),
                    (re.compile('zc.buildout[.]egg-link'),
                     'zc.buildout.egg'),
+                   # Distribute unzips eggs by default.
+                   (re.compile('\-  demoneeded'), 'd  demoneeded'),
                    ]),
                 ),
             )