Merge pull request #46 from reinout/reinout-versions

Merging buildout-versions' functionality into buildout

.travis.yml

@@ -5,10 +5,6 @@ env:
     - PYTHON_VER=3.2
     - PYTHON_VER=3.3
 
-branches:
-  only:
-    - master
-
 notifications:
   email:
     - buildout-development@googlegroups.com

CHANGES.rst

 Change History
 **************
 
+2.0.0b2 (unreleased)
+====================
+
+- Integrated the `buildout-versions
+  <http://packages.python.org/buildout-versions/>`_ extension into buildout
+  itself. For this, a few options were added to buildout:
+
+  - If ``show-picked-versions`` is set to true, all picked versions are
+    printed at the end of the buildout run. This saves you from running
+    buildout in verbose mode and extracting the picked versions from the
+    output.
+
+  - If ``update-versions-file`` is set to a filename (relative to the buildout
+    directory), the ``show-picked-versions`` output is appended to that file.
+
+
 2.0.0b1 (2013-01-21)
 ====================
 
-- buildout options can be given on the command line using the form::
+- Buildout options can be given on the command line using the form::
 
     option_name=value
 

src/zc/buildout/buildout.py

@@ -29,6 +29,7 @@ except ImportError:
 
 import zc.buildout.configparser
 import copy
+import datetime
 import distutils.errors
 import glob
 import itertools
@@ -136,7 +137,9 @@ _buildout_default_options = _annotate_section({
     'parts-directory': 'parts',
     'prefer-final': 'true',
     'python': 'buildout',
+    'show-picked-versions': 'false',
     'socket-timeout': '',
+    'update-versions-file': '',
     'use-dependency-links': 'true',
     }, 'DEFAULT_VALUE')
 
@@ -206,7 +209,7 @@ class Buildout(DictMixin):
 
         # Set up versions section, if necessary
         if 'versions' not in data['buildout']:
-            data['buildout']['versions'] = 'versions', 'DEFAULT_VALUE'
+            data['buildout']['versions'] = ('versions', 'DEFAULT_VALUE')
             if 'versions' not in data:
                 data['versions'] = {}
 
@@ -304,7 +307,7 @@ class Buildout(DictMixin):
 
         # finish w versions
         if versions_section_name:
-            # refetching section name just to avoid a warnin
+            # refetching section name just to avoid a warning
             versions = self[versions_section_name]
         else:
             # remove annotations
@@ -319,6 +322,11 @@ class Buildout(DictMixin):
             bool_option(options, 'use-dependency-links'))
         zc.buildout.easy_install.allow_picked_versions(
                 bool_option(options, 'allow-picked-versions'))
+        self.show_picked_versions = bool_option(options,
+                                                'show-picked-versions')
+        self.update_versions_file = options['update-versions-file']
+        zc.buildout.easy_install.show_picked_versions(
+            self.show_picked_versions)
 
         download_cache = options.get('download-cache')
         if download_cache:
@@ -631,6 +639,7 @@ class Buildout(DictMixin):
         elif (not installed_parts) and installed_exists:
             os.remove(self['buildout']['installed'])
 
+        self._print_picked_versions()
         self._unload_extensions()
 
     def _update_installed(self, **buildout_options):
@@ -944,6 +953,14 @@ class Buildout(DictMixin):
     def _load_extensions(self):
         __doing__ = 'Loading extensions.'
         specs = self['buildout'].get('extensions', '').split()
+        for superceded_extension in ['buildout-versions',
+                                     'buildout.dumppickedversions']:
+            if superceded_extension in specs:
+                msg = ("The extension %s is now included in buildout itself.\n"
+                       "Remove the extension from your configuration and "
+                       "look at the `show-picked-versions`\n"
+                       "option in buildout's documentation.")
+                raise zc.buildout.UserError(msg % superceded_extension)
         if specs:
             path = [self['buildout']['develop-eggs-directory']]
             if self.offline:
@@ -977,6 +994,45 @@ class Buildout(DictMixin):
                 'zc.buildout.unloadextension'):
                 ep.load()(self)
 
+    def _print_picked_versions(self):
+        Installer = zc.buildout.easy_install.Installer
+        if not self.show_picked_versions:
+            return
+        if not Installer._picked_versions:
+            # Don't print empty output.
+            return
+        output = ['[versions]']
+        required_output = []
+        for dist_, version in sorted(Installer._picked_versions.items()):
+            if dist_ in Installer._required_by:
+                required_output.append('')
+                required_output.append('# Required by:')
+                for req_ in sorted(Installer._required_by[dist_]):
+                    required_output.append('# '+req_)
+                target = required_output
+            else:
+                target = output
+            target.append("%s = %s" % (dist_, version))
+
+        output.extend(required_output)
+
+        print_("Versions had to be automatically picked.")
+        print_("The following part definition lists the versions picked:")
+        print_('\n'.join(output))
+        if self.update_versions_file:
+            # Also write to the versions file.
+            if os.path.exists(self.update_versions_file):
+                output[:1] = [
+                    '',
+                    '# Added by buildout at %s' % datetime.datetime.now(),
+                    ]
+            output.append('')
+            f = open(self.update_versions_file, 'a')
+            f.write('\n'.join(output))
+            f.close()
+            print_("This information has been written to " +
+                   self.update_versions_file)
+
     def setup(self, args):
         if not args:
             raise zc.buildout.UserError(

src/zc/buildout/buildout.txt

@@ -833,8 +833,12 @@ COMMAND_LINE_VALUE).
         DEFAULT_VALUE
     python= buildout
         DEFAULT_VALUE
+    show-picked-versions= false
+        DEFAULT_VALUE
     socket-timeout=
         DEFAULT_VALUE
+    update-versions-file=
+        DEFAULT_VALUE
     use-dependency-links= true
         DEFAULT_VALUE
     versions= versions
@@ -2373,7 +2377,9 @@ database is shown.
     parts-directory = /sample-buildout/parts
     prefer-final = true
     python = buildout
+    show-picked-versions = false
     socket-timeout =
+    update-versions-file =
     use-dependency-links = true
     verbosity = 20
     versions = versions

src/zc/buildout/easy_install.py

@@ -69,6 +69,7 @@ buildout_and_distribute_path = [
 
 FILE_SCHEME = re.compile('file://', re.I).match
 
+
 class AllowHostsPackageIndex(setuptools.package_index.PackageIndex):
     """Will allow urls that are local to the system.
 
@@ -120,11 +121,14 @@ _easy_install_cmd = 'from setuptools.command.easy_install import main; main()'
 class Installer:
 
     _versions = {}
+    _required_by = {}
+    _picked_versions = {}
     _download_cache = None
     _install_from_cache = False
     _prefer_final = True
     _use_dependency_links = True
     _allow_picked_versions = True
+    _show_picked_versions = False
 
     def __init__(self,
                  dest=None,
@@ -167,7 +171,7 @@ class Installer:
         self._index = _get_index(index, links, self._allow_hosts)
 
         if versions is not None:
-            self._versions = versions
+            self._versions = normalize_versions(versions)
 
     def _satisfied(self, req, source=None):
         dists = [dist for dist in self._env[req.project_name] if dist in req]
@@ -516,6 +520,8 @@ class Installer:
                 ):
                 logger.debug('Picked: %s = %s',
                              dist.project_name, dist.version)
+                self._picked_versions[dist.project_name] = dist.version
+
                 if not self._allow_picked_versions:
                     raise zc.buildout.UserError(
                         'Picked: %s = %s' % (dist.project_name, dist.version)
@@ -545,10 +551,9 @@ class Installer:
 
 
     def _constrain(self, requirement):
-        constraint = self._versions.get(requirement.project_name)
+        constraint = self._versions.get(requirement.project_name.lower())
         if constraint:
             requirement = _constrained_requirement(constraint, requirement)
-
         return requirement
 
     def install(self, specs, working_set=None):
@@ -680,10 +685,20 @@ class Installer:
             if tmp != self._download_cache:
                 shutil.rmtree(tmp)
 
+
+def normalize_versions(versions):
+    """Return version dict with keys normalized to lowercase.
+
+    PyPI is case-insensitive and not all distributions are consistent in
+    their own naming.
+    """
+    return dict([(k.lower(), v) for (k, v) in versions.items()])
+
+
 def default_versions(versions=None):
     old = Installer._versions
     if versions is not None:
-        Installer._versions = versions
+        Installer._versions = normalize_versions(versions)
     return old
 
 def download_cache(path=-1):
@@ -718,6 +733,13 @@ def allow_picked_versions(setting=None):
         Installer._allow_picked_versions = bool(setting)
     return old
 
+def show_picked_versions(setting=None):
+    old = Installer._show_picked_versions
+    if setting is not None:
+        Installer._show_picked_versions = bool(setting)
+    return old
+
+
 def install(specs, dest,
             links=(), index=None,
             executable=sys.executable,
@@ -1226,9 +1248,10 @@ class MissingDistribution(zc.buildout.UserError):
         return "Couldn't find a distribution for %r." % str(req)
 
 def _log_requirement(ws, req):
-    if not logger.isEnabledFor(logging.DEBUG):
+    if (not logger.isEnabledFor(logging.DEBUG) and
+        not Installer._show_picked_versions):
         # Sorting the working set and iterating over it's requirements
-        # is expensive, so short cirtuit the work if it won't even be
+        # is expensive, so short circuit the work if it won't even be
         # logged.  When profiling a simple buildout with 10 parts with
         # identical and large working sets, this resulted in a
         # decrease of run time from 93.411 to 15.068 seconds, about a
@@ -1240,6 +1263,10 @@ def _log_requirement(ws, req):
     for dist in ws:
         if req in dist.requires():
             logger.debug("  required by %s." % dist)
+            req_ = str(req)
+            if req_ not in Installer._required_by:
+                Installer._required_by[req_] = set()
+            Installer._required_by[req_].add(str(dist.as_requirement()))
 
 def _fix_file_links(links):
     for link in links:

src/zc/buildout/repeatable.txt

@@ -227,3 +227,166 @@ We can also disable checking versions:
     Uninstalling foo.
     Installing foo.
     recipe v2
+
+
+Easier reporting and managing of versions (new in buildout 2.0)
+---------------------------------------------------------------
+
+Since buildout 2.0, the functionality of the `buildout-versions
+<http://packages.python.org/buildout-versions/>`_ extension is part of
+buildout itself. This makes reporting and managing versions easier.
+
+If you set the ``show-picked-versions`` option, buildout will print
+versions it picked at the end of its run:
+
+    >>> write('buildout.cfg',
+    ... '''
+    ... [buildout]
+    ... parts = foo
+    ... find-links = %s
+    ... show-picked-versions = true
+    ...
+    ... [versions]
+    ...
+    ... [foo]
+    ... recipe = spam
+    ... ''' % join('recipe', 'dist'))
+    >>> print_(system(buildout), end='') # doctest: +ELLIPSIS
+    Updating foo.
+    recipe v2
+    Versions had to be automatically picked.
+    The following part definition lists the versions picked:
+    [versions]
+    distribute = 0.6.99
+    spam = 2
+
+When everything is pinned, no output is generated:
+
+    >>> write('buildout.cfg',
+    ... '''
+    ... [buildout]
+    ... parts = foo
+    ... find-links = %s
+    ... show-picked-versions = true
+    ...
+    ... [versions]
+    ... distribute = 0.6.34
+    ... spam = 2
+    ...
+    ... [foo]
+    ... recipe = spam
+    ... ''' % join('recipe', 'dist'))
+    >>> print_(system(buildout), end='') # doctest: +ELLIPSIS
+    Updating foo.
+    recipe v2
+
+The Python package index is case-insensitive. Both
+http://pypi.python.org/simple/Django/ and
+http://pypi.python.org/simple/dJaNgO/ work. And distributions aren't always
+naming themselves consistently case-wise. So all version names are normalized
+and case differences won't impact the pinning:
+
+    >>> write('buildout.cfg',
+    ... '''
+    ... [buildout]
+    ... parts = foo
+    ... find-links = %s
+    ... show-picked-versions = true
+    ...
+    ... [versions]
+    ... distriBUTE = 0.6.34
+    ... Spam = 2
+    ...
+    ... [foo]
+    ... recipe = spam
+    ... ''' % join('recipe', 'dist'))
+    >>> print_(system(buildout), end='') # doctest: +ELLIPSIS
+    Updating foo.
+    recipe v2
+
+Sometimes it is handy to have a separate file with versions. This is a regular
+buildout file with a single ``[versions]`` section. You include it by
+extending from that versions file:
+
+    >>> write('my_versions.cfg',
+    ... '''
+    ... [versions]
+    ... distribute = 0.6.34
+    ... spam = 2
+    ... ''')
+    >>> write('buildout.cfg',
+    ... '''
+    ... [buildout]
+    ... parts = foo
+    ... extends = my_versions.cfg
+    ... find-links = %s
+    ... show-picked-versions = true
+    ...
+    ... [foo]
+    ... recipe = spam
+    ... ''' % join('recipe', 'dist'))
+    >>> print_(system(buildout), end='') # doctest: +ELLIPSIS
+    Updating foo.
+    recipe v2
+
+If not everything is pinned and buildout has to pick versions, you can tell
+buildout to append the versions to your versions file. It simply appends them
+at the end.
+
+    >>> write('my_versions.cfg',
+    ... '''
+    ... [versions]
+    ... distribute = 0.6.34
+    ... ''')
+    >>> write('buildout.cfg',
+    ... '''
+    ... [buildout]
+    ... parts = foo
+    ... extends = my_versions.cfg
+    ... update-versions-file = my_versions.cfg
+    ... find-links = %s
+    ... show-picked-versions = true
+    ...
+    ... [foo]
+    ... recipe = spam
+    ... ''' % join('recipe', 'dist'))
+    >>> print_(system(buildout), end='') # doctest: +ELLIPSIS
+    Updating foo.
+    recipe v2
+    Versions had to be automatically picked.
+    The following part definition lists the versions picked:
+    [versions]
+    spam = 2
+    This information has been written to my_versions.cfg
+
+The versions file now contains the extra pin:
+
+    >>> 'spam = 2' in open('my_versions.cfg').read()
+    True
+
+And re-running buildout doesn't report any picked versions anymore:
+
+    >>> print_(system(buildout), end='') # doctest: +ELLIPSIS
+    Updating foo.
+    recipe v2
+
+Because buildout now includes buildout-versions' (and the older
+buildout.dumppickedversions') functionality, it warns if these extensions are
+configured.
+
+    >>> write(sample_buildout, 'buildout.cfg',
+    ... """
+    ... [buildout]
+    ... parts = foo
+    ... extensions = buildout-versions
+    ...
+    ... [foo]
+    ... recipe = spam
+    ... """)
+    >>> print_(system(buildout), end='') # doctest: +ELLIPSIS
+    While:
+      Installing.
+      Loading extensions.
+    Error: The extension buildout-versions is now included in buildout itself.
+    Remove the extension from your configuration and look at the `show-picked-versions`
+    option in buildout's documentation.

src/zc/buildout/tests.py

@@ -404,6 +404,41 @@ that we can't find. when run in verbose mode
     Error: Couldn't find a distribution for 'demoneeded'.
     """
 
+def show_who_requires_picked_versions():
+    """
+
+The show-picked-versions prints the versions, but it also prints who
+required the picked distributions.
+We do not need to run in verbose mode for that to work:
+
+    >>> make_dist_that_requires(sample_buildout, 'sampley', ['distribute'])
+    >>> make_dist_that_requires(sample_buildout, 'samplea', ['sampleb'])
+    >>> make_dist_that_requires(sample_buildout, 'sampleb',
+    ...                         ['sampley', 'samplea'])
+    >>> write('buildout.cfg',
+    ... '''
+    ... [buildout]
+    ... parts = eggs
+    ... show-picked-versions = true
+    ... develop = sampley samplea sampleb
+    ...
+    ... [eggs]
+    ... recipe = zc.recipe.egg
+    ... eggs = samplea
+    ... ''')
+
+    >>> print_(system(buildout), end='') # doctest: +ELLIPSIS
+    Develop: ...
+    Installing eggs.
+    Versions had to be automatically picked.
+    The following part definition lists the versions picked:
+    [versions]
+    <BLANKLINE>
+    # Required by:
+    # sampley==1
+    distribute = 0.6.99
+    """
+
 def test_comparing_saved_options_with_funny_characters():
     """
 If an option has newlines, extra/odd spaces or a %, we need to make sure
@@ -3205,6 +3240,7 @@ def test_suite():
                 'We have a develop egg: zc.buildout X.X.'),
                (re.compile(r'\\[\\]?'), '/'),
                (re.compile('WindowsError'), 'OSError'),
+               (re.compile('distribute = \S+'), 'distribute = 0.6.99'),
                (re.compile(r'\[Error 17\] Cannot create a file '
                            r'when that file already exists: '),
                 '[Errno 17] File exists: '
@@ -3320,6 +3356,7 @@ def test_suite():
                  'distribute.egg'),
                 (re.compile('zc.buildout-\S+-'),
                  'zc.buildout.egg'),
+                (re.compile('distribute = \S+'), 'distribute = 0.6.99'),
                 (re.compile('File "\S+one.py"'),
                  'File "one.py"'),
                 (re.compile(r'We have a develop egg: (\S+) (\S+)'),
@@ -3391,4 +3428,3 @@ def test_suite():
             ))
 
     return unittest.TestSuite(test_suite)
-