merged the tlotze-download-api branch

CHANGES.txt

 Change History
 **************
 
+1.4.0 (unreleased)
+==================
+
 - Added annotate command for annotated sections. Displays sections key-value pairs
   along with the value origin.
 
+- Added a download API that handles the download cache, offline mode etc and
+  is meant to be reused by recipes.
+
+- Used the download API to allow caching of base configurations (specified by
+  the buildout section's 'extends' option).
+
 1.3.0 (2009-06-22)
 ==================
 

setup.py

 ##############################################################################
 #
-# Copyright (c) 2006 Zope Corporation and Contributors.
+# Copyright (c) 2006-2009 Zope Corporation and Contributors.
 # All Rights Reserved.
 #
 # This software is subject to the provisions of the Zope Public License,
@@ -12,7 +12,7 @@
 #
 ##############################################################################
 
-version = "1.3.1dev"
+version = "1.4.0dev"
 
 import os
 from setuptools import setup, find_packages
@@ -32,8 +32,12 @@ long_description=(
         + '\n' +
         read('src', 'zc', 'buildout', 'repeatable.txt')
         + '\n' +
+        read('src', 'zc', 'buildout', 'download.txt')
+        + '\n' +
         read('src', 'zc', 'buildout', 'downloadcache.txt')
         + '\n' +
+        read('src', 'zc', 'buildout', 'extends-cache.txt')
+        + '\n' +
         read('src', 'zc', 'buildout', 'setup.txt')
         + '\n' +
         read('src', 'zc', 'buildout', 'update.txt')

src/zc/buildout/buildout.py

 ##############################################################################
 #
-# Copyright (c) 2005 Zope Corporation and Contributors.
+# Copyright (c) 2005-2009 Zope Corporation and Contributors.
 # All Rights Reserved.
 #
 # This software is subject to the provisions of the Zope Public License,
@@ -25,7 +25,6 @@ import shutil
 import cStringIO
 import sys
 import tempfile
-import urllib2
 import ConfigParser
 import UserDict
 import glob
@@ -34,6 +33,7 @@ import copy
 
 import pkg_resources
 import zc.buildout
+import zc.buildout.download
 import zc.buildout.easy_install
 
 from rmtree import rmtree
@@ -156,17 +156,22 @@ class Buildout(UserDict.DictMixin):
         else:
             base = None
 
+        override = dict((option, (value, 'COMMAND_LINE_VALUE'))
+                        for section, option, value in cloptions
+                        if section == 'buildout')
+
         # load user defaults, which override defaults
         if user_defaults:
             user_config = os.path.join(os.path.expanduser('~'),
                                        '.buildout', 'default.cfg')
             if os.path.exists(user_config):
                 _update(data, _open(os.path.dirname(user_config), user_config,
-                                    []))
+                                    [], data['buildout'].copy(), override))
 
         # load configuration files
         if config_file:
-            _update(data, _open(os.path.dirname(config_file), config_file, []))
+            _update(data, _open(os.path.dirname(config_file), config_file, [],
+                                data['buildout'].copy(), override))
 
         # apply command-line options
         for (section, option, value) in cloptions:
@@ -174,7 +179,6 @@ class Buildout(UserDict.DictMixin):
             if options is None:
                 options = data[section] = {}
             options[option] = value, "COMMAND_LINE_VALUE"
-                # The egg dire
 
         self._annotated = copy.deepcopy(data)
         self._raw = _unannotate(data)
@@ -313,6 +317,11 @@ class Buildout(UserDict.DictMixin):
         for name in _buildout_default_options:
             options[name]
 
+        # Do the same for extends-cache which is not among the defaults but
+        # wasn't recognized as having been used since it was used before
+        # tracking was turned on.
+        options.get('extends-cache')
+
         os.chdir(options['directory'])
 
     def _buildout_path(self, name):
@@ -1214,14 +1223,18 @@ def _save_options(section, options, f):
     for option, value in items:
         _save_option(option, value, f)
 
-def _open(base, filename, seen):
+def _open(base, filename, seen, dl_options, override):
     """Open a configuration file and return the result as a dictionary,
 
     Recursively open other files based on buildout options found.
     """
-
+    _update_section(dl_options, override)
+    _dl_options = _unannotate_section(dl_options.copy())
+    download = zc.buildout.download.Download(
+        _dl_options, cache=_dl_options.get('extends-cache'), fallback=True,
+        hash_name=True)
     if _isurl(filename):
-        fp = urllib2.urlopen(filename)
+        fp = open(download(filename))
         base = filename[:filename.rfind('/')]
     elif _isurl(base):
         if os.path.isabs(filename):
@@ -1229,7 +1242,7 @@ def _open(base, filename, seen):
             base = os.path.dirname(filename)
         else:
             filename = base + '/' + filename
-            fp = urllib2.urlopen(filename)
+            fp = open(download(filename))
             base = filename[:filename.rfind('/')]
     else:
         filename = os.path.join(base, filename)
@@ -1239,6 +1252,7 @@ def _open(base, filename, seen):
     if filename in seen:
         raise zc.buildout.UserError("Recursive file include", seen, filename)
 
+    root_config_file = not seen
     seen.append(filename)
 
     result = {}
@@ -1256,18 +1270,23 @@ def _open(base, filename, seen):
 
     result = _annotate(result, filename)
 
+    if root_config_file and 'buildout' in result:
+        dl_options = _update_section(dl_options, result['buildout'])
+
     if extends:
         extends = extends.split()
         extends.reverse()
         for fname in extends:
-            result = _update(_open(base, fname, seen), result)
+            result = _update(_open(base, fname, seen, dl_options, override),
+                             result)
 
     if extended_by:
         self._logger.warn(
             "The extendedBy option is deprecated.  Stop using it."
             )
         for fname in extended_by.split():
-            result = _update(result, _open(base, fname, seen))
+            result = _update(result,
+                             _open(base, fname, seen, dl_options, override))
 
     seen.pop()
     return result

src/zc/buildout/download.py

+##############################################################################
+#
+# Copyright (c) 2009 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Buildout download infrastructure"""
+
+try:
+    from hashlib import md5
+except ImportError:
+    from md5 import new as md5
+from zc.buildout.easy_install import realpath
+import atexit
+import logging
+import os
+import os.path
+import shutil
+import tempfile
+import urllib
+import urlparse
+import zc.buildout
+
+
+class URLOpener(urllib.FancyURLopener):
+    http_error_default = urllib.URLopener.http_error_default
+
+
+class ChecksumError(zc.buildout.UserError):
+    pass
+
+
+url_opener = URLOpener()
+
+
+class Download(object):
+    """Configurable download utility.
+
+    Handles the download cache and offline mode.
+
+    Download(options=None, cache=None, namespace=None, hash_name=False)
+
+    options: mapping of buildout options (e.g. a ``buildout`` config section)
+    cache: path to the download cache (excluding namespaces)
+    namespace: namespace directory to use inside the cache
+    hash_name: whether to use a hash of the URL as cache file name
+    logger: an optional logger to receive download-related log messages
+
+    """
+
+    def __init__(self, options={}, cache=-1, namespace=None,
+                 offline=-1, fallback=False, hash_name=False, logger=None):
+        self.cache = cache
+        if cache == -1:
+            self.cache = options.get('download-cache')
+        self.namespace = namespace
+        self.offline = offline
+        if offline == -1:
+            self.offline = (options.get('offline') == 'true'
+                            or options.get('install-from-cache') == 'true')
+        self.fallback = fallback
+        self.hash_name = hash_name
+        self.logger = logger or logging.getLogger('zc.buildout')
+
+    @property
+    def cache_dir(self):
+        if self.cache is not None:
+            return os.path.join(realpath(self.cache), self.namespace or '')
+
+    def __call__(self, url, md5sum=None, path=None):
+        """Download a file according to the utility's configuration.
+
+        url: URL to download
+        md5sum: MD5 checksum to match
+        path: where to place the downloaded file
+
+        Returns the path to the downloaded file.
+
+        """
+        if self.cache:
+            local_path = self.download_cached(url, md5sum)
+        else:
+            local_path = self.download(url, md5sum, path)
+
+        return locate_at(local_path, path)
+
+    def download_cached(self, url, md5sum=None):
+        """Download a file from a URL using the cache.
+
+        This method assumes that the cache has been configured. Optionally, it
+        raises a ChecksumError if a cached copy of a file has an MD5 mismatch,
+        but will not remove the copy in that case.
+
+        """
+        cache_dir = self.cache_dir
+        if not os.path.exists(cache_dir):
+            os.makedirs(cache_dir)
+        cache_key = self.filename(url)
+        cached_path = os.path.join(cache_dir, cache_key)
+
+        self.logger.debug('Searching cache at %s' % cache_dir)
+        if os.path.isfile(cached_path):
+            if self.fallback:
+                try:
+                    self.download(url, md5sum, cached_path)
+                except ChecksumError:
+                    raise
+                except Exception:
+                    pass
+
+            if not check_md5sum(cached_path, md5sum):
+                raise ChecksumError(
+                    'MD5 checksum mismatch for cached download '
+                    'from %r at %r' % (url, cached_path))
+            self.logger.debug('Using cache file %s' % cached_path)
+        else:
+            self.logger.debug('Cache miss; will cache %s as %s' %
+                              (url, cached_path))
+            self.download(url, md5sum, cached_path)
+
+        return cached_path
+
+    def download(self, url, md5sum=None, path=None):
+        """Download a file from a URL to a given or temporary path.
+
+        An online resource is always downloaded to a temporary file and moved
+        to the specified path only after the download is complete and the
+        checksum (if given) matches. If path is None, the temporary file is
+        returned and scheduled for deletion at process exit.
+
+        """
+        parsed_url = urlparse.urlparse(url, 'file')
+        if parsed_url.scheme == 'file':
+            self.logger.debug('Using local resource %s' % url)
+            if not check_md5sum(parsed_url.path, md5sum):
+                raise ChecksumError(
+                    'MD5 checksum mismatch for local resource at %r.' %
+                    parsed_url.path)
+            return locate_at(parsed_url.path, path)
+
+        if self.offline:
+            raise zc.buildout.UserError(
+                "Couldn't download %r in offline mode." % url)
+
+        self.logger.info('Downloading %s' % url)
+        urllib._urlopener = url_opener
+        handle, tmp_path = tempfile.mkstemp(prefix='buildout-')
+        tmp_path, headers = urllib.urlretrieve(url, tmp_path)
+        if not check_md5sum(tmp_path, md5sum):
+            os.remove(tmp_path)
+            raise ChecksumError(
+                'MD5 checksum mismatch downloading %r' % url)
+
+        if path:
+            shutil.move(tmp_path, path)
+            return path
+        else:
+            atexit.register(remove, tmp_path)
+            return tmp_path
+
+    def filename(self, url):
+        """Determine a file name from a URL according to the configuration.
+
+        """
+        if self.hash_name:
+            return md5(url).hexdigest()
+        else:
+            parsed = urlparse.urlparse(url)
+            for name in reversed(parsed.path.split('/')):
+                if name:
+                    return name
+            else:
+                return '%s:%s' % (parsed.host, parsed.port)
+
+
+def check_md5sum(path, md5sum):
+    """Tell whether the MD5 checksum of the file at path matches.
+
+    No checksum being given is considered a match.
+
+    """
+    if md5sum is None:
+        return True
+
+    f = open(path)
+    checksum = md5()
+    try:
+        chunk = f.read(2**16)
+        while chunk:
+            checksum.update(chunk)
+            chunk = f.read(2**16)
+        return checksum.hexdigest() == md5sum
+    finally:
+        f.close()
+
+
+def remove(path):
+    if os.path.exists(path):
+        os.remove(path)
+
+
+def locate_at(source, dest):
+    if dest is None or realpath(dest) == realpath(source):
+        return source
+
+    try:
+        os.link(source, dest)
+    except (AttributeError, OSError):
+        shutil.copyfile(source, dest)
+    return dest

src/zc/buildout/testing.py

 #############################################################################
 #
-# Copyright (c) 2004 Zope Corporation and Contributors.
+# Copyright (c) 2004-2009 Zope Corporation and Contributors.
 # All Rights Reserved.
 #
 # This software is subject to the provisions of the Zope Public License,
@@ -197,7 +197,7 @@ def wait_until(label, func, *args, **kw):
     while time.time() < deadline:
         if func(*args, **kw):
             return
-        time.sleep('.01')
+        time.sleep(0.01)
     raise ValueError('Timed out waiting for: '+label)
 
 def buildoutSetUp(test):

src/zc/buildout/tests.py

 ##############################################################################
 #
-# Copyright (c) 2004 Zope Corporation and Contributors.
+# Copyright (c) 2004-2009 Zope Corporation and Contributors.
 # All Rights Reserved.
 #
 # This software is subject to the provisions of the Zope Public License,
@@ -2811,6 +2811,21 @@ def test_suite():
                (re.compile(r'\\[\\]?'), '/'),
                ]),
             ),
+
+        doctest.DocFileSuite(
+            'download.txt', 'extends-cache.txt',
+            setUp=easy_install_SetUp,
+            tearDown=zc.buildout.testing.buildoutTearDown,
+            optionflags=doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS,
+            checker=renormalizing.RENormalizing([
+               (re.compile('0x[0-9a-f]+'), '<MEM ADDRESS>'),
+               (re.compile('http://localhost:[0-9]{4,5}/'),
+                'http://localhost/'),
+               (re.compile('[0-9a-f]{32}'), '<MD5 CHECKSUM>'),
+               zc.buildout.testing.normalize_path,
+               ]),
+            ),
+
         doctest.DocTestSuite(
             setUp=easy_install_SetUp,
             tearDown=zc.buildout.testing.buildoutTearDown,