Merge branch 'bug_778748'

buildout.cfg

@@ -27,4 +27,3 @@ defaults =
   '-t',
   '!(bootstrap|selectingpython|selecting-python)',
   ]
-

setup.py

@@ -82,7 +82,7 @@ setup(
     packages = ['zc', 'zc.buildout'],
     package_dir = {'': 'src'},
     namespace_packages = ['zc'],
-    install_requires = 'setuptools',
+    install_requires = ['setuptools'],
     include_package_data = True,
     entry_points = entry_points,
     extras_require = dict(test=['zope.testing']),

src/zc/buildout/buildout.py

@@ -362,6 +362,8 @@ class Buildout(UserDict.DictMixin):
                 newest=self.newest,
                 allow_hosts=self._allow_hosts,
                 prefer_final=not self.accept_buildout_test_releases,
+                sha_cache=options.get('sha-cache'),
+                sha_dir=options.get('sha-dir'),
                 )
 
         # Now copy buildout and setuptools eggs, and record destination eggs:
@@ -860,6 +862,8 @@ class Buildout(UserDict.DictMixin):
             path = [options['develop-eggs-directory']],
             allow_hosts = self._allow_hosts,
             prefer_final=not self.accept_buildout_test_releases,
+            sha_cache=options.get('sha-cache'),
+            sha_dir=options.get('sha-cache'),
             )
 
         upgraded = []
@@ -1087,7 +1091,9 @@ def _install_and_load(spec, group, entry, buildout):
                 working_set=pkg_resources.working_set,
                 newest=buildout.newest,
                 allow_hosts=buildout._allow_hosts,
-                prefer_final=not buildout.accept_buildout_test_releases)
+                prefer_final=not buildout.accept_buildout_test_releases,
+                sha_dir=buildout_options.get('sha-dir'),
+                sha_cache=buildout_options.get('sha-cache'))
 
         __doing__ = 'Loading %s recipe entry %s:%s.', group, spec, entry
         return pkg_resources.load_entry_point(

src/zc/buildout/download.py

@@ -17,7 +17,6 @@ try:
     from hashlib import md5
 except ImportError:
     from md5 import new as md5
-from zc.buildout.easy_install import realpath
 import logging
 import os
 import os.path
@@ -72,6 +71,8 @@ class Download(object):
         self.fallback = fallback
         self.hash_name = hash_name
         self.logger = logger or logging.getLogger('zc.buildout')
+        self.sha_dir = options.get('sha-dir')
+        self.sha_cache = options.get('sha-cache')
 
     @property
     def download_cache(self):
@@ -179,20 +180,25 @@ class Download(object):
         self.logger.info('Downloading %s' % url)
         urllib._urlopener = url_opener
         handle, tmp_path = tempfile.mkstemp(prefix='buildout-')
+
         try:
-            tmp_path, headers = urllib.urlretrieve(url, tmp_path)
-            if not check_md5sum(tmp_path, md5sum):
-                raise ChecksumError(
-                    'MD5 checksum mismatch downloading %r' % url)
-        except IOError, e:
-            os.remove(tmp_path)
-            raise zc.buildout.UserError("Error downloading extends for URL "
-                              "%s: %r" % (url, e[1:3]))
-        except Exception, e:
+            try:
+                if not download_network_cached(self.sha_dir, self.sha_cache,
+                                          tmp_path, url, self.logger, md5sum):
+
+                    # Download from original url
+                    tmp_path, headers = urllib.urlretrieve(url, tmp_path)
+                    if not check_md5sum(tmp_path, md5sum):
+                        raise ChecksumError(
+                            'MD5 checksum mismatch downloading %r' % url)
+                    # Upload the file to networkcached.
+                    upload_network_cached(self.sha_cache, self.sha_dir, url,
+                                                        tmp_path, self.logger)
+            finally:
+                os.close(handle)
+        except:
             os.remove(tmp_path)
             raise
-        finally:
-            os.close(handle)
 
         if path:
             shutil.move(tmp_path, path)
@@ -253,6 +259,9 @@ def remove(path):
     if os.path.exists(path):
         os.remove(path)
 
+from zc.buildout.networkcache import download_network_cached, \
+                                     upload_network_cached
+from zc.buildout.easy_install import realpath
 
 def locate_at(source, dest):
     if dest is None or realpath(dest) == realpath(source):

src/zc/buildout/easy_install.py

@@ -41,6 +41,12 @@ _oprp = getattr(os.path, 'realpath', lambda path: path)
 def realpath(path):
     return os.path.normcase(os.path.abspath(_oprp(path)))
 
+
+from zc.buildout.networkcache import get_filename_from_url, \
+                                     upload_network_cached, \
+                                     download_network_cached
+
+
 default_index_url = os.environ.get(
     'buildout-testing-index-url',
     'http://pypi.python.org/simple',
@@ -335,6 +341,8 @@ class Installer:
                  include_site_packages=None,
                  allowed_eggs_from_site_packages=None,
                  prefer_final=None,
+                 sha_dir=None,
+                 sha_cache=None
                  ):
         self._dest = dest
         self._allow_hosts = allow_hosts
@@ -403,6 +411,9 @@ class Installer:
         if versions is not None:
             self._versions = versions
 
+        self._sha_dir = sha_dir
+        self._sha_cache = sha_cache
+
     _allowed_eggs_from_site_packages_regex = None
     def allow_site_package_egg(self, name):
         if (not self._include_site_packages or
@@ -702,7 +713,14 @@ class Installer:
             ):
             return dist
 
-        new_location = self._index.download(dist.location, tmp)
+        filename = get_filename_from_url(dist.location)
+        new_location = os.path.join(tmp, filename)
+        if not download_network_cached(self._sha_dir, self._sha_cache,
+                                         new_location, dist.location, logger):
+            new_location = self._index.download(dist.location, tmp)
+            upload_network_cached(self._sha_cache, self._sha_dir,
+                                      dist.location, new_location, logger)
+
         if (download_cache
             and (realpath(new_location) == realpath(dist.location))
             and os.path.isfile(new_location)
@@ -1080,13 +1098,13 @@ def install(specs, dest,
             path=None, working_set=None, newest=True, versions=None,
             use_dependency_links=None, allow_hosts=('*',),
             include_site_packages=None, allowed_eggs_from_site_packages=None,
-            prefer_final=None):
+            prefer_final=None, sha_dir=None, sha_cache=None):
     installer = Installer(
         dest, links, index, executable, always_unzip, path, newest,
         versions, use_dependency_links, allow_hosts=allow_hosts,
         include_site_packages=include_site_packages,
         allowed_eggs_from_site_packages=allowed_eggs_from_site_packages,
-        prefer_final=prefer_final)
+        prefer_final=prefer_final, sha_dir=sha_dir, sha_cache=sha_cache)
     return installer.install(specs, working_set)
 
 

src/zc/buildout/networkcache.py

+##############################################################################
+#
+# Copyright (c) 2010 ViFiB SARL and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+
+
+import hashlib
+import os
+import posixpath
+import re
+import urllib
+import urlparse
+
+
+_md5_re = re.compile(r'md5=([a-f0-9]+)')
+
+
+def _get_md5_from_url(url):
+  match = _md5_re.search(url)
+  if match:
+    return match.group(1)
+  return None
+
+
+def get_directory_key(url):
+    """It must retorn the directory key based on the URL.
+
+    Basically check if the url belongs to pypi:
+      - if yes, the directory key will be pypi-buildout-urlmd5
+      - if not, the directory key will be slapos-buildout-urlmd5
+    """
+    urlmd5 = hashlib.md5(url).hexdigest()
+    if 'pypi' in url:
+      return 'pypi-buildout-%s' % urlmd5
+    return 'slapos-buildout-%s' % urlmd5
+
+
+def download_network_cached(sha_dir, sha_cache, path, url, logger, md5sum=None):
+    """Download from a network cache provider
+
+    If something fail (providor be offline, or hash_string fail), we ignore
+    network cached files.
+
+    return True if download succeeded.
+    """
+    try:
+      from slapos.libnetworkcache import NetworkcacheClient, UploadError, \
+                                   DirectoryNotFound
+    except:
+      return False
+
+
+    if sha_cache in (None, '',):
+        # Not able to use network cache
+        return False
+    if md5sum is None:
+        md5sum = _get_md5_from_url(url)
+
+
+    directory_key = get_directory_key(url)
+    file_name = os.path.basename(url)
+
+    logger.info('Downloading %s from network cache.' % file_name)
+    try:
+        nc = NetworkcacheClient(shacache=sha_cache, shadir=sha_dir)
+        file_content = nc.select(directory_key)
+
+        f = open(path, 'w+b')
+        try:
+          f.seek(0)
+          f.write(file_content)
+        finally:
+          f.close()
+
+        if not check_md5sum(path, md5sum):
+            logger.info('MD5 checksum mismatch downloading %s' % file_name)
+            return False
+
+    except (IOError, DirectoryNotFound), e:
+        logger.info('Fail to download from network cache %s: %s' % \
+                                                       (file_name, str(e)))
+        return False
+    return True
+
+
+def upload_network_cached(sha_cache, sha_dir, external_url, path, logger):
+    """Upload file to a shacache server"""
+    try:
+      from slapos.libnetworkcache import NetworkcacheClient, UploadError, \
+                                   DirectoryNotFound
+    except:
+      return False
+
+    if sha_cache in [None, ''] or sha_dir in [None, '']:
+        logger.debug(
+             'Upload cache ignored, shacache or shadir was not provided!')
+        return False
+
+    file_name = external_url.split('/')[-1].split('#')[0].split('?')[0]
+    directory_key = get_directory_key(external_url)
+    kw = dict(file=file_name,
+              urlmd5=hashlib.md5(external_url).hexdigest())
+
+    f = open(path, 'r')
+    try:
+        nc = NetworkcacheClient(shacache=sha_cache,
+                                shadir=sha_dir)
+        data = nc.upload(f, directory_key, **kw)
+    except (IOError, UploadError), e:
+        logger.info('Fail to upload file. %s' % \
+                                                  (str(e)))
+        return False
+
+    finally:
+      f.close()
+
+    return True
+
+
+def get_filename_from_url(url):
+    """Inspired how pip get filename from url.
+    """
+    url = url.split('#', 1)[0]
+    url = url.split('?', 1)[0]
+    url = url.rstrip('/')
+    name = posixpath.basename(url)
+    assert name, (
+           'URL %r produced no filename' % url)
+    return name
+
+
+from download import check_md5sum

src/zc/buildout/networkcache.txt

+Using buildout with network-cache
+=================================
+
+NOTE: You need to install the networkcached into your system to run this
+      test, otherwise it will not be possible to get proper results.
+
+The buildout itself must be able to cache everything which is downloadble
+into an external and universal cache server.
+
+Such feature is not enabled by default, but you can enable defining the
+option called ```network-cache```.
+
+   >>> write(sample_buildout, 'buildout.cfg',
+   ... '''
+   ... [buildout]
+   ... network-cache = http://127.0.0.1:5001/
+   ... find-links = %(link_server)s
+   ... parts = eggs
+   ...
+   ... [eggs]
+   ... recipe = zc.recipe.egg:eggs
+   ... eggs = demo ==0.2
+   ... ''' % globals())
+
+   >>> print system(buildout)
+   Installing eggs.
+   Getting distribution for 'demo==0.2'.
+   The url is not cached yet: http://localhost/demo-0.2-py2.6.egg
+   Got demo 0.2.
+   Getting distribution for 'demoneeded'.
+   The url is not cached yet: http://localhost/demoneeded-1.2c1.zip
+   Got demoneeded 1.2c1.
+   ...
+
+   >>> from glob import glob
+   >>> from os.path import join
+   >>> for egg in glob(join(sample_buildout, 'eggs', 'demoneeded*.egg')):
+   ...     remove(sample_buildout, 'eggs', egg)
+
+   >>> print system(buildout)
+   Updating eggs.
+   Getting distribution for 'demoneeded'.
+   Downloading from network cache http://127.0.0.1:5001/105e4a9ba3f9c7f45b46cc8c47ac2d2ca6440ae564538090c6d390a8959dd03e
+   Got demoneeded 1.2c1.
+   ...

src/zc/buildout/tests.py

@@ -4053,7 +4053,7 @@ def test_suite():
             ),
 
         doctest.DocFileSuite(
-            'download.txt', 'extends-cache.txt',
+            'download.txt', 'extends-cache.txt', 'networkcache.txt',
             setUp=easy_install_SetUp,
             tearDown=zc.buildout.testing.buildoutTearDown,
             optionflags=doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS,

zc.recipe.egg_/src/zc/recipe/egg/egg.py

@@ -88,6 +88,11 @@ class Eggs(object):
             kw = {}
             if 'unzip' in options:
                 kw['always_unzip'] = options.query_bool('unzip', None)
+            if 'sha-cache' in b_options:
+                kw['sha_cache'] = b_options.get('sha-cache')
+            if 'sha-dir' in b_options:
+                kw['sha_dir'] = b_options.get('sha-dir')
+
             ws = zc.buildout.easy_install.install(
                 distributions, options['eggs-directory'],
                 links=self.links,