Merge branch 'bug_778748'

.gitignore

+*pyc
+*pyo
+.installed.cfg
+bin/
+develop-eggs/
+dist/
+eggs/
+parts/
+src/zc.buildout.egg-info/
+z3c.recipe.scripts_/src/z3c.recipe.scripts.egg-info/
+zc.recipe.egg_/src/zc.recipe.egg.egg-info/

networkcached.cfg

+[buildout]
+extends = buildout.cfg
+
+#develop += path_to_slapos.libnetworkcache
+
+parts += cached
+
+[cached]
+recipe = z3c.recipe.scripts
+eggs =
+  slapos.libnetworkcache
+  zc.buildout
+scripts =
+  buildout=buildout_cached
+
+[test]
+eggs +=
+  slapos.libnetworkcache

setup.py

@@ -82,7 +82,7 @@ setup(
     packages = ['zc', 'zc.buildout'],
     package_dir = {'': 'src'},
     namespace_packages = ['zc'],
-    install_requires = ['setuptools'],
+    install_requires = 'setuptools',
     include_package_data = True,
     entry_points = entry_points,
     extras_require = dict(test=['zope.testing']),

src/zc/buildout/buildout.py

@@ -40,6 +40,12 @@ import zc.buildout
 import zc.buildout.download
 import zc.buildout.easy_install
 
+try:
+  import slapos.libnetworkcache
+except ImportError:
+  LIBNETWORKCACHE_ENABLED = False
+else:
+  LIBNETWORKCACHE_ENABLED = True
 
 realpath = zc.buildout.easy_install.realpath
 
@@ -330,6 +336,45 @@ class Buildout(UserDict.DictMixin):
 
         os.chdir(options['directory'])
 
+        if not LIBNETWORKCACHE_ENABLED:
+            self._logger.warning('Networkache disabled, no '
+                'slapos.libnetworkache available.')
+            self.download_cache_url = None
+            self.download_dir_url = None
+            self.upload_cache_url = None
+            self.upload_dir_url = None
+        else:
+            # support networkcache
+            networkcache_section_name = options.get('networkcache-section')
+            if networkcache_section_name:
+                networkcache_section = self[networkcache_section_name]
+                self.download_cache_url = networkcache_section[
+                    'download-cache-url']
+                self.download_dir_url = networkcache_section[
+                    'download-dir-url']
+                self.upload_cache_url = networkcache_section.get(
+                    'upload-cache-url', '')
+                self.upload_dir_url = networkcache_section.get(
+                    'upload-dir-url', '')
+                self._logger.info('Networkcache enabled.')
+                self._logger.info('Networkcache download cache: %r, directory '
+                    '%r' % (self.download_cache_url, self.download_dir_url))
+                self._logger.info('Networkcache upload cache: %r, directory %r'%
+                    (self.upload_cache_url, self.upload_dir_url))
+                # expose __networkcache__* in options for easier integration
+                options['__networkcache__download-cache-url'] = self.download_cache_url
+                options['__networkcache__download-dir-url'] = self.download_dir_url
+                options['__networkcache__upload-cache-url'] = self.upload_cache_url
+                options['__networkcache__upload-dir-url'] = self.upload_dir_url
+            else:
+                self._logger.warning('Networkcache functionality not enabled.'
+                    'In order to activate use networkcache-section.')
+                self.download_cache_url = None
+                self.download_dir_url = None
+                self.upload_cache_url = None
+                self.upload_dir_url = None
+
+
     def _buildout_path(self, name):
         if '${' in name:
             return name
@@ -345,6 +390,8 @@ class Buildout(UserDict.DictMixin):
         # Get a base working set for our distributions that corresponds to the
         # stated desires in the configuration.
         distributions = ['setuptools', 'zc.buildout']
+        if LIBNETWORKCACHE_ENABLED:
+            distributions.append('slapos.libnetworkcache')
         if options.get('offline') == 'true':
             ws = zc.buildout.easy_install.working_set(
                 distributions, options['executable'],
@@ -362,13 +409,18 @@ class Buildout(UserDict.DictMixin):
                 newest=self.newest,
                 allow_hosts=self._allow_hosts,
                 prefer_final=not self.accept_buildout_test_releases,
-                sha_cache=options.get('sha-cache'),
-                sha_dir=options.get('sha-dir'),
+                download_cache_url=self.download_cache_url,
+                download_dir_url=self.download_dir_url,
+                upload_cache_url=self.upload_cache_url,
+                upload_dir_url=self.upload_dir_url
                 )
 
         # Now copy buildout and setuptools eggs, and record destination eggs:
         entries = []
-        for name in 'setuptools', 'zc.buildout':
+        copy_list = ['setuptools', 'zc.buildout']
+        if LIBNETWORKCACHE_ENABLED:
+          copy_list.append('slapos.libnetworkcache')
+        for name in copy_list:
             r = pkg_resources.Requirement.parse(name)
             dist = ws.find(r)
             if dist.precedence == pkg_resources.DEVELOP_DIST:
@@ -862,8 +914,10 @@ class Buildout(UserDict.DictMixin):
             path = [options['develop-eggs-directory']],
             allow_hosts = self._allow_hosts,
             prefer_final=not self.accept_buildout_test_releases,
-            sha_cache=options.get('sha-cache'),
-            sha_dir=options.get('sha-cache'),
+            download_cache_url=self.download_cache_url,
+            download_dir_url=self.download_dir_url,
+            upload_cache_url=self.upload_cache_url,
+            upload_dir_url=self.upload_dir_url
             )
 
         upgraded = []
@@ -1092,8 +1146,14 @@ def _install_and_load(spec, group, entry, buildout):
                 newest=buildout.newest,
                 allow_hosts=buildout._allow_hosts,
                 prefer_final=not buildout.accept_buildout_test_releases,
-                sha_dir=buildout_options.get('sha-dir'),
-                sha_cache=buildout_options.get('sha-cache'))
+                download_cache_url=buildout_options.get(
+                    '__networkcache__download-cache-url'),
+                download_dir_url=buildout_options.get(
+                    '__networkcache__download-dir-url'),
+                upload_cache_url=buildout_options.get(
+                    '__networkcache__upload-cache-url'),
+                upload_dir_url=buildout_options.get('__networkcache__upload-dir-url')
+                )
 
         __doing__ = 'Loading %s recipe entry %s:%s.', group, spec, entry
         return pkg_resources.load_entry_point(
@@ -1454,22 +1514,6 @@ def _open(base, filename, seen, dl_options, override, downloaded):
                     filename)
         result[section] = options
 
-    # find and expose _profile_base_location_ and _profile_location_
-    for section, value in result.iteritems():
-        _profile_base_location_ = None
-        _profile_location_ = None
-        for k,v in value.iteritems():
-          if '${:_profile_base_location_}' in v:
-            _profile_base_location_ = base
-          if '${:_profile_location_}' in v:
-            _profile_location_ = filename
-          if _profile_base_location_ is not None and _profile_location_ is not None:
-            break
-        if _profile_base_location_ is not None:
-          value['_profile_base_location_'] = _profile_base_location_
-        if _profile_location_ is not None:
-          value['_profile_location_'] = _profile_location_
-
     result = _annotate(result, filename)
 
     if root_config_file and 'buildout' in result:

src/zc/buildout/download.py

@@ -71,8 +71,12 @@ class Download(object):
         self.fallback = fallback
         self.hash_name = hash_name
         self.logger = logger or logging.getLogger('zc.buildout')
-        self.sha_dir = options.get('sha-dir')
-        self.sha_cache = options.get('sha-cache')
+        self.download_dir_url = options.get('__networkcache__download-dir-url')
+        self.download_cache_url = options.get(
+            '__networkcache__download-cache-url')
+        self.upload_dir_url = options.get('__networkcache__upload-dir-url')
+        self.upload_cache_url = options.get(
+            '__networkcache__upload-cache-url')
 
     @property
     def download_cache(self):
@@ -181,7 +185,7 @@ class Download(object):
 
         try:
             try:
-                if not download_network_cached(self.sha_dir, self.sha_cache,
+                if not download_network_cached(self.download_dir_url, self.download_cache_url,
                                           tmp_path, url, self.logger, md5sum):
 
                     # Download from original url
@@ -190,8 +194,9 @@ class Download(object):
                         raise ChecksumError(
                             'MD5 checksum mismatch downloading %r' % url)
                     # Upload the file to networkcached.
-                    upload_network_cached(self.sha_cache, self.sha_dir, url,
-                                                        tmp_path, self.logger)
+                    if self.upload_cache_url and self.upload_dir_url:
+                        upload_network_cached(self.upload_dir_url,
+                            self.upload_cache_url, url, tmp_path, self.logger)
             finally:
                 os.close(handle)
         except:

src/zc/buildout/easy_install.py

@@ -341,8 +341,10 @@ class Installer:
                  include_site_packages=None,
                  allowed_eggs_from_site_packages=None,
                  prefer_final=None,
-                 sha_dir=None,
-                 sha_cache=None
+                 download_dir_url=None,
+                 download_cache_url=None,
+                 upload_dir_url=None,
+                 upload_cache_url=None
                  ):
         self._dest = dest
         self._allow_hosts = allow_hosts
@@ -411,8 +413,10 @@ class Installer:
         if versions is not None:
             self._versions = versions
 
-        self._sha_dir = sha_dir
-        self._sha_cache = sha_cache
+        self._download_dir_url = download_dir_url
+        self._download_cache_url = download_cache_url
+        self._upload_dir_url = upload_dir_url
+        self._upload_cache_url = upload_cache_url
 
     _allowed_eggs_from_site_packages_regex = None
     def allow_site_package_egg(self, name):
@@ -715,11 +719,12 @@ class Installer:
 
         filename = get_filename_from_url(dist.location)
         new_location = os.path.join(tmp, filename)
-        if not download_network_cached(self._sha_dir, self._sha_cache,
+        if not download_network_cached(self._download_dir_url, self._download_cache_url,
                                          new_location, dist.location, logger):
             new_location = self._index.download(dist.location, tmp)
-            upload_network_cached(self._sha_cache, self._sha_dir,
-                                      dist.location, new_location, logger)
+            if self._upload_cache_url and self._upload_dir_url:
+                upload_network_cached(self._upload_dir_url,
+                   self._upload_cache_url, dist.location, new_location, logger)
 
         if (download_cache
             and (realpath(new_location) == realpath(dist.location))
@@ -1098,13 +1103,16 @@ def install(specs, dest,
             path=None, working_set=None, newest=True, versions=None,
             use_dependency_links=None, allow_hosts=('*',),
             include_site_packages=None, allowed_eggs_from_site_packages=None,
-            prefer_final=None, sha_dir=None, sha_cache=None):
+            prefer_final=None, download_dir_url=None, download_cache_url=None,
+            upload_dir_url=None, upload_cache_url=None):
     installer = Installer(
         dest, links, index, executable, always_unzip, path, newest,
         versions, use_dependency_links, allow_hosts=allow_hosts,
         include_site_packages=include_site_packages,
         allowed_eggs_from_site_packages=allowed_eggs_from_site_packages,
-        prefer_final=prefer_final, sha_dir=sha_dir, sha_cache=sha_cache)
+        prefer_final=prefer_final, download_dir_url=download_dir_url,
+        download_cache_url=download_cache_url, upload_dir_url=upload_dir_url,
+        upload_cache_url=upload_cache_url)
     return installer.install(specs, working_set)
 
 

src/zc/buildout/networkcache.py

@@ -17,9 +17,14 @@ import hashlib
 import os
 import posixpath
 import re
-import urllib
-import urlparse
 
+try:
+    from slapos.libnetworkcache import NetworkcacheClient, UploadError, \
+      DirectoryNotFound
+except ImportError:
+    LIBNETWORKCACHE_ENABLED = False
+else:
+    LIBNETWORKCACHE_ENABLED = True
 
 _md5_re = re.compile(r'md5=([a-f0-9]+)')
 
@@ -32,7 +37,7 @@ def _get_md5_from_url(url):
 
 
 def get_directory_key(url):
-    """It must retorn the directory key based on the URL.
+    """Returns directory hash based on url.
 
     Basically check if the url belongs to pypi:
       - if yes, the directory key will be pypi-buildout-urlmd5
@@ -44,23 +49,18 @@ def get_directory_key(url):
     return 'slapos-buildout-%s' % urlmd5
 
 
-def download_network_cached(sha_dir, sha_cache, path, url, logger, md5sum=None):
-    """Download from a network cache provider
+def download_network_cached(dir_url, cache_url, path, url, logger, md5sum=None):
+    """Downloads from a network cache provider
 
     If something fail (providor be offline, or hash_string fail), we ignore
     network cached files.
 
     return True if download succeeded.
     """
-    try:
-      from slapos.libnetworkcache import NetworkcacheClient, UploadError, \
-                                   DirectoryNotFound
-    except:
+    if not LIBNETWORKCACHE_ENABLED:
         return False
 
-
-    if sha_cache in (None, '',):
-        # Not able to use network cache
+    if not(dir_url and cache_url):
         return False
     if md5sum is None:
         md5sum = _get_md5_from_url(url)
@@ -71,7 +71,7 @@ def download_network_cached(sha_dir, sha_cache, path, url, logger, md5sum=None):
 
     logger.info('Downloading %s from network cache.' % file_name)
     try:
-        nc = NetworkcacheClient(shacache=sha_cache, shadir=sha_dir)
+        nc = NetworkcacheClient(shacache=cache_url, shadir=dir_url)
         file_content = nc.select(directory_key)
 
         f = open(path, 'w+b')
@@ -86,25 +86,22 @@ def download_network_cached(sha_dir, sha_cache, path, url, logger, md5sum=None):
             return False
 
     except (IOError, DirectoryNotFound), e:
-        logger.info('Fail to download from network cache %s: %s' % \
+        logger.info('Failed to download from network cache %s: %s' % \
                                                        (file_name, str(e)))
         return False
     return True
 
 
-def upload_network_cached(sha_cache, sha_dir, external_url, path, logger):
-    """Upload file to a shacache server"""
-    try:
-      from slapos.libnetworkcache import NetworkcacheClient, UploadError, \
-                                   DirectoryNotFound
-    except:
+def upload_network_cached(dir_url, cache_url, external_url, path, logger):
+    """Upload file to a network cache server"""
+    if not LIBNETWORKCACHE_ENABLED:
         return False
 
-    if sha_cache in [None, ''] or sha_dir in [None, '']:
-        logger.debug(
-             'Upload cache ignored, shacache or shadir was not provided!')
+    if not (dir_url and cache_url):
         return False
 
+    logger.info('Uploading %s into network cache.' % external_url)
+
     file_name = external_url.split('/')[-1].split('#')[0].split('?')[0]
     directory_key = get_directory_key(external_url)
     kw = dict(file=file_name,
@@ -112,9 +109,9 @@ def upload_network_cached(sha_cache, sha_dir, external_url, path, logger):
 
     f = open(path, 'r')
     try:
-        nc = NetworkcacheClient(shacache=sha_cache,
-                                shadir=sha_dir)
-        data = nc.upload(f, directory_key, **kw)
+        nc = NetworkcacheClient(shacache=cache_url,
+                                shadir=dir_url)
+        return nc.upload(f, directory_key, **kw)
     except (IOError, UploadError), e:
         logger.info('Fail to upload file. %s' % \
                                                   (str(e)))

src/zc/buildout/networkcache.txt

-Using buildout with network-cache
-=================================
+Using buildout with cache over network
+======================================
 
-NOTE: You need to install the networkcached into your system to run this
-      test, otherwise it will not be possible to get proper results.
-
-The buildout itself must be able to cache everything which is downloadble
-into an external and universal cache server.
-
-Such feature is not enabled by default, but you can enable defining the
-option called ```network-cache```.
+Note: We are in process of preparing docs. This is only showcase.
 
    >>> write(sample_buildout, 'buildout.cfg',
    ... '''
    ... [buildout]
-   ... network-cache = http://127.0.0.1:5001/
+   ...
+   ... networkcache-section = networkcache
    ... find-links = %(link_server)s
    ... parts = eggs
    ...
    ... [eggs]
    ... recipe = zc.recipe.egg:eggs
    ... eggs = demo ==0.2
-   ... ''' % globals())
-
-   >>> print system(buildout)
-   Installing eggs.
-   Getting distribution for 'demo==0.2'.
-   The url is not cached yet: http://localhost/demo-0.2-py2.6.egg
-   Got demo 0.2.
-   Getting distribution for 'demoneeded'.
-   The url is not cached yet: http://localhost/demoneeded-1.2c1.zip
-   Got demoneeded 1.2c1.
    ...
+   ... [networkcache]
+   ... download-cache-url = http://127.0.0.1:1/
+   ... download-dir-url = http://127.0.0.1:1/
+   ... upload-cache-url = http://127.0.0.1:1/
+   ... upload-dir-url = http://127.0.0.1:1/
+   ... ''' % globals())
 
    >>> from glob import glob
    >>> from os.path import join
@@ -38,8 +28,20 @@ option called ```network-cache```.
    ...     remove(sample_buildout, 'eggs', egg)
 
    >>> print system(buildout)
-   Updating eggs.
+   Networkcache enabled.
+   Networkcache download cache: 'http://127.0.0.1:1/', directory 'http://127.0.0.1:1/'
+   Networkcache upload cache: 'http://127.0.0.1:1/', directory 'http://127.0.0.1:1/'
+   Installing eggs.
+   Getting distribution for 'demo==0.2'.
+   Downloading demo-0.2-py2.7.egg from network cache.
+   Failed to download from network cache demo-0.2-py2.7.egg: [Errno 111] Connection refused
+   Uploading http://localhost/demo-0.2-py2.7.egg into network cache.
+   Fail to upload file. [Errno 111] Connection refused
+   Got demo 0.2.
    Getting distribution for 'demoneeded'.
-   Downloading from network cache http://127.0.0.1:5001/105e4a9ba3f9c7f45b46cc8c47ac2d2ca6440ae564538090c6d390a8959dd03e
+   Downloading demoneeded-1.2c1.zip from network cache.
+   Failed to download from network cache demoneeded-1.2c1.zip: [Errno 111] Connection refused
+   Uploading http://localhost/demoneeded-1.2c1.zip into network cache.
+   Fail to upload file. [Errno 111] Connection refused
    Got demoneeded 1.2c1.
-   ...
+   <BLANKLINE>

zc.recipe.egg_/src/zc/recipe/egg/egg.py

@@ -88,10 +88,13 @@ class Eggs(object):
             kw = {}
             if 'unzip' in options:
                 kw['always_unzip'] = options.query_bool('unzip', None)
-            if 'sha-cache' in b_options:
-                kw['sha_cache'] = b_options.get('sha-cache')
-            if 'sha-dir' in b_options:
-                kw['sha_dir'] = b_options.get('sha-dir')
+            for option_key, kw_key in (
+                ('__networkcache__download-cache-url', 'download_cache_url'),
+                ('__networkcache__download-dir-url', 'download_dir_url'),
+                ('__networkcache__upload-cache-url', 'upload_cache_url'),
+                ('__networkcache__upload-dir-url', 'upload_dir_url')):
+                if option_key in b_options:
+                    kw[kw_key] = b_options[option_key]
 
             ws = zc.buildout.easy_install.install(
                 distributions, options['eggs-directory'],