Cache pypi index, extensions and recipes using buildout internals.

src/zc/buildout/buildout.py

@@ -1193,7 +1193,17 @@ class Buildout(UserDict.DictMixin):
                 links = self['buildout'].get('find-links', '').split(),
                 index = self['buildout'].get('index'),
                 newest=self.newest, allow_hosts=self._allow_hosts,
-                prefer_final=not self.accept_buildout_test_releases)
+                prefer_final=not self.accept_buildout_test_releases,
+                download_cache_url=self.download_cache_url,
+                download_dir_url=self.download_dir_url,
+                upload_cache_url=self.upload_cache_url,
+                upload_dir_url=self.upload_dir_url,
+                signature_private_key_file=self.signature_private_key_file,
+                signature_certificate_list=self.signature_certificate_list,
+                shacache_cert_file=self.shacache_cert_file,
+                shacache_key_file=self.shacache_key_file,
+                shadir_cert_file=self.shadir_cert_file,
+                shadir_key_file=self.shadir_key_file,)
 
             # Clear cache because extensions might now let us read pages we
             # couldn't read before.

src/zc/buildout/easy_install.py

@@ -44,7 +44,9 @@ def realpath(path):
 
 from zc.buildout.networkcache import get_filename_from_url, \
                                      upload_network_cached, \
-                                     download_network_cached
+                                     download_network_cached, \
+                                     download_index_network_cached, \
+                                     upload_index_network_cached
 
 
 default_index_url = os.environ.get(
@@ -199,22 +201,137 @@ def _get_version(executable):
         return version
 
 FILE_SCHEME = re.compile('file://', re.I).match
-
+URL_SCHEME = re.compile('([-+.a-z0-9]{2,}):',re.I).match
+HREF = re.compile("""href\\s*=\\s*['"]?([^'"> ]+)""", re.I)
+from setuptools.package_index import distros_for_url, htmldecode
+import urlparse
 
 class AllowHostsPackageIndex(setuptools.package_index.PackageIndex):
     """Will allow urls that are local to the system.
 
     No matter what is allow_hosts.
     """
+
+    def __init__(self, index_url, hosts, search_path, python,):
+        setuptools.package_index.PackageIndex.__init__(self,
+           index_url, hosts=hosts, search_path=search_path,
+           python=python)
+
     def url_ok(self, url, fatal=False):
         if FILE_SCHEME(url):
             return True
         return setuptools.package_index.PackageIndex.url_ok(self, url, False)
 
+    def obtain(self, requirement, installer=None):
+        # XXX BEGIN HARDCODE
+        self._current_requirement = requirement.__str__()
+        # XXX END HARDCODE
+
+        self.prescan(); self.find_packages(requirement)
+        for dist in self[requirement.key]:
+            if dist in requirement:
+                return dist
+            self.debug("%s does not match %s", requirement, dist)
+        return super(PackageIndex, self).obtain(requirement,installer)
+
+    def find_packages(self, requirement):
+        # XXX BEGIN HARDCODE
+        self._current_requirement = requirement.__str__()
+        # XXX END HARDCODE
+
+        self.scan_url(self.index_url + requirement.unsafe_name+'/')
+
+        if not self.package_pages.get(requirement.key):
+            # Fall back to safe version of the name
+            self.scan_url(self.index_url + requirement.project_name+'/')
+
+        if not self.package_pages.get(requirement.key):
+            # We couldn't find the target package, so search the index page too
+            self.not_found_in_index(requirement)
+
+        for url in list(self.package_pages.get(requirement.key,())):
+            # scan each page that might be related to the desired package
+            self.scan_url(url)
+
+    def process_url(self, url, retrieve=False):
+        """Evaluate a URL as a possible download, and maybe retrieve it"""
+        if url in self.scanned_urls and not retrieve:
+            return
+        self.scanned_urls[url] = True
+        if not URL_SCHEME(url):
+            self.process_filename(url)
+            return
+        else:
+            dists = list(distros_for_url(url))
+            if dists:
+                if not self.url_ok(url):
+                    return
+                self.debug("Found link: %s", url)
+
+        if dists or not retrieve or url in self.fetched_urls:
+            map(self.add, dists)
+            return  # don't need the actual page
+
+        if not self.url_ok(url):
+            self.fetched_urls[url] = True
+            return
+
+        self.info("Reading %s", url)
+        self.fetched_urls[url] = True   # prevent multiple fetch attempts
+
+        # We download from cache only if we have a specific version for egg requirement
+        # Reason: If no specific version, we don't want to be blocked with an old index.
+        #         If there is specific version, we want index to freeze forever.
+        download_result = False
+        if getattr(self, '_current_requirement', None) \
+          and "==" in self._current_requirement \
+          and globals().get('_download_dir_url') \
+          and globals().get('_download_dir_url') \
+          and globals().get('_signature_certificate_list'):
+            download_result = download_index_network_cached(_download_dir_url,
+               _download_cache_url, url, self._current_requirement, logger,
+               _signature_certificate_list)
+        if download_result:
+            # Successfully fetched from cache. Hardcode some things...
+            page, base = download_result
+            f = None
+            self.fetched_urls[base] = True
+        else:
+            f = self.open_url(url, "Download error: %s -- Some packages may not be found!")
+            if f is None: return
+            self.fetched_urls[f.url] = True
+            if 'html' not in f.headers.get('content-type', '').lower():
+                f.close()   # not html, we can't process it
+                return
+
+            base = f.url     # handle redirects
+            page = f.read()
+            f.close()
+
+            # Check if shacache options exist in global vars, then check if
+            # it is not null
+            if globals().get('_upload_cache_url') \
+               and globals().get('_upload_dir_url') \
+               and getattr(self, '_current_requirement', None) \
+               and "==" in self._current_requirement \
+               and getattr(f,'code',None)!=404:
+                upload_index_network_cached(_upload_dir_url,
+                   _upload_cache_url, url, base, self._current_requirement, page, logger,
+                   globals().get('_signature_private_key_file'),
+                   globals().get('_shacache_cert_file'),
+                   globals().get('_shacache_key_file'),
+                   globals().get('_shadir_cert_file'),
+                   globals().get('_shadir_key_file'))
+
+        if url.startswith(self.index_url) and getattr(f,'code',None)!=404:
+            page = self.process_index(url, page)
+        for match in HREF.finditer(page):
+            link = urlparse.urljoin(base, htmldecode(match.group(1)))
+            self.process_url(link)
 
 _indexes = {}
 def _get_index(executable, index_url, find_links, allow_hosts=('*',),
-               path=None):
+               path=None,):
     # If path is None, the index will use sys.path.  If you provide an empty
     # path ([]), it will complain uselessly about missing index pages for
     # packages found in the paths that you expect to use.  Therefore, this path
@@ -228,7 +345,7 @@ def _get_index(executable, index_url, find_links, allow_hosts=('*',),
         index_url = default_index_url
     index = AllowHostsPackageIndex(
         index_url, hosts=allow_hosts, search_path=path,
-        python=_get_version(executable)
+        python=_get_version(executable),
         )
 
     if find_links:
@@ -352,6 +469,39 @@ class Installer:
                  shadir_cert_file=None,
                  shadir_key_file=None,
                  ):
+        # XXX Setting shacahe options to global variables in order to be able
+        # to use them from recipes not aware of it. I now hate myself.
+        if download_dir_url:
+            global _download_dir_url
+            _download_dir_url = download_dir_url
+        if download_cache_url:
+            global _download_cache_url
+            _download_cache_url = download_cache_url
+        if upload_dir_url:
+            global _upload_dir_url
+            _upload_dir_url = upload_dir_url
+        if upload_cache_url:
+            global _upload_cache_url
+            _upload_cache_url = upload_cache_url
+        if signature_certificate_list:
+            global _signature_certificate_list
+            _signature_certificate_list = signature_certificate_list
+        if signature_private_key_file:
+            global _signature_private_key_file
+            _signature_private_key_file = signature_private_key_file
+        if shacache_cert_file:
+            global _shacache_cert_file
+            _shacache_cert_file = shacache_cert_file
+        if shacache_key_file:
+            global _shacache_key_file
+            _shacache_key_file = shacache_key_file
+        if shadir_cert_file:
+            global _shadir_cert_file
+            _shadir_cert_file = shadir_cert_file
+        if shadir_key_file:
+            global _shadir_key_file
+            _shadir_key_file = shadir_key_file
+
         self._dest = dest
         self._allow_hosts = allow_hosts
 
@@ -414,22 +564,11 @@ class Installer:
         self._env = pkg_resources.Environment(path,
                                               python=_get_version(executable))
         self._index = _get_index(executable, index, links, self._allow_hosts,
-                                 self._path)
+                                 self._path,)
 
         if versions is not None:
             self._versions = versions
 
-        self._download_dir_url = download_dir_url
-        self._download_cache_url = download_cache_url
-        self._upload_dir_url = upload_dir_url
-        self._upload_cache_url = upload_cache_url
-        self._signature_certificate_list = signature_certificate_list
-        self._signature_private_key_file = signature_private_key_file
-        self._shacache_cert_file = shacache_cert_file
-        self._shacache_key_file = shacache_key_file
-        self._shadir_cert_file = shadir_cert_file
-        self._shadir_key_file = shadir_key_file
-
     _allowed_eggs_from_site_packages_regex = None
     def allow_site_package_egg(self, name):
         if (not self._include_site_packages or
@@ -731,16 +870,25 @@ class Installer:
 
         filename = get_filename_from_url(dist.location)
         new_location = os.path.join(tmp, filename)
-        if not download_network_cached(self._download_dir_url,
-                 self._download_cache_url, new_location, dist.location, logger,
-                 self._signature_certificate_list):
+        # Try to download from shacache first. If not possible, downloads from
+        # Original location and uploads it to shacache.
+        downloaded_from_cache = False
+        if globals().get('_download_dir_url') \
+           and globals().get('_download_dir_url') \
+           and globals().get('_signature_certificate_list'):
+            downloaded_from_cache = download_network_cached(_download_dir_url,
+               _download_cache_url, new_location, dist.location, logger,
+               _signature_certificate_list)
+        if not downloaded_from_cache:
             new_location = self._index.download(dist.location, tmp)
-            if self._upload_cache_url and self._upload_dir_url:
-                upload_network_cached(self._upload_dir_url,
-                   self._upload_cache_url, dist.location, new_location, logger,
-                   self._signature_private_key_file,
-                   self._shacache_cert_file, self._shacache_key_file,
-                   self._shadir_cert_file, self._shadir_key_file)
+            # XXX Better check for unbound global variables
+            if globals().get('_upload_cache_url') \
+               and globals().get('_upload_dir_url'):
+                upload_network_cached(_upload_dir_url,
+                   _upload_cache_url, dist.location, new_location, logger,
+                   _signature_private_key_file,
+                   _shacache_cert_file, _shacache_key_file,
+                   _shadir_cert_file, _shadir_key_file)
 
         if (download_cache
             and (realpath(new_location) == realpath(dist.location))
@@ -854,7 +1002,7 @@ class Installer:
                         self._links.append(link)
                         self._index = _get_index(self._executable,
                                                  self._index_url, self._links,
-                                                 self._allow_hosts, self._path)
+                                                 self._allow_hosts, self._path,)
 
         for dist in dists:
             # Check whether we picked a version and, if we did, report it:

src/zc/buildout/networkcache.py

@@ -12,12 +12,12 @@
 #
 ##############################################################################
 
+#XXX factor with slapos/grid/networkcache.py and use libnetworkcache helpers
 
 import hashlib
 import os
 import posixpath
 import re
-import shutil
 import urllib2
 import urlparse
 import traceback
@@ -26,6 +26,10 @@ try:
     try:
         from slapos.libnetworkcache import NetworkcacheClient, UploadError, \
            DirectoryNotFound
+        from slapos.networkcachehelper import \
+           helper_download_network_cached, \
+           helper_upload_network_cached_from_file, \
+           helper_download_network_cached_to_file
     except ImportError:
         LIBNETWORKCACHE_ENABLED = False
     else:
@@ -68,12 +72,19 @@ def get_directory_key(url):
     Basically check if the url belongs to pypi:
       - if yes, the directory key will be pypi-buildout-urlmd5
       - if not, the directory key will be slapos-buildout-urlmd5
+    # XXX why is that?
     """
     urlmd5 = hashlib.md5(url).hexdigest()
     if 'pypi' in url:
       return 'pypi-buildout-%s' % urlmd5
     return 'slapos-buildout-%s' % urlmd5
 
+@fallback_call
+def get_index_directory_key(url, requirement):
+    """Returns directory hash based on egg requirement.
+    """
+    return 'pypi-index-%s-%s' % (hashlib.md5(url).hexdigest(), requirement)
+
 
 @fallback_call
 def download_network_cached(dir_url, cache_url, path, url, logger,
@@ -88,41 +99,60 @@ def download_network_cached(dir_url, cache_url, path, url, logger,
     if not LIBNETWORKCACHE_ENABLED:
         return False
 
-    if not(dir_url and cache_url):
-        return False
     if md5sum is None:
         md5sum = _get_md5_from_url(url)
 
-
     directory_key = get_directory_key(url)
     url = os.path.basename(url)
 
-    if len(signature_certificate_list) == 0:
-        # convert [] into None in order to call nc nicely
-        signature_certificate_list = None
-    try:
-        nc = NetworkcacheClient(cache_url, dir_url,
-            signature_certificate_list=signature_certificate_list)
-    except TypeError:
-      logger.warning('Incompatible version of networkcache, not using it.')
-      return False
-
     logger.debug('Trying to download %s from network cache...' % url)
-    try:
-        file_descriptor = nc.select(directory_key)
 
-        f = open(path, 'w+b')
-        try:
-            shutil.copyfileobj(file_descriptor, f)
-        finally:
-            f.close()
-            file_descriptor.close()
+    if helper_download_network_cached_to_file(
+       dir_url=dir_url,
+       cache_url=cache_url,
+       signature_certificate_list=signature_certificate_list,
+       directory_key=directory_key,
+       path=path):
         logger.info('Downloaded %s from network cache.' % url)
 
         if not check_md5sum(path, md5sum):
             logger.info('MD5 checksum mismatch downloading %s' % url)
             return False
+        return True
+    return False
+
+@fallback_call
+def download_index_network_cached(dir_url, cache_url, url, requirement, logger,
+                            signature_certificate_list):
+    """
+    XXX description
+    Downloads pypi index from a network cache provider
 
+    If something fail (providor be offline, or hash_string fail), we ignore
+    network cached index.
+
+    return index if succeeded, False otherwise.
+    """
+    if not LIBNETWORKCACHE_ENABLED:
+        return False
+
+    directory_key = get_index_directory_key(url, requirement)
+
+    wanted_metadata_dict = {
+       'urlmd5':hashlib.md5(url).hexdigest(),
+       'requirement':requirement
+    }
+    required_key_list = ['base']
+
+    result = helper_download_network_cached(dir_url, cache_url,
+       signature_certificate_list,
+       directory_key, wanted_metadata_dict, required_key_list)
+    if result:
+        file_descriptor, metadata = result
+        try:
+            content = file_descriptor.read()
+            logger.info('Downloaded %s from network cache.' % url)
+            return content, metadata['base']
         except (IOError, DirectoryNotFound), e:
             if isinstance(e, urllib2.HTTPError) and e.code == 404:
               logger.debug('%s does not exist in network cache.' % url)
@@ -130,14 +160,13 @@ def download_network_cached(dir_url, cache_url, path, url, logger,
               logger.debug('Failed to download from network cache %s: %s' % \
                                                              (url, str(e)))
     return False
-    return True
-
 
 @fallback_call
 def upload_network_cached(dir_url, cache_url, external_url, path, logger,
    signature_private_key_file, shacache_cert_file, shacache_key_file,
    shadir_cert_file, shadir_key_file):
     """Upload file to a network cache server"""
+    # XXX use helper and FACTOR code
     if not LIBNETWORKCACHE_ENABLED:
         return False
 
@@ -187,6 +216,64 @@ def upload_network_cached(dir_url, cache_url, external_url, path, logger,
 
     return True
 
+@fallback_call
+def upload_index_network_cached(dir_url, cache_url, external_url, base, requirement, content, logger,
+   signature_private_key_file, shacache_cert_file, shacache_key_file,
+   shadir_cert_file, shadir_key_file):
+    # XXX use helper and FACTOR code
+    """Upload content of a web page to a network cache server"""
+    if not LIBNETWORKCACHE_ENABLED:
+        return False
+
+    if not (dir_url and cache_url):
+        return False
+
+    logger.info('Uploading %s content into network cache.' % external_url)
+
+    directory_key = get_index_directory_key(external_url, requirement)
+    kw = dict(file="file",
+              base=base,
+              urlmd5=hashlib.md5(external_url).hexdigest(),
+              requirement=requirement)
+
+    import tempfile
+    f = tempfile.TemporaryFile()
+    f.write(content)
+
+    # convert '' into None in order to call nc nicely
+    if not signature_private_key_file:
+        signature_private_key_file = None
+    if not shacache_cert_file:
+        shacache_cert_file = None
+    if not shacache_key_file:
+        shacache_key_file = None
+    if not shadir_cert_file:
+        shadir_cert_file = None
+    if not shadir_key_file:
+        shadir_key_file = None
+    try:
+        nc = NetworkcacheClient(cache_url, dir_url,
+            signature_private_key_file=signature_private_key_file,
+            shacache_cert_file=shacache_cert_file,
+            shacache_key_file=shacache_key_file,
+            shadir_cert_file=shadir_cert_file,
+            shadir_key_file=shadir_key_file)
+    except TypeError:
+        logger.warning('Incompatible version of networkcache, not using it.')
+        return False
+
+    try:
+        return nc.upload_generic(f, directory_key, **kw)
+    except (IOError, UploadError), e:
+        logger.info('Fail to upload file. %s' % \
+                                                  (str(e)))
+        return False
+
+    finally:
+      f.close()
+
+    return True
+
 
 @fallback_call
 def get_filename_from_url(url):