easy_install: add slapos.libnetworkcache support

src/zc/buildout/easy_install.py

@@ -38,6 +38,8 @@ import sys
 import tempfile
 import zc.buildout
 import warnings
+from contextlib import closing
+from setuptools.package_index import distros_for_location, URL_SCHEME
 
 WHL_DIST = pkg_resources.EGG_DIST + 1
 
@@ -97,6 +99,9 @@ python_lib = distutils.sysconfig.get_python_lib()
 FILE_SCHEME = re.compile('file://', re.I).match
 DUNDER_FILE_PATTERN = re.compile(r"__file__ = '(?P<filename>.+)'$")
 
+networkcache_key = 'pypi:{}={}'.format
+
+
 class _Monkey(object):
     def __init__(self, module, **kw):
         mdict = self._mdict = module.__dict__
@@ -441,29 +446,62 @@ class Installer:
         finally:
             shutil.rmtree(tmp)
 
-    def _obtain(self, requirement, source=None):
+    def _obtain(self, requirement, source=None, networkcache_failed=False):
         # get the non-patched version
         req = str(requirement)
         if PATCH_MARKER in req:
             requirement = pkg_resources.Requirement.parse(re.sub(orig_versions_re, '', req))
 
-        # initialize out index for this project:
+        wheel = getattr(requirement, 'wheel', False)
+        def filter_precedence(dist):
+            return (dist.precedence == WHL_DIST) == wheel and (
+                not source or dist.precedence == pkg_resources.SOURCE_DIST)
+
         index = self._index
+        if not networkcache_failed:
+            try:
+                (operator, version,), = requirement.specs
+            except ValueError:
+                pass
+            else:
+                # Network cache is not expected to contain all versions so it
+                # couldn't tell whether a found version is the best existing
+                # one. Therefore, it's only accessed when we have a
+                # specification for a single version, which is anyway enough
+                # for our usage (picked versions not allowed).
+                if operator == '==':
+                    # But first, avoid any network access by checking local
+                    # urls. PackageIndex.add_find_links scans them immediately.
+                    dists = [dist for dist in index[requirement.project_name]
+                        if dist in requirement and filter_precedence(dist) and (
+                            FILE_SCHEME(dist.location) or
+                            not URL_SCHEME(dist.location))]
+                    if dists:
+                        return max(dists)
+                    from .buildout import networkcache_client as nc
+                    if nc:
+                        key = networkcache_key(requirement.key, version)
+                        if nc.tryDownload(key):
+                            with nc:
+                                for entry in nc.select(key):
+                                    basename = entry['basename']
+                                    dist = next(iter(distros_for_location(
+                                        entry['sha512'], basename)))
+                                    if filter_precedence(dist):
+                                        dist.networkcache = (
+                                            basename, requirement, source)
+                                        dists.append(dist)
+                            if dists:
+                                return max(dists)
 
+        # initialize out index for this project:
         if index.obtain(requirement) is None:
             # Nothing is available.
             return None
 
         # Filter the available dists for the requirement and source flag
-        wheel = getattr(requirement, 'wheel', False)
         dists = [dist for dist in index[requirement.project_name]
-                 if ((dist in requirement)
-                     and (dist.precedence == WHL_DIST) == wheel and
-                     ((not source) or
-                      (dist.precedence == pkg_resources.SOURCE_DIST)
-                      )
-                     )
-                 ]
+                      if dist in requirement and filter_precedence(dist)]
 
         # If we prefer final dists, filter for final and use the
         # result if it is non empty.
@@ -500,16 +538,38 @@ class Installer:
                     ):
                     return dist
 
-        best.sort()
-        return best[-1]
+        return max(best)
 
     def _fetch(self, dist, tmp, download_cache):
-        if (download_cache
-            and (realpath(os.path.dirname(dist.location)) == download_cache)
-            ):
-            return dist
+        from .buildout import networkcache_client as nc
+        while hasattr(dist, 'networkcache'):
+            basename, requirement, source = dist.networkcache
+            new_location = os.path.join(tmp, basename)
+            with nc, closing(nc.download(dist.location)) as src, \
+                     open(new_location, 'wb') as dst:
+                shutil.copyfileobj(src, dst)
+                break
+            # Downloading content from network cache failed: let's resume index
+            # lookup to get a fallback url. This will respect _satisfied()
+            # decision because the specification is for a single version.
+            dist = self._obtain(requirement, source, networkcache_failed=True)
+            if dist is None:
+                raise zc.buildout.UserError(
+                    "Couldn't find a distribution for %r."
+                    % str(requirement))
+        else:
+            if download_cache and (
+                    realpath(os.path.dirname(dist.location)) == download_cache):
+                return dist
+
+            new_location = self._index.download(dist.location, tmp)
+            if nc:
+                key = networkcache_key(dist.key, dist.version)
+                if nc.tryUpload(key):
+                    with nc, open(new_location, 'rb') as f:
+                        nc.upload(f, key,
+                            basename=os.path.basename(new_location))
 
-        new_location = self._index.download(dist.location, tmp)
         if (download_cache
             and (realpath(new_location) == realpath(dist.location))
             and os.path.isfile(new_location)

src/zc/buildout/easy_install.txt

@@ -1362,9 +1362,8 @@ Now when we install the distributions:
     ...     ['demo==0.2'], dest,
     ...     links=[link_server], index=link_server+'index/')
     GET 200 /
-    GET 404 /index/demo/
-    GET 200 /index/
     GET 404 /index/demoneeded/
+    GET 200 /index/
 
     >>> zc.buildout.easy_install.build(
     ...   'extdemo', dest,
@@ -1386,6 +1385,7 @@ from the link server:
     >>> ws = zc.buildout.easy_install.install(
     ...     ['demo'], dest,
     ...     links=[link_server], index=link_server+'index/')
+    GET 404 /index/demo/
     GET 200 /demo-0.3-py2.4.egg
 
 Normally, the download cache is the preferred source of downloads, but