[feat] easy_install: add slapos.libnetworkcache support

src/zc/buildout/easy_install.py

@@ -43,6 +43,8 @@ import zc.buildout.rmtree
 from zc.buildout import WINDOWS
 from zc.buildout import PY3
 import warnings
+from contextlib import closing
+from setuptools.package_index import distros_for_location, URL_SCHEME
 import csv
 
 try:
@@ -113,6 +115,9 @@ python_lib = distutils.sysconfig.get_python_lib()
 FILE_SCHEME = re.compile('file://', re.I).match
 DUNDER_FILE_PATTERN = re.compile(r"__file__ = '(?P<filename>.+)'$")
 
+networkcache_key = 'pypi:{}={}'.format
+
+
 class _Monkey(object):
     def __init__(self, module, **kw):
         mdict = self._mdict = module.__dict__
@@ -466,32 +471,71 @@ class Installer(object):
         finally:
             zc.buildout.rmtree.rmtree(tmp)
 
-    def _obtain(self, requirement, source=None):
+    def _obtain(self, requirement, source=None, networkcache_failed=False):
         # get the non-patched version
         req = str(requirement)
         if PATCH_MARKER in req:
             requirement = pkg_resources.Requirement.parse(re.sub(orig_versions_re, '', req))
 
-        # initialize out index for this project:
+        wheel = getattr(requirement, 'wheel', False)
+        def filter_precedence(dist):
+            return (dist.precedence == WHL_DIST) == wheel and (
+                dist.precedence == pkg_resources.SOURCE_DIST if source
+                else not (dist.precedence == pkg_resources.DEVELOP_DIST
+                          and {'setup.py', 'pyproject.toml'}.isdisjoint(
+                              os.listdir(dist.location)))
+                )
+
         index = self._index
+        if not networkcache_failed:
+            try:
+                (operator, version,), = requirement.specs
+            except ValueError:
+                pass
+            else:
+                # Network cache is not expected to contain all versions so it
+                # couldn't tell whether a found version is the best existing
+                # one. Therefore, it's only accessed when we have a
+                # specification for a single version, which is anyway enough
+                # for our usage (picked versions not allowed).
+                if operator == '==':
+                    # But first, avoid any network access by checking local
+                    # urls. PackageIndex.add_find_links scans them immediately.
+                    dists = [dist for dist in index[requirement.project_name]
+                        if dist in requirement and filter_precedence(dist) and (
+                            FILE_SCHEME(dist.location) or
+                            not URL_SCHEME(dist.location))]
+                    if dists:
+                        return max(dists)
+                    from .buildout import networkcache_client as nc
+                    if nc:
+                        key = networkcache_key(requirement.key, version)
+                        if nc.tryDownload(key):
+                            with nc:
+                                for entry in nc.select(key):
+                                    basename = entry['basename']
+                                    for dist in distros_for_location(
+                                            entry['sha512'], basename):
+                                        # The version comparison is to keep
+                                        # the one that's correctly parsed by
+                                        # distros_for_location.
+                                        if (dist.version == version and
+                                                self._env.can_add(dist) and
+                                                filter_precedence(dist)):
+                                            dist.networkcache = (
+                                                basename, requirement, source)
+                                            dists.append(dist)
+                            if dists:
+                                return max(dists)
 
+        # initialize out index for this project:
         if index.obtain(requirement) is None:
             # Nothing is available.
             return None
 
         # Filter the available dists for the requirement and source flag
-        wheel = getattr(requirement, 'wheel', False)
         dists = [dist for dist in index[requirement.project_name]
-                 if ((dist in requirement)
-                     and (dist.precedence == WHL_DIST) == wheel and
-                     (dist.precedence == pkg_resources.SOURCE_DIST if source
-                      else not (dist.precedence == pkg_resources.DEVELOP_DIST
-                                and {'setup.py', 'pyproject.toml'}.isdisjoint(
-                                    os.listdir(dist.location))
-                                )
-                      )
-                     )
-                 ]
+                      if dist in requirement and filter_precedence(dist)]
 
         # If we prefer final dists, filter for final and use the
         # result if it is non empty.
@@ -528,18 +572,41 @@ class Installer(object):
                     ):
                     return dist
 
-        best.sort()
-        return best[-1]
+        return max(best)
 
     def _fetch(self, dist, tmp, download_cache):
-        if (download_cache
-            and (realpath(os.path.dirname(dist.location)) == download_cache)
-            ):
-            logger.debug("Download cache has %s at: %s", dist, dist.location)
-            return dist
+        from .buildout import networkcache_client as nc
+        while hasattr(dist, 'networkcache'):
+            basename, requirement, source = dist.networkcache
+            new_location = os.path.join(tmp, basename)
+            with nc, closing(nc.download(dist.location)) as src, \
+                     open(new_location, 'wb') as dst:
+                shutil.copyfileobj(src, dst)
+                break
+            # Downloading content from network cache failed: let's resume index
+            # lookup to get a fallback url. This will respect _satisfied()
+            # decision because the specification is for a single version.
+            dist = self._obtain(requirement, source, networkcache_failed=True)
+            if dist is None:
+                raise zc.buildout.UserError(
+                    "Couldn't find a distribution for %r."
+                    % str(requirement))
+        else:
+            if (download_cache
+                and (realpath(os.path.dirname(dist.location)) == download_cache)
+                ):
+                logger.debug("Download cache has %s at: %s", dist, dist.location)
+                return dist
+
+            logger.debug("Fetching %s from: %s", dist, dist.location)
+            new_location = self._index.download(dist.location, tmp)
+            if nc:
+                key = networkcache_key(dist.key, dist.version)
+                if nc.tryUpload(key):
+                    with nc, open(new_location, 'rb') as f:
+                        nc.upload(f, key,
+                            basename=os.path.basename(new_location))
 
-        logger.debug("Fetching %s from: %s", dist, dist.location)
-        new_location = self._index.download(dist.location, tmp)
         if (download_cache
             and (realpath(new_location) == realpath(dist.location))
             and os.path.isfile(new_location)

src/zc/buildout/tests/easy_install.txt

@@ -1451,9 +1451,8 @@ Now when we install the distributions:
     ...     ['demo==0.2'], dest,
     ...     links=[link_server], index=link_server+'index/')
     GET 200 /
-    GET 404 /index/demo/
-    GET 200 /index/
     GET 404 /index/demoneeded/
+    GET 200 /index/
 
     >>> zc.buildout.easy_install.build(
     ...   'extdemo', dest,
@@ -1475,6 +1474,7 @@ from the link server:
     >>> ws = zc.buildout.easy_install.install(
     ...     ['demo'], dest,
     ...     links=[link_server], index=link_server+'index/')
+    GET 404 /index/demo/
     GET 200 /demo-0.3-py2.4.egg
 
 Normally, the download cache is the preferred source of downloads, but