zc/buildout/easy_install.py: Buildout did not use find-links properly because...

zc/buildout/easy_install.py: Buildout did not use find-links properly because of networkcache. Fix this problem.

src/zc/buildout/easy_install.py

@@ -128,6 +128,7 @@ class _NoWarn(object):
 
 _no_warn = _NoWarn()
 
+to_scan_page_cache = {}
 class AllowHostsPackageIndex(setuptools.package_index.PackageIndex):
     """Will allow urls that are local to the system.
 
@@ -143,14 +144,45 @@ class AllowHostsPackageIndex(setuptools.package_index.PackageIndex):
             return setuptools.package_index.PackageIndex.url_ok(
                                                 self, url, False)
 
-    def obtain(self, requirement, installer=None):
+    def obtain(self, requirement, installer=None, source=None):
+        """Our strategy is:
+        1. Use the default index. Mostly it is enough.
+        2. If a requirement was not met, then use find-links one by one.
+        A new parameter `source` was added to find a source distribution
+        for slapos.rebootstrap.
+        """
         self._current_requirement = str(requirement)
-        return super(AllowHostsPackageIndex, self).obtain(requirement, installer)
+        self.find_packages(requirement)
+        for dist in self[requirement.key]:
+            if dist in requirement:
+                if source is not None:
+                    if dist.precedence == pkg_resources.SOURCE_DIST:
+                      return dist
+                else:
+                    return dist
+            self.debug("%s does not match %s", requirement, dist)
+
+        for find_link in self.to_scan:
+            self.scan_url(find_link)
+            for dist in self[requirement.key]:
+                if dist in requirement:
+                    if source is not None:
+                        if dist.precedence == pkg_resources.SOURCE_DIST:
+                            return dist
+                    else:
+                        return dist
+                self.debug("%s does not match %s", requirement, dist)
+
+        return super(PackageIndex, self).obtain(requirement, installer)
 
     def find_packages(self, requirement):
         self._current_requirement = str(requirement)
         return super(AllowHostsPackageIndex, self).find_packages(requirement)
 
+    def prescan(self):
+        """We do not use prescan because it does not make sense with networkcache."""
+        pass
+
     def process_url(self, url, retrieve=False):
         """Evaluate a URL as a possible download, and maybe retrieve it"""
         if url in self.scanned_urls and not retrieve:
@@ -166,7 +198,7 @@ class AllowHostsPackageIndex(setuptools.package_index.PackageIndex):
                     return
                 self.debug("Found link: %s", url)
 
-        if dists or not retrieve or url in self.fetched_urls:
+        if dists or not retrieve or (not url in self.to_scan and url in self.fetched_urls):
             list(map(self.add, dists))
             return  # don't need the actual page
 
@@ -182,6 +214,7 @@ class AllowHostsPackageIndex(setuptools.package_index.PackageIndex):
         #         If there is specific version, we want index to freeze forever.
         download_result = False
         requirement = getattr(self, '_current_requirement', '')
+        requirement_name = requirement.split('==')[0]
         from .buildout import network_cache_parameter_dict as nc
         if '==' in requirement:
             download_result = download_index_network_cached(
@@ -194,12 +227,39 @@ class AllowHostsPackageIndex(setuptools.package_index.PackageIndex):
             page, base = download_result
             f = None
             self.fetched_urls[base] = True
+        elif url in to_scan_page_cache:
+            page_cache = to_scan_page_cache[url]
+            if page_cache is None:
+                return
+            page, base = page_cache
+            if (requirement_name
+                and requirement_name in page
+                and nc.get('upload-cache-url')
+                and nc.get('upload-dir-url')
+                and '==' in requirement):
+                upload_index_network_cached(
+                    nc.get('upload-dir-url'),
+                    nc.get('upload-cache-url'),
+                    url, base, requirement, page, logger,
+                    nc.get('signature-private-key-file'),
+                    nc.get('shacache-ca-file'),
+                    nc.get('shacache-cert-file'),
+                    nc.get('shacache-key-file'),
+                    nc.get('shadir-ca-file'),
+                    nc.get('shadir-cert-file'),
+                    nc.get('shadir-key-file'))
+            return
         else:
             f = self.open_url(url, "Download error on %s: %%s -- Some packages may not be found!" % url)
-            if f is None: return
+            if f is None:
+                if url in self.to_scan:
+                    to_scan_page_cache[url] = None
+                return
             self.fetched_urls[f.url] = True
             if 'html' not in f.headers.get('content-type', '').lower():
                 f.close()   # not html, we can't process it
+                if url in self.to_scan:
+                    to_scan_page_cache[url] = None
                 return
 
             base = f.url     # handle redirects
@@ -212,8 +272,16 @@ class AllowHostsPackageIndex(setuptools.package_index.PackageIndex):
                     charset = f.headers.get_param('charset') or 'latin-1'
                 page = page.decode(charset, "ignore")
             f.close()
+
+            if url in self.to_scan:
+                if getattr(f, 'code', None) != 404:
+                    to_scan_page_cache[url] = page, base
+                else:
+                    to_scan_page_cache[url] = None
+
             # Upload the index to network cache.
-            if nc.get('upload-cache-url') \
+            if  (not url in self.to_scan or (requirement_name and requirement_name in page)) \
+                and nc.get('upload-cache-url') \
                 and nc.get('upload-dir-url') \
                 and '==' in requirement \
                 and getattr(f, 'code', None) != 404:
@@ -541,7 +609,7 @@ class Installer:
         # initialize out index for this project:
         index = self._index
 
-        if index.obtain(requirement) is None:
+        if index.obtain(requirement, source=source) is None:
             # Nothing is available.
             return None