[wkrd] Workaround package index & PEP 625

Buildout relies on setuptools to fetch dists from package indexes like
PyPI. Setuptools infers the project name of source distributions found
in the index from their filename, as derived from their download link.

This works on the assumption the sdist filename corresponds losslessly
to the project name, and in the way setuptools expects.

Setuptools normalizes any occurrence of a non alphanumeric and non '.'
character into a '-'. This is actually already lossy for project names
that contain other characters; therefore the assumption should instead
be that the sdist filename is lossless with regards to this normalized
project name - in general project names should respect this normalized
form.

Since PEP 625, that assumption is definitely broken, i.e. the filename
is lossy with regards to the normalized project name:
- in the sdist filename, characters . and - must both be replaced by _
- in the sdist filename, all characters must be lowercased

Internally, setuptools lowercases the normalized project name in order
to compare dists. The possibles consequences of that broken assumption
are thus:
1. the sdist is discarded from the search
2. the sdist is found but the project name may be cased differently

For example, zc.buildout==3.0.1 yields zc_buildout-3.0.1.tar.gz, which
results in setuptools interpreting the project name as zc-buildout and
discarding it from the search.

Workaround 1. by also looking up the name with . replaced by _.
Workaround 2. by fixing the project names back to the looked-up name.

src/zc/buildout/easy_install.py

@@ -45,7 +45,7 @@ from zc.buildout import WINDOWS
 from zc.buildout import PY3
 import warnings
 from contextlib import closing
-from setuptools.package_index import distros_for_location, URL_SCHEME
+from setuptools.package_index import URL_SCHEME
 import csv
 
 from setuptools import __version__ as setuptools_version
@@ -147,10 +147,23 @@ class _NoWarn(object):
 
 _no_warn = _NoWarn()
 
-class AllowHostsPackageIndex(setuptools.package_index.PackageIndex):
-    """Will allow urls that are local to the system.
 
-    No matter what is allow_hosts.
+
+def dists_according_to_requirement(dists, requirement):
+  project_name = requirement.project_name
+  dists = [
+    d if d.project_name == project_name else
+    # Fix the dist's project name to match the original requirement
+    d.clone(project_name=project_name)
+    for d in dists if d in requirement
+  ]
+  return dists
+
+
+class WkrdPackageIndex(setuptools.package_index.PackageIndex):
+    """
+    Allow urls that are local to the system,cno matter what is allow_hosts.
+    Workaround PEP 625 sdist filenames being parsed to wrong project name.
     """
     def url_ok(self, url, fatal=False):
         if FILE_SCHEME(url):
@@ -159,8 +172,32 @@ class AllowHostsPackageIndex(setuptools.package_index.PackageIndex):
         # so we monkey-patch the 'log' submodule to suppress the stupid
         # "Link to <URL> ***BLOCKED*** by --allow-hosts" message.
         with _Monkey(setuptools.package_index, log=_no_warn):
-            return setuptools.package_index.PackageIndex.url_ok(
-                                                self, url, False)
+            return super(WkrdPackageIndex, self).url_ok(url, False)
+
+    def getdists(self, requirement):
+      project_name = requirement.project_name
+      dists = dists_according_to_requirement(self[project_name], requirement)
+      # setuptools may get mixed-up between . and - in project names
+      # due to . and - being turned into _ in sdist archive names, but
+      # _ in archive names being translated to - during index lookup.
+      # See https://github.com/buildout/buildout/issues/647
+      # Workaround by also looking up the name with - instead of .
+      # and fixing the name of dists found back to . instead of -.
+      wkrd_name = project_name.replace('.', '-')
+      if wkrd_name != project_name:
+          dists.extend(
+              d.clone(project_name=project_name)
+              for d in self[wkrd_name]
+              if d.version in requirement
+          )
+      return dists
+
+
+def required_distros_for_location(requirement, *args, **kwargs):
+    return dists_according_to_requirement(
+      setuptools.package_index.distros_for_location(*args, **kwargs),
+      requirement,
+    )
 
 
 _indexes = {}
@@ -174,7 +211,7 @@ def _get_index(index_url, find_links, allow_hosts=('*',)):
         index_url = default_index_url
     if index_url.startswith('file://'):
         index_url = index_url[7:]
-    index = AllowHostsPackageIndex(index_url, hosts=allow_hosts)
+    index = WkrdPackageIndex(index_url, hosts=allow_hosts)
 
     if find_links:
         index.add_find_links(find_links)
@@ -510,8 +547,8 @@ class Installer(object):
                 if operator == '==':
                     # But first, avoid any network access by checking local
                     # urls. PackageIndex.add_find_links scans them immediately.
-                    dists = [dist for dist in index[requirement.project_name]
-                        if dist in requirement and filter_precedence(dist) and (
+                    dists = [dist for dist in index.getdists(requirement)
+                        if filter_precedence(dist) and (
                             FILE_SCHEME(dist.location) or
                             not URL_SCHEME(dist.location))]
                     if dists:
@@ -523,11 +560,12 @@ class Installer(object):
                             with nc:
                                 for entry in nc.select(key):
                                     basename = entry['basename']
-                                    for dist in distros_for_location(
+                                    for dist in required_distros_for_location(
+                                            requirement,
                                             entry['sha512'], basename):
                                         # The version comparison is to keep
                                         # the one that's correctly parsed by
-                                        # distros_for_location.
+                                        # required_distros_for_location.
                                         if (dist.version == version and
                                                 self._env.can_add(dist) and
                                                 filter_precedence(dist)):
@@ -538,13 +576,20 @@ class Installer(object):
                                 return max(dists)
 
         # initialize out index for this project:
-        if index.obtain(requirement) is None:
+        index.obtain(requirement) # ignore result
+        # setuptools may get mixed-up between . and - in project names
+        # due to . and - being turned into _ in sdist archive names, but
+        # _ in archive names being translated to - during index lookup.
+        # See https://github.com/buildout/buildout/issues/647
+
+        dists = index.getdists(requirement)
+
+        if not dists:
             # Nothing is available.
             return None
 
-        # Filter the available dists for the requirement and source flag
-        dists = [dist for dist in index[requirement.project_name]
-                      if dist in requirement and filter_precedence(dist)]
+        # Filter the available dists for the source flag.
+        dists = [dist for dist in dists if filter_precedence(dist)]
 
         # If we prefer final dists, filter for final and use the
         # result if it is non empty.