[feat] Use pip wheel + Wheel.install_as_egg

`pip install <package>` produces a `<package-name>` package folder
and a `<package-name>.dist-info` metadata folder, which is another
format than eggs. Then buildout bundles both folders into a parent
folder `<package.egg>` and tries to act as though it were an egg.

Instead, use `pip wheel` to produce a wheel - which `pip install`
does internally anyway - and `setuptools.Wheel.install_as_egg` to
produce a genuine egg.

This is much cleaner: it consistently produces genuine eggs instead
of sometimes true eggs, sometimes `.dist-info` bundles depending on
whether `pip install` is called or the package was installed from a
`.whl` or `.egg` archive directly.

The only downside it this requires setuptools >= 38.2.3.

src/zc/buildout/easy_install.py

@@ -295,7 +295,7 @@ class Installer(object):
             self._versions = normalize_versions(versions)
 
     def _make_env(self):
-        full_path = self._get_dest_dist_paths() + self._get_path_dist_paths() + self._path
+        full_path = self._get_dest_dist_paths() + self._path
         env = pkg_resources.Environment(full_path)
         # this needs to be called whenever self._env is modified (or we could
         # make an Environment subclass):
@@ -310,18 +310,9 @@ class Installer(object):
         dest = self._dest
         if dest is None:
             return []
-        return self._get_dist_paths(dest)
-
-    def _get_path_dist_paths(self):
-      dist_paths = []
-      for path in self._path:
-          dist_paths.extend(self._get_dist_paths(path))
-      return dist_paths
-
-    def _get_dist_paths(self, path):
-        eggs = glob.glob(os.path.join(path, '*.egg'))
+        eggs = glob.glob(os.path.join(dest, '*.egg'))
         dists = [os.path.dirname(dist_info) for dist_info in
-                 glob.glob(os.path.join(path, '*', '*.dist-info'))]
+                 glob.glob(os.path.join(dest, '*', '*.dist-info'))]
         return list(set(eggs + dists))
 
     @staticmethod
@@ -438,11 +429,11 @@ class Installer(object):
             str(req))
         return best_we_have, None
 
-    def _call_pip_install(self, spec, dest, dist):
+    def _call_pip_wheel(self, spec, dest, dist):
 
         tmp = tempfile.mkdtemp(dir=dest)
         try:
-            paths = call_pip_install(spec, tmp)
+            paths = call_pip_wheel(spec, tmp)
 
             dists = []
             env = pkg_resources.Environment(paths)
@@ -867,7 +858,7 @@ class Installer(object):
                 setuptools.command.setopt.edit_config(
                     setup_cfg, dict(build_ext=build_ext))
 
-                dists = self._call_pip_install(base, self._dest, dist)
+                dists = self._call_pip_wheel(base, self._dest, dist)
 
                 return [dist.location for dist in dists]
             finally:
@@ -1669,13 +1660,13 @@ class IncompatibleConstraintError(zc.buildout.UserError):
 IncompatibleVersionError = IncompatibleConstraintError # Backward compatibility
 
 
-def call_pip_install(spec, dest):
+def call_pip_wheel(spec, dest):
     """
-    Call `pip install` from a subprocess to install a
+    Call `pip wheel` from a subprocess to install a
     distribution specified by `spec` into `dest`.
     Returns all the paths inside `dest` created by the above.
     """
-    args = [sys.executable, '-m', 'pip', 'install', '--no-deps', '-t', dest]
+    args = [sys.executable, '-m', 'pip', 'wheel', '--no-deps', '-w', dest]
     level = logger.getEffectiveLevel()
     if level >= logging.INFO:
         args.append('-q')
@@ -1690,8 +1681,8 @@ def call_pip_install(spec, dest):
     except ImportError:
         HAS_WARNING_OPTION = False
     if HAS_WARNING_OPTION:
-        if not hasattr(call_pip_install, 'displayed'):
-            call_pip_install.displayed = True
+        if not hasattr(call_pip_wheel, 'displayed'):
+            call_pip_wheel.displayed = True
         else:
             args.append('--no-python-version-warning')
 
@@ -1716,128 +1707,24 @@ def call_pip_install(spec, dest):
             spec)
         sys.exit(1)
 
-    split_entries = [os.path.splitext(entry) for entry in os.listdir(dest)]
+    entries = os.listdir(dest)
     try:
-        distinfo_dir = [
-            base + ext for base, ext in split_entries if ext == ".dist-info"
-        ][0]
-    except IndexError:
+        assert len(entries) == 1, "Got multiple entries afer pip wheel"
+        wheel = entries[0]
+        assert os.path.splitext(wheel)[1] == '.whl', "Expected a .whl"
+    except AssertionError:
         logger.error(
-            "No .dist-info directory after successful pip install of %s",
+            "No .whl after successful pip wheel of %s",
             spec)
         raise
 
-    return make_egg_after_pip_install(dest, distinfo_dir)
-
-
-def make_egg_after_pip_install(dest, distinfo_dir):
-    """build properly named egg directory"""
-
-    # `pip install` does not build the namespace aware __init__.py files
-    # but they are needed in egg directories.
-    # Add them before moving files setup by pip
-    namespace_packages_file = os.path.join(
-        dest, distinfo_dir,
-        'namespace_packages.txt'
-    )
-    if os.path.isfile(namespace_packages_file):
-        with open(namespace_packages_file) as f:
-            namespace_packages = [
-                line.strip().replace('.', os.path.sep)
-                for line in f.readlines()
-            ]
-
-        for namespace_package in namespace_packages:
-            namespace_package_dir = os.path.join(dest, namespace_package)
-            if os.path.isdir(namespace_package_dir):
-                init_py_file = os.path.join(
-                    namespace_package_dir, '__init__.py')
-                with open(init_py_file, 'w') as f:
-                    f.write(
-                        "__import__('pkg_resources')."
-                        "declare_namespace(__name__)"
-                    )
-
-    # Remove `bin` directory if needed
-    # as there is no way to avoid script installation
-    # when running `pip install`
-    entry_points_file = os.path.join(dest, distinfo_dir, 'entry_points.txt')
-    if os.path.isfile(entry_points_file):
-        with open(entry_points_file) as f:
-            content = f.read()
-            if "console_scripts" in content or "gui_scripts" in content:
-                bin_dir = os.path.join(dest, BIN_SCRIPTS)
-                if os.path.exists(bin_dir):
-                    shutil.rmtree(bin_dir)
-
-    # Make properly named new egg dir
-    distro = list(pkg_resources.find_distributions(dest))[0]
-    base = "{}-{}".format(
-        distro.egg_name(), pkg_resources.get_supported_platform()
-    )
-    egg_name = base + '.egg'
-    new_distinfo_dir = base + '.dist-info'
-    egg_dir = os.path.join(dest, egg_name)
-    os.mkdir(egg_dir)
-
-    # Move ".dist-info" dir into new egg dir
-    os.rename(
-        os.path.join(dest, distinfo_dir),
-        os.path.join(egg_dir, new_distinfo_dir)
-    )
+    return make_egg_after_pip_wheel(dest, wheel)
 
-    top_level_file = os.path.join(egg_dir, new_distinfo_dir, 'top_level.txt')
-    if os.path.isfile(top_level_file):
-        with open(top_level_file) as f:
-            top_levels = filter(
-                (lambda x: len(x) != 0),
-                [line.strip() for line in f.readlines()]
-                )
-    else:
-        top_levels = ()
-
-    # Move all top_level modules or packages
-    for top_level in top_levels:
-        # as package
-        top_level_dir = os.path.join(dest, top_level)
-        if os.path.exists(top_level_dir):
-            shutil.move(top_level_dir, egg_dir)
-            continue
-        # as module
-        top_level_py = top_level_dir + '.py'
-        if os.path.exists(top_level_py):
-            shutil.move(top_level_py, egg_dir)
-            top_level_pyc = top_level_dir + '.pyc'
-            if os.path.exists(top_level_pyc):
-                shutil.move(top_level_pyc, egg_dir)
-            continue
-
-    record_file = os.path.join(egg_dir, new_distinfo_dir, 'RECORD')
-    if os.path.isfile(record_file):
-        if PY3:
-            with open(record_file, newline='') as f:
-                all_files = [row[0] for row in csv.reader(f)]
-        else:
-            with open(record_file, 'rb') as f:
-                all_files = [row[0] for row in csv.reader(f)]
-
-    # There might be some c extensions left over
-    for entry in all_files:
-        if entry.endswith(('.pyc', '.pyo')):
-            continue
-        dest_entry = os.path.join(dest, entry)
-        # work around pip install -t bug that leaves entries in RECORD
-        # that starts with '../../'
-        if not os.path.abspath(dest_entry).startswith(dest):
-            continue
-        egg_entry = os.path.join(egg_dir, entry)
-        if os.path.exists(dest_entry) and not os.path.exists(egg_entry):
-            egg_entry_dir = os.path.dirname(egg_entry)
-            if not os.path.exists(egg_entry_dir):
-                os.makedirs(egg_entry_dir)
-            os.rename(dest_entry, egg_entry)
 
-    return [egg_dir]
+def make_egg_after_pip_wheel(dest, wheel):
+    unpack_wheel(os.path.join(dest, wheel), dest)
+    assert len(os.listdir(dest)) == 2
+    return glob.glob(os.path.join(dest, '*.egg'))
 
 
 def unpack_egg(location, dest):
@@ -1925,7 +1812,7 @@ def _move_to_eggs_dir_and_compile(dist, dest):
                 unpacker(dist.location, tmp_dest)
                 [tmp_loc] = glob.glob(os.path.join(tmp_dest, '*'))
             else:
-                [tmp_loc] = call_pip_install(dist.location, tmp_dest)
+                [tmp_loc] = call_pip_wheel(dist.location, tmp_dest)
                 installed_with_pip = True
 
         # We have installed the dist. Now try to rename/move it.