[feat] Use pip install --editable in easy_install.develop

Instead of running python setup.py develop directly. This will allow
using zc.buildout.easy_install.develop on recent projects that have
only a pyproject.toml. It also fixes develop leaving build artifacts
in the source directory that caused later runs to do nothing, e.g.
preventing develop-eggs to be rebuilt when a build dependency passed
in setup-eggs option of zc.recipe.egg:develop changed.

A verbosity parameter to tune verbosity of pip is added, with adjusted
values for the case of buildout:develop and of zc.recipe.egg:develop,
so as to remain close to the previous behavior with regards to logs.

Technical details:

For packages using PEP-660-style editable installs, supported by more
recent versions of pip, pip will not delegate to `setup.py develop` -
enabling editable installs for pure pyproject.toml projects - and will
instead generate a .dist-info metadata folder but not a .egg-link.

Since buildout currently requires a .egg-link, as it does not support
PEP 660's mechanism that relies on having a sites-packages directory,
we need to create this .egg-link after the fact. The tricky part is
finding out where the .egg-link should point: the path containing the
pyproject.toml, or a subdirectory?

For this we make use of *.dist-info/top_level.txt to first determine
the list of top-level packages, and then importlib to extract info
from the PEP-660-style install.

If top_level.txt does not exist, is empty, or otherwise does not list
any package that resolves to an import path, fallback to the path of
the folder that contains the pyproject.toml as the .egg-link target.
If it lists multiple packages that resolve to different import paths,
arbitrarily use the first one and emit a warning.

Support namespace packages where `spec.submodule_search_locations` is
a `_NamespacePath` object instead of a simple `list` and also support
cases where the layout of the source project does not follow the same
structure as the package tree - meaning some custom magic might be
involved in making editable imports work as intended.

src/zc/buildout/buildout.py

@@ -1131,7 +1131,8 @@ class Buildout(DictMixin):
                     for setup in files:
                         self._logger.info("Develop: %r", setup)
                         __doing__ = 'Processing develop directory %r.', setup
-                        zc.buildout.easy_install.develop(setup, dest)
+                        zc.buildout.easy_install.develop(setup, dest,
+                                                         verbosity=-20)
             except:
                 # if we had an error, we need to roll back changes, by
                 # removing any files we created.

src/zc/buildout/easy_install.py

@@ -23,6 +23,7 @@ import distutils.errors
 import distutils.sysconfig
 import errno
 import glob
+import json
 import logging
 import os
 import pkg_resources
@@ -1113,22 +1114,6 @@ def _rm(*paths):
             os.remove(path)
 
 
-def _copyeggs(src, dest, suffix, undo):
-    result = []
-    undo.append(lambda : _rm(*result))
-    for name in os.listdir(src):
-        if name.endswith(suffix):
-            new = os.path.join(dest, name)
-            _rm(new)
-            os.rename(os.path.join(src, name), new)
-            result.append(new)
-
-    assert len(result) == 1, str(result)
-    undo.pop()
-
-    return result[0]
-
-
 _develop_distutils_scripts = {}
 
 
@@ -1175,13 +1160,11 @@ def _detect_distutils_scripts(directory):
         _develop_distutils_scripts[egg_name] = scripts_found
 
 
-def develop(setup, dest,
-            build_ext=None,
-            executable=sys.executable):
-    assert executable == sys.executable, (executable, sys.executable)
+def develop(setup, dest, build_ext=None, executable=None, verbosity=-10):
+    if executable is not None: # BBB
+        assert executable == sys.executable, (executable, sys.executable)
     if os.path.isdir(setup):
         directory = setup
-        setup = os.path.join(directory, 'setup.py')
     else:
         directory = os.path.dirname(setup)
 
@@ -1203,43 +1186,123 @@ def develop(setup, dest,
             setuptools.command.setopt.edit_config(
                 setup_cfg, dict(build_ext=build_ext))
 
-        fd, tsetup = tempfile.mkstemp()
-        undo.append(lambda: os.remove(tsetup))
-        undo.append(lambda: os.close(fd))
+        tmp3 = tempfile.mkdtemp('build', dir=dest)
+        undo.append(lambda : zc.buildout.rmtree.rmtree(tmp3))
 
-        extra_path = os.environ.get('PYTHONEXTRAPATH')
-        extra_path_list = []
-        if extra_path:
-          extra_path_list = extra_path.split(os.pathsep)
+        class options: pass
+        options._allow_picked_versions = allow_picked_versions()
 
-        os.write(fd, (runsetup_template % dict(
-            setupdir=directory,
-            setup=setup,
-            path_list=extra_path_list,
-            __file__ = setup,
-            )).encode())
+        call_pip_editable(directory, tmp3, options, verbosity=verbosity)
 
-        tmp3 = tempfile.mkdtemp('build', dir=dest)
-        undo.append(lambda : zc.buildout.rmtree.rmtree(tmp3))
+        _detect_distutils_scripts(tmp3)
 
-        args = [executable,  tsetup, '-q', 'develop', '-mN', '-d', tmp3]
+        egg_link, dist_info = [], []
+        for entry in os.listdir(tmp3):
+            if entry.endswith('.egg-link'):
+              egg_link.append(entry)
+            if entry.endswith('.dist-info') or entry.endswith('.egg-info'):
+              dist_info.append(entry)
 
-        log_level = logger.getEffectiveLevel()
-        if log_level <= 0:
-            if log_level == 0:
-                del args[2]
-            else:
-                args[2] == '-v'
-        if log_level < logging.DEBUG:
-            logger.debug("in: %r\n%s", directory, ' '.join(args))
+        assert len(egg_link) + len(dist_info) == 1, str(egg_link + dist_info)
+        entry, = (egg_link or dist_info)
+
+        entry_path = os.path.join(tmp3, entry)
 
-        if SETUPTOOLS_IGNORE_WARNINGS:
-            env = dict(os.environ, PYTHONWARNINGS='ignore')
+        entry_filename, ext = os.path.splitext(entry)
+        project_name = entry_filename.split('-', 1)[0]
+
+        def move(src, dst):
+            _rm(dst)
+            os.rename(src, dst)
+            return dst
+
+        if egg_link:
+            # case: legacy setup.py
+            return move(entry_path, os.path.join(dest, entry))
         else:
-            env=None
-        call_subprocess(args, env=env)
-        _detect_distutils_scripts(tmp3)
-        return _copyeggs(tmp3, dest, '.egg-link', undo)
+            # case: PEP 660 pyproject.toml
+            # This case can only happen in Python3
+            # (because pip versions that support PEP 660 are Python3-only).
+
+            # Resolve the package path(s) and import (parent) path(s).
+            # This may be a subpath of the project directory,
+            # e.g. zc.buildout/src.
+            try:
+                with open(os.path.join(entry_path, 'top_level.txt')) as f:
+                    top_level = f.read().splitlines()
+                packages = tuple(p for p in top_level if p)
+            except IOError as e:
+                if e.errno != errno.ENOENT:
+                    raise
+                packages = ()
+            try:
+                package_paths = json.loads(subprocess.check_output([
+                    sys.executable, '-S', '-c',
+                    "import importlib.util, json, site; "
+                    "site.addsitedir(%(site)r); "
+                    "packages = %(packages)r; "
+                    "specs = [importlib.util.find_spec(p) for p in packages]; "
+                    "paths = {p: list(s.submodule_search_locations)"
+                    " for p, s in zip(packages, specs) "
+                    " if s.submodule_search_locations is not None}; "
+                    "print(json.dumps(paths))"  % {
+                        'site': tmp3,
+                        'packages': packages,
+                        }
+                    ],
+                    env=dict(os.environ, PYTHONPATH=''),
+                    stderr=subprocess.STDOUT,
+                    universal_newlines=True,
+                ))
+            except subprocess.CalledProcessError as e:
+                logger.error(e.output)
+                raise
+            # Resolve import path.
+            directory = os.path.normpath(directory)
+            import_paths = []
+            for package, paths in package_paths.items():
+                for path in paths:
+                  # Filter out magic hook values from setuptools PEP 660.
+                  if (path.startswith('__editable__')
+                      and path.endswith('.__path_hook__')): continue
+                  path = os.path.normpath(path)
+                  # Do not parent-path out of the source repository.
+                  if path == directory:
+                      import_paths.append(path)
+                      continue
+                  parent, folder = os.path.split(path)
+                  # Only parent-path if the folder has the name of the package.
+                  import_paths.append(parent if folder == package else path)
+            # Warn if there are several and arbitrarily take the first one.
+            if import_paths:
+                import_path = import_paths[0]
+                unique_import_paths = set(import_paths)
+                if len(unique_import_paths) > 1:
+                    logger.warning(
+                        "Found multiple package import paths"
+                        " in develop project %s\n(\n  %s\n)\n"
+                        "Selected %s" % (
+                          directory,
+                          '\n  '.join(unique_import_paths),
+                          import_path
+                        ))
+            # Warn if no import path is found; fallback to source directory.
+            else:
+                logger.warning(
+                    "Found no package import path in develop project %s "
+                    "for packages %r\n"
+                    "Falling back to project's root directory"
+                    % (directory, packages),)
+                import_path = directory
+            # Move the .dist-info folder to the import path like setuptools.
+            move(entry_path, os.path.join(import_path, project_name + ext))
+            # Create a temporary .egg-link.
+            tmp_egg_link = os.path.join(tmp3, project_name + '.egg-link')
+            with open(tmp_egg_link, 'w') as f:
+                f.write(import_path)
+            # Move the .egg-link to the destination directory.
+            egg_link_path = os.path.join(dest, project_name + '.egg-link')
+            return move(tmp_egg_link, egg_link_path)
 
     finally:
         undo.reverse()
@@ -1806,63 +1869,50 @@ class IncompatibleConstraintError(zc.buildout.UserError):
 IncompatibleVersionError = IncompatibleConstraintError # Backward compatibility
 
 
-def call_pip_wheel(spec, dest, options):
+try:
+    from pip._internal.cli.cmdoptions import no_python_version_warning
+    PIP_HAS_PYTHON_VERSION_WARNING_OPTION = True
+except ImportError:
+    PIP_HAS_PYTHON_VERSION_WARNING_OPTION = False
+
+
+def call_pip_command(command, operand, options, verbosity=-10):
     """
-    Call `pip wheel` from a subprocess to install a
-    distribution specified by `spec` into `dest`.
-    Returns all the paths inside `dest` created by the above.
+    Call `pip <command...> <operand...>` from a subprocess
+    with appropriate options and environment.
     """
     cleanup = []
     try:
-        args = [sys.executable, '-m', 'pip', 'wheel', '--no-deps', '-w', dest]
-        level = logger.getEffectiveLevel()
-        if level >= logging.INFO:
-            args.append('-q')
-        else:
-            args.append('-v')
+        env = os.environ.copy()
+        pythonpath = pip_path[:]
+        pythonpath.extend(
+            env.get(k) for k in ('PYTHONPATH', 'PYTHONEXTRAPATH'))
+        env['PYTHONPATH'] = os.pathsep.join(p for p in pythonpath if p)
+
+        args = [sys.executable, '-m', 'pip']
+        args.extend(command)
+        args.append('--no-deps')
+
+        log_level = logger.getEffectiveLevel()
+        pip_level = log_level - verbosity
+        if pip_level >= logging.WARNING:
+            args.append('-' + 'q' * (1 + (pip_level >= logging.ERROR)))
+        elif pip_level < logging.INFO:
+            args.append('-' + 'v' * (1 + (pip_level < logging.DEBUG)))
+        # Note: more recent pip accepts even -vvv and -qqq.
 
-        # Prevent pip from installing build dependencies on the fly
-        # without respecting pinned versions. This only works for
-        # PEP 517 specifications using pyproject.toml and not for
-        # dependencies in setup_requires option in legacy setup.py
         if not options._allow_picked_versions:
+            # Prevent pip from installing build dependencies on the fly
+            # without respecting pinned versions. This only works for
+            # PEP 517 specifications using pyproject.toml and not for
+            # dependencies in setup_requires option in legacy setup.py
             args.append('--no-index')
             args.append('--no-build-isolation')
 
-        args.append(spec)
-
-        try:
-            from pip._internal.cli.cmdoptions import no_python_version_warning
-            HAS_WARNING_OPTION = True
-        except ImportError:
-            HAS_WARNING_OPTION = False
-        if HAS_WARNING_OPTION:
-            if not hasattr(call_pip_wheel, 'displayed'):
-                call_pip_wheel.displayed = True
-            else:
-                args.append('--no-python-version-warning')
-
-        env = os.environ.copy()
-        python_path = pip_path[:]
-        env_paths = env.get('PYTHONPATH')
-        if env_paths:
-            python_path.append(env_paths)
-        extra_env_path = env.get('PYTHONEXTRAPATH')
-        if extra_env_path:
-            python_path.append(extra_env_path)
-        env['PYTHONPATH'] = os.pathsep.join(python_path)
-
-        if level <= logging.DEBUG:
-            logger.debug('Running pip install:\n"%s"\npath=%s\n',
-                            '" "'.join(args), pip_path)
-
-        sys.stdout.flush() # We want any pending output first
-
-        # Prevent setuptools from downloading and thus installing
-        # build dependencies specified in setup_requires option of
-        # legacy setup.py by providing a crafted .pydistutils.cfg.
-        # This is used in complement to --no-build-isolation.
-        if not options._allow_picked_versions:
+            # Prevent setuptools from downloading and thus installing
+            # build dependencies specified in setup_requires option of
+            # legacy setup.py by providing a crafted .pydistutils.cfg.
+            # This is used in complement to --no-build-isolation.
             pip_home = tempfile.mkdtemp('pip-pydistutils-home')
             cleanup.append(lambda: zc.buildout.rmtree.rmtree(pip_home))
             with open(os.path.join(pip_home, '.pydistutils.cfg'), 'w') as f:
@@ -1870,24 +1920,60 @@ def call_pip_wheel(spec, dest, options):
                         "index_url = file:///dev/null")
             env['HOME'] = pip_home
 
-        subprocess.check_call(args, env=env)
+        if PIP_HAS_PYTHON_VERSION_WARNING_OPTION:
+            # Let pip display Python warnings only on first run.
+            if not hasattr(call_pip_command, 'displayed'):
+                call_pip_command.displayed = True
+            else:
+                args.append('--no-python-version-warning')
 
-        entries = os.listdir(dest)
-        try:
-            assert len(entries) == 1, "Got multiple entries afer pip wheel"
-            wheel = entries[0]
-            assert os.path.splitext(wheel)[1] == '.whl', "Expected a .whl"
-        except AssertionError:
-            logger.error(
-                "No .whl after successful pip wheel of %s",
-                spec)
-            raise
+        args.extend(operand)
+
+        if log_level < logging.DEBUG:
+            # Log this only when buildout log level is quite low
+            logger.debug('Running pip %s', ' '.join(command[0:1] + operand))
+        if log_level < 0:
+            # Log this only when buildout log level is even lower
+            logger.debug(
+                '%s\nPYTHONPATH=%s\n', ' '.join(args), env['PYTHONPATH'])
 
-        return make_egg_after_pip_wheel(dest, wheel)
+        sys.stdout.flush() # We want any pending output first
+        subprocess.check_call(args, env=env)
     finally:
         for f in cleanup:
             f()
 
+
+def call_pip_editable(path, dest, options, verbosity=-10):
+    """
+    Call `pip install --editable` from a subprocess to install a
+    the project in `path` into `dest` in editable mode.
+    """
+    call_pip_command(
+        ['install', '-t', dest], ['--editable', path],
+        options, verbosity)
+
+
+def call_pip_wheel(spec, dest, options):
+    """
+    Call `pip wheel` from a subprocess to install a
+    distribution specified by `spec` into `dest`.
+    Returns all the paths inside `dest` created by the above.
+    """
+    call_pip_command(['wheel', '-w', dest], [spec], options)
+
+    entries = os.listdir(dest)
+    try:
+        assert len(entries) == 1, "Got multiple entries afer pip wheel"
+        wheel = entries[0]
+        assert os.path.splitext(wheel)[1] == '.whl', "Expected a .whl"
+    except AssertionError:
+        logger.error("No .whl after successful pip wheel of %s", spec)
+        raise
+
+    return make_egg_after_pip_wheel(dest, wheel)
+
+
 def make_egg_after_pip_wheel(dest, wheel):
     unpack_wheel(os.path.join(dest, wheel), dest)
     assert len(os.listdir(dest)) == 2

src/zc/buildout/tests/easy_install.txt

@@ -348,7 +348,7 @@ reporting that a version was picked automatically:
     zc.buildout.easy_install DEBUG
       Fetching demoneeded 1.1 from: http://.../demoneeded-1.1.zip
     zc.buildout.easy_install DEBUG
-      Running pip install:...
+      Running pip wheel...
     zc.buildout.easy_install INFO
       Got demoneeded 1.1.
     zc.buildout.easy_install DEBUG

src/zc/buildout/tests/test_all.py

@@ -204,8 +204,7 @@ We should be able to deal with setup scripts that aren't setuptools based.
     ... # doctest: +ELLIPSIS
     Installing...
     Develop: '/sample-buildout/foo'
-    ...
-    Installed /sample-buildout/foo
+    Running pip install --editable /sample-buildout/foo
     ...
 
     >>> ls('develop-eggs')
@@ -216,10 +215,9 @@ We should be able to deal with setup scripts that aren't setuptools based.
     ... # doctest: +ELLIPSIS
     Installing...
     Develop: '/sample-buildout/foo'
-    in: '/sample-buildout/foo'
-    ... -q develop -mN -d /sample-buildout/develop-eggs/...
-
-
+    Running pip install --editable /sample-buildout/foo
+    ... -m pip install -t ... --editable /sample-buildout/foo
+    ...
     """
 
 def buildout_error_handling():