[feat] Use pip install --editable in easy_install.develop

Instead of running python setup.py develop directly. This will allow
using zc.buildout.easy_install.develop on recent projects that have
only a pyproject.toml. It also fixes develop leaving build artifacts
in the source directory that caused later runs to do nothing, e.g.
preventing develop-eggs to be rebuilt when a build dependency passed
in setup-eggs option of zc.recipe.egg:develop changed.

A verbosity parameter to tune verbosity of pip is added, with adjusted
values for the case of buildout:develop and of zc.recipe.egg:develop,
so as to remain close to the previous behavior with regards to logs.

Technical details:

For packages using PEP-660-style editable installs, supported by more
recent versions of pip, pip will not delegate to `setup.py develop` -
enabling editable installs for pure pyproject.toml projects - and will
instead generate a .dist-info metadata folder but not a .egg-link.

Since buildout currently requires a .egg-link, as it does not support
PEP 660's mechanism that relies on having a sites-packages directory,
we need to create this .egg-link after the fact. The tricky part is
finding out where the .egg-link should point: the path containing the
pyproject.toml, or a subdirectory?

For this we make use of *.dist-info/top_level.txt to first determine
the list of top-level packages, and then importlib to extract info
from the PEP-660-style install.

If top_level.txt does not exist or is empty, fallback to the path of
the folder that contains the pyproject.toml as the .egg-link target.
If it lists multiple packages that resolve to different import paths,
arbitrarily use the first one and emit a warning.

Support namespace packages where `spec.submodule_search_locations` is
a `_NamespacePath` object instead of a simple `list` and also support
cases where the layout of the source project does not follow the same
structure as the package tree - meaning some custom magic might be
involved in making editable imports work as intended.

src/zc/buildout/buildout.py

@@ -1131,7 +1131,8 @@ class Buildout(DictMixin):
                     for setup in files:
                         self._logger.info("Develop: %r", setup)
                         __doing__ = 'Processing develop directory %r.', setup
-                        zc.buildout.easy_install.develop(setup, dest)
+                        zc.buildout.easy_install.develop(setup, dest,
+                                                         verbosity=-20)
             except:
                 # if we had an error, we need to roll back changes, by
                 # removing any files we created.

src/zc/buildout/easy_install.py

@@ -23,6 +23,7 @@ import distutils.errors
 import distutils.sysconfig
 import errno
 import glob
+import json
 import logging
 import os
 import pkg_resources
@@ -1113,22 +1114,6 @@ def _rm(*paths):
             os.remove(path)
 
 
-def _copyeggs(src, dest, suffix, undo):
-    result = []
-    undo.append(lambda : _rm(*result))
-    for name in os.listdir(src):
-        if name.endswith(suffix):
-            new = os.path.join(dest, name)
-            _rm(new)
-            os.rename(os.path.join(src, name), new)
-            result.append(new)
-
-    assert len(result) == 1, str(result)
-    undo.pop()
-
-    return result[0]
-
-
 _develop_distutils_scripts = {}
 
 
@@ -1175,13 +1160,11 @@ def _detect_distutils_scripts(directory):
         _develop_distutils_scripts[egg_name] = scripts_found
 
 
-def develop(setup, dest,
-            build_ext=None,
-            executable=sys.executable):
-    assert executable == sys.executable, (executable, sys.executable)
+def develop(setup, dest, build_ext=None, executable=None, verbosity=-10):
+    if executable is not None: # BBB
+        assert executable == sys.executable, (executable, sys.executable)
     if os.path.isdir(setup):
         directory = setup
-        setup = os.path.join(directory, 'setup.py')
     else:
         directory = os.path.dirname(setup)
 
@@ -1203,43 +1186,122 @@ def develop(setup, dest,
             setuptools.command.setopt.edit_config(
                 setup_cfg, dict(build_ext=build_ext))
 
-        fd, tsetup = tempfile.mkstemp()
-        undo.append(lambda: os.remove(tsetup))
-        undo.append(lambda: os.close(fd))
+        tmp3 = tempfile.mkdtemp('build', dir=dest)
+        undo.append(lambda : zc.buildout.rmtree.rmtree(tmp3))
 
-        extra_path = os.environ.get('PYTHONEXTRAPATH')
-        extra_path_list = []
-        if extra_path:
-          extra_path_list = extra_path.split(os.pathsep)
+        class options: pass
+        options._allow_picked_versions = allow_picked_versions()
 
-        os.write(fd, (runsetup_template % dict(
-            setupdir=directory,
-            setup=setup,
-            path_list=extra_path_list,
-            __file__ = setup,
-            )).encode())
+        call_pip_editable(directory, tmp3, options, verbosity=verbosity)
 
-        tmp3 = tempfile.mkdtemp('build', dir=dest)
-        undo.append(lambda : zc.buildout.rmtree.rmtree(tmp3))
+        _detect_distutils_scripts(tmp3)
 
-        args = [executable,  tsetup, '-q', 'develop', '-mN', '-d', tmp3]
+        egg_link, dist_info = [], []
+        for entry in os.listdir(tmp3):
+            if entry.endswith('.egg-link'):
+              egg_link.append(entry)
+            if entry.endswith('.dist-info') or entry.endswith('.egg-info'):
+              dist_info.append(entry)
 
-        log_level = logger.getEffectiveLevel()
-        if log_level <= 0:
-            if log_level == 0:
-                del args[2]
-            else:
-                args[2] == '-v'
-        if log_level < logging.DEBUG:
-            logger.debug("in: %r\n%s", directory, ' '.join(args))
+        assert len(egg_link) + len(dist_info) == 1, str(egg_link + dist_info)
+        entry, = (egg_link or dist_info)
+
+        entry_path = os.path.join(tmp3, entry)
 
-        if SETUPTOOLS_IGNORE_WARNINGS:
-            env = dict(os.environ, PYTHONWARNINGS='ignore')
+        entry_filename, ext = os.path.splitext(entry)
+        project_name = entry_filename.split('-', 1)[0]
+
+        def move(src, dst):
+            _rm(dst)
+            os.rename(src, dst)
+            return dst
+
+        if egg_link:
+            # case: legacy setup.py
+            return move(entry_path, os.path.join(dest, entry))
         else:
-            env=None
-        call_subprocess(args, env=env)
-        _detect_distutils_scripts(tmp3)
-        return _copyeggs(tmp3, dest, '.egg-link', undo)
+            # case: PEP 660 pyproject.toml
+            # This case can only happen in Python3
+            # (because pip versions that support PEP 660 are Python3-only).
+
+            # Resolve the package path(s) and import (parent) path(s).
+            # This may be a subpath of the project directory,
+            # e.g. zc.buildout/src.
+            try:
+                with open(os.path.join(entry_path, 'top_level.txt')) as f:
+                    top_level = f.read().splitlines()
+                packages = tuple(p for p in top_level if p)
+            except IOError as e:
+                if e.errno != errno.ENOENT:
+                    raise
+                packages = ()
+            try:
+                package_paths = json.loads(subprocess.check_output([
+                    sys.executable, '-S', '-c',
+                    "import importlib.util, json, site; "
+                    "site.addsitedir(%(site)r); "
+                    "packages = %(packages)r; "
+                    "specs = [importlib.util.find_spec(p) for p in packages]; "
+                    "paths = {p: list(s.submodule_search_locations)"
+                    " for p, s in zip(packages, specs)}; "
+                    "print(json.dumps(paths))"  % {
+                        'site': tmp3,
+                        'packages': packages,
+                        }
+                    ],
+                    env=dict(os.environ, PYTHONPATH=''),
+                    stderr=subprocess.STDOUT,
+                    universal_newlines=True,
+                ))
+            except subprocess.CalledProcessError as e:
+                logger.error(e.output)
+                raise
+            # Resolve import path.
+            directory = os.path.normpath(directory)
+            import_paths = []
+            for package, paths in package_paths.items():
+                for path in paths:
+                  # Filter out magic hook values from setuptools PEP 660.
+                  if (path.startswith('__editable__')
+                      and path.endswith('.__path_hook__')): continue
+                  path = os.path.normpath(path)
+                  # Do not parent-path out of the source repository.
+                  if path == directory:
+                      import_paths.append(path)
+                      continue
+                  parent, folder = os.path.split(path)
+                  # Only parent-path if the folder has the name of the package.
+                  import_paths.append(parent if folder == package else path)
+            # Warn if there are several and arbitrarily take the first one.
+            if import_paths:
+                import_path = import_paths[0]
+                unique_import_paths = set(import_paths)
+                if len(unique_import_paths) > 1:
+                    logger.warning(
+                        "Found multiple package import paths"
+                        " in develop project %s\n(\n  %s\n)\n"
+                        "Selected %s" % (
+                          directory,
+                          '\n  '.join(unique_import_paths),
+                          import_path
+                        ))
+            # Warn if no import path is found; fallback to source directory.
+            else:
+                logger.warning(
+                    "Found no package import path in develop project %s "
+                    "for packages %r\n"
+                    "Falling back to project's root directory"
+                    % (directory, packages),)
+                import_path = directory
+            # Move the .dist-info folder to the import path like setuptools.
+            move(entry_path, os.path.join(import_path, project_name + ext))
+            # Create a temporary .egg-link.
+            tmp_egg_link = os.path.join(tmp3, project_name + '.egg-link')
+            with open(tmp_egg_link, 'w') as f:
+                f.write(import_path)
+            # Move the .egg-link to the destination directory.
+            egg_link_path = os.path.join(dest, project_name + '.egg-link')
+            return move(tmp_egg_link, egg_link_path)
 
     finally:
         undo.reverse()
@@ -1806,63 +1868,50 @@ class IncompatibleConstraintError(zc.buildout.UserError):
 IncompatibleVersionError = IncompatibleConstraintError # Backward compatibility
 
 
-def call_pip_wheel(spec, dest, options):
+try:
+    from pip._internal.cli.cmdoptions import no_python_version_warning
+    PIP_HAS_PYTHON_VERSION_WARNING_OPTION = True
+except ImportError:
+    PIP_HAS_PYTHON_VERSION_WARNING_OPTION = False
+
+
+def call_pip_command(command, operand, options, verbosity=-10):
     """
-    Call `pip wheel` from a subprocess to install a
-    distribution specified by `spec` into `dest`.
-    Returns all the paths inside `dest` created by the above.
+    Call `pip <command...> <operand...>` from a subprocess
+    with appropriate options and environment.
     """
     cleanup = []
     try:
-        args = [sys.executable, '-m', 'pip', 'wheel', '--no-deps', '-w', dest]
-        level = logger.getEffectiveLevel()
-        if level >= logging.INFO:
-            args.append('-q')
-        else:
-            args.append('-v')
+        env = os.environ.copy()
+        pythonpath = pip_path[:]
+        pythonpath.extend(
+            env.get(k) for k in ('PYTHONPATH', 'PYTHONEXTRAPATH'))
+        env['PYTHONPATH'] = os.pathsep.join(p for p in pythonpath if p)
+
+        args = [sys.executable, '-m', 'pip']
+        args.extend(command)
+        args.append('--no-deps')
+
+        log_level = logger.getEffectiveLevel()
+        pip_level = log_level - verbosity
+        if pip_level >= logging.WARNING:
+            args.append('-' + 'q' * (1 + (pip_level >= logging.ERROR)))
+        elif pip_level < logging.INFO:
+            args.append('-' + 'v' * (1 + (pip_level < logging.DEBUG)))
+        # Note: more recent pip accepts even -vvv and -qqq.
 
-        # Prevent pip from installing build dependencies on the fly
-        # without respecting pinned versions. This only works for
-        # PEP 517 specifications using pyproject.toml and not for
-        # dependencies in setup_requires option in legacy setup.py
         if not options._allow_picked_versions:
+            # Prevent pip from installing build dependencies on the fly
+            # without respecting pinned versions. This only works for
+            # PEP 517 specifications using pyproject.toml and not for
+            # dependencies in setup_requires option in legacy setup.py
             args.append('--no-index')
             args.append('--no-build-isolation')
 
-        args.append(spec)
-
-        try:
-            from pip._internal.cli.cmdoptions import no_python_version_warning
-            HAS_WARNING_OPTION = True
-        except ImportError:
-            HAS_WARNING_OPTION = False
-        if HAS_WARNING_OPTION:
-            if not hasattr(call_pip_wheel, 'displayed'):
-                call_pip_wheel.displayed = True
-            else:
-                args.append('--no-python-version-warning')
-
-        env = os.environ.copy()
-        python_path = pip_path[:]
-        env_paths = env.get('PYTHONPATH')
-        if env_paths:
-            python_path.append(env_paths)
-        extra_env_path = env.get('PYTHONEXTRAPATH')
-        if extra_env_path:
-            python_path.append(extra_env_path)
-        env['PYTHONPATH'] = os.pathsep.join(python_path)
-
-        if level <= logging.DEBUG:
-            logger.debug('Running pip install:\n"%s"\npath=%s\n',
-                            '" "'.join(args), pip_path)
-
-        sys.stdout.flush() # We want any pending output first
-
-        # Prevent setuptools from downloading and thus installing
-        # build dependencies specified in setup_requires option of
-        # legacy setup.py by providing a crafted .pydistutils.cfg.
-        # This is used in complement to --no-build-isolation.
-        if not options._allow_picked_versions:
+            # Prevent setuptools from downloading and thus installing
+            # build dependencies specified in setup_requires option of
+            # legacy setup.py by providing a crafted .pydistutils.cfg.
+            # This is used in complement to --no-build-isolation.
             pip_home = tempfile.mkdtemp('pip-pydistutils-home')
             cleanup.append(lambda: zc.buildout.rmtree.rmtree(pip_home))
             with open(os.path.join(pip_home, '.pydistutils.cfg'), 'w') as f:
@@ -1870,24 +1919,60 @@ def call_pip_wheel(spec, dest, options):
                         "index_url = file:///dev/null")
             env['HOME'] = pip_home
 
-        subprocess.check_call(args, env=env)
+        if PIP_HAS_PYTHON_VERSION_WARNING_OPTION:
+            # Let pip display Python warnings only on first run.
+            if not hasattr(call_pip_command, 'displayed'):
+                call_pip_command.displayed = True
+            else:
+                args.append('--no-python-version-warning')
 
-        entries = os.listdir(dest)
-        try:
-            assert len(entries) == 1, "Got multiple entries afer pip wheel"
-            wheel = entries[0]
-            assert os.path.splitext(wheel)[1] == '.whl', "Expected a .whl"
-        except AssertionError:
-            logger.error(
-                "No .whl after successful pip wheel of %s",
-                spec)
-            raise
+        args.extend(operand)
+
+        if log_level < logging.DEBUG:
+            # Log this only when buildout log level is quite low
+            logger.debug('Running pip %s', ' '.join(command[0:1] + operand))
+        if log_level < 0:
+            # Log this only when buildout log level is even lower
+            logger.debug(
+                '%s\nPYTHONPATH=%s\n', ' '.join(args), env['PYTHONPATH'])
 
-        return make_egg_after_pip_wheel(dest, wheel)
+        sys.stdout.flush() # We want any pending output first
+        subprocess.check_call(args, env=env)
     finally:
         for f in cleanup:
             f()
 
+
+def call_pip_editable(path, dest, options, verbosity=-10):
+    """
+    Call `pip install --editable` from a subprocess to install a
+    the project in `path` into `dest` in editable mode.
+    """
+    call_pip_command(
+        ['install', '-t', dest], ['--editable', path],
+        options, verbosity)
+
+
+def call_pip_wheel(spec, dest, options):
+    """
+    Call `pip wheel` from a subprocess to install a
+    distribution specified by `spec` into `dest`.
+    Returns all the paths inside `dest` created by the above.
+    """
+    call_pip_command(['wheel', '-w', dest], [spec], options)
+
+    entries = os.listdir(dest)
+    try:
+        assert len(entries) == 1, "Got multiple entries afer pip wheel"
+        wheel = entries[0]
+        assert os.path.splitext(wheel)[1] == '.whl', "Expected a .whl"
+    except AssertionError:
+        logger.error("No .whl after successful pip wheel of %s", spec)
+        raise
+
+    return make_egg_after_pip_wheel(dest, wheel)
+
+
 def make_egg_after_pip_wheel(dest, wheel):
     unpack_wheel(os.path.join(dest, wheel), dest)
     assert len(os.listdir(dest)) == 2

src/zc/buildout/tests/easy_install.txt

@@ -348,7 +348,7 @@ reporting that a version was picked automatically:
     zc.buildout.easy_install DEBUG
       Fetching demoneeded 1.1 from: http://.../demoneeded-1.1.zip
     zc.buildout.easy_install DEBUG
-      Running pip install:...
+      Running pip wheel...
     zc.buildout.easy_install INFO
       Got demoneeded 1.1.
     zc.buildout.easy_install DEBUG

src/zc/buildout/tests/test_all.py

@@ -204,8 +204,7 @@ We should be able to deal with setup scripts that aren't setuptools based.
     ... # doctest: +ELLIPSIS
     Installing...
     Develop: '/sample-buildout/foo'
-    ...
-    Installed /sample-buildout/foo
+    Running pip install --editable /sample-buildout/foo
     ...
 
     >>> ls('develop-eggs')
@@ -216,10 +215,9 @@ We should be able to deal with setup scripts that aren't setuptools based.
     ... # doctest: +ELLIPSIS
     Installing...
     Develop: '/sample-buildout/foo'
-    in: '/sample-buildout/foo'
-    ... -q develop -mN -d /sample-buildout/develop-eggs/...
-
-
+    Running pip install --editable /sample-buildout/foo
+    ... -m pip install -t ... --editable /sample-buildout/foo
+    ...
     """
 
 def buildout_error_handling():