[feat] Use pip install --editable in easy_install.develop

Instead of running python setup.py develop directly. This will allow
using zc.buildout.easy_install.develop on recent projects that have
only a pyproject.toml. It also fixes develop leaving build artifacts
in the source directory that caused later runs to do nothing, e.g.
preventing develop-eggs to be rebuilt when a build dependency passed
in setup-eggs option of zc.recipe.egg:develop changed.

A verbosity parameter to tune verbosity of pip is added, with adjusted
values for the case of buildout:develop and of zc.recipe.egg:develop,
so as to remain close to the previous behavior with regards to logs.

src/zc/buildout/buildout.py

@@ -1131,7 +1131,8 @@ class Buildout(DictMixin):
                     for setup in files:
                         self._logger.info("Develop: %r", setup)
                         __doing__ = 'Processing develop directory %r.', setup
-                        zc.buildout.easy_install.develop(setup, dest)
+                        zc.buildout.easy_install.develop(setup, dest,
+                                                         verbosity=-20)
             except:
                 # if we had an error, we need to roll back changes, by
                 # removing any files we created.

src/zc/buildout/easy_install.py

@@ -1114,22 +1114,6 @@ def _rm(*paths):
             os.remove(path)
 
 
-def _copyeggs(src, dest, suffix, undo):
-    result = []
-    undo.append(lambda : _rm(*result))
-    for name in os.listdir(src):
-        if name.endswith(suffix):
-            new = os.path.join(dest, name)
-            _rm(new)
-            os.rename(os.path.join(src, name), new)
-            result.append(new)
-
-    assert len(result) == 1, str(result)
-    undo.pop()
-
-    return result[0]
-
-
 _develop_distutils_scripts = {}
 
 
@@ -1176,13 +1160,11 @@ def _detect_distutils_scripts(directory):
         _develop_distutils_scripts[egg_name] = scripts_found
 
 
-def develop(setup, dest,
-            build_ext=None,
-            executable=sys.executable):
-    assert executable == sys.executable, (executable, sys.executable)
+def develop(setup, dest, build_ext=None, executable=None, verbosity=-10):
+    if executable is not None: # BBB
+        assert executable == sys.executable, (executable, sys.executable)
     if os.path.isdir(setup):
         directory = setup
-        setup = os.path.join(directory, 'setup.py')
     else:
         directory = os.path.dirname(setup)
 
@@ -1204,43 +1186,55 @@ def develop(setup, dest,
             setuptools.command.setopt.edit_config(
                 setup_cfg, dict(build_ext=build_ext))
 
-        fd, tsetup = tempfile.mkstemp()
-        undo.append(lambda: os.remove(tsetup))
-        undo.append(lambda: os.close(fd))
+        tmp3 = tempfile.mkdtemp('build', dir=dest)
+        undo.append(lambda : zc.buildout.rmtree.rmtree(tmp3))
 
-        extra_path = os.environ.get('PYTHONEXTRAPATH')
-        extra_path_list = []
-        if extra_path:
-          extra_path_list = extra_path.split(os.pathsep)
+        class options: pass
+        options._allow_picked_versions = allow_picked_versions()
 
-        os.write(fd, (runsetup_template % dict(
-            setupdir=directory,
-            setup=setup,
-            path_list=extra_path_list,
-            __file__ = setup,
-            )).encode())
+        call_pip_editable(directory, tmp3, options, verbosity=verbosity)
 
-        tmp3 = tempfile.mkdtemp('build', dir=dest)
-        undo.append(lambda : zc.buildout.rmtree.rmtree(tmp3))
+        _detect_distutils_scripts(tmp3)
 
-        args = [executable,  tsetup, '-q', 'develop', '-mN', '-d', tmp3]
+        egg_link, dist_info = [], []
+        for entry in os.listdir(tmp3):
+            if entry.endswith('.egg-link'):
+              egg_link.append(entry)
+            if entry.endswith('.dist-info') or entry.endswith('.egg-info'):
+              dist_info.append(entry)
 
-        log_level = logger.getEffectiveLevel()
-        if log_level <= 0:
-            if log_level == 0:
-                del args[2]
-            else:
-                args[2] == '-v'
-        if log_level < logging.DEBUG:
-            logger.debug("in: %r\n%s", directory, ' '.join(args))
+        assert len(egg_link) + len(dist_info) == 1, str(egg_link + dist_info)
+        entry, = (egg_link or dist_info)
+
+        entry_path = os.path.join(tmp3, entry)
+
+        entry_filename, ext = os.path.splitext(entry)
+        project_name = entry_filename.split('-', 1)[0]
 
-        if SETUPTOOLS_IGNORE_WARNINGS:
-            env = dict(os.environ, PYTHONWARNINGS='ignore')
+        def move(src, dst):
+            _rm(dst)
+            os.rename(src, dst)
+            return dst
+
+        if egg_link:
+            # case: setup.py
+            return move(entry_path, os.path.join(dest, entry))
         else:
-            env=None
-        call_subprocess(args, env=env)
-        _detect_distutils_scripts(tmp3)
-        return _copyeggs(tmp3, dest, '.egg-link', undo)
+            # case: pyproject.toml
+            editable_pth = '__editable__.%s.pth' % entry_filename
+            editable_pth_path = os.path.join(tmp3, editable_pth)
+            assert os.path.isfile(editable_pth_path), editable_pth_path
+            # Resolve package path (import path to be referenced by .egg-link)
+            # This may be a subpath of the source path (e.g. zc.buildout/src).
+            with open(editable_pth_path) as f:
+                package_paths = f.read().splitlines()
+            assert len(package_paths) == 1, package_paths
+            package_path, = package_paths
+            # Move the metadata folder into the package path like setuptools.
+            move(entry_path, os.path.join(package_path, project_name + ext))
+            # Create the .egg-link by renaming the __editable__.*.pth
+            egg_link_path = os.path.join(dest, project_name + '.egg-link')
+            return move(editable_pth_path, egg_link_path)
 
     finally:
         undo.reverse()
@@ -1807,6 +1801,12 @@ class IncompatibleConstraintError(zc.buildout.UserError):
 IncompatibleVersionError = IncompatibleConstraintError # Backward compatibility
 
 
+try:
+    from pip._internal.cli.cmdoptions import no_python_version_warning
+    PIP_HAS_PYTHON_VERSION_WARNING_OPTION = True
+except ImportError:
+    PIP_HAS_PYTHON_VERSION_WARNING_OPTION = False
+
 # Temporary HOME with .pydistutils.cfg to disable setup_requires
 pip_pydistutils_home = tempfile.mkdtemp('pip-pydistutils-home')
 with open(os.path.join(pip_pydistutils_home, '.pydistutils.cfg'), 'w') as f:
@@ -1814,64 +1814,79 @@ with open(os.path.join(pip_pydistutils_home, '.pydistutils.cfg'), 'w') as f:
             "index-url = file:///dev/null")
 atexit.register(zc.buildout.rmtree.rmtree, pip_pydistutils_home)
 
-def call_pip_wheel(spec, dest, options):
+def call_pip_command(command, operand, options, verbosity=-10):
     """
-    Call `pip wheel` from a subprocess to install a
-    distribution specified by `spec` into `dest`.
-    Returns all the paths inside `dest` created by the above.
+    Call `pip <command...> <operand...>` from a subprocess
+    with appropriate options and environment.
     """
-    args = [sys.executable, '-m', 'pip', 'wheel', '--no-deps', '-w', dest]
-    level = logger.getEffectiveLevel()
-    if level >= logging.INFO:
-        args.append('-q')
-    else:
-        args.append('-v')
+    env = os.environ.copy()
+    pythonpath = pip_path[:]
+    pythonpath.extend(env.get(k) for k in ('PYTHONPATH', 'PYTHONEXTRAPATH'))
+    env['PYTHONPATH'] = os.pathsep.join(p for p in pythonpath if p)
+
+    args = [sys.executable, '-m', 'pip']
+    args.extend(command)
+    args.append('--no-deps')
+
+    log_level = logger.getEffectiveLevel()
+    pip_level = log_level - verbosity
+    if pip_level >= logging.WARNING:
+        args.append('-' + 'q' * (1 + (pip_level >= logging.ERROR))) # -q or -qq
+    elif pip_level < logging.INFO:
+        args.append('-' + 'v' * (1 + (pip_level < logging.DEBUG))) # -v or -vv
+    # Note: more recent pip accepts even -vvv and -qqq.
 
-    # Prevent pip from installing build dependencies on the fly
-    # without respecting pinned versions. This only works for
-    # PEP 517 specifications using pyproject.toml and not for
-    # dependencies in setup_requires option in legacy setup.py
     if not options._allow_picked_versions:
+        # Prevent pip from installing build dependencies on the fly
+        # without respecting pinned versions. This only works for
+        # PEP 517 specifications using pyproject.toml and not for
+        # dependencies in setup_requires option in legacy setup.py
         args.append('--no-index')
         args.append('--no-build-isolation')
 
-    args.append(spec)
+        # Prevent setuptools from downloading and thus installing
+        # build dependencies specified in setup_requires option of
+        # legacy setup.py by providing a crafted .pydistutils.cfg.
+        # This is used in complement to --no-build-isolation.
+        env['HOME'] = pip_pydistutils_home
 
-    try:
-        from pip._internal.cli.cmdoptions import no_python_version_warning
-        HAS_WARNING_OPTION = True
-    except ImportError:
-        HAS_WARNING_OPTION = False
-    if HAS_WARNING_OPTION:
-        if not hasattr(call_pip_wheel, 'displayed'):
-            call_pip_wheel.displayed = True
+    if PIP_HAS_PYTHON_VERSION_WARNING_OPTION:
+        # Let pip display Python warnings only on first run.
+        if not hasattr(call_pip_command, 'displayed'):
+            call_pip_command.displayed = True
         else:
             args.append('--no-python-version-warning')
 
-    env = os.environ.copy()
-    python_path = pip_path[:]
-    env_paths = env.get('PYTHONPATH')
-    if env_paths:
-        python_path.append(env_paths)
-    extra_env_path = env.get('PYTHONEXTRAPATH')
-    if extra_env_path:
-        python_path.append(extra_env_path)
-    env['PYTHONPATH'] = os.pathsep.join(python_path)
-
-    if level <= logging.DEBUG:
-        logger.debug('Running pip install:\n"%s"\npath=%s\n',
-                        '" "'.join(args), pip_path)
+    args.extend(operand)
+
+    if log_level < logging.DEBUG:
+        # Log this only when buildout log level is quite low
+        logger.debug('Running pip %s', ' '.join(command[0:1] + operand))
+    if log_level < 0:
+        # Log this only when buildout log level is even lower
+        logger.debug('%s\nPYTHONPATH=%s\n', ' '.join(args), env['PYTHONPATH'])
 
     sys.stdout.flush() # We want any pending output first
+    subprocess.check_call(args, env=env)
 
-    # Prevent setuptools from downloading and thus installing
-    # build dependencies specified in setup_requires option of
-    # legacy setup.py by providing a crafted .pydistutils.cfg.
-    # This is used in complement to --no-build-isolation.
-    if not options._allow_picked_versions:
-        env['HOME'] = pip_pydistutils_home
 
-    subprocess.check_call(args, env=env)
+def call_pip_editable(path, dest, options, verbosity=-10):
+    """
+    Call `pip install --editable` from a subprocess to install a
+    the project in `path` into `dest` in editable mode.
+    """
+    call_pip_command(
+        ['install', '-t', dest], ['--editable', path],
+        options, verbosity)
+
+
+def call_pip_wheel(spec, dest, options):
+    """
+    Call `pip wheel` from a subprocess to install a
+    distribution specified by `spec` into `dest`.
+    Returns all the paths inside `dest` created by the above.
+    """
+    call_pip_command(['wheel', '-w', dest], [spec], options)
 
     entries = os.listdir(dest)
     try:
@@ -1879,9 +1894,7 @@ def call_pip_wheel(spec, dest, options):
         wheel = entries[0]
         assert os.path.splitext(wheel)[1] == '.whl', "Expected a .whl"
     except AssertionError:
-        logger.error(
-            "No .whl after successful pip wheel of %s",
-            spec)
+        logger.error("No .whl after successful pip wheel of %s", spec)
         raise
 
     return make_egg_after_pip_wheel(dest, wheel)

src/zc/buildout/tests/easy_install.txt

@@ -348,7 +348,7 @@ reporting that a version was picked automatically:
     zc.buildout.easy_install DEBUG
       Fetching demoneeded 1.1 from: http://.../demoneeded-1.1.zip
     zc.buildout.easy_install DEBUG
-      Running pip install:...
+      Running pip wheel...
     zc.buildout.easy_install INFO
       Got demoneeded 1.1.
     zc.buildout.easy_install DEBUG

src/zc/buildout/tests/test_all.py

@@ -204,8 +204,7 @@ We should be able to deal with setup scripts that aren't setuptools based.
     ... # doctest: +ELLIPSIS
     Installing...
     Develop: '/sample-buildout/foo'
-    ...
-    Installed /sample-buildout/foo
+    Running pip install --editable /sample-buildout/foo
     ...
 
     >>> ls('develop-eggs')
@@ -216,10 +215,9 @@ We should be able to deal with setup scripts that aren't setuptools based.
     ... # doctest: +ELLIPSIS
     Installing...
     Develop: '/sample-buildout/foo'
-    in: '/sample-buildout/foo'
-    ... -q develop -mN -d /sample-buildout/develop-eggs/...
-
-
+    Running pip install --editable /sample-buildout/foo
+    ... -m pip install -t ... --editable /sample-buildout/foo
+    ...
     """
 
 def buildout_error_handling():