Commit ee5fded5 authored by Xavier Thompson's avatar Xavier Thompson

[feat] Use pip wheel + Wheel.install_as_egg

`pip install <package>` produces a `<package-name>` package folder
and a `<package-name>.dist-info` metadata folder, which is another
format than eggs. Then buildout bundles both folders into a parent
folder `<package.egg>` and tries to act as though it were an egg.

Instead, use `pip wheel` to produce a wheel - which `pip install`
does internally anyway - and `setuptools.Wheel.install_as_egg` to
produce a genuine egg.

This is much cleaner: it consistently produces genuine eggs instead
of sometimes true eggs, sometimes `.dist-info` bundles depending on
whether `pip install` is called or the package was installed from a
`.whl` or `.egg` archive directly.

The only downside it this requires setuptools >= 38.2.3.
parent 0bb13e03
...@@ -441,11 +441,11 @@ class Installer(object): ...@@ -441,11 +441,11 @@ class Installer(object):
str(req)) str(req))
return best_we_have, None return best_we_have, None
def _call_pip_install(self, spec, dest, dist): def _call_pip_wheel(self, spec, dest, dist):
tmp = tempfile.mkdtemp(dir=dest) tmp = tempfile.mkdtemp(dir=dest)
try: try:
paths = call_pip_install(spec, tmp, self) paths = call_pip_wheel(spec, tmp, self)
dists = [] dists = []
env = pkg_resources.Environment(paths) env = pkg_resources.Environment(paths)
...@@ -900,7 +900,7 @@ class Installer(object): ...@@ -900,7 +900,7 @@ class Installer(object):
setuptools.command.setopt.edit_config( setuptools.command.setopt.edit_config(
setup_cfg, setup_cfg_dict) setup_cfg, setup_cfg_dict)
dists = self._call_pip_install(base, self._dest, dist) dists = self._call_pip_wheel(base, self._dest, dist)
return [dist.location for dist in dists] return [dist.location for dist in dists]
finally: finally:
...@@ -1749,13 +1749,13 @@ class IncompatibleConstraintError(zc.buildout.UserError): ...@@ -1749,13 +1749,13 @@ class IncompatibleConstraintError(zc.buildout.UserError):
IncompatibleVersionError = IncompatibleConstraintError # Backward compatibility IncompatibleVersionError = IncompatibleConstraintError # Backward compatibility
def call_pip_install(spec, dest, options): def call_pip_wheel(spec, dest, options):
""" """
Call `pip install` from a subprocess to install a Call `pip wheel` from a subprocess to install a
distribution specified by `spec` into `dest`. distribution specified by `spec` into `dest`.
Returns all the paths inside `dest` created by the above. Returns all the paths inside `dest` created by the above.
""" """
args = [sys.executable, '-m', 'pip', 'install', '--no-deps', '-t', dest] args = [sys.executable, '-m', 'pip', 'wheel', '--no-deps', '-w', dest]
level = logger.getEffectiveLevel() level = logger.getEffectiveLevel()
if level >= logging.INFO: if level >= logging.INFO:
args.append('-q') args.append('-q')
...@@ -1776,8 +1776,8 @@ def call_pip_install(spec, dest, options): ...@@ -1776,8 +1776,8 @@ def call_pip_install(spec, dest, options):
except ImportError: except ImportError:
HAS_WARNING_OPTION = False HAS_WARNING_OPTION = False
if HAS_WARNING_OPTION: if HAS_WARNING_OPTION:
if not hasattr(call_pip_install, 'displayed'): if not hasattr(call_pip_wheel, 'displayed'):
call_pip_install.displayed = True call_pip_wheel.displayed = True
else: else:
args.append('--no-python-version-warning') args.append('--no-python-version-warning')
...@@ -1799,128 +1799,24 @@ def call_pip_install(spec, dest, options): ...@@ -1799,128 +1799,24 @@ def call_pip_install(spec, dest, options):
subprocess.check_call(list(args), env=env) subprocess.check_call(list(args), env=env)
split_entries = [os.path.splitext(entry) for entry in os.listdir(dest)] entries = os.listdir(dest)
try: try:
distinfo_dir = [ assert len(entries) == 1, "Got multiple entries afer pip wheel"
base + ext for base, ext in split_entries if ext == ".dist-info" wheel = entries[0]
][0] assert os.path.splitext(wheel)[1] == '.whl', "Expected a .whl"
except IndexError: except AssertionError:
logger.error( logger.error(
"No .dist-info directory after successful pip install of %s", "No .whl after successful pip wheel of %s",
spec) spec)
raise raise
return make_egg_after_pip_install(dest, distinfo_dir) return make_egg_after_pip_wheel(dest, wheel)
def make_egg_after_pip_install(dest, distinfo_dir): def make_egg_after_pip_wheel(dest, wheel):
"""build properly named egg directory""" unpack_wheel(os.path.join(dest, wheel), dest)
assert len(os.listdir(dest)) == 2
# `pip install` does not build the namespace aware __init__.py files return glob.glob(os.path.join(dest, '*.egg'))
# but they are needed in egg directories.
# Add them before moving files setup by pip
namespace_packages_file = os.path.join(
dest, distinfo_dir,
'namespace_packages.txt'
)
if os.path.isfile(namespace_packages_file):
with open(namespace_packages_file) as f:
namespace_packages = [
line.strip().replace('.', os.path.sep)
for line in f.readlines()
]
for namespace_package in namespace_packages:
namespace_package_dir = os.path.join(dest, namespace_package)
if os.path.isdir(namespace_package_dir):
init_py_file = os.path.join(
namespace_package_dir, '__init__.py')
with open(init_py_file, 'w') as f:
f.write(
"__import__('pkg_resources')."
"declare_namespace(__name__)"
)
# Remove `bin` directory if needed
# as there is no way to avoid script installation
# when running `pip install`
entry_points_file = os.path.join(dest, distinfo_dir, 'entry_points.txt')
if os.path.isfile(entry_points_file):
with open(entry_points_file) as f:
content = f.read()
if "console_scripts" in content or "gui_scripts" in content:
bin_dir = os.path.join(dest, BIN_SCRIPTS)
if os.path.exists(bin_dir):
shutil.rmtree(bin_dir)
# Make properly named new egg dir
distro = list(pkg_resources.find_distributions(dest))[0]
base = "{}-{}".format(
distro.egg_name(), pkg_resources.get_supported_platform()
)
egg_name = base + '.egg'
new_distinfo_dir = base + '.dist-info'
egg_dir = os.path.join(dest, egg_name)
os.mkdir(egg_dir)
# Move ".dist-info" dir into new egg dir
os.rename(
os.path.join(dest, distinfo_dir),
os.path.join(egg_dir, new_distinfo_dir)
)
top_level_file = os.path.join(egg_dir, new_distinfo_dir, 'top_level.txt')
if os.path.isfile(top_level_file):
with open(top_level_file) as f:
top_levels = filter(
(lambda x: len(x) != 0),
[line.strip() for line in f.readlines()]
)
else:
top_levels = ()
# Move all top_level modules or packages
for top_level in top_levels:
# as package
top_level_dir = os.path.join(dest, top_level)
if os.path.exists(top_level_dir):
shutil.move(top_level_dir, egg_dir)
continue
# as module
top_level_py = top_level_dir + '.py'
if os.path.exists(top_level_py):
shutil.move(top_level_py, egg_dir)
top_level_pyc = top_level_dir + '.pyc'
if os.path.exists(top_level_pyc):
shutil.move(top_level_pyc, egg_dir)
continue
record_file = os.path.join(egg_dir, new_distinfo_dir, 'RECORD')
if os.path.isfile(record_file):
if PY3:
with open(record_file, newline='') as f:
all_files = [row[0] for row in csv.reader(f)]
else:
with open(record_file, 'rb') as f:
all_files = [row[0] for row in csv.reader(f)]
# There might be some c extensions left over
for entry in all_files:
if entry.endswith(('.pyc', '.pyo')):
continue
dest_entry = os.path.join(dest, entry)
# work around pip install -t bug that leaves entries in RECORD
# that starts with '../../'
if not os.path.abspath(dest_entry).startswith(dest):
continue
egg_entry = os.path.join(egg_dir, entry)
if os.path.exists(dest_entry) and not os.path.exists(egg_entry):
egg_entry_dir = os.path.dirname(egg_entry)
if not os.path.exists(egg_entry_dir):
os.makedirs(egg_entry_dir)
os.rename(dest_entry, egg_entry)
return [egg_dir]
def unpack_egg(location, dest): def unpack_egg(location, dest):
...@@ -2008,7 +1904,7 @@ def _move_to_eggs_dir_and_compile(dist, dest, options): ...@@ -2008,7 +1904,7 @@ def _move_to_eggs_dir_and_compile(dist, dest, options):
unpacker(dist.location, tmp_dest) unpacker(dist.location, tmp_dest)
[tmp_loc] = glob.glob(os.path.join(tmp_dest, '*')) [tmp_loc] = glob.glob(os.path.join(tmp_dest, '*'))
else: else:
[tmp_loc] = call_pip_install(dist.location, tmp_dest, options) [tmp_loc] = call_pip_wheel(dist.location, tmp_dest, options)
installed_with_pip = True installed_with_pip = True
# We have installed the dist. Now try to rename/move it. # We have installed the dist. Now try to rename/move it.
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment