Commit dcdc34d1 authored by Godefroid Chapelle's avatar Godefroid Chapelle

Problem: there is no support for `Requires-Python`

Solution: monkeypatch `setuptools.packageIndex`
with code from `pip._internal`.

Because it depends on `pip._internal`, it is fragile.
As a first defense against breakage, imports are protected.
In case they would fail, buildout still works.
parent 768502ce
......@@ -9,7 +9,7 @@ jobs:
strategy:
fail-fast: false
matrix:
python-version: [3.5, 3.6, 3.7, 3.8]
python-version: [2.7, 3.5, 3.6, 3.7, 3.8]
steps:
- uses: actions/checkout@v2
......
......@@ -9,7 +9,7 @@ jobs:
strategy:
max-parallel: 4
matrix:
python-version: [3.5, 3.6, 3.7, 3.8]
python-version: [2.7, 3.5, 3.6, 3.7, 3.8]
package: [zest.releaser, pyspf]
steps:
......
......@@ -11,7 +11,7 @@ jobs:
strategy:
max-parallel: 4
matrix:
python-version: [3.5, 3.6, 3.7, 3.8]
python-version: [2.7, 3.5, 3.6, 3.7, 3.8]
package: [zest.releaser, pyspf]
steps:
......
language: minimal
env:
- PYTHON_VER=2.7
- PYTHON_VER=3.5
- PYTHON_VER=3.6
- PYTHON_VER=3.7
......
......@@ -4,6 +4,10 @@ Change History
3.0.0 (unreleased)
==================
- Add support for ``Requires-Python`` metadata.
Fragile monkeypatch that relies on ``pip._internal``.
Emits a warning when support is disabled due to changes in ``pip``.
- Use ``pip install`` instead of deprecated ``setuptools.easy_install``.
- Patch ``pkg_resources.Distribution`` to make install of unpinned versions quicker.
......
......@@ -100,12 +100,14 @@ test: $(VENV)/bin/test
$(VENV)/bin/test -c -vvv $(testargs)
all_pythons:
$(MAKE) PYTHON_VER=2.7 python
$(MAKE) PYTHON_VER=3.5 python
$(MAKE) PYTHON_VER=3.6 python
$(MAKE) PYTHON_VER=3.7 python
$(MAKE) PYTHON_VER=3.8 python
all_test:
$(MAKE) PYTHON_VER=2.7 test
$(MAKE) PYTHON_VER=3.5 test
$(MAKE) PYTHON_VER=3.6 test
$(MAKE) PYTHON_VER=3.7 test
......
......@@ -16,8 +16,16 @@
This is different from a normal boostrapping process because the
buildout egg itself is installed as a develop egg.
"""
import sys
import os, shutil, sys, subprocess
if sys.version_info < (2, 7):
raise SystemError("Outside Python 2.7, no support for Python 2.x.")
if sys.version_info > (3, ) and sys.version_info < (3, 5):
raise SystemError("No support for Python 3.x under 3.5.")
import os, shutil, sys, subprocess, tempfile
for d in 'eggs', 'develop-eggs', 'bin', 'parts':
if not os.path.exists(d):
......@@ -30,6 +38,40 @@ if os.path.isfile(bin_buildout):
if os.path.isdir('build'):
shutil.rmtree('build')
#######################################################################
def install_pip():
print('')
print('Install pip')
print('')
try:
from urllib.request import urlopen
except ImportError:
from urllib2 import urlopen
tmp = tempfile.mkdtemp(prefix='buildout-dev-')
try:
get_pip = os.path.join(tmp, 'get-pip.py')
with open(get_pip, 'wb') as f:
f.write(urlopen('https://bootstrap.pypa.io/get-pip.py').read())
sys.stdout.flush()
if subprocess.call([sys.executable, get_pip]):
raise RuntimeError("Failed to install pip.")
finally:
shutil.rmtree(tmp)
print("Restart")
sys.stdout.flush()
return_code = subprocess.call(
[sys.executable] + sys.argv
)
sys.exit(return_code)
try:
import pip
except ImportError:
install_pip()
######################################################################
def check_upgrade(package):
print('')
......@@ -76,7 +118,7 @@ pkg_resources.working_set.add_entry('src')
import zc.buildout.easy_install
zc.buildout.easy_install.scripts(
['zc.buildout'], pkg_resources.working_set , sys.executable, 'bin')
['zc.buildout'], pkg_resources.working_set, sys.executable, 'bin')
######################################################################
print('')
......
......@@ -56,3 +56,124 @@ def patch_Distribution():
patch_Distribution()
def patch_PackageIndex():
try:
from setuptools.package_index import PackageIndex
from setuptools.package_index import URL_SCHEME
from setuptools.package_index import HREF
from setuptools.package_index import distros_for_url
from setuptools.package_index import htmldecode
from pip._internal.index.collector import HTMLPage
from pip._internal.index.collector import parse_links
from pip._internal.index.collector import _clean_link
from pip._internal.index.package_finder import _check_link_requires_python
from pip._internal.models.target_python import TargetPython
from pip._vendor import six
from pip._vendor.six.moves import urllib
PY_VERSION_INFO = TargetPython().py_version_info
# method copied over from setuptools 46.1.3
def process_url(self, url, retrieve=False):
"""Evaluate a URL as a possible download, and maybe retrieve it"""
if url in self.scanned_urls and not retrieve:
return
self.scanned_urls[url] = True
if not URL_SCHEME(url):
self.process_filename(url)
return
else:
dists = list(distros_for_url(url))
if dists:
if not self.url_ok(url):
return
self.debug("Found link: %s", url)
if dists or not retrieve or url in self.fetched_urls:
list(map(self.add, dists))
return # don't need the actual page
if not self.url_ok(url):
self.fetched_urls[url] = True
return
self.info("Reading %s", url)
self.fetched_urls[url] = True # prevent multiple fetch attempts
tmpl = "Download error on %s: %%s -- Some packages may not be found!"
f = self.open_url(url, tmpl % url)
if f is None:
return
if isinstance(f, urllib.error.HTTPError) and f.code == 401:
self.info("Authentication error: %s" % f.msg)
self.fetched_urls[f.url] = True
if 'html' not in f.headers.get('content-type', '').lower():
f.close() # not html, we can't process it
return
base = f.url # handle redirects
page = f.read()
# --- LOCAL CHANGES MADE HERE: ---
if isinstance(page, six.text_type):
page = page.encode('utf8')
charset = 'utf8'
else:
if isinstance(f, urllib.error.HTTPError):
# Errors have no charset, assume latin1:
charset = 'latin-1'
else:
try:
charset = f.headers.get_param('charset') or 'latin-1'
except AttributeError:
# Python 2
charset = f.headers.getparam('charset') or 'latin-1'
try:
html_page = HTMLPage(page, charset, base, cache_link_parsing=False)
except TypeError:
html_page = HTMLPage(page, charset, base)
plinks = list(parse_links(html_page))
pip_links = [l.url for l in plinks]
# --- END OF LOCAL CHANGES ---
if not isinstance(page, str):
# In Python 3 and got bytes but want str.
page = page.decode(charset, "ignore")
f.close()
# --- LOCAL CHANGES MADE HERE: ---
links = []
for match in HREF.finditer(page):
link = urllib.parse.urljoin(base, htmldecode(match.group(1)))
links.append(_clean_link(link))
# TODO: remove assertion and double index page parsing before releasing.
assert set(pip_links) == set(links)
for link in plinks:
if _check_link_requires_python(link, PY_VERSION_INFO):
self.process_url(link.url)
# --- END OF LOCAL CHANGES ---
if url.startswith(self.index_url) and getattr(f, 'code', None) != 404:
page = self.process_index(url, page)
setattr(PackageIndex, 'process_url', process_url)
except ImportError:
import logging
logger = logging.getLogger('zc.buildout.patches')
logger.warning('Requires-Python support missing. \n\n',
exc_info=True
)
return
patch_PackageIndex()
......@@ -25,7 +25,7 @@ except ImportError:
import errno
import logging
from multiprocessing import get_context
import multiprocessing
import os
import pkg_resources
import random
......@@ -592,6 +592,20 @@ normalize_open_in_generated_script = (
not_found = (re.compile(r'Not found: [^\n]+/(\w|\.)+/\r?\n'), '')
python27_warning = (re.compile(r'DEPRECATION: Python 2.7 reached the end of its '
'life on January 1st, 2020. Please upgrade your Python as Python 2.7 is no '
'longer maintained. A future version of pip will drop support for Python '
'2.7. More details about Python 2 support in pip, can be found at '
'https://pip.pypa.io/en/latest/development/release-process/#python-2-support\n'),
'')
python27_warning_2 = (re.compile(r'DEPRECATION: Python 2.7 reached the end of its '
'life on January 1st, 2020. Please upgrade your Python as Python 2.7 is no '
'longer maintained. pip 21.0 will drop support for Python 2.7 in January 2021. '
'More details about Python 2 support in pip, can be found at '
'https://pip.pypa.io/en/latest/development/release-process/#python-2-support\n'),
'')
# Setuptools now pulls in dependencies when installed.
adding_find_link = (re.compile(r"Adding find link '[^']+'"
r" from setuptools .*\r?\n"), '')
......@@ -615,8 +629,11 @@ def run_from_process(target, *args, **kw):
target(*args, **kw)
def run_in_process(*args, **kwargs):
ctx = get_context('fork')
try:
ctx = multiprocessing.get_context('fork')
process = ctx.Process(target=run_from_process, args=args, kwargs=kwargs)
except AttributeError:
process = multiprocessing.Process(target=run_from_process, args=args, kwargs=kwargs)
process.daemon = True
process.start()
process.join(99)
......
......@@ -3611,6 +3611,8 @@ def test_suite():
zc.buildout.testing.normalize_egg_py,
zc.buildout.testing.not_found,
zc.buildout.testing.adding_find_link,
zc.buildout.testing.python27_warning,
zc.buildout.testing.python27_warning_2,
# (re.compile(r"Installing 'zc.buildout >=\S+"), ''),
(re.compile(r'__buildout_signature__ = recipes-\S+'),
'__buildout_signature__ = recipes-SSSSSSSSSSS'),
......@@ -3748,6 +3750,8 @@ def test_suite():
zc.buildout.testing.normalize_open_in_generated_script,
zc.buildout.testing.adding_find_link,
zc.buildout.testing.not_found,
zc.buildout.testing.python27_warning,
zc.buildout.testing.python27_warning_2,
normalize_bang,
normalize_S,
(re.compile(r'[-d] setuptools-\S+[.]egg'), 'setuptools.egg'),
......@@ -3793,6 +3797,8 @@ def test_suite():
zc.buildout.testing.not_found,
zc.buildout.testing.normalize_exception_type_for_python_2_and_3,
zc.buildout.testing.adding_find_link,
zc.buildout.testing.python27_warning,
zc.buildout.testing.python27_warning_2,
normalize_bang,
(re.compile(r'^(\w+\.)*(Missing\w+: )'), '\2'),
(re.compile(r"buildout: Running \S*setup.py"),
......
......@@ -50,6 +50,8 @@ def test_suite():
zc.buildout.tests.normalize_bang,
zc.buildout.tests.normalize_S,
zc.buildout.testing.not_found,
zc.buildout.testing.python27_warning,
zc.buildout.testing.python27_warning_2,
(re.compile(r'[d-] zc.buildout(-\S+)?[.]egg(-link)?'),
'zc.buildout.egg'),
(re.compile(r'[d-] setuptools-[^-]+-'), 'setuptools-X-'),
......@@ -66,6 +68,8 @@ def test_suite():
zc.buildout.testing.normalize_path,
zc.buildout.testing.normalize_endings,
zc.buildout.testing.not_found,
zc.buildout.testing.python27_warning,
zc.buildout.testing.python27_warning_2,
(re.compile('__buildout_signature__ = '
r'sample-\S+\s+'
r'zc.recipe.egg-\S+\s+'
......@@ -87,6 +91,8 @@ def test_suite():
zc.buildout.testing.normalize_path,
zc.buildout.testing.normalize_endings,
zc.buildout.testing.not_found,
zc.buildout.testing.python27_warning,
zc.buildout.testing.python27_warning_2,
(re.compile("(d ((ext)?demo(needed)?|other)"
r"-\d[.]\d-py)\d[.]\d(-\S+)?[.]egg"),
'\\1V.V.egg'),
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment