Commit 842d801d authored by Jason Madden's avatar Jason Madden Committed by GitHub

Merge pull request #1401 from gevent/stages

Refactor the Travis build to make the matrix explicit and faster, simplify Python installs 
parents 110b8263 b5581241
...@@ -9,20 +9,20 @@ parallel = True ...@@ -9,20 +9,20 @@ parallel = True
source = gevent source = gevent
omit = omit =
# This is for <= 2.7.8, which we don't test # This is for <= 2.7.8, which we don't test
src/gevent/_ssl2.py */gevent/_ssl2.py
src/gevent/libev/_corecffi_build.py */gevent/libev/_corecffi_build.py
src/gevent/libuv/_corecffi_build.py */gevent/libuv/_corecffi_build.py
src/gevent/win32util.py */gevent/win32util.py
# having concurrency=greenlet means that the Queue class # having concurrency=greenlet means that the Queue class
# which is used from multiple real threads doesn't # which is used from multiple real threads doesn't
# properly get covered. # properly get covered.
src/gevent/_threading.py */gevent/_threading.py
# local.so sometimes gets included, and it can't be parsed # local.so sometimes gets included, and it can't be parsed
# as source, so it fails the whole process. # as source, so it fails the whole process.
*.so *.so
src/gevent/libev/*.so */gevent/libev/*.so
src/gevent/libuv/*.so */gevent/libuv/*.so
src/gevent/resolver/*.so */gevent/resolver/*.so
[report] [report]
...@@ -47,3 +47,17 @@ omit = ...@@ -47,3 +47,17 @@ omit =
/tmp/test_* /tmp/test_*
# Third-party vendored code # Third-party vendored code
src/gevent/_tblib.py src/gevent/_tblib.py
[paths]
# Combine source and paths from the Travis CI installs so they all get
# collapsed during combining. Otherwise, coveralls.io reports
# many different files (/lib/pythonX.Y/site-packages/gevent/...) and we don't
# get a good aggregate number.
source =
src/
*/lib/*/site-packages/
*/pypy*/site-packages/
# Local Variables:
# mode: conf
# End:
...@@ -91,8 +91,3 @@ deps/libuv/.libs ...@@ -91,8 +91,3 @@ deps/libuv/.libs
deps/libuv/*.lo deps/libuv/*.lo
deps/libuv/*.la deps/libuv/*.la
deps/libuv/*.o deps/libuv/*.o
# running setup.py on PyPy
config.h
configure-output.txt
This diff is collapsed.
...@@ -78,6 +78,12 @@ ...@@ -78,6 +78,12 @@
- Spawning greenlets can be up to 10% faster. - Spawning greenlets can be up to 10% faster.
- Remove the ``Makefile``. Its most useful commands, ``make clean``
and ``make distclean``, can now be accomplished in a cross-platform
way using ``python setup.py clean`` and ``python setup.py clean
-a``, respectively. The remainder of the ``Makefile`` contained
Travis CI commands that have been moved to ``.travis.yml``.
1.4.0 (2019-01-04) 1.4.0 (2019-01-04)
================== ==================
......
# This file is renamed to "Makefile.ext" in release tarballs so that setup.py won't try to
# run it. If you want setup.py to run "make" automatically, rename it back to "Makefile".
# The pyvenv multiple runtime support is based on https://github.com/DRMacIver/hypothesis/blob/master/Makefile
PYTHON?=python${TRAVIS_PYTHON_VERSION}
CYTHON?=cython
export PATH:=$(BUILD_RUNTIMES)/snakepit:$(PATH)
export LC_ALL=C.UTF-8
export GEVENT_RESOLVER_NAMESERVERS=8.8.8.8
clean:
rm -f src/gevent/libev/corecext.c src/gevent/libev/corecext.h
rm -f src/gevent/resolver/cares.c src/gevent/resolver/cares.h
rm -f src/gevent/_semaphore.c src/gevent/_semaphore.h
rm -f src/gevent/local.c src/gevent/local.h
rm -f src/gevent/*.so src/gevent/*.pyd src/gevent/libev/*.so src/gevent/libuv/*.so src/gevent/libev/*.pyd src/gevent/libuv/*.pyd
rm -rf src/gevent/libev/*.o src/gevent/libuv/*.o src/gevent/*.o
rm -rf src/gevent/__pycache__ src/greentest/__pycache__ src/greentest/greentest/__pycache__ src/gevent/libev/__pycache__
rm -rf src/gevent/*.pyc src/greentest/*.pyc src/gevent/libev/*.pyc
rm -rf htmlcov .coverage
rm -rf build
distclean: clean
rm -rf dist
rm -rf deps/libev/config.h deps/libev/config.log deps/libev/config.status deps/libev/.deps deps/libev/.libs
rm -rf deps/c-ares/config.h deps/c-ares/config.log deps/c-ares/config.status deps/c-ares/.deps deps/c-ares/.libs
doc:
cd doc && PYTHONPATH=.. make html
prospector:
which pylint
pylint --rcfile=.pylintrc gevent
# debugging
# pylint --rcfile=.pylintrc --init-hook="import sys, code; sys.excepthook = lambda exc, exc_type, tb: print(tb.tb_next.tb_next.tb_next.tb_next.tb_next.tb_next.tb_next.tb_next.tb_next.tb_next.tb_frame.f_locals['self'])" gevent src/greentest/* || true
# XXX: prospector is failing right now. I can't reproduce locally:
# https://travis-ci.org/gevent/gevent/jobs/345474139
# ${PYTHON} scripts/gprospector.py -X
lint: prospector
test_prelim:
@which ${PYTHON}
@${PYTHON} --version
@${PYTHON} -c 'import greenlet; print(greenlet, greenlet.__version__)'
@${PYTHON} -c 'import gevent.core; print(gevent.core.loop)'
@${PYTHON} -c 'import gevent.ares; print(gevent.ares)'
@make bench
# Folding from https://github.com/travis-ci/travis-rubies/blob/9f7962a881c55d32da7c76baefc58b89e3941d91/build.sh#L38-L44
basictest: test_prelim
@${PYTHON} scripts/travis.py fold_start basictest "Running basic tests"
GEVENT_RESOLVER=thread ${PYTHON} -mgevent.tests --config known_failures.py --quiet
@${PYTHON} scripts/travis.py fold_end basictest
alltest: basictest
@${PYTHON} scripts/travis.py fold_start ares "Running c-ares tests"
GEVENT_RESOLVER=ares ${PYTHON} -mgevent.tests --config known_failures.py --ignore tests_that_dont_use_resolver.txt --quiet
@${PYTHON} scripts/travis.py fold_end ares
@${PYTHON} scripts/travis.py fold_start dnspython "Running dnspython tests"
GEVENT_RESOLVER=dnspython ${PYTHON} -mgevent.tests --config known_failures.py --ignore tests_that_dont_use_resolver.txt --quiet
@${PYTHON} scripts/travis.py fold_end dnspython
# In the past, we included all test files that had a reference to 'subprocess'' somewhere in their
# text. The monkey-patched stdlib tests were specifically included here.
# However, we now always also test on AppVeyor (Windows) which only has GEVENT_FILE=thread,
# so we can save a lot of CI time by reducing the set and excluding the stdlib tests without
# losing any coverage.
@${PYTHON} scripts/travis.py fold_start thread "Running GEVENT_FILE=thread tests"
cd src/gevent/tests && GEVENT_FILE=thread ${PYTHON} -mgevent.tests --config known_failures.py test__*subprocess*.py --quiet
@${PYTHON} scripts/travis.py fold_end thread
allbackendtest:
@${PYTHON} scripts/travis.py fold_start default "Testing default backend"
GEVENTTEST_COVERAGE=1 make alltest
@${PYTHON} scripts/travis.py fold_end default
GEVENTTEST_COVERAGE=1 make cffibackendtest
# because we set parallel=true, each run produces new and different coverage files; they all need
# to be combined
make coverage_combine
cffibackendtest:
@${PYTHON} scripts/travis.py fold_start libuv "Testing libuv backend"
GEVENT_LOOP=libuv make alltest
@${PYTHON} scripts/travis.py fold_end libuv
@${PYTHON} scripts/travis.py fold_start libev "Testing libev CFFI backend"
GEVENT_LOOP=libev-cffi make alltest
@${PYTHON} scripts/travis.py fold_end libev
leaktest: test_prelim
@${PYTHON} scripts/travis.py fold_start leaktest "Running leak tests"
GEVENT_RESOLVER=thread GEVENTTEST_LEAKCHECK=1 ${PYTHON} -mgevent.tests --config known_failures.py --quiet --ignore tests_that_dont_do_leakchecks.txt
@${PYTHON} scripts/travis.py fold_end leaktest
@${PYTHON} scripts/travis.py fold_start default "Testing default backend pure python"
PURE_PYTHON=1 GEVENTTEST_COVERAGE=1 make basictest
@${PYTHON} scripts/travis.py fold_end default
bench:
time ${PYTHON} benchmarks/bench_sendall.py --loops 3 --processes 2 --values 2 --warmups 2 --quiet
travis_test_linters:
make lint
make leaktest
make cffibackendtest
coverage_combine:
coverage combine .
coverage report -i
-coveralls
.PHONY: clean doc prospector lint travistest travis
# Managing runtimes
BUILD_RUNTIMES?=$(PWD)/.runtimes
PY27=$(BUILD_RUNTIMES)/snakepit/python2.7.16
PY35=$(BUILD_RUNTIMES)/snakepit/python3.5.6
PY36=$(BUILD_RUNTIMES)/snakepit/python3.6.8
PY37=$(BUILD_RUNTIMES)/snakepit/python3.7.2
PYPY=$(BUILD_RUNTIMES)/snakepit/pypy710
PYPY3=$(BUILD_RUNTIMES)/snakepit/pypy3.6_710
$(PY27):
scripts/install.sh 2.7
$(PY35):
scripts/install.sh 3.5
$(PY36):
scripts/install.sh 3.6
$(PY37):
scripts/install.sh 3.7
$(PYPY):
scripts/install.sh pypy
$(PYPY3):
scripts/install.sh pypy3
develop:
@${PYTHON} scripts/travis.py fold_start install "Installing gevent"
@echo python is at `which $(PYTHON)`
# First install a newer pip so that it can use the wheel cache
# (only needed until travis upgrades pip to 7.x; note that the 3.5
# environment uses pip 7.1 by default)
${PYTHON} -m pip install -U pip setuptools
# Then start installing our deps so they can be cached. Note that use of --build-options / --global-options / --install-options
# disables the cache.
# We need wheel>=0.26 on Python 3.5. See previous revisions.
GEVENTSETUP_EV_VERIFY=3 time ${PYTHON} -m pip install -U --upgrade-strategy=eager -r dev-requirements.txt
${PYTHON} -m pip freeze
ccache -s
@${PYTHON} scripts/travis.py fold_end install
test-py27: $(PY27)
PYTHON=python2.7.16 PATH=$(BUILD_RUNTIMES)/versions/python2.7.16/bin:$(PATH) make develop leaktest cffibackendtest coverage_combine
test-py35: $(PY35)
PYTHON=python3.5.6 PATH=$(BUILD_RUNTIMES)/versions/python3.5.6/bin:$(PATH) make develop basictest
test-py36: $(PY36)
PYTHON=python3.6.8 PATH=$(BUILD_RUNTIMES)/versions/python3.6.8/bin:$(PATH) make develop lint basictest
test-py37: $(PY37)
PYTHON=python3.7.2 PATH=$(BUILD_RUNTIMES)/versions/python3.7.2/bin:$(PATH) make develop leaktest cffibackendtest coverage_combine
test-pypy: $(PYPY)
PYTHON=$(PYPY) PATH=$(BUILD_RUNTIMES)/versions/pypy710/bin:$(PATH) make develop cffibackendtest
test-pypy3: $(PYPY3)
PYTHON=$(PYPY3) PATH=$(BUILD_RUNTIMES)/versions/pypy3.6_710/bin:$(PATH) make develop basictest
test-py27-noembed: $(PY27)
@python2.7.16 scripts/travis.py fold_start conf_libev "Configuring libev"
cd deps/libev && ./configure --disable-dependency-tracking && make
@python2.7.16 scripts/travis.py fold_end conf_libev
@python2.7.16 scripts/travis.py fold_start conf_cares "Configuring cares"
cd deps/c-ares && ./configure --disable-dependency-tracking && make
@python2.7.16 scripts/travis.py fold_end conf_cares
@python2.7.16 scripts/travis.py fold_start conf_libuv "Configuring libuv"
cd deps/libuv && ./autogen.sh && ./configure --disable-static && make
@python2.7.16 scripts/travis.py fold_end conf_libuv
CPPFLAGS="-Ideps/libev -Ideps/c-ares -Ideps/libuv/include" LDFLAGS="-Ldeps/libev/.libs -Ldeps/c-ares/.libs -Ldeps/libuv/.libs" LD_LIBRARY_PATH="$(PWD)/deps/libev/.libs:$(PWD)/deps/c-ares/.libs:$(PWD)/deps/libuv/.libs" EMBED=0 PYTHON=python2.7.16 PATH=$(BUILD_RUNTIMES)/versions/python2.7.16/bin:$(PATH) make develop alltest cffibackendtest
...@@ -221,15 +221,11 @@ class ConfiguringBuildExt(build_ext): ...@@ -221,15 +221,11 @@ class ConfiguringBuildExt(build_ext):
def build_extension(self, ext): def build_extension(self, ext):
self.gevent_prepare(ext) self.gevent_prepare(ext)
try: try:
result = build_ext.build_extension(self, ext) return build_ext.build_extension(self, ext)
except ext_errors: except ext_errors:
if getattr(ext, 'optional', False): if getattr(ext, 'optional', False):
raise BuildFailed() raise BuildFailed()
else:
raise raise
return result
class Extension(_Extension): class Extension(_Extension):
...@@ -242,3 +238,137 @@ class Extension(_Extension): ...@@ -242,3 +238,137 @@ class Extension(_Extension):
# Python 2 has this as an old-style class for some reason # Python 2 has this as an old-style class for some reason
# so super() doesn't work. # so super() doesn't work.
_Extension.__init__(self, *args, **kwargs) # pylint:disable=no-member,non-parent-init-called _Extension.__init__(self, *args, **kwargs) # pylint:disable=no-member,non-parent-init-called
from distutils.command.clean import clean # pylint:disable=no-name-in-module,import-error
from distutils import log # pylint:disable=no-name-in-module
from distutils.dir_util import remove_tree # pylint:disable=no-name-in-module,import-error
class GeventClean(clean):
BASE_GEVENT_SRC = os.path.join('src', 'gevent')
def __find_directories_in(self, top, named=None):
"""
Iterate directories, beneath and including *top* ignoring '.'
entries.
"""
for dirpath, dirnames, _ in os.walk(top):
# Modify dirnames in place to prevent walk from
# recursing into hidden directories.
dirnames[:] = [x for x in dirnames if not x.startswith('.')]
for dirname in dirnames:
if named is None or named == dirname:
yield os.path.join(dirpath, dirname)
def __glob_under(self, base, file_pat):
return glob_many(
os.path.join(base, file_pat),
*(os.path.join(x, file_pat)
for x in
self.__find_directories_in(base)))
def __remove_dirs(self, remove_file):
dirs_to_remove = [
'htmlcov',
'.eggs',
]
if self.all:
dirs_to_remove += [
# tox
'.tox',
# instal.sh for pyenv
'.runtimes',
# Built wheels from manylinux
'wheelhouse',
# Doc build
os.path.join('.', 'doc', '_build'),
]
dir_finders = [
# All python cache dirs
(self.__find_directories_in, '.', '__pycache__'),
]
for finder in dir_finders:
func = finder[0]
args = finder[1:]
dirs_to_remove.extend(func(*args))
for f in sorted(dirs_to_remove):
remove_file(f)
def run(self):
clean.run(self)
if self.dry_run:
def remove_file(f):
if os.path.isdir(f):
remove_tree(f, dry_run=self.dry_run)
elif os.path.exists(f):
log.info("Would remove '%s'", f)
else:
def remove_file(f):
if os.path.isdir(f):
remove_tree(f, dry_run=self.dry_run)
elif os.path.exists(f):
log.info("Removing '%s'", f)
os.remove(f)
# Remove directories first before searching for individual files
self.__remove_dirs(remove_file)
def glob_gevent(file_path):
return glob(os.path.join(self.BASE_GEVENT_SRC, file_path))
def glob_gevent_and_under(file_pat):
return self.__glob_under(self.BASE_GEVENT_SRC, file_pat)
def glob_root_and_under(file_pat):
return self.__glob_under('.', file_pat)
files_to_remove = [
'.coverage',
# One-off cython-generated code that doesn't
# follow a globbale-pattern
os.path.join(self.BASE_GEVENT_SRC, 'libev', 'corecext.c'),
os.path.join(self.BASE_GEVENT_SRC, 'libev', 'corecext.h'),
os.path.join(self.BASE_GEVENT_SRC, 'resolver', 'cares.c'),
os.path.join(self.BASE_GEVENT_SRC, 'resolver', 'cares.c'),
]
def dep_configure_artifacts(dep):
for f in (
'config.h',
'config.log',
'config.status',
'.libs'
):
yield os.path.join('deps', dep, f)
file_finders = [
# The base gevent directory contains
# only generated .c code. Remove it.
(glob_gevent, "*.c"),
# Any .html files found in the gevent directory
# are the result of Cython annotations. Remove them.
(glob_gevent_and_under, "*.html"),
# Any compiled binaries have to go
(glob_gevent_and_under, "*.so"),
(glob_gevent_and_under, "*.pyd"),
(glob_root_and_under, "*.o"),
# Compiled python files too
(glob_gevent_and_under, "*.pyc"),
(glob_gevent_and_under, "*.pyo"),
# Artifacts of building dependencies in place
(dep_configure_artifacts, 'libev'),
(dep_configure_artifacts, 'libuv'),
(dep_configure_artifacts, 'c-ares'),
]
for func, pat in file_finders:
files_to_remove.extend(func(pat))
for f in sorted(files_to_remove):
remove_file(f)
...@@ -2,10 +2,6 @@ ...@@ -2,10 +2,6 @@
Managing Embedded Dependencies Managing Embedded Dependencies
================================ ================================
- Modify the c-ares Makefile.in[c] to empty out the MANPAGES variables
so that we don't have to ship those in the sdist.
XXX: We need a patch for that.
Updating libev Updating libev
============== ==============
...@@ -30,6 +26,17 @@ Check if 'config.guess' and/or 'config.sub' went backwards in time ...@@ -30,6 +26,17 @@ Check if 'config.guess' and/or 'config.sub' went backwards in time
from the latest source from the latest source
http://git.savannah.gnu.org/gitweb/?p=config.git;a=tree ) http://git.savannah.gnu.org/gitweb/?p=config.git;a=tree )
Updating c-ares
===============
- Modify the c-ares Makefile.in[c] to empty out the MANPAGES variables
so that we don't have to ship those in the sdist.
XXX: We need a patch for that.
- Follow the same 'config.guess' and 'config.sub' steps as libev.
Updating libuv Updating libuv
============== ==============
...@@ -57,3 +64,5 @@ and the build process. Evaluate those and add them to git and to ...@@ -57,3 +64,5 @@ and the build process. Evaluate those and add them to git and to
``src/gevent/libuv/_corecffi_build.py`` as needed. Then check if there ``src/gevent/libuv/_corecffi_build.py`` as needed. Then check if there
are changes to the build system (e.g., the .gyp files) that need to be are changes to the build system (e.g., the .gyp files) that need to be
accounted for in our build file. accounted for in our build file.
- Follow the same 'config.guess' and 'config.sub' steps as libev.
This diff is collapsed.
This diff is collapsed.
...@@ -9,6 +9,9 @@ pylint>=1.8.0 ; python_version < "3.4" ...@@ -9,6 +9,9 @@ pylint>=1.8.0 ; python_version < "3.4"
pylint >= 2.3.1 ; python_version >= "3.4" and platform_python_implementation == "CPython" pylint >= 2.3.1 ; python_version >= "3.4" and platform_python_implementation == "CPython"
astroid >= 2.2.5 ; python_version >= "3.4" and platform_python_implementation == "CPython" astroid >= 2.2.5 ; python_version >= "3.4" and platform_python_implementation == "CPython"
# We need this at runtime to use the libev-CFFI and libuv backends
cffi >= 1.12.2 ; platform_python_implementation == 'CPython'
# benchmarks use this # benchmarks use this
perf >= 1.6.0 perf >= 1.6.0
......
...@@ -10,41 +10,13 @@ ...@@ -10,41 +10,13 @@
set -e set -e
set -x set -x
# This is to guard against multiple builds in parallel. The various installers will tend
# to stomp all over eachother if you do this and they haven't previously successfully
# succeeded. We use a lock file to block progress so only one install runs at a time.
# This script should be pretty fast once files are cached, so the lost of concurrency
# is not a major problem.
# This should be using the lockfile command, but that's not available on the
# containerized travis and we can't install it without sudo.
# Is is unclear if this is actually useful. I was seeing behaviour that suggested
# concurrent runs of the installer, but I can't seem to find any evidence of this lock
# ever not being acquired.
# Where installations go
BASE=${BUILD_RUNTIMES-$PWD/.runtimes} BASE=${BUILD_RUNTIMES-$PWD/.runtimes}
PYENV=$BASE/pyenv
echo $BASE echo $BASE
mkdir -p $BASE mkdir -p $BASE
LOCKFILE="$BASE/.install-lockfile"
while true; do
if mkdir $LOCKFILE 2>/dev/null; then
echo "Successfully acquired installer."
break
else
echo "Failed to acquire lock. Is another installer running? Waiting a bit."
fi
sleep $[ ( $RANDOM % 10) + 1 ].$[ ( $RANDOM % 100) ]s
if (( $(date '+%s') > 300 + $(stat --format=%X $LOCKFILE) )); then
echo "We've waited long enough"
rm -rf $LOCKFILE
fi
done
trap "rm -rf $LOCKFILE" EXIT
PYENV=$BASE/pyenv
if [ ! -d "$PYENV/.git" ]; then if [ ! -d "$PYENV/.git" ]; then
rm -rf $PYENV rm -rf $PYENV
...@@ -52,56 +24,86 @@ if [ ! -d "$PYENV/.git" ]; then ...@@ -52,56 +24,86 @@ if [ ! -d "$PYENV/.git" ]; then
else else
back=$PWD back=$PWD
cd $PYENV cd $PYENV
git fetch || echo "Fetch failed to complete. Ignoring" # We don't fetch or reset after the initial creation;
git reset --hard origin/master # doing so causes the Travis cache to need re-packed and uploaded,
# and it's pretty large.
# So if we need to incorporate changes from pyenv, either temporarily
# turn this back on, or remove the Travis caches.
# git fetch || echo "Fetch failed to complete. Ignoring"
# git reset --hard origin/master
cd $back cd $back
fi fi
SNAKEPIT=$BASE/snakepit SNAKEPIT=$BASE/snakepit
##
# install(exact-version, bin-alias, dir-alias)
#
# Produce a python executable at $SNAKEPIT/bin-alias
# having the exact version given as exact-version.
#
# Also produces a $SNAKEPIT/dir-alias/ pointing to the root
# of the python install.
##
install () { install () {
VERSION="$1" VERSION="$1"
ALIAS="$2" ALIAS="$2"
DIR_ALIAS="$3"
DESTINATION=$BASE/versions/$VERSION
mkdir -p $BASE/versions mkdir -p $BASE/versions
SOURCE=$BASE/versions/$ALIAS mkdir -p $SNAKEPIT
if [ ! -e "$SOURCE" ]; then if [ ! -e "$DESTINATION" ]; then
mkdir -p $SNAKEPIT mkdir -p $SNAKEPIT
mkdir -p $BASE/versions mkdir -p $BASE/versions
$BASE/pyenv/plugins/python-build/bin/python-build $VERSION $SOURCE $BASE/pyenv/plugins/python-build/bin/python-build $VERSION $DESTINATION
fi fi
rm -f $SNAKEPIT/$ALIAS
mkdir -p $SNAKEPIT # Travis CI doesn't take symlink changes (or creation!) into
# account on its caching, So we need to write an actual file if we
# actually changed something. For python version upgrades, this is
# usually handled automatically (obviously) because we installed
# python. But if we make changes *just* to symlink locations above,
# nothing happens. So for every symlink, write a file...with identical contents,
# so that we don't get *spurious* caching. (Travis doesn't check for mod times,
# just contents, so echoing each time doesn't cause it to re-cache.)
# Overwrite an existing alias
ln -sf $DESTINATION/bin/python $SNAKEPIT/$ALIAS
ln -sf $DESTINATION $SNAKEPIT/$DIR_ALIAS
echo $VERSION $ALIAS $DIR_ALIAS > $SNAKEPIT/$ALIAS.installed
$SNAKEPIT/$ALIAS --version
# Set the PATH to include the install's bin directory so pip
# doesn't nag.
PATH="$DESTINATION/bin/:$PATH" $SNAKEPIT/$ALIAS -m pip install --upgrade pip wheel virtualenv
ls -l $SNAKEPIT ls -l $SNAKEPIT
ls -l $BASE/versions
ls -l $SOURCE/
ls -l $SOURCE/bin
ln -s $SOURCE/bin/python $SNAKEPIT/$ALIAS
$SOURCE/bin/python -m pip.__main__ install --upgrade pip wheel virtualenv
} }
for var in "$@"; do for var in "$@"; do
case "${var}" in case "${var}" in
2.7) 2.7)
install 2.7.16 python2.7.16 install 2.7.16 python2.7 2.7.d
;; ;;
3.5) 3.5)
install 3.5.6 python3.5.6 install 3.5.6 python3.5 3.5.d
;; ;;
3.6) 3.6)
install 3.6.8 python3.6.8 install 3.6.8 python3.6 3.6.d
;; ;;
3.7) 3.7)
install 3.7.2 python3.7.2 install 3.7.2 python3.7 3.7.d
;; ;;
pypy) pypy2.7)
install pypy2.7-7.1.0 pypy710 install pypy2.7-7.1.0 pypy2.7 pypy2.7.d
;; ;;
pypy3) pypy3.6)
install pypy3.6-7.1.0 pypy3.6_710 install pypy3.6-7.1.0 pypy3.6 pypy3.6.d
;; ;;
esac esac
done done
...@@ -20,6 +20,7 @@ from _setuputils import PYPY, WIN ...@@ -20,6 +20,7 @@ from _setuputils import PYPY, WIN
from _setuputils import IGNORE_CFFI from _setuputils import IGNORE_CFFI
from _setuputils import SKIP_LIBUV from _setuputils import SKIP_LIBUV
from _setuputils import ConfiguringBuildExt from _setuputils import ConfiguringBuildExt
from _setuputils import GeventClean
from _setuputils import BuildFailed from _setuputils import BuildFailed
from _setuputils import cythonize1 from _setuputils import cythonize1
...@@ -325,6 +326,17 @@ def run_setup(ext_modules, run_make): ...@@ -325,6 +326,17 @@ def run_setup(ext_modules, run_make):
# TODO: Generalize this. # TODO: Generalize this.
if LIBEV_CFFI_MODULE in cffi_modules and not WIN: if LIBEV_CFFI_MODULE in cffi_modules and not WIN:
system(libev_configure_command) system(libev_configure_command)
# This changed to the libev directory, and ran configure .
# It then copied the generated config.h back to the previous
# directory, which happened to be beside us. In the embedded case,
# we're building in a different directory, so it copied it back to build
# directory, but here, we're building in the embedded directory, so
# it gave us useless files.
bad_file = None
for bad_file in ('config.h', 'configure-output.txt'):
if os.path.exists(bad_file):
os.remove(bad_file)
del bad_file
setup( setup(
name='gevent', name='gevent',
...@@ -347,7 +359,10 @@ def run_setup(ext_modules, run_make): ...@@ -347,7 +359,10 @@ def run_setup(ext_modules, run_make):
packages=find_packages('src'), packages=find_packages('src'),
include_package_data=True, include_package_data=True,
ext_modules=ext_modules, ext_modules=ext_modules,
cmdclass=dict(build_ext=ConfiguringBuildExt), cmdclass={
'build_ext': ConfiguringBuildExt,
'clean': GeventClean,
},
install_requires=install_requires, install_requires=install_requires,
setup_requires=setup_requires, setup_requires=setup_requires,
extras_require={ extras_require={
......
...@@ -18,6 +18,7 @@ from .sysinfo import PY2 ...@@ -18,6 +18,7 @@ from .sysinfo import PY2
from .sysinfo import RESOLVER_ARES from .sysinfo import RESOLVER_ARES
from .sysinfo import RUN_LEAKCHECKS from .sysinfo import RUN_LEAKCHECKS
from . import six from . import six
from . import travis
# Import this while we're probably single-threaded/single-processed # Import this while we're probably single-threaded/single-processed
# to try to avoid issues with PyPy 5.10. # to try to avoid issues with PyPy 5.10.
...@@ -63,103 +64,172 @@ def _dir_from_package_name(package): ...@@ -63,103 +64,172 @@ def _dir_from_package_name(package):
return package_dir return package_dir
def run_many(tests, class ResultCollector(object):
def __init__(self):
self.total = 0
self.failed = {}
self.passed = {}
self.total_cases = 0
self.total_skipped = 0
def __iadd__(self, result):
if not result:
self.failed[result.name] = result #[cmd, kwargs]
else:
self.passed[result.name] = True
self.total_cases += result.run_count
self.total_skipped += result.skipped_count
return self
class Runner(object):
TIME_WAIT_REAP = 0.1
TIME_WAIT_SPAWN = 0.05
def __init__(self,
tests,
configured_failing_tests=(), configured_failing_tests=(),
failfast=False, failfast=False,
quiet=False, quiet=False,
configured_run_alone_tests=()): configured_run_alone_tests=()):
# pylint:disable=too-many-locals,too-many-statements self._tests = tests
global NWORKERS self._configured_failing_tests = configured_failing_tests
start = time.time() self._failfast = failfast
total = 0 self._quiet = quiet
failed = {} self._configured_run_alone_tests = configured_run_alone_tests
passed = {}
total_cases = [0]
total_skipped = [0]
NWORKERS = min(len(tests), NWORKERS) or 1 self.results = ResultCollector()
self.results.total = len(self._tests)
self._running_jobs = []
pool = ThreadPool(NWORKERS) self._worker_count = min(len(tests), NWORKERS) or 1
util.BUFFER_OUTPUT = NWORKERS > 1 or quiet
def run_one(cmd, **kwargs): def _run_one(self, cmd, **kwargs):
kwargs['quiet'] = quiet kwargs['quiet'] = self._quiet
result = util.run(cmd, **kwargs) result = util.run(cmd, **kwargs)
if result: if not result and self._failfast:
if failfast:
sys.exit(1) sys.exit(1)
failed[result.name] = [cmd, kwargs] self.results += result
else:
passed[result.name] = True
total_cases[0] += result.run_count
total_skipped[0] += result.skipped_count
results = [] def _reap(self):
"Clean up the list of running jobs, returning how many are still outstanding."
def reap(): for r in self._running_jobs[:]:
for r in results[:]:
if not r.ready(): if not r.ready():
continue continue
if r.successful(): if r.successful():
results.remove(r) self._running_jobs.remove(r)
else: else:
r.get() r.get()
sys.exit('Internal error in testrunner.py: %r' % (r, )) sys.exit('Internal error in testrunner.py: %r' % (r, ))
return len(results) return len(self._running_jobs)
def reap_all(): def _reap_all(self):
while reap() > 0: while self._reap() > 0:
time.sleep(0.1) time.sleep(self.TIME_WAIT_REAP)
def spawn(cmd, options): def _spawn(self, pool, cmd, options):
while True: while True:
if reap() < NWORKERS: if self._reap() < self._worker_count:
r = pool.apply_async(run_one, (cmd, ), options or {}) job = pool.apply_async(self._run_one, (cmd, ), options or {})
results.append(r) self._running_jobs.append(job)
return return
time.sleep(0.05) time.sleep(self.TIME_WAIT_SPAWN)
run_alone = [] def __call__(self):
util.log("Running tests in parallel with concurrency %s" % (self._worker_count,),)
# Setting global state, in theory we can be used multiple times.
# This is fine as long as we are single threaded and call these
# sequentially.
util.BUFFER_OUTPUT = self._worker_count > 1 or self._quiet
start = time.time()
try: try:
self._run_tests()
except KeyboardInterrupt:
self._report(time.time() - start, exit=False)
util.log('(partial results)\n')
raise
except:
traceback.print_exc()
raise
self._reap_all()
self._report(time.time() - start, exit=True)
def _run_tests(self):
"Runs the tests, produces no report."
run_alone = []
tests = self._tests
pool = ThreadPool(self._worker_count)
try: try:
util.log("Running tests in parallel with concurrency %s" % (NWORKERS,),)
for cmd, options in tests: for cmd, options in tests:
total += 1
options = options or {} options = options or {}
if matches(configured_run_alone_tests, cmd): if matches(self._configured_run_alone_tests, cmd):
run_alone.append((cmd, options)) run_alone.append((cmd, options))
else: else:
spawn(cmd, options) self._spawn(pool, cmd, options)
pool.close() pool.close()
pool.join() pool.join()
if run_alone: if run_alone:
util.log("Running tests marked standalone") util.log("Running tests marked standalone")
for cmd, options in run_alone: for cmd, options in run_alone:
run_one(cmd, **options) self._run_one(cmd, **options)
except KeyboardInterrupt: except KeyboardInterrupt:
try: try:
util.log('Waiting for currently running to finish...') util.log('Waiting for currently running to finish...')
reap_all() self._reap_all()
except KeyboardInterrupt: except KeyboardInterrupt:
pool.terminate() pool.terminate()
report(total, failed, passed, exit=False, took=time.time() - start,
configured_failing_tests=configured_failing_tests,
total_cases=total_cases[0], total_skipped=total_skipped[0])
util.log('(partial results)\n')
raise raise
except: except:
traceback.print_exc()
pool.terminate() pool.terminate()
raise raise
reap_all() def _report(self, elapsed_time, exit=False):
report(total, failed, passed, took=time.time() - start, results = self.results
configured_failing_tests=configured_failing_tests, report(
total_cases=total_cases[0], total_skipped=total_skipped[0]) results.total, results.failed, results.passed,
exit=exit,
took=elapsed_time,
configured_failing_tests=self._configured_failing_tests,
total_cases=results.total_cases,
total_skipped=results.total_skipped
)
class TravisFoldingRunner(object):
def __init__(self, runner, travis_fold_msg):
self._runner = runner
self._travis_fold_msg = travis_fold_msg
self._travis_fold_name = str(int(time.time()))
# A zope-style acquisition proxy would be convenient here.
run_tests = runner._run_tests
def _run_tests():
self._begin_fold()
try:
run_tests()
finally:
self._end_fold()
runner._run_tests = _run_tests
def _begin_fold(self):
travis.fold_start(self._travis_fold_name,
self._travis_fold_msg)
def _end_fold(self):
travis.fold_end(self._travis_fold_name)
def __call__(self):
return self._runner()
def discover( def discover(
tests=None, ignore_files=None, tests=None, ignore_files=None,
...@@ -296,7 +366,7 @@ def report(total, failed, passed, exit=True, took=None, ...@@ -296,7 +366,7 @@ def report(total, failed, passed, exit=True, took=None,
configured_failing_tests=(), configured_failing_tests=(),
total_cases=0, total_skipped=0): total_cases=0, total_skipped=0):
# pylint:disable=redefined-builtin,too-many-branches,too-many-locals # pylint:disable=redefined-builtin,too-many-branches,too-many-locals
runtimelog = util.runtimelog runtimelog = util.runtimelog # XXX: Global state!
if runtimelog: if runtimelog:
util.log('\nLongest-running tests:') util.log('\nLongest-running tests:')
runtimelog.sort() runtimelog.sort()
...@@ -422,6 +492,8 @@ def main(): ...@@ -422,6 +492,8 @@ def main():
parser.add_argument("--verbose", action="store_false", dest='quiet') parser.add_argument("--verbose", action="store_false", dest='quiet')
parser.add_argument("--debug", action="store_true", default=False) parser.add_argument("--debug", action="store_true", default=False)
parser.add_argument("--package", default="gevent.tests") parser.add_argument("--package", default="gevent.tests")
parser.add_argument("--travis-fold", metavar="MSG",
help="Emit Travis CI log fold markers around the output.")
parser.add_argument('tests', nargs='*') parser.add_argument('tests', nargs='*')
options = parser.parse_args() options = parser.parse_args()
...@@ -439,6 +511,9 @@ def main(): ...@@ -439,6 +511,9 @@ def main():
coverage = False coverage = False
if options.coverage or os.environ.get("GEVENTTEST_COVERAGE"): if options.coverage or os.environ.get("GEVENTTEST_COVERAGE"):
if PYPY and RUNNING_ON_CI:
print("Ignoring coverage option on PyPy on CI; slow")
else:
coverage = True coverage = True
os.environ['COVERAGE_PROCESS_START'] = os.path.abspath(".coveragerc") os.environ['COVERAGE_PROCESS_START'] = os.path.abspath(".coveragerc")
if PYPY: if PYPY:
...@@ -490,7 +565,7 @@ def main(): ...@@ -490,7 +565,7 @@ def main():
# Put this directory on the path so relative imports work. # Put this directory on the path so relative imports work.
package_dir = _dir_from_package_name(options.package) package_dir = _dir_from_package_name(options.package)
os.environ['PYTHONPATH'] = os.environ.get('PYTHONPATH', "") + os.pathsep + package_dir os.environ['PYTHONPATH'] = os.environ.get('PYTHONPATH', "") + os.pathsep + package_dir
run_many( runner = Runner(
tests, tests,
configured_failing_tests=FAILING_TESTS, configured_failing_tests=FAILING_TESTS,
failfast=options.failfast, failfast=options.failfast,
...@@ -498,6 +573,11 @@ def main(): ...@@ -498,6 +573,11 @@ def main():
configured_run_alone_tests=RUN_ALONE, configured_run_alone_tests=RUN_ALONE,
) )
if options.travis_fold:
runner = TravisFoldingRunner(runner, options.travis_fold)
runner()
if __name__ == '__main__': if __name__ == '__main__':
main() main()
...@@ -8,15 +8,15 @@ from __future__ import print_function ...@@ -8,15 +8,15 @@ from __future__ import print_function
import sys import sys
commands = {} commands = {}
def command(func): def command(func):
commands[func.__name__] = func commands[func.__name__] = lambda: func(*sys.argv[2:])
return func
@command @command
def fold_start(): def fold_start(name, msg):
name = sys.argv[2]
msg = sys.argv[3]
sys.stdout.write('travis_fold:start:') sys.stdout.write('travis_fold:start:')
sys.stdout.write(name) sys.stdout.write(name)
sys.stdout.write(chr(0o33)) sys.stdout.write(chr(0o33))
...@@ -26,8 +26,7 @@ def fold_start(): ...@@ -26,8 +26,7 @@ def fold_start():
sys.stdout.write('[33;0m\n') sys.stdout.write('[33;0m\n')
@command @command
def fold_end(): def fold_end(name):
name = sys.argv[2]
sys.stdout.write("\ntravis_fold:end:") sys.stdout.write("\ntravis_fold:end:")
sys.stdout.write(name) sys.stdout.write(name)
sys.stdout.write("\r\n") sys.stdout.write("\r\n")
......
...@@ -257,10 +257,24 @@ def start(command, quiet=False, **kwargs): ...@@ -257,10 +257,24 @@ def start(command, quiet=False, **kwargs):
class RunResult(object): class RunResult(object):
"""
The results of running an external command.
If the command was successful, this has a boolean
value of True; otherwise, a boolean value of false.
The integer value of this object is the command's exit code.
"""
def __init__(self, code, def __init__(self,
command,
run_kwargs,
code,
output=None, name=None, output=None, name=None,
run_count=0, skipped_count=0): run_count=0, skipped_count=0):
self.command = command
self.run_kwargs = run_kwargs
self.code = code self.code = code
self.output = output self.output = output
self.name = name self.name = name
...@@ -269,7 +283,7 @@ class RunResult(object): ...@@ -269,7 +283,7 @@ class RunResult(object):
def __bool__(self): def __bool__(self):
return bool(self.code) return not bool(self.code)
__nonzero__ = __bool__ __nonzero__ = __bool__
...@@ -322,6 +336,7 @@ def _find_test_status(took, out): ...@@ -322,6 +336,7 @@ def _find_test_status(took, out):
def run(command, **kwargs): # pylint:disable=too-many-locals def run(command, **kwargs): # pylint:disable=too-many-locals
"Execute *command*, returning a `RunResult`"
buffer_output = kwargs.pop('buffer_output', BUFFER_OUTPUT) buffer_output = kwargs.pop('buffer_output', BUFFER_OUTPUT)
quiet = kwargs.pop('quiet', QUIET) quiet = kwargs.pop('quiet', QUIET)
verbose = not quiet verbose = not quiet
...@@ -361,7 +376,8 @@ def run(command, **kwargs): # pylint:disable=too-many-locals ...@@ -361,7 +376,8 @@ def run(command, **kwargs): # pylint:disable=too-many-locals
log('- %s %s', name, status) log('- %s %s', name, status)
if took >= MIN_RUNTIME: if took >= MIN_RUNTIME:
runtimelog.append((-took, name)) runtimelog.append((-took, name))
return RunResult(result, out, name, run_count, skipped_count) return RunResult(command, kwargs, result,
output=out, name=name, run_count=run_count, skipped_count=skipped_count)
class NoSetupPyFound(Exception): class NoSetupPyFound(Exception):
......
...@@ -21,10 +21,6 @@ from gevent.testing.sysinfo import LIBUV ...@@ -21,10 +21,6 @@ from gevent.testing.sysinfo import LIBUV
IGNORED_TESTS = [] IGNORED_TESTS = []
FAILING_TESTS = [ FAILING_TESTS = [
# Sometimes fails with AssertionError: ...\nIOError: close() called during concurrent operation on the same file object.\n'
# Sometimes it contains "\nUnhandled exception in thread started by \nsys.excepthook is missing\nlost sys.stderr\n"
"FLAKY test__subprocess_interrupted.py",
# test__issue6 (see comments in test file) is really flaky on both Travis and Appveyor; # test__issue6 (see comments in test file) is really flaky on both Travis and Appveyor;
# on Travis we could just run the test again (but that gets old fast), but on appveyor # on Travis we could just run the test again (but that gets old fast), but on appveyor
# we don't have that option without a new commit---and sometimes we really need a build # we don't have that option without a new commit---and sometimes we really need a build
...@@ -233,6 +229,7 @@ if PYPY: ...@@ -233,6 +229,7 @@ if PYPY:
'FLAKY test__backdoor.py', 'FLAKY test__backdoor.py',
] ]
if RESOLVER_NOT_SYSTEM: if RESOLVER_NOT_SYSTEM:
FAILING_TESTS += [ FAILING_TESTS += [
...@@ -243,7 +240,10 @@ if PYPY: ...@@ -243,7 +240,10 @@ if PYPY:
# AssertionError: Lists differ: # AssertionError: Lists differ:
# (10, 1, 6, '', ('2607:f8b0:4004:810::200e', 80, 0L, 0L)) # (10, 1, 6, '', ('2607:f8b0:4004:810::200e', 80, 0L, 0L))
# (10, 1, 6, '', ('2607:f8b0:4004:805::200e', 80, 0, 0)) # (10, 1, 6, '', ('2607:f8b0:4004:805::200e', 80, 0, 0))
'test__socket_dns.py', #
# Somehow it seems most of these are fixed with PyPy3.6-7 under dnspython,
# (once we commented out TestHostname)?
'FLAKY test__socket_dns.py',
] ]
if LIBUV: if LIBUV:
...@@ -260,10 +260,14 @@ if PYPY: ...@@ -260,10 +260,14 @@ if PYPY:
# This fails to get the correct results, sometimes. I can't reproduce locally # This fails to get the correct results, sometimes. I can't reproduce locally
'FLAKY test__example_udp_server.py', 'FLAKY test__example_udp_server.py',
'FLAKY test__example_udp_client.py', 'FLAKY test__example_udp_client.py',
]
IGNORED_TESTS += [
# PyPy 7.0 and 7.1 on Travis with Ubunto Xenial 16.04 # PyPy 7.0 and 7.1 on Travis with Ubunto Xenial 16.04
# can't allocate SSL Context objects, either in Python 2.7 # can't allocate SSL Context objects, either in Python 2.7
# or 3.6. There must be some library incompatibility. # or 3.6. There must be some library incompatibility.
# No point even running them.
# XXX: Remember to turn this back on.
'test_ssl.py', 'test_ssl.py',
] ]
......
...@@ -106,7 +106,7 @@ def TESTRUNNER(tests=None): ...@@ -106,7 +106,7 @@ def TESTRUNNER(tests=None):
def main(): def main():
from gevent.testing import testrunner from gevent.testing import testrunner
return testrunner.run_many(list(TESTRUNNER(sys.argv[1:]))) return testrunner.Runner(list(TESTRUNNER(sys.argv[1:])))()
if __name__ == '__main__': if __name__ == '__main__':
......
...@@ -48,7 +48,7 @@ class _AbstractTestMixin(util.ExampleMixin): ...@@ -48,7 +48,7 @@ class _AbstractTestMixin(util.ExampleMixin):
def test_runs(self): def test_runs(self):
start = time.time() start = time.time()
min_time, max_time = self.time_range min_time, max_time = self.time_range
if util.run([sys.executable, '-u', self.filename], if not util.run([sys.executable, '-u', self.filename],
timeout=max_time, timeout=max_time,
cwd=self.cwd, cwd=self.cwd,
quiet=True, quiet=True,
......
...@@ -380,7 +380,12 @@ add(TestTypeError, 25) ...@@ -380,7 +380,12 @@ add(TestTypeError, 25)
class TestHostname(TestCase): class TestHostname(TestCase):
pass pass
add(TestHostname, socket.gethostname) add(
TestHostname,
socket.gethostname,
skip=greentest.RUNNING_ON_TRAVIS and greentest.RESOLVER_DNSPYTHON,
skip_reason="Sometimes get a different result for getaddrinfo",
)
class TestLocalhost(TestCase): class TestLocalhost(TestCase):
......
...@@ -14,7 +14,21 @@ else: ...@@ -14,7 +14,21 @@ else:
out, err = subprocess.Popen([sys.executable, '-W', 'ignore', out, err = subprocess.Popen([sys.executable, '-W', 'ignore',
__file__, 'runtestcase'], __file__, 'runtestcase'],
stderr=subprocess.PIPE).communicate() stderr=subprocess.PIPE).communicate()
if b'refs' in err: # Something to do with debug mode python builds? # We've seen three. unexpected forms out output.
#
# The first involves 'refs'; I don't remember what that was
# about, but I think it had to do with debug builds of Python.
#
# The second is the classic "Unhandled exception in thread
# started by \nsys.excepthook is missing\nlost sys.stderr".
# This is a race condition between closing sys.stderr and
# writing buffered data to a pipe that hasn't been read. We
# only see this using GEVENT_FILE=thread (which makes sense).
#
# The third is similar to the second: "AssertionError:
# ...\nIOError: close() called during concurrent operation on
# the same file object.\n"
if b'refs' in err or b'sys.excepthook' in err or b'concurrent' in err:
assert err.startswith(b'bye'), repr(err) # pragma: no cover assert err.startswith(b'bye'), repr(err) # pragma: no cover
else: else:
assert err.strip() == b'bye', repr(err) assert err.strip() == b'bye', repr(err)
...@@ -14,13 +14,6 @@ whitelist_externals = ...@@ -14,13 +14,6 @@ whitelist_externals =
commands = commands =
python -m gevent.tests python -m gevent.tests
[testenv:py27-full]
basepython = python2.7
commands =
make allbackendtest
[testenv:lint] [testenv:lint]
skip_install = true skip_install = true
skipsdist = true skipsdist = true
...@@ -49,4 +42,4 @@ setenv = ...@@ -49,4 +42,4 @@ setenv =
basepython = basepython =
python2.7 python2.7
commands = commands =
make leaktest GEVENTTEST_LEAKCHECK=1 python -mgevent.tests --config known_failures.py --quiet --ignore tests_that_dont_do_leakchecks.txt
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment