Handle corecext and corecffi backends in the same way.

This allows building, using and testing corecffi on CPython, which can
make debugging easier. It also simplifies the build steps for PyPy.

Start testing corecffi on CPython too for comparison with PyPy and speed
comparisons with CPython. Currently getting a few unexpected failures
that need investigation; could be bugs in corecffi.

.gitignore

@@ -4,7 +4,7 @@ build/
 *.pyd
 .runtimes
 gevent.*.[ch]
-gevent/core.pyx
+gevent/corecext.pyx
 gevent/__pycache__
 gevent/libev
 gevent/_corecffi.c
@@ -37,9 +37,14 @@ c-ares/config.status
 c-ares/stamp-h1
 c-ares/stamp-h2
 c-ares/ares_build.h.orig
+c-ares/ares_config.h
 libev/.deps
 libev/Makefile
 libev/config.log
+libev/config.h
 libev/config.status
 libev/libtool
 libev/stamp-h1
+# running setup.py on PyPy
+config.h
+configure-output.txt

.travis.yml

@@ -16,6 +16,7 @@ env:
       - TASK=test-py27
       - TASK=test-py33
       - TASK=test-py34
+      - TASK=test-py27-cffi
 
 
 matrix:

MANIFEST.in

@@ -26,7 +26,7 @@ recursive-include scripts *.sh
 
 ### Artifacts of configuring/building in place
 # These we want, they come from the Makefile step
-#- recursive-exclude gevent core.pyx *.c *.h
+#- recursive-exclude gevent corecext.pyx *.c *.h
 # This we want if we're on PyPy it's moved there ahead of time
 # by setup.py
 #- prune gevent/libev

Makefile

@@ -14,8 +14,8 @@ export LC_ALL=C.UTF-8
 
 all: gevent/gevent.corecext.c gevent/gevent.ares.c gevent/gevent._semaphore.c gevent/gevent._util.c
 
-gevent/gevent.corecext.c: gevent/core.ppyx gevent/libev.pxd
-	$(PYTHON) util/cythonpp.py -o gevent.corecext.c gevent/core.ppyx
+gevent/gevent.corecext.c: gevent/corecext.ppyx gevent/libev.pxd
+	$(PYTHON) util/cythonpp.py -o gevent.corecext.c gevent/corecext.ppyx
 	echo '#include "callbacks.c"' >> gevent.corecext.c
 	mv gevent.corecext.* gevent/
 
@@ -38,7 +38,7 @@ gevent/gevent._util.c: gevent/_util.pyx
 	mv gevent._util.* gevent/
 
 clean:
-	rm -f gevent.core.c gevent.core.h core.pyx gevent/gevent.core.c gevent/gevent.core.h gevent/core.pyx
+	rm -f corecext.pyx gevent/corecext.pyx
 	rm -f gevent.corecext.c gevent.corecext.h gevent/gevent.corecext.c gevent/gevent.corecext.h
 	rm -f gevent.ares.c gevent.ares.h gevent/gevent.ares.c gevent/gevent.ares.h
 	rm -f gevent._semaphore.c gevent._semaphore.h gevent/gevent._semaphore.c gevent/gevent._semaphore.h
@@ -48,7 +48,7 @@ doc:
 	cd doc && PYTHONPATH=.. make html
 
 whitespace:
-	! find . -not -path "./.eggs/*" -not -path "./greentest/htmlcov/*" -not -path "./greentest/.coverage.*" -not -path "./.tox/*" -not -path "*/__pycache__/*" -not -path "*.so" -not -path "*.pyc" -not -path "./.git/*" -not -path "./build/*" -not -path "./libev/*" -not -path "./gevent/libev/*" -not -path "./gevent.egg-info/*" -not -path "./dist/*" -not -path "./.DS_Store" -not -path "./c-ares/*" -not -path "./gevent/gevent.*.[ch]" -not -path "./gevent/core.pyx" -not -path "./doc/_build/*" -not -path "./doc/mytheme/static/*" -type f | xargs egrep -l " $$"
+	! find . -not -path "./.eggs/*" -not -path "./greentest/htmlcov/*" -not -path "./greentest/.coverage.*" -not -path "./.tox/*" -not -path "*/__pycache__/*" -not -path "*.so" -not -path "*.pyc" -not -path "./.git/*" -not -path "./build/*" -not -path "./libev/*" -not -path "./gevent/libev/*" -not -path "./gevent.egg-info/*" -not -path "./dist/*" -not -path "./.DS_Store" -not -path "./c-ares/*" -not -path "./gevent/gevent.*.[ch]" -not -path "./gevent/corecext.pyx" -not -path "./doc/_build/*" -not -path "./doc/mytheme/static/*" -type f | xargs egrep -l " $$"
 
 pep8:
 	${PYTHON} `which pep8` .
@@ -58,29 +58,17 @@ pyflakes:
 
 lint: whitespace pyflakes pep8
 
-travistest:
+test_prelim:
 	which ${PYTHON}
 	${PYTHON} --version
-
 	${PYTHON} -c 'import greenlet; print(greenlet, greenlet.__version__)'
-
-# develop, not install, so that coverage doesn't think it's part of the stdlib
-	${PYTHON} setup.py develop
+	${PYTHON} -c 'import gevent.core; print(gevent.core.loop)'
 	make bench
 
-	cd greentest && GEVENT_RESOLVER=thread ${PYTHON} testrunner.py --config ../known_failures.py
-	cd greentest && GEVENT_RESOLVER=ares GEVENTARES_SERVERS=8.8.8.8 ${PYTHON} testrunner.py --config ../known_failures.py --ignore tests_that_dont_use_resolver.txt
-	cd greentest && GEVENT_FILE=thread ${PYTHON} testrunner.py --config ../known_failures.py `grep -l subprocess test_*.py`
-# because we set parallel=true, each run produces new and different coverage files; they all need
-# to be combined
-	coverage combine . greentest/
-
-toxtest:
+toxtest: test_prelim
 	cd greentest && GEVENT_RESOLVER=thread python testrunner.py --config ../known_failures.py
 
-fulltoxtest:
-	which ${PYTHON}
-	${PYTHON} --version
+fulltoxtest: test_prelim
 	cd greentest && GEVENT_RESOLVER=thread ${PYTHON} testrunner.py --config ../known_failures.py
 	cd greentest && GEVENT_RESOLVER=ares GEVENTARES_SERVERS=8.8.8.8 ${PYTHON} testrunner.py --config ../known_failures.py --ignore tests_that_dont_use_resolver.txt
 	cd greentest && GEVENT_FILE=thread ${PYTHON} testrunner.py --config ../known_failures.py `grep -l subprocess test_*.py`
@@ -91,21 +79,14 @@ leaktest:
 bench:
 	${PYTHON} greentest/bench_sendall.py
 
-travis_pypy:
-	which ${PYTHON}
-	${PYTHON} --version
-	${PYTHON} setup.py install
-	make bench
-	cd greentest && ${PYTHON} testrunner.py --config ../known_failures.py
-
-travis_cpython:
-	pip install cython greenlet
-
-	make travistest
 
 travis_test_linters:
 	make lint
 	GEVENTTEST_COVERAGE=1 make leaktest
+# because we set parallel=true, each run produces new and different coverage files; they all need
+# to be combined
+	coverage combine . greentest/
+
 	coveralls --rcfile=greentest/.coveragerc
 
 
@@ -162,7 +143,7 @@ develop:
 # disables the cache.
 # We need wheel>=0.26 on Python 3.5. See previous revisions.
 	${PIP} install -U wheel
-	${PIP} install -U tox cython greenlet pep8 pyflakes "coverage>=4.0" "coveralls>=1.0"
+	${PIP} install -U tox cython cffi greenlet pep8 pyflakes "coverage>=4.0" "coveralls>=1.0"
 	${PYTHON} setup.py develop
 
 lint-py27: $(PY27)
@@ -185,3 +166,6 @@ test-py35: $(PY35)
 
 test-pypy: $(PYPY)
 	PYTHON=pypy PATH=$(BUILD_RUNTIMES)/versions/pypy/bin:$(PATH) make develop toxtest
+
+test-py27-cffi: $(PY27)
+	GEVENT_CORE_CFFI_ONLY=1 PYTHON=python2.7 PATH=$(BUILD_RUNTIMES)/versions/python2.7/bin:$(PATH) make develop toxtest

appveyor/make.cmd

 IF "%PYTHON_EXE%" == "python" (
-    %PYEXE% util/cythonpp.py -o gevent.corecext.c gevent/core.ppyx
+    %PYEXE% util/cythonpp.py -o gevent.corecext.c gevent/corecext.ppyx
 	type gevent\\callbacks.c >> gevent.corecext.c
     move gevent.corecext.* gevent
 )

gevent/_corecffi_build.py

@@ -308,8 +308,11 @@ for _watcher_type in _watcher_types:
     }
     """ % (_watcher_type, _watcher_type, _watcher_type)
 
-thisdir = os.path.dirname(os.path.realpath(__file__))
-include_dirs = [thisdir, os.path.join(thisdir, 'libev')]
+thisdir = os.path.dirname(os.path.abspath(__file__))
+include_dirs = [
+    thisdir, # libev_vfd.h
+    os.path.abspath(os.path.join(thisdir, '..', 'libev')),
+]
 ffi.cdef(_cdef)
 ffi.set_source('gevent._corecffi', _source, include_dirs=include_dirs)
 

gevent/callbacks.c

@@ -19,7 +19,7 @@ static void gevent_handle_error(struct PyGeventLoopObject* loop, PyObject* conte
 
     PyErr_Clear();
 
-    result = ((struct __pyx_vtabstruct_6gevent_4core_loop *)loop->__pyx_vtab)->handle_error(loop, context, type, value, traceback, 0);
+    result = ((struct __pyx_vtabstruct_6gevent_8corecext_loop *)loop->__pyx_vtab)->handle_error(loop, context, type, value, traceback, 0);
 
     if (result) {
         Py_DECREF(result);
@@ -195,7 +195,7 @@ static void gevent_run_callbacks(struct ev_loop *_loop, void *watcher, int reven
     loop = GET_OBJECT(PyGeventLoopObject, watcher, _prepare);
     Py_INCREF(loop);
     gevent_check_signals(loop);
-    result = ((struct __pyx_vtabstruct_6gevent_4core_loop *)loop->__pyx_vtab)->_run_callbacks(loop);
+    result = ((struct __pyx_vtabstruct_6gevent_8corecext_loop *)loop->__pyx_vtab)->_run_callbacks(loop);
     if (result) {
         Py_DECREF(result);
     }

gevent/core.py

-from gevent.hub import PYPY
+# Copyright (c) 2009-2015 Denis Bilenko and gevent contributors. See LICENSE for details.
+from __future__ import absolute_import
 
-if PYPY:
-    from gevent import corecffi as _core
-else:
-    # NOTE: On CPython, this file is never imported (and there is no
-    # corecext module). Instead, the core.so file that should be build
-    # is imported in preference.
-    # NOTE: CFFI is now usable on CPython, and the performance is
-    # mostly comparable, so this could be refactored to allow that
-    # (along with the makefile, etc)
-    from gevent import corecext as _core
+import os
+
+try:
+    if os.environ.get('GEVENT_CORE_CFFI_ONLY'):
+        raise ImportError("Not attempting corecext")
 
+    from gevent import corecext as _core
+except ImportError:
+    # CFFI/PyPy
+    from gevent import corecffi as _core
 
 for item in dir(_core):
     if item.startswith('__'):

gevent/corecffi.py

@@ -17,6 +17,7 @@ __all__ = ['get_version',
 try:
     import gevent._corecffi
 except ImportError:
+    traceback.print_exc()
     # Not built yet
     import gevent._corecffi_build
     gevent._corecffi_build.ffi.compile()

setup.py

@@ -12,8 +12,9 @@ from glob import glob
 
 PYPY = hasattr(sys, 'pypy_version_info')
 WIN = sys.platform.startswith('win')
+CFFI_WIN_BUILD_ANYWAY = os.environ.get("PYPY_WIN_BUILD_ANYWAY")
 
-if PYPY and WIN and not os.environ.get("PYPY_WIN_BUILD_ANYWAY"):
+if PYPY and WIN and not CFFI_WIN_BUILD_ANYWAY:
     # We can't properly handle (hah!) file-descriptors and
     # handle mapping on Windows/CFFI, because the file needed,
     # libev_vfd.h, can't be included, linked, and used: it uses
@@ -85,10 +86,18 @@ CARES_EMBED = get_config_value('CARES_EMBED', 'EMBED', 'c-ares')
 
 define_macros = []
 libraries = []
-libev_configure_command = ' '.join(["(cd ", _quoted_abspath('libev/'),
+# Configure libev in place; but cp the config.h to the old directory;
+# if we're building a CPython extension, the old directory will be
+# the build/temp.XXX/libev/ directory. If we're building from a
+# source checkout on pypy, OLDPWD will be the location of setup.py
+# and the PyPy branch will clean it up.
+libev_configure_command = ' '.join([
+    "(cd ", _quoted_abspath('libev/'),
     " && /bin/sh ./configure ",
-                                    " && mv config.h \"$OLDPWD\")",
-                                    '> configure-output.txt'])
+    " && cp config.h \"$OLDPWD\"",
+    ")",
+    '> configure-output.txt'
+])
 
 # See #616, trouble building for a 32-bit python against a 64-bit platform
 _config_vars = distutils.sysconfig.get_config_var("CFLAGS")
@@ -118,7 +127,7 @@ def expand(*lst):
     return result
 
 
-CORE = Extension(name='gevent.core',
+CORE = Extension(name='gevent.corecext',
                  sources=['gevent/gevent.corecext.c'],
                  include_dirs=['libev'] if LIBEV_EMBED else [],
                  libraries=libraries,
@@ -341,14 +350,29 @@ def read(name, *args):
     except OSError:
         return ''
 
+cffi_modules = ['gevent/_corecffi_build.py:ffi']
+
 if PYPY:
     install_requires = []
-    cffi_modules = ['gevent/_corecffi_build.py:ffi']
-    setup_kwds = {'cffi_modules': cffi_modules}
 else:
     install_requires = ['greenlet >= 0.4.9']
     setup_kwds = {}
 
+try:
+    __import__('cffi')
+except NameError:
+    setup_kwds = {}
+else:
+    _kwds = {'cffi_modules': cffi_modules}
+    # We already checked for PyPy on Windows above and excluded it
+    if PYPY:
+        setup_kwds = _kwds
+    elif LIBEV_EMBED and (not WIN or CFFI_WIN_BUILD_ANYWAY):
+        # If we're on CPython, we can only reliably build
+        # the CFFI module if we're embedding libev (in some cases
+        # we wind up embedding it anyway, which may not be what the
+        # distributor wanted).
+        setup_kwds = _kwds
 
 # If we are running info / help commands, or we're being imported by
 # tools like pyroma, we don't need to build anything
@@ -364,18 +388,12 @@ if ((len(sys.argv) >= 2
     include_package_data = PYPY
     run_make = False
 elif PYPY:
-    sys.path.insert(0, '.')
-    # XXX ugly - need to find a better way
-    cp_cmd = 'cp -r'
-    if WIN:
-        cp_cmd = "copy"
-    system(cp_cmd + ' libev gevent/libev')
-    if WIN:
-        system('echo > gevent/libev/__init__.py')
-    else:
-        system('touch gevent/libev/__init__.py')
-    if sys.platform != 'win32':
-        system('cd gevent/libev && ./configure > configure_output.txt')
+    if not WIN:
+        # We need to configure libev because the CORE Extension
+        # won't do it (since we're not building it)
+        system(libev_configure_command)
+        # Then get rid of the extra copy created in place
+        system('rm config.h')
     # NOTE that we're NOT adding the distutils extension module, as
     # doing so compiles the module already: import gevent._corecffi_build
     # imports gevent, which imports the hub, which imports the core,

tox.ini

 [tox]
 envlist =
-    py26,py27,pypy,py33,py34,py35,lint
+    py26,py27,py27-cffi,pypy,py33,py34,py35,lint
 
 [testenv]
 deps =
@@ -31,13 +31,16 @@ deps =
 commands =
     make lint
 
-[testenv:travis-lint]
+[testenv:py27-cffi]
 basepython =
     python2.7
 deps =
-    {[testenv:lint]deps}
+    {[testenv]deps}
+    cffi
+setenv =
+	GEVENT_CORE_CFFI_ONLY=1
 commands =
-    make travis_test_linters
+    make toxtest
 
 [testenv:leak]
 basepython =