Simplify the logic for CFFI and PyPy.

sdists built on PyPy should be the same as those built elsewhere.

Also fix the module name of gevent.libev._corecffi.

Makefile

@@ -44,6 +44,12 @@ clean:
 	rm -rf src/gevent/__pycache__ src/greentest/__pycache__ src/gevent/libev/__pycache__
 	rm -rf src/gevent/*.pyc src/greentest/*.pyc src/gevent/libev/*.pyc
 	rm -rf src/greentest/htmlcov src/greentest/.coverage
+	rm -rf build
+
+distclean: clean
+	rm -rf dist
+	rm -rf deps/libev/config.h deps/libev/config.log deps/libev/config.status deps/libev/.deps deps/libev/.libs
+	rm -rf deps/c-ares/config.h deps/c-ares/config.log deps/c-ares/config.status deps/c-ares/.deps deps/c-ares/.libs
 
 doc:
 	cd doc && PYTHONPATH=.. make html

_setuplibev.py

@@ -20,6 +20,7 @@ from _setuputils import glob_many
 from _setuputils import dep_abspath
 from _setuputils import should_embed
 
+
 LIBEV_EMBED = should_embed('libev')
 
 # Configure libev in place; but cp the config.h to the old directory;

_setuputils.py

@@ -15,6 +15,7 @@ from subprocess import check_call
 from glob import glob
 
 from setuptools.command.build_ext import build_ext
+from setuptools.command.sdist import sdist
 
 ## Exported configurations
 
@@ -78,6 +79,7 @@ def _parse_environ(key):
     raise ValueError('Environment variable %r has invalid value %r. '
                      'Please set it to 1, 0 or an empty string' % (key, value))
 
+IGNORE_CFFI = _parse_environ("GEVENT_NO_CFFI_BUILD")
 
 def _get_config_value(key, defkey, path=None):
     """
@@ -164,6 +166,52 @@ class ConfiguringBuildExt(build_ext):
                 raise
         return result
 
+
+class MakeSdist(sdist):
+    """
+    An sdist that runs make if needed, and makes sure
+    that the Makefile doesn't make it into the dist
+    archive.
+    """
+
+    _ran_make = False
+
+    @classmethod
+    def make(cls, targets=''):
+        # NOTE: We have two copies of the makefile, one
+        # for posix, one for windows. Our sdist command takes
+        # care of renaming the posix one so it doesn't get into
+        # the .tar.gz file (we don't want to re-run make in a released
+        # file). We trigger off the presence/absence of that file altogether
+        # to skip both posix and unix branches.
+        # See https://github.com/gevent/gevent/issues/757
+        if cls._ran_make:
+            return
+
+        if os.path.exists('Makefile'):
+            if WIN:
+                # make.cmd handles checking for PyPy and only making the
+                # right things, so we can ignore the targets
+                system("appveyor\\make.cmd")
+            else:
+                if "PYTHON" not in os.environ:
+                    os.environ["PYTHON"] = sys.executable
+                system('make ' + targets)
+        cls._ran_make = True
+
+    def run(self):
+        renamed = False
+        if os.path.exists('Makefile'):
+            self.make()
+            os.rename('Makefile', 'Makefile.ext')
+            renamed = True
+        try:
+            return sdist.run(self)
+        finally:
+            if renamed:
+                os.rename('Makefile.ext', 'Makefile')
+
+
 from setuptools import Extension as _Extension
 
 class Extension(_Extension):

setup.py

@@ -8,7 +8,9 @@ from _setuputils import read
 from _setuputils import read_version
 from _setuputils import system
 from _setuputils import PYPY, WIN, CFFI_WIN_BUILD_ANYWAY
+from _setuputils import IGNORE_CFFI
 from _setuputils import ConfiguringBuildExt
+from _setuputils import MakeSdist
 from _setuputils import BuildFailed
 
 # setuptools is *required* on Windows
@@ -36,7 +38,12 @@ if WIN:
 if sys.version_info[:2] < (2, 7):
     raise Exception("Please install gevent 1.1 for Python 2.6")
 
-from distutils.command.sdist import sdist as _sdist
+if PYPY and sys.pypy_version_info[:3] < (2, 6, 1): # pylint:disable=no-member
+    # We have to have CFFI >= 1.3.0, and this platform cannot upgrade
+    # it.
+    raise Exception("PyPy >= 2.6.1 is required")
+
+
 
 __version__ = read_version()
 
@@ -47,77 +54,52 @@ from _setuplibev import CORE
 
 from _setupares import ARES
 
+SEMAPHORE = Extension(name="gevent._semaphore",
+                      sources=["src/gevent/gevent._semaphore.c"])
 
-_ran_make = []
-
-
-def make(targets=''):
-    # NOTE: We have two copies of the makefile, one
-    # for posix, one for windows. Our sdist command takes
-    # care of renaming the posix one so it doesn't get into
-    # the .tar.gz file (we don't want to re-run make in a released
-    # file). We trigger off the presence/absence of that file altogether
-    # to skip both posix and unix branches.
-    # See https://github.com/gevent/gevent/issues/757
-    if not _ran_make:
-        if os.path.exists('Makefile'):
-            if WIN:
-                # make.cmd handles checking for PyPy and only making the
-                # right things, so we can ignore the targets
-                system("appveyor\\make.cmd")
-            else:
-                if "PYTHON" not in os.environ:
-                    os.environ["PYTHON"] = sys.executable
-                system('make ' + targets)
-        _ran_make.append(1)
-
-
-class sdist(_sdist):
-
-    def run(self):
-        renamed = False
-        if os.path.exists('Makefile'):
-            make()
-            os.rename('Makefile', 'Makefile.ext')
-            renamed = True
-        try:
-            return _sdist.run(self)
-        finally:
-            if renamed:
-                os.rename('Makefile.ext', 'Makefile')
-
-
+EXT_MODULES = [
+    CORE,
+    ARES,
+    SEMAPHORE,
+]
 
 cffi_modules = ['src/gevent/libev/_corecffi_build.py:ffi']
 
 if PYPY:
     install_requires = []
+    setup_requires = []
+    EXT_MODULES.remove(CORE)
+    EXT_MODULES.remove(SEMAPHORE)
+    # By building the semaphore with Cython under PyPy, we get
+    # atomic operations (specifically, exiting/releasing), at the
+    # cost of some speed (one trivial semaphore micro-benchmark put the pure-python version
+    # at around 1s and the compiled version at around 4s). Some clever subclassing
+    # and having only the bare minimum be in cython might help reduce that penalty.
+    # NOTE: You must use version 0.23.4 or later to avoid a memory leak.
+    # https://mail.python.org/pipermail/cython-devel/2015-October/004571.html
+    # However, that's all for naught on up to and including PyPy 4.0.1 which
+    # have some serious crashing bugs with GC interacting with cython,
+    # so this is disabled
 else:
     install_requires = ['greenlet >= 0.4.9']
-    setup_kwds = {}
+    setup_requires = []
+
 
 try:
     cffi = __import__('cffi')
 except ImportError:
-    setup_kwds = {}
+    pass
 else:
-    _min_cffi_version = (1, 3, 0)
-    _cffi_version_is_supported = cffi.__version_info__ >= _min_cffi_version
-    _kwds = {'cffi_modules': cffi_modules}
-    # We already checked for PyPy on Windows above and excluded it
-    if PYPY:
-        if not _cffi_version_is_supported:
-            raise Exception("PyPy 2.6.1 or higher is required")
-        setup_kwds = _kwds
-    elif LIBEV_EMBED and (not WIN or CFFI_WIN_BUILD_ANYWAY):
-        if not _cffi_version_is_supported:
-            print("WARNING: CFFI version 1.3.0 is required to build CFFI backend", file=sys.stderr)
-        else:
-            # If we're on CPython, we can only reliably build
-            # the CFFI module if we're embedding libev (in some cases
-            # we wind up embedding it anyway, which may not be what the
-            # distributor wanted).
-            setup_kwds = _kwds
+    if IGNORE_CFFI and not PYPY:
+        # Allow distributors to turn off CFFI builds
+        # even if it's available, because CFFI always embeds
+        # our copy of libev and they may not want that.
+        del cffi_modules[:]
+    # Note that we don't add cffi to install_requires, it's
+    # optional. We tend to build and distribute wheels with the CFFI
+    # modules built and they can be imported if CFFI is installed.
+    # install_requires.append('cffi >= 1.3.0')
+
 
 # If we are running info / help commands, or we're being imported by
 # tools like pyroma, we don't need to build anything
@@ -131,67 +113,19 @@ if ((len(sys.argv) >= 2
                              '--long-description')))
     or __name__ != '__main__'):
     _BUILDING = False
-    ext_modules = []
-    include_package_data = PYPY # XXX look into this. we're excluding c files? Why? Old pypy builds? not needed anymore.
-    run_make = False
-elif PYPY:
-    if not WIN:
-        # We need to configure libev because the CORE Extension
-        # won't do it (since we're not building it)
-        system(libev_configure_command)
-
-    # NOTE that we're NOT adding the distutils extension module, as
-    # doing so compiles the module already: import gevent._corecffi_build
-    # imports gevent, which imports the hub, which imports the core,
-    # which compiles the module in-place. Instead we use the setup-time
-    # support of cffi_modules
-    ext_modules = [
-        #_corecffi_build.ffi.distutils_extension(),
-        ARES,
-        # By building the semaphore with Cython under PyPy, we get
-        # atomic operations (specifically, exiting/releasing), at the
-        # cost of some speed (one trivial semaphore micro-benchmark put the pure-python version
-        # at around 1s and the compiled version at around 4s). Some clever subclassing
-        # and having only the bare minimum be in cython might help reduce that penalty.
-        # NOTE: You must use version 0.23.4 or later to avoid a memory leak.
-        # https://mail.python.org/pipermail/cython-devel/2015-October/004571.html
-        # However, that's all for naught on up to and including PyPy 4.0.1 which
-        # have some serious crashing bugs with GC interacting with cython,
-        # so this is disabled (would need to add gevent/gevent._semaphore.c back to
-        # the run_make line)
-        #Extension(name="gevent._semaphore",
-        #          sources=["gevent/gevent._semaphore.c"]),
-    ]
-    include_package_data = True
-    run_make = 'src/gevent/gevent.ares.c'
-else:
-    ext_modules = [
-        CORE,
-        ARES,
-        Extension(name="gevent._semaphore",
-                  sources=["src/gevent/gevent._semaphore.c"]),
-    ]
-    include_package_data = False
-    run_make = True
-
-if run_make and os.path.exists("Makefile"):
-    # The 'sdist' command renames our makefile after it
-    # runs so we don't try to use it from a release tarball.
-
-    # NOTE: This is effectively pointless and serves only for
-    # documentation/metadata, because we run 'make' *before* we run
-    # setup(), so installing cython happens too late.
-    setup_requires = ['cython >= 0.24']
-else:
-    setup_requires = []
 
 
 def run_setup(ext_modules, run_make):
     if run_make:
-        if isinstance(run_make, str):
-            make(run_make)
-        else:
-            make()
+        if (not LIBEV_EMBED and not WIN and cffi_modules) or PYPY:
+            # We're not embedding libev but we do want
+            # to build the CFFI module. We need to configure libev
+            # because the CORE Extension won't.
+            # TODO: Generalize this.
+            system(libev_configure_command)
+
+        MakeSdist.make()
+
     setup(
         name='gevent',
         version=__version__,
@@ -206,11 +140,14 @@ def run_setup(ext_modules, run_make):
         url='http://www.gevent.org/',
         package_dir={'': 'src'},
         packages=find_packages('src'),
-        include_package_data=include_package_data,
+        include_package_data=True,
         ext_modules=ext_modules,
-        cmdclass=dict(build_ext=ConfiguringBuildExt, sdist=sdist),
+        cmdclass=dict(build_ext=ConfiguringBuildExt, sdist=MakeSdist),
         install_requires=install_requires,
         setup_requires=setup_requires,
+        # It's always safe to pass the CFFI keyword, even if
+        # cffi is not installed: it's just ignored in that case.
+        cffi_modules=cffi_modules,
         zip_safe=False,
         test_suite="greentest.testrunner",
         classifiers=[
@@ -227,8 +164,8 @@ def run_setup(ext_modules, run_make):
             "Topic :: Internet",
             "Topic :: Software Development :: Libraries :: Python Modules",
             "Intended Audience :: Developers",
-            "Development Status :: 4 - Beta"],
-        **setup_kwds
+            "Development Status :: 4 - Beta"
+        ],
     )
 
 # Tools like pyroma expect the actual call to `setup` to be performed
@@ -242,11 +179,11 @@ if os.getenv('READTHEDOCS'):
     os.environ['PATH'] = new_path
 
 try:
-    run_setup(ext_modules, run_make=run_make)
+    run_setup(EXT_MODULES, run_make=_BUILDING)
 except BuildFailed:
-    if ARES not in ext_modules:
+    if ARES not in EXT_MODULES:
         raise
-    ext_modules.remove(ARES)
-    run_setup(ext_modules, run_make=run_make)
-if ARES not in ext_modules and __name__ == '__main__' and _BUILDING:
+    EXT_MODULES.remove(ARES)
+    run_setup(EXT_MODULES, run_make=_BUILDING)
+if ARES not in EXT_MODULES and __name__ == '__main__' and _BUILDING:
     sys.stderr.write('\nWARNING: The gevent.ares extension has been disabled.\n')

src/gevent/libev/_corecffi_build.py

@@ -64,7 +64,7 @@ include_dirs = [
     os.path.abspath(os.path.join(thisdir, '..', '..', '..', 'deps', 'libev')),
 ]
 ffi.cdef(_cdef)
-ffi.set_source('gevent._corecffi', _source, include_dirs=include_dirs)
+ffi.set_source('gevent.libev._corecffi', _source, include_dirs=include_dirs)
 
 if __name__ == '__main__':
     # XXX: Note, on Windows, we would need to specify the external libraries

src/gevent/libev/corecffi.py

@@ -16,10 +16,10 @@ __all__ = [
     'loop',
 ]
 
-import gevent._corecffi # pylint:disable=no-name-in-module
+import gevent.libev._corecffi as _corecffi # pylint:disable=no-name-in-module
 
-ffi = gevent._corecffi.ffi # pylint:disable=no-member
-libev = gevent._corecffi.lib # pylint:disable=no-member
+ffi = _corecffi.ffi # pylint:disable=no-member
+libev = _corecffi.lib # pylint:disable=no-member
 
 if hasattr(libev, 'vfd_open'):
     # Must be on windows