checkpoint on breaking the setup.py into manageable pieces.

MANIFEST.in

@@ -17,6 +17,7 @@ include Makefile.ext
 include known_failures.py
 include *.yml
 include *.txt
+include _setup*.py
 
 include tox.ini
 include .pep8

_setupares.py

+# -*- coding: utf-8 -*-
+"""
+setup helpers for c-ares.
+"""
+
+from __future__ import print_function, absolute_import, division
+
+import os
+import os.path
+import shutil
+import sys
+
+from _setuputils import Extension
+
+import distutils.sysconfig  # to get CFLAGS to pass into c-ares configure script pylint:disable=import-error
+
+from _setuputils import WIN
+from _setuputils import quoted_dep_abspath
+from _setuputils import system
+from _setuputils import make_universal_header
+from _setuputils import should_embed
+from _setuputils import LIBRARIES
+from _setuputils import DEFINE_MACROS
+from _setuputils import glob_many
+from _setuputils import dep_abspath
+
+
+CARES_EMBED = should_embed('c-ares')
+
+
+# See #616, trouble building for a 32-bit python against a 64-bit platform
+_config_vars = distutils.sysconfig.get_config_var("CFLAGS")
+if _config_vars and "m32" in _config_vars:
+    _m32 = 'CFLAGS="' + os.getenv('CFLAGS', '') + ' -m32" '
+else:
+    _m32 = ''
+
+# Use -r, not -e, for support of old solaris. See
+# https://github.com/gevent/gevent/issues/777
+ares_configure_command = ' '.join([
+    "(cd ", quoted_dep_abspath('c-ares'),
+    " && if [ -r ares_build.h ]; then cp ares_build.h ares_build.h.orig; fi ",
+    " && /bin/sh ./configure --disable-dependency-tracking " + _m32 + "CONFIG_COMMANDS= ",
+    " && cp ares_config.h ares_build.h \"$OLDPWD\" ",
+    " && mv ares_build.h.orig ares_build.h)",
+    "> configure-output.txt"])
+
+
+
+def configure_ares(bext, ext):
+    bdir = os.path.join(bext.build_temp, 'c-ares')
+    ext.include_dirs.insert(0, bdir)
+
+    if not os.path.isdir(bdir):
+        os.makedirs(bdir)
+
+    if WIN:
+        shutil.copy("deps\\c-ares\\ares_build.h.dist", os.path.join(bdir, "ares_build.h"))
+        return
+
+    cwd = os.getcwd()
+    os.chdir(bdir)
+    try:
+        if os.path.exists('ares_config.h') and os.path.exists('ares_build.h'):
+            return
+        try:
+            system(ares_configure_command)
+        except:
+            with open('configure-output.txt', 'r') as t:
+                print(t.read(), file=sys.stderr)
+            raise
+        if sys.platform == 'darwin':
+            make_universal_header('ares_build.h', 'CARES_SIZEOF_LONG')
+            make_universal_header('ares_config.h', 'SIZEOF_LONG', 'SIZEOF_SIZE_T', 'SIZEOF_TIME_T')
+    finally:
+        os.chdir(cwd)
+
+
+ARES = Extension(name='gevent.ares',
+                 sources=['src/gevent/gevent.ares.c'],
+                 include_dirs=[dep_abspath('c-ares')] if CARES_EMBED else [],
+                 libraries=list(LIBRARIES),
+                 define_macros=list(DEFINE_MACROS),
+                 depends=glob_many('src/gevent/dnshelper.c',
+                                   'src/gevent/cares_*.[ch]'))
+ARES.optional = True
+
+
+if CARES_EMBED:
+    ARES.sources += glob_many('deps/c-ares/*.c')
+    # Strip the standalone binaries that would otherwise
+    # cause linking issues
+    for bin_c in ('acountry', 'adig', 'ahost'):
+        try:
+            ARES.sources.remove('deps/c-ares/' + bin_c + '.c')
+        except ValueError:
+            pass
+    ARES.configure = configure_ares
+    if WIN:
+        ARES.libraries += ['advapi32']
+        ARES.define_macros += [('CARES_STATICLIB', '')]
+    else:
+        ARES.define_macros += [('HAVE_CONFIG_H', '')]
+        if sys.platform != 'darwin':
+            ARES.libraries += ['rt']
+    ARES.define_macros += [('CARES_EMBED', '1')]
+else:
+    ARES.libraries.append('cares')
+    ARES.define_macros += [('HAVE_NETDB_H', '')]

_setuplibev.py

+# -*- coding: utf-8 -*-
+"""
+setup helpers for libev.
+"""
+
+from __future__ import print_function, absolute_import, division
+
+import sys
+import os.path
+
+from _setuputils import Extension
+
+from _setuputils import system
+from _setuputils import quoted_dep_abspath
+from _setuputils import WIN
+from _setuputils import make_universal_header
+from _setuputils import LIBRARIES
+from _setuputils import DEFINE_MACROS
+from _setuputils import glob_many
+from _setuputils import dep_abspath
+from _setuputils import should_embed
+
+LIBEV_EMBED = should_embed('libev')
+
+# Configure libev in place; but cp the config.h to the old directory;
+# if we're building a CPython extension, the old directory will be
+# the build/temp.XXX/libev/ directory. If we're building from a
+# source checkout on pypy, OLDPWD will be the location of setup.py
+# and the PyPy branch will clean it up.
+libev_configure_command = ' '.join([
+    "(cd ", quoted_dep_abspath('libev'),
+    " && /bin/sh ./configure ",
+    " && cp config.h \"$OLDPWD\"",
+    ")",
+    '> configure-output.txt'
+])
+
+
+def configure_libev(bext, ext):
+    if WIN:
+        return
+
+    bdir = os.path.join(bext.build_temp, 'libev')
+    ext.include_dirs.insert(0, bdir)
+
+    if not os.path.isdir(bdir):
+        os.makedirs(bdir)
+
+    cwd = os.getcwd()
+    os.chdir(bdir)
+    try:
+        if os.path.exists('config.h'):
+            return
+        system(libev_configure_command)
+        if sys.platform == 'darwin':
+            make_universal_header('config.h',
+                                  'SIZEOF_LONG', 'SIZEOF_SIZE_T', 'SIZEOF_TIME_T')
+    finally:
+        os.chdir(cwd)
+
+CORE = Extension(name='gevent.libev.corecext',
+                 sources=['src/gevent/libev/gevent.corecext.c'],
+                 include_dirs=[dep_abspath('libev')] if LIBEV_EMBED else [],
+                 libraries=list(LIBRARIES),
+                 define_macros=list(DEFINE_MACROS),
+                 depends=glob_many('src/gevent/libev/callbacks.*',
+                                   'src/gevent/libev/stathelper.c',
+                                   'src/gevent/libev/libev*.h',
+                                   'deps/libev/*.[ch]'))
+if WIN:
+    CORE.define_macros.append(('EV_STANDALONE', '1'))
+# QQQ libev can also use -lm, however it seems to be added implicitly
+
+if LIBEV_EMBED:
+    CORE.define_macros += [('LIBEV_EMBED', '1'),
+                           ('EV_COMMON', ''),  # we don't use void* data
+                           # libev watchers that we don't use currently:
+                           ('EV_CLEANUP_ENABLE', '0'),
+                           ('EV_EMBED_ENABLE', '0'),
+                           ("EV_PERIODIC_ENABLE", '0')]
+    CORE.configure = configure_libev
+    if sys.platform == "darwin":
+        os.environ["CPPFLAGS"] = ("%s %s" % (os.environ.get("CPPFLAGS", ""), "-U__llvm__")).lstrip()
+    if os.environ.get('GEVENTSETUP_EV_VERIFY') is not None:
+        CORE.define_macros.append(('EV_VERIFY', os.environ['GEVENTSETUP_EV_VERIFY']))
+else:
+    CORE.libraries.append('ev')

_setuputils.py

+# -*- coding: utf-8 -*-
+"""
+gevent build utilities.
+
+.. $Id$
+"""
+
+from __future__ import print_function, absolute_import, division
+
+import re
+import os
+import os.path
+import sys
+from subprocess import check_call
+from glob import glob
+
+from setuptools.command.build_ext import build_ext
+
+## Exported configurations
+
+PYPY = hasattr(sys, 'pypy_version_info')
+WIN = sys.platform.startswith('win')
+CFFI_WIN_BUILD_ANYWAY = os.environ.get("PYPY_WIN_BUILD_ANYWAY")
+
+LIBRARIES = []
+DEFINE_MACROS = []
+
+
+if WIN:
+    LIBRARIES += ['ws2_32']
+    DEFINE_MACROS += [('FD_SETSIZE', '1024'), ('_WIN32', '1')]
+
+### File handling
+
+THIS_DIR = os.path.dirname(__file__)
+
+def quoted_abspath(*segments):
+    return '"' + os.path.abspath(os.path.join(*segments)) + '"'
+
+def read(name, *args):
+    """Read a file path relative to this file."""
+    with open(os.path.join(THIS_DIR, name)) as f:
+        return f.read(*args)
+
+def read_version(name="src/gevent/__init__.py"):
+    contents = read(name)
+    version = re.search(r"__version__\s*=\s*'(.*)'", contents, re.M).group(1)
+    assert version, "could not read version"
+    return version
+
+def dep_abspath(depname, *extra):
+    return os.path.abspath(os.path.join('deps', depname, *extra))
+
+def quoted_dep_abspath(depname):
+    return quoted_abspath(dep_abspath(depname))
+
+def glob_many(*globs):
+    """
+    Return a list of all the glob patterns expanded.
+    """
+    result = []
+    for pattern in globs:
+        result.extend(glob(pattern))
+    return sorted(result)
+
+
+## Configuration
+
+def _parse_environ(key):
+    value = os.environ.get(key)
+    if not value:
+        return
+    value = value.lower().strip()
+    if value in ('1', 'true', 'on', 'yes'):
+        return True
+    elif value in ('0', 'false', 'off', 'no'):
+        return False
+    raise ValueError('Environment variable %r has invalid value %r. '
+                     'Please set it to 1, 0 or an empty string' % (key, value))
+
+
+def _get_config_value(key, defkey, path=None):
+    """
+    Find a boolean value, configured in the environment at *key* or
+    *defkey* (typically, *defkey* will be shared by several calls). If
+    those don't exist, then check for the existence of *path* and return
+    that (if path is given)
+    """
+    value = _parse_environ(key)
+    if value is None:
+        value = _parse_environ(defkey)
+    if value is not None:
+        return value
+    return os.path.exists(path) if path is not None else False
+
+def should_embed(dep_name):
+    """
+    Check the configuration for the dep_name and see if it
+    should be embedded. Environment keys are derived from the
+    dep name: libev becomes LIBEV_EMBED and c-ares becomes CARES_EMBED.
+    """
+    path = dep_abspath(dep_name)
+    defkey = 'EMBED'
+    key = dep_name.replace('-', '').upper() + '_' + defkey
+
+    return _get_config_value(key, defkey, path)
+
+## Headers
+
+def make_universal_header(filename, *defines):
+    defines = [('#define %s ' % define, define) for define in defines]
+    with open(filename, 'r') as f:
+        lines = f.read().split('\n')
+    ifdef = 0
+    with open(filename, 'w') as f:
+        for line in lines:
+            if line.startswith('#ifdef'):
+                ifdef += 1
+            elif line.startswith('#endif'):
+                ifdef -= 1
+            elif not ifdef:
+                for prefix, define in defines:
+                    if line.startswith(prefix):
+                        line = '#ifdef __LP64__\n#define %s 8\n#else\n#define %s 4\n#endif' % (define, define)
+                        break
+            print(line, file=f)
+
+# Processes
+
+
+def _system(cmd):
+    sys.stdout.write('Running %r in %s\n' % (cmd, os.getcwd()))
+    return check_call(cmd, shell=True)
+
+
+def system(cmd):
+    if _system(cmd):
+        sys.exit(1)
+
+
+## Distutils extensions
+class BuildFailed(Exception):
+    pass
+
+from distutils.errors import CCompilerError, DistutilsExecError, DistutilsPlatformError # pylint:disable=no-name-in-module,import-error
+ext_errors = (CCompilerError, DistutilsExecError, DistutilsPlatformError, IOError)
+
+
+class ConfiguringBuildExt(build_ext):
+
+    def gevent_prepare(self, ext):
+        configure = getattr(ext, 'configure', None)
+        if configure:
+            configure(self, ext)
+
+    def build_extension(self, ext):
+        self.gevent_prepare(ext)
+        try:
+            result = build_ext.build_extension(self, ext)
+        except ext_errors:
+            if getattr(ext, 'optional', False):
+                raise BuildFailed()
+            else:
+                raise
+        return result
+
+from setuptools import Extension as _Extension
+
+class Extension(_Extension):
+    # This class exists currently mostly to make pylint
+    # happy in terms of attributes we use.
+
+    def __init__(self, *args, **kwargs):
+        self.libraries = []
+        self.define_macros = []
+        super(Extension, self).__init__(*args, **kwargs)