Tweak include paths.

Fixes #1461 (hopefully).

CHANGES.rst

@@ -14,6 +14,10 @@ Platform and Packaging Updates
   3.7.6 and 3.8.1. It is also tested with PyPy2 7.3 and PyPy 3.6
   7.3.
 
+- The include directories used to compile the C extensions have been
+  tweaked with the intent of making it easier to use older debug
+  versions of Python. See :issue:`1461`.
+
 Library and Dependency Updates
 ------------------------------
 

_setuplibev.py

@@ -22,6 +22,7 @@ from _setuputils import LIBRARIES
 from _setuputils import DEFINE_MACROS
 from _setuputils import glob_many
 from _setuputils import should_embed
+from _setuputils import get_include_dirs
 
 
 LIBEV_EMBED = should_embed('libev')
@@ -63,12 +64,16 @@ def build_extension():
     # Return the un-cythonized extension.
     # This can be used to access things like `libraries` and `include_dirs`
     # and `define_macros` so we DRY.
+    include_dirs = get_include_dirs()
+    include_dirs.append(os.path.abspath(os.path.join('src', 'gevent', 'libev')))
+    if LIBEV_EMBED:
+        include_dirs.append(dep_abspath('libev'))
     CORE = Extension(name='gevent.libev.corecext',
                      sources=[
                          'src/gevent/libev/corecext.pyx',
                          'src/gevent/libev/callbacks.c',
                      ],
-                     include_dirs=['src/gevent/libev'] + [dep_abspath('libev')] if LIBEV_EMBED else [],
+                     include_dirs=include_dirs,
                      libraries=list(LIBRARIES),
                      define_macros=list(DEFINE_MACROS),
                      depends=glob_many('src/gevent/libev/callbacks.*',

_setuputils.py

@@ -9,6 +9,8 @@ import re
 import os
 import os.path
 import sys
+import sysconfig
+from distutils import sysconfig as dist_sysconfig
 from subprocess import check_call
 from glob import glob
 
@@ -151,8 +153,51 @@ def make_universal_header(filename, *defines):
                         break
             print(line, file=f)
 
-# Processes
-
+def get_include_dirs():
+    """
+    Return additional include directories that might be needed to
+    compile extensions. Specifically, we need the greenlet.h header
+    in many of our extensions.
+    """
+    # setuptools will put the normal include directory for Python.h on the
+    # include path automatically. We don't want to override that with
+    # a different Python3h if we can avoid it: On older versions of Python,
+    # that can cause issues with debug builds (see https://github.com/gevent/gevent/issues/1461)
+    # so order matters here.
+    #
+    # sysconfig.get_path('include') will return the path to the main include
+    # directory. In a virtual environment, that's a symlink to the main
+    # Python installation include directory:
+    #   sysconfig.get_path('include') -> /path/to/venv/include/python3.8
+    #   /path/to/venv/include/python3.7 -> /pythondir/include/python3.8
+    #
+    # distutils.sysconfig.get_python_inc() returns the main Python installation
+    # include directory:
+    #   distutils.sysconfig.get_python_inc() -> /pythondir/include/python3.8
+    #
+    # Neither sysconfig dir is not enough if we're in a virtualenv; the greenlet.h
+    # header goes into a site/ subdir. See https://github.com/pypa/pip/issues/4610
+    dist_inc_dir = os.path.abspath(dist_sysconfig.get_python_inc()) # 1
+    sys_inc_dir = os.path.abspath(sysconfig.get_path("include")) # 2
+    venv_include_dir = os.path.join(
+        sys.prefix, 'include', 'site',
+        'python' + sysconfig.get_python_version()
+    )
+    venv_include_dir = os.path.abspath(venv_include_dir)
+
+    # If we're installed via buildout, and buildout also installs
+    # greenlet, we have *NO* access to greenlet.h at all. So include
+    # our own copy as a fallback.
+    dep_inc_dir = os.path.abspath('deps') # 3
+
+    return [
+        p
+        for p in (dist_inc_dir, sys_inc_dir, dep_inc_dir)
+        if os.path.exists(p)
+    ]
+
+
+## Processes
 
 def _system(cmd, cwd=None, env=None, **kwargs):
     sys.stdout.write('Running %r in %s\n' % (cmd, cwd or os.getcwd()))

setup.py

@@ -4,7 +4,7 @@ from __future__ import print_function
 import sys
 import os
 import os.path
-import sysconfig
+
 
 # setuptools is *required* on Windows
 # (https://bugs.python.org/issue23246) and for PyPy. No reason not to
@@ -20,6 +20,7 @@ from _setuputils import ConfiguringBuildExt
 from _setuputils import GeventClean
 from _setuputils import BuildFailed
 from _setuputils import cythonize1
+from _setuputils import get_include_dirs
 
 # Environment variables that are intended to be used outside of our own
 # CI should be documented in ``installing_from_source.rst``
@@ -48,30 +49,17 @@ from _setupares import ARES
 CORE = cythonize1(build_libev_extension())
 
 # Get access to the greenlet header file.
-# The sysconfig dir is not enough if we're in a virtualenv
-# See https://github.com/pypa/pip/issues/4610
-include_dirs = [sysconfig.get_path("include")]
-venv_include_dir = os.path.join(sys.prefix, 'include', 'site',
-                                'python' + sysconfig.get_python_version())
-venv_include_dir = os.path.abspath(venv_include_dir)
-if os.path.exists(venv_include_dir):
-    include_dirs.append(venv_include_dir)
-
-# If we're installed via buildout, and buildout also installs
-# greenlet, we have *NO* access to greenlet.h at all. So include
-# our own copy as a fallback.
-include_dirs.append('deps')
 
 SEMAPHORE = Extension(name="gevent.__semaphore",
                       sources=["src/gevent/_semaphore.py"],
                       depends=['src/gevent/__semaphore.pxd'],
-                      include_dirs=include_dirs)
+                      include_dirs=get_include_dirs())
 
 
 LOCAL = Extension(name="gevent._local",
                   sources=["src/gevent/local.py"],
                   depends=['src/gevent/_local.pxd'],
-                  include_dirs=include_dirs)
+                  include_dirs=get_include_dirs())
 
 
 GREENLET = Extension(name="gevent._greenlet",
@@ -83,59 +71,59 @@ GREENLET = Extension(name="gevent._greenlet",
                          'src/gevent/__ident.pxd',
                          'src/gevent/_ident.py'
                      ],
-                     include_dirs=include_dirs)
+                     include_dirs=get_include_dirs())
 
 ABSTRACT_LINKABLE = Extension(name="gevent.__abstract_linkable",
                               sources=["src/gevent/_abstract_linkable.py"],
                               depends=['src/gevent/__abstract_linkable.pxd'],
-                              include_dirs=include_dirs)
+                              include_dirs=get_include_dirs())
 
 
 IDENT = Extension(name="gevent.__ident",
                   sources=["src/gevent/_ident.py"],
                   depends=['src/gevent/__ident.pxd'],
-                  include_dirs=include_dirs)
+                  include_dirs=get_include_dirs())
 
 
 IMAP = Extension(name="gevent.__imap",
                  sources=["src/gevent/_imap.py"],
                  depends=['src/gevent/__imap.pxd'],
-                 include_dirs=include_dirs)
+                 include_dirs=get_include_dirs())
 
 EVENT = Extension(name="gevent._event",
                   sources=["src/gevent/event.py"],
                   depends=['src/gevent/_event.pxd'],
-                  include_dirs=include_dirs)
+                  include_dirs=get_include_dirs())
 
 QUEUE = Extension(name="gevent._queue",
                   sources=["src/gevent/queue.py"],
                   depends=['src/gevent/_queue.pxd'],
-                  include_dirs=include_dirs)
+                  include_dirs=get_include_dirs())
 
 HUB_LOCAL = Extension(name="gevent.__hub_local",
                       sources=["src/gevent/_hub_local.py"],
                       depends=['src/gevent/__hub_local.pxd'],
-                      include_dirs=include_dirs)
+                      include_dirs=get_include_dirs())
 
 WAITER = Extension(name="gevent.__waiter",
                    sources=["src/gevent/_waiter.py"],
                    depends=['src/gevent/__waiter.pxd'],
-                   include_dirs=include_dirs)
+                   include_dirs=get_include_dirs())
 
 HUB_PRIMITIVES = Extension(name="gevent.__hub_primitives",
                            sources=["src/gevent/_hub_primitives.py"],
                            depends=['src/gevent/__hub_primitives.pxd'],
-                           include_dirs=include_dirs)
+                           include_dirs=get_include_dirs())
 
 GLT_PRIMITIVES = Extension(name="gevent.__greenlet_primitives",
                            sources=["src/gevent/_greenlet_primitives.py"],
                            depends=['src/gevent/__greenlet_primitives.pxd'],
-                           include_dirs=include_dirs)
+                           include_dirs=get_include_dirs())
 
 TRACER = Extension(name="gevent.__tracer",
                    sources=["src/gevent/_tracer.py"],
                    depends=['src/gevent/__tracer.pxd'],
-                   include_dirs=include_dirs)
+                   include_dirs=get_include_dirs())
 
 
 _to_cythonize = [

src/gevent/testing/monkey_test.py

@@ -2,21 +2,14 @@ import sys
 import os
 
 
-kwargs = {}
-
-if sys.argv[1] == '--Event':
-    kwargs['Event'] = True
-    del sys.argv[1]
-else:
-    kwargs['Event'] = False
-
 test_filename = sys.argv[1]
 del sys.argv[1]
 
-print('Running with patch_all(%s): %s' % (','.join('%s=%r' % x for x in kwargs.items()), test_filename))
+print('Running with patch_all(): %s' % (test_filename,))
 
 from gevent import monkey
-monkey.patch_all(**kwargs)
+# Only test the default set of patch arguments.
+monkey.patch_all()
 
 from .sysinfo import RUNNING_ON_APPVEYOR
 from .sysinfo import PY37
@@ -87,7 +80,7 @@ try:
 except SkipTest as e:
     # Some tests can raise test.support.ResourceDenied
     # in their main method before the testrunner takes over.
-    # That's a kind of SkipTest. we can't get  a skip count because it
+    # That's a kind of SkipTest. we can't get a true skip count because it
     # hasn't run, though.
     print(e)
     # Match the regular unittest output, including ending with skipped