prefer sys.path before Cython/Include when searching for pxd files (GH-2993)

* REF: make Utils dependency-free by moving search_include_directories to Main
* BUG: prefer sys.path before Cython/Include when searching for pxd files

CHANGES.rst

@@ -14,6 +14,12 @@ Bugs fixed
 * Support slice handling in newer Pythran versions.
   (Github issue #2989)
 
+* The search order for include files was changed. Previously it was
+  ``include_directories``, ``Cython/Includes``, ``sys.path``. Now it is
+  ``include_directories``, ``sys.path``, ``Cython/Includes``. This was done to
+  allow third-party ``*.pxd`` files to override the ones in Cython.
+  (Github issue #2905)
+
 
 0.29.10 (2019-06-02)
 ====================

Cython/Compiler/Main.py

@@ -38,6 +38,8 @@ module_name_pattern = re.compile(r"[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_
 
 verbose = 0
 
+standard_include_path = os.path.abspath(os.path.join(os.path.dirname(__file__),
+                                        os.path.pardir, 'Includes'))
 
 class CompilationData(object):
     #  Bundles the information that is passed from transform to transform.
@@ -88,10 +90,6 @@ class Context(object):
         self.pxds = {}  # full name -> node tree
         self._interned = {}  # (type(value), value, *key_args) -> interned_value
 
-        standard_include_path = os.path.abspath(os.path.normpath(
-            os.path.join(os.path.dirname(__file__), os.path.pardir, 'Includes')))
-        self.include_directories = include_directories + [standard_include_path]
-
         if language_level is not None:
             self.set_language_level(language_level)
 
@@ -290,8 +288,13 @@ class Context(object):
 
     def search_include_directories(self, qualified_name, suffix, pos,
                                    include=False, sys_path=False):
-        return Utils.search_include_directories(
-            tuple(self.include_directories), qualified_name, suffix, pos, include, sys_path)
+        include_dirs = self.include_directories
+        if sys_path:
+            include_dirs = include_dirs + sys.path
+        # include_dirs must be hashable for caching in @cached_function
+        include_dirs = tuple(include_dirs + [standard_include_path])
+        return search_include_directories(include_dirs, qualified_name,
+                                          suffix, pos, include)
 
     def find_root_package_dir(self, file_path):
         return Utils.find_root_package_dir(file_path)
@@ -778,6 +781,56 @@ def compile(source, options = None, full_module_name = None, **kwds):
         return compile_multiple(source, options)
 
 
+@Utils.cached_function
+def search_include_directories(dirs, qualified_name, suffix, pos, include=False):
+    """
+    Search the list of include directories for the given file name.
+
+    If a source file position is given, first searches the directory
+    containing that file. Returns None if not found, but does not
+    report an error.
+
+    The 'include' option will disable package dereferencing.
+    """
+
+    if pos:
+        file_desc = pos[0]
+        if not isinstance(file_desc, FileSourceDescriptor):
+            raise RuntimeError("Only file sources for code supported")
+        if include:
+            dirs = (os.path.dirname(file_desc.filename),) + dirs
+        else:
+            dirs = (Utils.find_root_package_dir(file_desc.filename),) + dirs
+
+    dotted_filename = qualified_name
+    if suffix:
+        dotted_filename += suffix
+
+    if not include:
+        names = qualified_name.split('.')
+        package_names = tuple(names[:-1])
+        module_name = names[-1]
+        module_filename = module_name + suffix
+        package_filename = "__init__" + suffix
+
+    for dirname in dirs:
+        path = os.path.join(dirname, dotted_filename)
+        if os.path.exists(path):
+            return path
+
+        if not include:
+            package_dir = Utils.check_package_dir(dirname, package_names)
+            if package_dir is not None:
+                path = os.path.join(package_dir, module_filename)
+                if os.path.exists(path):
+                    return path
+                path = os.path.join(package_dir, module_name,
+                                    package_filename)
+                if os.path.exists(path):
+                    return path
+    return None
+
+
 # ------------------------------------------------------------------------
 #
 #  Main command-line entry point

Cython/Utils.py

@@ -123,54 +123,6 @@ def copy_file_to_dir_if_newer(sourcefile, destdir):
     shutil.copy2(sourcefile, destfile)
 
 
-@cached_function
-def search_include_directories(dirs, qualified_name, suffix, pos,
-                               include=False, sys_path=False):
-    # Search the list of include directories for the given
-    # file name. If a source file position is given, first
-    # searches the directory containing that file. Returns
-    # None if not found, but does not report an error.
-    # The 'include' option will disable package dereferencing.
-    # If 'sys_path' is True, also search sys.path.
-    if sys_path:
-        dirs = dirs + tuple(sys.path)
-    if pos:
-        file_desc = pos[0]
-        from Cython.Compiler.Scanning import FileSourceDescriptor
-        if not isinstance(file_desc, FileSourceDescriptor):
-            raise RuntimeError("Only file sources for code supported")
-        if include:
-            dirs = (os.path.dirname(file_desc.filename),) + dirs
-        else:
-            dirs = (find_root_package_dir(file_desc.filename),) + dirs
-
-    dotted_filename = qualified_name
-    if suffix:
-        dotted_filename += suffix
-    if not include:
-        names = qualified_name.split('.')
-        package_names = tuple(names[:-1])
-        module_name = names[-1]
-        module_filename = module_name + suffix
-        package_filename = "__init__" + suffix
-
-    for dir in dirs:
-        path = os.path.join(dir, dotted_filename)
-        if path_exists(path):
-            return path
-        if not include:
-            package_dir = check_package_dir(dir, package_names)
-            if package_dir is not None:
-                path = os.path.join(package_dir, module_filename)
-                if path_exists(path):
-                    return path
-                path = os.path.join(dir, package_dir, module_name,
-                                    package_filename)
-                if path_exists(path):
-                    return path
-    return None
-
-
 @cached_function
 def find_root_package_dir(file_path):
     dir = os.path.dirname(file_path)

tests/compile/find_pxd.srctree

@@ -6,12 +6,13 @@ from Cython.Build import cythonize
 from Cython.Distutils.extension import Extension
 
 import sys
-sys.path.append("path")
+sys.path.insert(0, "path")
 
 ext_modules = [
     Extension("a", ["a.pyx"]),
     Extension("b", ["b.pyx"]),
     Extension("c", ["c.pyx"]),
+    Extension("d", ["d.pyx"]),
 ]
 
 ext_modules = cythonize(ext_modules, include_path=["include"])
@@ -37,3 +38,15 @@ ctypedef int my_type
 
 ######## path/c.pxd ########
 +++syntax error just to show that this file is not actually cimported+++
+
+######## path/numpy/__init__.pxd ########
+
+# gh-2905: This should be found before Cython/Inlude/numpy/__init__.pxd
+
+ctypedef int my_type
+
+######## d.pyx ########
+
+cimport numpy
+
+cdef numpy.my_type foo