Add depfile generation support to cythonize (GH-4563)

Add a new command line option so that
$ cythonize -M foo.pyx
produces a file `foo.c.dep` with dependencies of foo.pyx, in addition to `foo.c`.
Try to write relative paths as much as possible.

Closes https://github.com/cython/cython/issues/1214

Cython/Build/Cythonize.py

@@ -74,6 +74,7 @@ def cython_compile(path_pattern, options):
                 compile_time_env=options.compile_time_env,
                 force=options.force,
                 quiet=options.quiet,
+                depfile=options.depfile,
                 **options.options)
 
             if ext_modules and options.build:
@@ -171,6 +172,7 @@ def create_args_parser():
                       help='compile as much as possible, ignore compilation failures')
     parser.add_argument('--no-docstrings', dest='no_docstrings', action='store_true', default=None,
                       help='strip docstrings')
+    parser.add_argument('-M', '--depfile', action='store_true', help='produce depfiles for the sources')
     parser.add_argument('sources', nargs='*')
     return parser
 

Cython/Build/Dependencies.py

@@ -957,6 +957,8 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False,
     :param compiler_directives: Allow to set compiler directives in the ``setup.py`` like this:
                                 ``compiler_directives={'embedsignature': True}``.
                                 See :ref:`compiler-directives`.
+
+    :param depfile: produce depfiles for the sources if True.
     """
     if exclude is None:
         exclude = []
@@ -965,6 +967,8 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False,
     if 'common_utility_include_dir' in options:
         safe_makedirs(options['common_utility_include_dir'])
 
+    depfile = options.pop('depfile', None)
+
     if pythran is None:
         pythran_options = None
     else:
@@ -1040,6 +1044,25 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False,
                     dir = os.path.dirname(c_file)
                     safe_makedirs_once(dir)
 
+                # write out the depfile, if requested
+                if depfile:
+                    dependencies = deps.all_dependencies(source)
+                    src_base_dir, _ = os.path.split(source)
+                    if not src_base_dir.endswith(os.sep):
+                        src_base_dir += os.sep
+                    # paths below the base_dir are relative, otherwise absolute
+                    paths = []
+                    for fname in dependencies:
+                        if fname.startswith(src_base_dir):
+                            paths.append(os.path.relpath(fname, src_base_dir))
+                        else:
+                            paths.append(os.path.abspath(fname))
+
+                    depline = os.path.split(c_file)[1] + ": \\\n  "
+                    depline += " \\\n  ".join(paths) + "\n"
+                    with open(c_file+'.dep', 'w') as outfile:
+                        outfile.write(depline)
+
                 if os.path.exists(c_file):
                     c_timestamp = os.path.getmtime(c_file)
                 else:

tests/build/depfile.srctree

+CYTHONIZE -M foo.pyx
+PYTHON check.py
+
+######## foo.pyx ########
+
+from bar cimport empty
+
+include "baz.pxi"
+
+empty()
+print(foo())
+
+
+######## baz.pxi ########
+
+def foo():
+    return "foo"
+
+
+######## bar.pxd ########
+
+cdef inline void empty():
+    print("empty")
+
+
+######## check.py ########
+
+import os
+
+with open("foo.c.dep", "r") as f:
+    contents = f.read().replace("\n", " ").replace("\\", "")
+
+assert sorted(contents.split()) == ['bar.pxd', 'baz.pxi', 'foo.c:', 'foo.pyx'], contents

tests/build/depfile_numpy.srctree

+# tag: numpy
+
+CYTHONIZE -M dep_np.pyx
+PYTHON check_np.py
+
+######## dep_np.pyx ########
+
+cimport numpy as np
+np.import_array()
+
+
+
+######## check_np.py ########
+
+import os
+import re
+
+import numpy as np
+import Cython
+
+with open("dep_np.c.dep", "r") as f:
+    contents = f.read().replace("\n", " ").replace("\\", "")
+
+contents = contents.split()
+
+cy_prefix, _ = os.path.split(Cython.__file__)
+contents = [fname.replace(cy_prefix, "cy_prefix") for fname in contents]
+
+np_prefix, _ = os.path.split(np.__file__)
+contents = [fname.replace(np_prefix, "np_prefix") for fname in contents]
+
+# filter out the version number from `np_prefix/__init__.cython-30.pxd`.
+contents = [re.sub('[.]cython-[0-9]+', '', entry) for entry in contents]
+
+expected = ['cy_prefix/Includes/cpython/object.pxd',
+    'cy_prefix/Includes/cpython/ref.pxd',
+    'cy_prefix/Includes/cpython/type.pxd',
+    'cy_prefix/Includes/libc/stdio.pxd',
+    'cy_prefix/Includes/libc/string.pxd',
+    'dep_np.c:',
+    'dep_np.pyx',]
+
+# Also account for legacy numpy versions, which do not ship
+# `__init__.pxd` hence the fallback is used:
+if 'cy_prefix/Includes/numpy/__init__.pxd' in contents:
+    expected.append('cy_prefix/Includes/numpy/__init__.pxd')
+else:
+    expected.append('np_prefix/__init__.pxd')
+
+assert sorted(contents) == sorted(expected), sorted(contents)

tests/build/depfile_package.srctree

+'''
+PYTHON -m Cython.Build.Cythonize -i pkg --depfile
+PYTHON package_test.py
+'''
+
+######## package_test.py ########
+
+import os
+
+with open("pkg/test.c.dep", "r") as f:
+    contents = f.read().replace("\n", " ").replace("\\", "")
+
+assert sorted(contents.split()) == sorted(['test.c:', 'sub/incl.pxi', 'test.pxd', 'test.pyx']), contents
+
+
+with open("pkg/sub/test.c.dep", "r") as f:
+    contents = f.read().replace("\n", " ").replace("\\", "")
+
+contents = [os.path.relpath(entry, '.')
+            if os.path.isabs(entry) else entry for entry in contents.split()]
+assert sorted(contents) == sorted(['test.c:', 'incl.pxi', 'test.pyx', 'pkg/test.pxd']), contents  # last is really one level up
+
+
+######## pkg/__init__.py ########
+
+
+######## pkg/test.pyx ########
+
+TEST = "pkg.test"
+
+include "sub/incl.pxi"
+
+cdef object get_str():
+   return TEST
+
+
+######## pkg/test.pxd ########
+
+cdef object get_str()
+
+
+######## pkg/sub/__init__.py ########
+
+
+######## pkg/sub/test.pyx ########
+# cython: language_level=3
+
+from ..test cimport get_str
+
+include 'incl.pxi'
+
+TEST = 'pkg.sub.test'
+
+
+######## pkg/sub/incl.pxi ########
+
+pass
+