Merge branch 'master' into 0.19.x

Conflicts:
	CHANGES.rst

CHANGES.rst

@@ -26,6 +26,9 @@ Features added
 
 * Constant folding for boolean expressions (and/or) was improved.
 
+* Added a build_dir option to cythonize() which allows one to place
+  the generated .c files outside the source tree.
+
 Bugs fixed
 ----------
 

Cython/Build/Dependencies.py

@@ -18,13 +18,28 @@ try:
 except ImportError:
     import md5 as hashlib
 
+try:
+    from os.path import relpath as _relpath
+except ImportError:
+    # Py<2.6
+    def _relpath(path, start=os.path.curdir):
+        if not path:
+            raise ValueError("no path specified")
+        start_list = os.path.abspath(start).split(os.path.sep)
+        path_list = os.path.abspath(path).split(os.path.sep)
+        i = len(os.path.commonprefix([start_list, path_list]))
+        rel_list = [os.path.pardir] * (len(start_list)-i) + path_list[i:]
+        if not rel_list:
+            return os.path.curdir
+        return os.path.join(*rel_list)
+
 
 from distutils.extension import Extension
 
 from Cython import Utils
-from Cython.Utils import cached_function, cached_method, path_exists
+from Cython.Utils import cached_function, cached_method, path_exists, find_root_package_dir
 from Cython.Compiler.Main import Context, CompilationOptions, default_options
-    
+
 join_path = cached_function(os.path.join)
 
 if sys.version_info[0] < 3:
@@ -208,7 +223,7 @@ def strip_string_literals(code, prefix='__Pyx_L'):
     in_quote = False
     hash_mark = single_q = double_q = -1
     code_len = len(code)
-    
+
     while True:
         if hash_mark < q:
             hash_mark = code.find('#', q)
@@ -374,7 +389,7 @@ class DependencyTree(object):
             elif not self.quiet:
                 print("Unable to locate '%s' referenced from '%s'" % (filename, include))
         return all
-    
+
     @cached_method
     def cimports_and_externs(self, filename):
         # This is really ugly. Nested cimports are resolved with respect to the
@@ -598,7 +613,7 @@ def create_extension_list(patterns, exclude=[], ctx=None, aliases=None, quiet=Fa
                             sources.append(source)
                     del kwds['sources']
                 if 'depends' in kwds:
-                    depends = resolve_depends(kwds['depends'], kwds.get('include_dirs') or [])
+                    depends = resolve_depends(kwds['depends'], (kwds.get('include_dirs') or []) + [find_root_package_dir(file)])
                     if template is not None:
                         # Always include everything from the template.
                         depends = list(set(template.depends).union(set(depends)))
@@ -640,6 +655,7 @@ def cythonize(module_list, exclude=[], nthreads=0, aliases=None, quiet=False, fo
     c_options = CompilationOptions(**options)
     cpp_options = CompilationOptions(**options); cpp_options.cplus = True
     ctx = c_options.create_context()
+    options = c_options
     module_list = create_extension_list(
         module_list,
         exclude=exclude,
@@ -648,9 +664,23 @@ def cythonize(module_list, exclude=[], nthreads=0, aliases=None, quiet=False, fo
         exclude_failures=exclude_failures,
         aliases=aliases)
     deps = create_dependency_tree(ctx, quiet=quiet)
+    build_dir = getattr(options, 'build_dir', None)
     modules_by_cfile = {}
     to_compile = []
     for m in module_list:
+        if build_dir:
+            root = os.path.realpath(os.path.abspath(find_root_package_dir(m.sources[0])))
+            def copy_to_build_dir(filepath, root=root):
+                filepath = os.path.abspath(filepath)
+                if os.path.realpath(filepath).startswith(root):
+                    mod_dir = os.path.join(
+                        build_dir, os.path.dirname(_relpath(filepath)))
+                    if not os.path.isdir(mod_dir):
+                        os.makedirs(mod_dir)
+                    shutil.copy(filepath, mod_dir)
+            for dep in m.depends:
+                copy_to_build_dir(dep)
+
         new_sources = []
         for source in m.sources:
             base, ext = os.path.splitext(source)
@@ -661,11 +691,19 @@ def cythonize(module_list, exclude=[], nthreads=0, aliases=None, quiet=False, fo
                 else:
                     c_file = base + '.c'
                     options = c_options
+
+                # setup for out of place build directory if enabled
+                if build_dir:
+                    c_file = os.path.join(build_dir, c_file)
+                    dir = os.path.dirname(c_file)
+                    if not os.path.isdir(dir):
+                        os.makedirs(dir)
+
                 if os.path.exists(c_file):
                     c_timestamp = os.path.getmtime(c_file)
                 else:
                     c_timestamp = -1
-                    
+
                 # Priority goes first to modified files, second to direct
                 # dependents, and finally to indirect dependents.
                 if c_timestamp < deps.timestamp(source):
@@ -694,6 +732,8 @@ def cythonize(module_list, exclude=[], nthreads=0, aliases=None, quiet=False, fo
                     modules_by_cfile[c_file].append(m)
             else:
                 new_sources.append(source)
+                if build_dir:
+                    copy_to_build_dir(source)
         m.sources = new_sources
     if hasattr(options, 'cache'):
         if not os.path.exists(options.cache):

Cython/Includes/cpython/buffer.pxd

@@ -56,7 +56,7 @@ cdef extern from "Python.h":
     void* PyBuffer_GetPointer(Py_buffer *view, Py_ssize_t *indices)
     # ??
 
-    int PyBuffer_SizeFromFormat(char *) # actually const char
+    Py_ssize_t PyBuffer_SizeFromFormat(char *) # actually const char
     # Return the implied ~Py_buffer.itemsize from the struct-stype
     # ~Py_buffer.format
 
@@ -90,7 +90,7 @@ cdef extern from "Python.h":
     void PyBuffer_FillContiguousStrides(int ndims,
                                         Py_ssize_t *shape,
                                         Py_ssize_t *strides,
-                                        int itemsize,
+                                        Py_ssize_t itemsize,
                                         char fort)
     # Fill the strides array with byte-strides of a contiguous
     # (Fortran-style if fort is 'F' or C-style otherwise) array of the

Cython/Includes/cpython/sequence.pxd

@@ -84,7 +84,7 @@ cdef extern from "Python.h":
     # value, return 1, otherwise return 0. On error, return -1. This
     # is equivalent to the Python expression "value in o".
 
-    int PySequence_Index(object o, object value) except -1
+    Py_ssize_t PySequence_Index(object o, object value) except -1
     # Return the first index i for which o[i] == value. On error,
     # return -1. This is equivalent to the Python expression
     # "o.index(value)".
@@ -126,7 +126,7 @@ cdef extern from "Python.h":
     # PySequence_Check(o) is true and without adjustment for negative
     # indices.
 
-    int PySequence_Fast_GET_SIZE(object o)
+    Py_ssize_t PySequence_Fast_GET_SIZE(object o)
     # Returns the length of o, assuming that o was returned by
     # PySequence_Fast() and that o is not NULL. The size can also be
     # gotten by calling PySequence_Size() on o, but

Cython/Includes/cpython/set.pxd

@@ -66,12 +66,12 @@ cdef extern from "Python.h":
     # The following functions and macros are available for instances
     # of set or frozenset or instances of their subtypes.
 
-    int PySet_Size(object anyset) except -1
+    Py_ssize_t PySet_Size(object anyset) except -1
     # Return the length of a set or frozenset object. Equivalent to
     # "len(anyset)". Raises a PyExc_SystemError if anyset is not a
     # set, frozenset, or an instance of a subtype.
 
-    int PySet_GET_SIZE(object anyset)
+    Py_ssize_t PySet_GET_SIZE(object anyset)
     # Macro form of PySet_Size() without error checking.
 
     bint PySet_Contains(object anyset, object key) except -1

Cython/Includes/cpython/tuple.pxd

@@ -24,10 +24,10 @@ cdef extern from "Python.h":
     # pointing to Python objects. "PyTuple_Pack(2, a, b)" is
     # equivalent to "Py_BuildValue("(OO)", a, b)".
 
-    int PyTuple_Size(object  p) except -1
+    Py_ssize_t PyTuple_Size(object  p) except -1
     # Take a pointer to a tuple object, and return the size of that tuple.
 
-    int PyTuple_GET_SIZE(object  p)
+    Py_ssize_t PyTuple_GET_SIZE(object  p)
     # Return the size of the tuple p, which must be non-NULL and point
     # to a tuple; no error checking is performed.
 

Cython/Includes/cpython/unicode.pxd

@@ -377,7 +377,8 @@ cdef extern from *:
     # consumed is not NULL, PyUnicode_DecodeMBCSStateful() will not
     # decode trailing lead byte and the number of bytes that have been
     # decoded will be stored in consumed. New in version 2.5.
-    object PyUnicode_DecodeMBCSStateful(char *s, int size, char *errors, int *consumed)
+    # NOTE: Python 2.x uses 'int' values for 'size' and 'consumed' (changed in 3.0)
+    object PyUnicode_DecodeMBCSStateful(char *s, Py_ssize_t size, char *errors, Py_ssize_t *consumed)
 
     # Encode the Py_UNICODE buffer of the given size using MBCS and
     # return a Python string object. Return NULL if an exception was

Cython/Utils.py

@@ -117,12 +117,12 @@ def search_include_directories(dirs, qualified_name, suffix, pos,
 @cached_function
 def find_root_package_dir(file_path):
     dir = os.path.dirname(file_path)
-    while is_package_dir(dir):
-        parent = os.path.dirname(dir)
-        if parent == dir:
-            break
-        dir = parent
-    return dir
+    if file_path == dir:
+        return dir
+    elif is_package_dir(dir):
+        return find_root_package_dir(dir)
+    else:
+        return dir
 
 @cached_function
 def check_package_dir(dir, package_names):

tests/build/build_dir.srctree

+PYTHON setup.py build_ext --inplace
+PYTHON -c "import a"
+PYTHON -c "import pkg.b"
+PYTHON check_paths.py
+
+######## setup.py ########
+
+
+# TODO: Better interface...
+from Cython.Build.Dependencies import cythonize
+
+from distutils.core import setup
+
+setup(
+  ext_modules = (cythonize("*.pyx", build_dir="scratchA") +
+                 cythonize("pkg/*.pyx", build_dir="scratchB")),
+)
+
+######## a.pyx ########
+
+cdef extern from "helper.h":
+    int value1
+    
+cdef extern from "subdir/helper.h":
+    int value2
+
+cdef extern from "pkg/pkg_helper.h":
+    int value3
+
+assert value1 == 100
+assert value2 == 200
+assert value3 == 300
+
+######## helper.h ########
+
+int value1 = 100;
+
+######## subdir/helper.h ########
+
+int value2 = 200;
+
+######## pkg/__init__.py ########
+
+######## pkg/b.pyx ########
+
+cdef extern from "pkg_helper.h":
+    int value3
+
+cdef extern from "subdir/pkg_helper.h":
+    int value4
+
+assert value3 == 300
+assert value4 == 400
+
+######## pkg/pkg_helper.h ########
+
+int value3 = 300;
+
+######## pkg/subdir/pkg_helper.h ########
+
+int value4 = 400;
+
+######## check_paths.py ########
+
+import os
+assert os.path.exists("scratchA/a.c")
+assert os.path.exists("scratchA/helper.h")
+assert os.path.exists("scratchA/subdir/helper.h")
+assert os.path.exists("scratchA/pkg/pkg_helper.h")
+assert not os.path.exists("a.c")
+
+assert os.path.exists("scratchB/pkg/b.c")
+assert os.path.exists("scratchB/pkg/pkg_helper.h")
+assert os.path.exists("scratchB/pkg/subdir/pkg_helper.h")
+assert not os.path.exists("b.c")
+assert not os.path.exists("pkg/b.c")