Merge branch 'master' into enable_tests_for_documentation

Cython/Compiler/ModuleNode.py

@@ -707,7 +707,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         code.putln('static PyObject *%s = NULL;' % env.module_cname)
         code.putln('static PyObject *%s;' % env.module_dict_cname)
         code.putln('static PyObject *%s;' % Naming.builtins_cname)
-        code.putln('static PyObject *%s;' % Naming.cython_runtime_cname)
+        code.putln('static PyObject *%s = NULL;' % Naming.cython_runtime_cname)
         code.putln('static PyObject *%s;' % Naming.empty_tuple)
         code.putln('static PyObject *%s;' % Naming.empty_bytes)
         code.putln('static PyObject *%s;' % Naming.empty_unicode)

Cython/Utility/Exceptions.c

@@ -617,6 +617,12 @@ static int __Pyx_CLineForTraceback(PyThreadState *tstate, int c_line) {
 #if CYTHON_COMPILING_IN_CPYTHON
     PyObject **cython_runtime_dict;
 #endif
+
+    if (unlikely(!${cython_runtime_cname})) {
+        // Very early error where the runtime module is not set up yet.
+        return c_line;
+    }
+
     __Pyx_ErrFetchInState(tstate, &ptype, &pvalue, &ptraceback);
 
 #if CYTHON_COMPILING_IN_CPYTHON

docs/examples/memoryviews/C_func_file.c

+void multiply_by_10_in_C(double arr[], unsigned int n)
+{
+    for (int i = 0; i < n; i++) {
+        arr[i] *= 10;
+    }
+}

docs/examples/memoryviews/memview_to_c.pyx

+# distutils: sources=./C_func_file.c
+# distutils: include_dirs=./
+
+cdef extern from "C_func_file.h":
+    void multiply_by_10_in_C(double *, unsigned int)
+
+import numpy as np
+
+def multiply_by_10(arr): # 'arr' is a one-dimensional numpy array
+
+    if not arr.flags['C_CONTIGUOUS']:
+        arr = np.ascontiguousarray(arr) # Makes a contiguous copy of the numpy array.
+
+    cdef double[::1] arr_memview = arr
+
+    multiply_by_10_in_C(&arr_memview[0], arr_memview.shape[0])
+
+    return arr
+
+
+a = np.ones(5, dtype=np.double)
+print(multiply_by_10(a))
+
+b = np.ones(10, dtype=np.double)
+b = b[::2]  # b is not contiguous.
+
+print(multiply_by_10(b))  # but our function still works as expected.

docs/src/tutorial/clibraries.rst

 
+.. _using_c_libraries:
+
 ******************
 Using C libraries
 ******************

docs/src/userguide/extension_types.rst

@@ -530,6 +530,106 @@ statically sized freelist of ``N`` instances for a given type.  Example::
     penguin = None
     penguin = Penguin('fish 2')  # does not need to allocate memory!
 
+.. _existing-pointers-instantiation:
+
+Instantiation from existing C/C++ pointers
+===========================================
+
+It is quite common to want to instantiate an extension class from an existing
+(pointer to a) data structure, often as returned by external C/C++ functions.
+
+As extension classes can only accept Python objects as arguments in their
+contructors, this necessitates the use of factory functions. For example, ::
+
+    from libc.stdlib cimport malloc, free
+
+    # Example C struct
+    ctypedef struct my_c_struct:
+        int a
+        int b
+
+
+    cdef class WrapperClass:
+        """A wrapper class for a C/C++ data structure"""
+        cdef my_c_struct *_ptr
+        cdef bint ptr_owner
+
+        def __cinit__(self):
+            self.ptr_owner = False
+
+        def __dealloc__(self):
+            # De-allocate if not null and flag is set
+            if self._ptr is not NULL and self.ptr_owner is True:
+                free(self._ptr)
+                self._ptr = NULL
+
+        # Extension class properties
+        @property
+        def a(self):
+            return self._ptr.a if self._ptr is not NULL else None
+
+        @property
+        def b(self):
+            return self._ptr.b if self._ptr is not NULL else None
+
+        @staticmethod
+        cdef WrapperClass from_ptr(my_c_struct *_ptr, bint owner=False):
+            """Factory function to create WrapperClass objects from
+            given my_c_struct pointer.
+
+            Setting ``owner`` flag to ``True`` causes
+            the extension type to ``free`` the structure pointed to by ``_ptr``
+            when the wrapper object is deallocated."""
+            # Call to __new__ bypasses __init__ constructor
+            cdef WrapperClass wrapper = WrapperClass.__new__(WrapperClass)
+            wrapper._ptr = _ptr
+            wrapper.ptr_owner = owner
+            return wrapper
+
+        @staticmethod
+        cdef WrapperClass new_struct():
+            """Factory function to create WrapperClass objects with
+            newly allocated my_c_struct"""
+            cdef my_c_struct *_ptr = <my_c_struct *>malloc(sizeof(my_c_struct))
+            if _ptr is NULL:
+                raise MemoryError
+            _ptr.a = 0
+            _ptr.b = 0
+            return WrapperClass.from_ptr(_ptr, owner=True)
+
+
+To then create a ``WrapperClass`` object from an existing ``my_c_struct``
+pointer, ``WrapperClass.from_ptr(ptr)`` can be used in Cython code. To allocate
+a new structure and wrap it at the same time, ``WrapperClass.new_struct`` can be
+used instead.
+
+It is possible to create multiple Python objects all from the same pointer
+which point to the same in-memory data, if that is wanted, though care must be
+taken when de-allocating as can be seen above.
+Additionally, the ``ptr_owner`` flag can be used to control which
+``WrapperClass`` object owns the pointer and is responsible for de-allocation -
+this is set to ``False`` by default in the example and can be enabled by calling
+``from_ptr(ptr, owner=True)``.
+
+The GIL must *not* be released in ``__dealloc__`` either, or another lock used
+if it is, in such cases or race conditions can occur with multiple
+de-allocations.
+
+Being a part of the object constructor, the ``__cinit__`` method has a Python
+signature, which makes it unable to accept a ``my_c_struct`` pointer as an
+argument.
+
+Attempts to use pointers in a Python signature will result in errors like::
+
+  Cannot convert 'my_c_struct *' to Python object
+
+This is because Cython cannot automatically convert a pointer to a Python
+object, unlike with native types like ``int``.
+
+Note that for native types, Cython will copy the value and create a new Python
+object while in the above case, data is not copied and deallocating memory is
+a responsibility of the extension class.
+
 .. _making_extension_types_weak_referenceable:
 
 Making extension types weak-referenceable

docs/src/userguide/memoryviews.rst

@@ -396,6 +396,8 @@ like::
 
     int [:, :, :] my_memoryview = obj
 
+.. _c_and_fortran_contiguous_memoryviews:
+
 C and Fortran contiguous memoryviews
 ------------------------------------
 
@@ -691,6 +693,50 @@ reject None input straight away in the signature, which is supported in Cython
 Unlike object attributes of extension classes, memoryview slices are not
 initialized to None.
 
+
+Pass data from a C function via pointer
+=======================================
+
+Since use of pointers in C is ubiquitous, here we give a quick example of how
+to call C functions whose arguments contain pointers. Let's suppose you want to
+manage an array (allocate and deallocate) with NumPy (it can also be Python arrays, or
+anything that supports the buffer interface), but you want to perform computation on this
+array with an external C function implemented in :file:`C_func_file.c`:
+
+.. literalinclude:: ../../examples/memoryviews/C_func_file.c
+    :linenos:
+
+This file comes with a header file called :file:`C_func_file.h` containing::
+
+    void multiply_by_10_in_C(double arr[], unsigned int n);
+
+where ``arr`` points to the array and ``n`` is its size.
+
+You can call the function in a Cython file in the following way:
+
+.. literalinclude:: ../../examples/memoryviews/memview_to_c.pyx
+    :linenos:
+
+Several things to note:
+ - ``::1`` requests a C contiguous view, and fails if the buffer is not C contiguous.
+   See :ref:`c_and_fortran_contiguous_memoryviews`.
+ - ``&arr_memview[0]`` can be understood as 'the adress of the first element of the
+   memoryview'. For contiguous arrays, this is equivalent to the
+   start address of the flat memory buffer.
+ - ``arr_memview.shape[0]`` could have been replaced by ``arr_memview.size``,
+   ``arr.shape[0]`` or ``arr.size``. But ``arr_memview.shape[0]`` is more efficient
+   because it doesn't require any Python interaction.
+ - ``multiply_by_10`` will perform computation in-place if the array passed is contiguous,
+   and will return a new numpy array if ``arr`` is not contiguous.
+ - If you are using Python arrays instead of numpy arrays, you don't need to check
+   if the data is stored contiguously as this is always the case. See :ref:`array-array`.
+
+This way, you can call the C function similar to a normal Python function,
+and leave all the memory management and cleanup to NumPy arrays and Python's
+object handling. For the details of how to compile and
+call functions in C files, see :ref:`using_c_libraries`.
+
+
 .. _GIL: http://docs.python.org/dev/glossary.html#term-global-interpreter-lock
 .. _new style buffers: http://docs.python.org/c-api/buffer.html
 .. _pep 3118: http://www.python.org/peps/pep-3118.html

docs/src/userguide/special_methods.rst

@@ -88,6 +88,8 @@ complaining about the signature mismatch.
     It often helps to directly call ``__new__()`` in this function to bypass the
     call to the ``__init__()`` constructor.
 
+    See :ref:`existing-pointers-instantiation` for an example.
+
 .. Note::
 
     Older Cython files may use :meth:`__new__` rather than :meth:`__cinit__`. The two are synonyms.

runtests.py

@@ -1863,7 +1863,8 @@ def main():
                       action="store_true",
                       help="stop on first failure or error")
     parser.add_option("--root-dir", dest="root_dir", default=os.path.join(DISTDIR, 'tests'),
-                      help="working directory")
+                      help=("Directory to look for the file based "
+                            "tests (the ones which are deactivated with '--no-file'."))
     parser.add_option("--examples-dir", dest="examples_dir",
                       default=os.path.join(DISTDIR, 'docs', 'examples'),
                       help="working directory")