Support casting pointers to cython.array

Cython/Compiler/Buffer.py

@@ -5,7 +5,7 @@ from ExprNodes import *
 from StringEncoding import EncodedString
 from Errors import CompileError
 from UtilityCode import CythonUtilityCode
-from Code import UtilityCode
+from Code import UtilityCode, ContentHashingUtilityCode
 import Cython.Compiler.Options
 import Interpreter
 import PyrexTypes
@@ -34,10 +34,11 @@ class IntroduceBufferAuxiliaryVars(CythonTransform):
         assert isinstance(node, ModuleNode)
         self.max_ndim = 0
         result = super(IntroduceBufferAuxiliaryVars, self).__call__(node)
-        if self.buffers_exists or self.using_memoryview:
+        if self.buffers_exists:
             use_bufstruct_declare_code(node.scope)
             use_py2_buffer_functions(node.scope)
-            use_empty_bufstruct_code(node.scope, self.max_ndim)
+            node.scope.use_utility_code(empty_bufstruct_utility)
+
         return result
 
 
@@ -476,13 +477,15 @@ def put_buffer_lookup_code(entry, index_signeds, index_cnames, directives,
 def use_bufstruct_declare_code(env):
     env.use_utility_code(buffer_struct_declare_code)
 
-def use_empty_bufstruct_code(env, max_ndim):
+
+def get_empty_bufstruct_code(max_ndim):
     code = dedent("""
         Py_ssize_t __Pyx_zeros[] = {%s};
         Py_ssize_t __Pyx_minusones[] = {%s};
     """) % (", ".join(["0"] * max_ndim), ", ".join(["-1"] * max_ndim))
-    env.use_utility_code(UtilityCode(proto=code))
+    return UtilityCode(proto=code)
 
+empty_bufstruct_utility = get_empty_bufstruct_code(Options.buffer_max_dims)
 
 def buf_lookup_full_code(proto, defin, name, nd):
     """
@@ -725,12 +728,18 @@ def get_type_information_cname(code, dtype, maxdepth=None):
             print dtype
             assert False
 
-        typecode.putln(('static __Pyx_TypeInfo %s = { "%s", %s, sizeof(%s), \'%s\' };'
+        if dtype.is_int:
+            is_unsigned = "IS_UNSIGNED(%s)" % declcode
+        else:
+            is_unsigned = "0"
+
+        typecode.putln(('static __Pyx_TypeInfo %s = { "%s", %s, sizeof(%s), \'%s\', %s };'
                         ) % (name,
                              rep,
                              structinfo_name,
                              declcode,
                              typegroup,
+                             is_unsigned,
                         ), safe=True)
     return name
 
@@ -753,5 +762,11 @@ impl = """
 """)
 
 raise_buffer_fallback_code = load_buffer_utility("BufferFallbackError")
+buffer_structs_code = load_buffer_utility("BufferFormatStructs")
 acquire_utility_code = load_buffer_utility("BufferFormatCheck",
-                                           context=context)
\ No newline at end of file
+                                           context=context,
+                                           requires=[buffer_structs_code])
+
+# See utility code BufferFormatFromTypeInfo
+_typeinfo_to_format_code = load_buffer_utility(
+        "TypeInfoToFormat", context={}, requires=[buffer_structs_code])
\ No newline at end of file

Cython/Compiler/Code.py

@@ -318,7 +318,12 @@ class ContentHashingUtilityCode(UtilityCode):
         return hash((self.proto, self.impl))
 
     def __eq__(self, other):
-        return (self.proto, self.impl) == (other.proto, other.impl)
+        if not isinstance(other, type(self)):
+            return False
+
+        self_proto = getattr(self, 'proto', None)
+        other_proto = getattr(other, 'proto', None)
+        return (self_proto, self.impl) == (other_proto, other.impl)
 
 
 class LazyUtilityCode(UtilityCodeBase):

Cython/Compiler/CythonScope.py

@@ -71,12 +71,14 @@ class CythonScope(ModuleScope):
 #        self.test_cythonscope()
 
     def test_cythonscope(self):
-        # A special function just to make it easy to test the scope and
-        # utility code functionality in isolation. It is available to
-        # "end-users" but nobody will know it is there anyway...
+        """
+        Creates some entries for testing purposes and entries for
+        cython.array() and for cython.view.*.
+        """
         cython_testscope_utility_code.declare_in_scope(self)
         cython_test_extclass_utility_code.declare_in_scope(self)
-        cython_array_utility_code.declare_in_scope(self)
+
+        MemoryView.cython_array_utility_code.declare_in_scope(self)
 
         #
         # The view sub-scope
@@ -92,7 +94,9 @@ class CythonScope(ModuleScope):
 
         cythonview_testscope_utility_code.declare_in_scope(viewscope)
 
-        view_utility_code.declare_in_scope(viewscope)
+        view_utility_scope = MemoryView.view_utility_code.declare_in_scope(viewscope)
+        MemoryView.memview_fromslice_utility_code.from_scope = view_utility_scope
+        MemoryView.memview_fromslice_utility_code.declare_in_scope(viewscope)
 
 
 def create_cython_scope(context, create_testscope):
@@ -132,15 +136,4 @@ cython_test_extclass_utility_code = \
                            requires=[undecorated_methods_protos,
                                      test_cython_utility_dep])
 
-cythonview_testscope_utility_code = load_testscope_utility("View.TestScope")
-
-view_utility_code = MemoryView.load_memview_cy_utility(
-        "View.MemoryView", context=MemoryView.context,
-        requires=[Buffer.GetAndReleaseBufferUtilityCode(),
-                  MemoryView.memviewslice_declare_code],
-)
-
-cython_array_utility_code = MemoryView.load_memview_cy_utility(
-        "CythonArray",
-        context=MemoryView.context,
-        requires=[view_utility_code])
+cythonview_testscope_utility_code = load_testscope_utility("View.TestScope")
\ No newline at end of file

Cython/Compiler/ExprNodes.py

@@ -584,6 +584,11 @@ class ExprNode(Node):
         if dst_type.is_reference:
             dst_type = dst_type.ref_base_type
 
+        if self.coercion_type is not None:
+            # This is purely for error checking purposes!
+            node = NameNode(self.pos, name='', type=self.coercion_type)
+            node.coerce_to(dst_type, env)
+
         if dst_type.is_memoryviewslice:
             import MemoryView
             if not src.type.is_memoryviewslice:
@@ -747,6 +752,7 @@ class PyConstNode(AtomicExprNode):
 class NoneNode(PyConstNode):
     #  The constant value None
 
+    is_none = 1
     value = "Py_None"
 
     constant_result = None
@@ -6051,6 +6057,157 @@ class TypecastNode(ExprNode):
             code.put_incref(self.result(), self.ctype())
 
 
+ERR_START = "Start may not be given"
+ERR_NOT_STOP = "Stop must be provided to indicate shape"
+ERR_STEPS = ("Strides may only be given to indicate contiguity. "
+             "Consider slicing it after conversion")
+ERR_NOT_POINTER = "Can only create cython.array from pointer"
+ERR_BASE_TYPE = "Pointer base type does not match cython.array base type"
+
+class CythonArrayNode(ExprNode):
+    """
+    Used when a pointer of base_type is cast to a memoryviewslice with that
+    base type. i.e.
+
+        <int[::1, :]> p
+
+    creates a fortran-contiguous cython.array.
+
+    We leave the type set to object so coercions to object are more efficient
+    and less work. Acquiring a memoryviewslice from this will be just as
+    efficient. ExprNode.coerce_to() will do the additional typecheck on
+    self.compile_time_type
+    """
+
+    subexprs = ['operand', 'shapes']
+
+    shapes = None
+    is_temp = True
+    mode = "c"
+
+    shape_type = PyrexTypes.c_py_ssize_t_type
+
+    def analyse_types(self, env):
+        import MemoryView
+
+        self.type = error_type
+        self.env = env
+
+        self.shapes = []
+
+        for axis_no, axis in enumerate(self.base_type_node.axes):
+            if not axis.start.is_none:
+                return error(axis.start.pos, ERR_START)
+
+            if axis.stop.is_none:
+                return error(axis.pos, ERR_NOT_STOP)
+
+            axis.stop.analyse_types(env)
+            shape = axis.stop.coerce_to(self.shape_type, env)
+            if not shape.is_literal:
+                shape.coerce_to_temp(env)
+
+            self.shapes.append(shape)
+
+            if not axis.stop.type.is_int:
+                return error(axis.stop.pos, "Expected an integer type")
+
+            first_or_last = axis_no in (0, len(self.base_type_node.axes) - 1)
+            if not axis.step.is_none and first_or_last:
+                axis.step.analyse_types(env)
+                if (not axis.step.type.is_int and axis.step.is_literal and not
+                        axis.step.type.is_error):
+                    return error(axis.step.pos, "Expected an integer literal")
+
+                if axis.step.compile_time_value(env) != 1:
+                    return error(axis.step.pos, ERR_STEPS)
+
+                if axis_no == 0:
+                    self.mode = "fortran"
+
+            elif axis.step and not first_or_last:
+                return error(axis.step.pos, ERR_STEPS)
+
+        self.operand.analyse_types(env)
+        array_dtype = self.base_type_node.base_type_node.analyse(env)
+
+        if not self.operand.type.is_ptr:
+            return error(self.operand.pos, ERR_NOT_POINTER)
+
+        elif not self.operand.type.base_type.same_as(array_dtype):
+            return error(self.operand.pos, ERR_BASE_TYPE)
+
+        #self.operand = self.operand.coerce_to(PyrexTypes.c_char_ptr_type, env)
+        if not self.operand.is_name:
+            self.operand = self.operand.coerce_to_temp(env)
+
+        axes = [('direct', 'follow')] * len(self.base_type_node.axes)
+        if self.mode == "fortran":
+            axes[0] = ('direct', 'contig')
+        else:
+            axes[-1] = ('direct', 'contig')
+
+        self.coercion_type = PyrexTypes.MemoryViewSliceType(array_dtype, axes)
+        #self.type = py_object_type
+        self.type = env.global_scope().context.cython_scope.lookup("array").type
+        assert self.type
+
+        env.use_utility_code(MemoryView.cython_array_utility_code)
+        env.use_utility_code(MemoryView.typeinfo_to_format_code)
+
+    def allocate_temp_result(self, code):
+        if self.temp_code:
+            raise RuntimeError("temp allocated mulitple times")
+
+        self.temp_code = code.funcstate.allocate_temp(self.type, True)
+
+    def generate_result_code(self, code):
+        import Buffer
+
+        shapes = [self.shape_type.cast_code(shape.result())
+                      for shape in self.shapes]
+        dtype = self.coercion_type.dtype
+
+        shapes_temp = code.funcstate.allocate_temp(py_object_type, True)
+        format_temp = code.funcstate.allocate_temp(py_object_type, True)
+
+        itemsize = "sizeof(%s)" % dtype.declaration_code("")
+        type_info = Buffer.get_type_information_cname(code, dtype)
+
+        code.putln("if (!%s) {" % self.operand.result())
+        code.putln(    'PyErr_SetString(PyExc_ValueError,'
+                            '"Cannot create cython.array from NULL pointer");')
+        code.putln(code.error_goto(self.operand.pos))
+        code.putln("}")
+
+        code.putln("%s = __pyx_format_from_typeinfo(&%s);" %
+                                                (format_temp, type_info))
+        code.putln('%s = Py_BuildValue("(%s)", %s);' % (shapes_temp,
+                                                        "n" * len(shapes),
+                                                        ", ".join(shapes)))
+
+        err = "!%s || !%s || !PyBytes_Check(%s)" % (format_temp, shapes_temp,
+                                                    format_temp)
+        code.putln(code.error_goto_if(err, self.pos))
+        code.put_gotref(format_temp)
+        code.put_gotref(shapes_temp)
+
+        tup = (self.result(), shapes_temp, itemsize, format_temp,
+               self.mode, self.operand.result())
+        code.putln('%s = __pyx_array_new('
+                            '%s, %s, PyBytes_AS_STRING(%s), '
+                            '"%s", (char *) %s);' % tup)
+        code.putln(code.error_goto_if_null(self.result(), self.pos))
+        code.put_gotref(self.result())
+
+        def dispose(temp):
+            code.put_decref_clear(temp, py_object_type)
+            code.funcstate.release_temp(temp)
+
+        dispose(shapes_temp)
+        dispose(format_temp)
+
+
 class SizeofNode(ExprNode):
     #  Abstract base class for sizeof(x) expression nodes.
 
@@ -7910,6 +8067,10 @@ class CoerceToPyTypeNode(CoercionNode):
             # FIXME: check that the target type and the resulting type are compatible
             pass
 
+        if arg.type.is_memoryviewslice:
+            # Register utility codes at this point
+            arg.type.get_to_py_function(env, arg)
+
         self.env = env
 
     gil_message = "Converting to Python object"

Cython/Compiler/MemoryView.py

@@ -66,8 +66,6 @@ memview_typeptr_cname = '__pyx_memoryview_type'
 memview_objstruct_cname = '__pyx_memoryview_obj'
 memviewslice_cname = u'__Pyx_memviewslice'
 
-
-
 def put_init_entry(mv_cname, code):
     code.putln("%s.data = NULL;" % mv_cname)
     code.putln("%s.memview = NULL;" % mv_cname)
@@ -129,18 +127,9 @@ def get_buf_flags(specs):
     else:
         return memview_strided_access
 
-def use_memview_util_code(env):
-    import CythonScope
-    env.use_utility_code(CythonScope.view_utility_code)
-    env.use_utility_code(memviewslice_declare_code)
-
-def use_memview_cwrap(env):
-    import CythonScope
-    env.use_utility_code(CythonScope.view_utility_code)
 
 def use_cython_array(env):
-    import CythonScope
-    env.use_utility_code(CythonScope.cython_array_utility_code)
+    env.use_utility_code(cython_array_utility_code)
 
 def src_conforms_to_dst(src, dst):
     '''
@@ -778,7 +767,31 @@ memviewslice_declare_code = load_memview_c_utility(
 memviewslice_init_code = load_memview_c_utility(
     "MemviewSliceInit",
     context=dict(context, BUF_MAX_NDIMS=Options.buffer_max_dims),
-    requires=[memviewslice_declare_code, Buffer.acquire_utility_code],
+    requires=[memviewslice_declare_code,
+              Buffer.acquire_utility_code],
+)
+
+memviewslice_index_helpers = load_memview_c_utility("MemviewSliceIndex")
+
+typeinfo_to_format_code = load_memview_cy_utility(
+        "BufferFormatFromTypeInfo", requires=[Buffer._typeinfo_to_format_code])
+
+view_utility_code = load_memview_cy_utility(
+        "View.MemoryView",
+        context=context,
+        requires=[Buffer.GetAndReleaseBufferUtilityCode(),
+                  Buffer.buffer_struct_declare_code,
+                  Buffer.empty_bufstruct_utility,
+                  memviewslice_init_code],
 )
 
-memviewslice_index_helpers = load_memview_c_utility("MemviewSliceIndex")
\ No newline at end of file
+cython_array_utility_code = load_memview_cy_utility(
+        "CythonArray",
+        context=context,
+        requires=[view_utility_code])
+
+memview_fromslice_utility_code = load_memview_cy_utility(
+        "MemviewFromSlice",
+        context=context,
+        requires=[view_utility_code],
+)
\ No newline at end of file

Cython/Compiler/ModuleNode.py

@@ -2440,8 +2440,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                     type.typeptr_cname, type.typeobj_cname))
 
 def generate_cfunction_declaration(entry, env, code, definition):
+    from_cy_utility = entry.used and entry.utility_code_definition
     if entry.inline_func_in_pxd or (not entry.in_cinclude and (definition
-            or entry.defined_in_pxd or entry.visibility == 'extern')):
+            or entry.defined_in_pxd or entry.visibility == 'extern' or from_cy_utility)):
         if entry.visibility == 'extern':
             storage_class = "%s " % Naming.extern_c_macro
             dll_linkage = "DL_IMPORT"

Cython/Compiler/Nodes.py

@@ -126,6 +126,7 @@ class Node(object):
         __metaclass__ = VerboseCodeWriter
 
     is_name = 0
+    is_none = 0
     is_literal = 0
     is_terminator = 0
     temps = None
@@ -137,6 +138,11 @@ class Node(object):
 
     cf_state = None
 
+    # This may be an additional (or 'actual') type that will be checked when
+    # this node is coerced to another type. This could be useful to set when
+    # the actual type to which it can coerce is known, but you want to leave
+    # the type a py_object_type
+    coercion_type = None
 
     def __init__(self, pos, **kw):
         self.pos = pos
@@ -823,12 +829,18 @@ class MemoryViewSliceTypeNode(CBaseTypeNode):
             return self.type
 
         self.type = PyrexTypes.MemoryViewSliceType(base_type, axes_specs)
-        MemoryView.use_memview_util_code(env)
-        MemoryView.use_cython_array(env)
-        MemoryView.use_memview_util_code(env)
-        env.use_utility_code(MemoryView.memviewslice_declare_code)
+        if self.type.dtype.is_memoryviewslice:
+            error(self.pos, "Memoryview slices may not be used as the "
+                            "base type for memoryview slices")
+
+        self.use_memview_utilities(env)
         return self.type
 
+    def use_memview_utilities(self, env):
+        import MemoryView
+        env.use_utility_code(MemoryView.view_utility_code)
+
+
 class CNestedBaseTypeNode(CBaseTypeNode):
     # For C++ classes that live inside other C++ classes.
 

Cython/Compiler/Parsing.py

@@ -297,7 +297,8 @@ def p_typecast(s):
     pos = s.position()
     s.next()
     base_type = p_c_base_type(s)
-    if base_type.name is None:
+    is_memslice = isinstance(base_type, Nodes.MemoryViewSliceTypeNode)
+    if not is_memslice and base_type.name is None:
         s.error("Unknown type")
     declarator = p_c_declarator(s, empty = 1)
     if s.sy == '?':
@@ -307,6 +308,10 @@ def p_typecast(s):
         typecheck = 0
     s.expect(">")
     operand = p_factor(s)
+    if is_memslice:
+        return ExprNodes.CythonArrayNode(pos, base_type_node=base_type,
+                                         operand=operand)
+
     return ExprNodes.TypecastNode(pos,
         base_type = base_type,
         declarator = declarator,

Cython/Compiler/PyrexTypes.py

@@ -553,14 +553,15 @@ class MemoryViewSliceType(PyrexType):
         return True
 
     def get_to_py_function(self, env, obj):
+        import MemoryView
+        env.use_utility_code(MemoryView.memview_fromslice_utility_code)
+
         to_py_func, from_py_func = self.dtype_object_conversion_funcs(env)
         to_py_func = "(PyObject *(*)(char *)) " + to_py_func
         from_py_func = "(int (*)(char *, PyObject *)) " + from_py_func
 
-        tup = (obj.result(), obj.result(), self.flags, self.ndim,
-               to_py_func, from_py_func)
-        return ("__pyx_memoryview_fromslice(&%s, %s.memview->obj, "
-                                            "%s, %s, %s, %s);" % tup)
+        tup = (obj.result(), self.ndim, to_py_func, from_py_func)
+        return "__pyx_memoryview_fromslice(&%s, %s, %s, %s);" % tup
 
     def dtype_object_conversion_funcs(self, env):
         import MemoryView, Code

Cython/Compiler/Symtab.py

@@ -291,22 +291,21 @@ class Scope(object):
         entries = [(name, entry)
                        for name, entry in other.entries.iteritems()
                            if entry.used or merge_unused]
-        # !@#$ py23
-        entries = dict(entries)
 
         self.entries.update(entries)
 
         for attr in ('const_entries',
-                  'type_entries',
-                  'sue_entries',
-                  'arg_entries',
-                  'var_entries',
-                  'pyfunc_entries',
-                  'cfunc_entries',
-                  'c_class_entries'):
+                     'type_entries',
+                     'sue_entries',
+                     'arg_entries',
+                     'var_entries',
+                     'pyfunc_entries',
+                     'cfunc_entries',
+                     'c_class_entries'):
             self_entries = getattr(self, attr)
+            names = set([e.name for e in self_entries])
             for entry in getattr(other, attr):
-                if entry.used or merge_unused:
+                if (entry.used or merge_unused) and entry.name not in names:
                     self_entries.append(entry)
 
     def __str__(self):

Cython/Compiler/UtilityCode.py

@@ -115,8 +115,8 @@ class CythonUtilityCode(Code.UtilityCodeBase):
                 return module_node
 
             transform = ParseTreeTransforms.AnalyseDeclarationsTransform
-            pipeline = Pipeline.insert_into_pipeline(pipeline, transform,
-                                                     before=scope_transform)
+            pipeline = Pipeline.insert_into_pipeline(pipeline, scope_transform,
+                                                     before=transform)
 
         (err, tree) = Pipeline.run_pipeline(pipeline, tree)
         assert not err, err
@@ -125,7 +125,7 @@ class CythonUtilityCode(Code.UtilityCodeBase):
     def put_code(self, output):
         pass
 
-    def declare_in_scope(self, dest_scope, used=False, modname=None):
+    def declare_in_scope(self, dest_scope, used=False):
         """
         Declare all entries from the utility code in dest_scope. Code will only
         be included for used entries. If module_name is given, declare the
@@ -143,13 +143,12 @@ class CythonUtilityCode(Code.UtilityCodeBase):
             entry.utility_code_definition = self
             entry.used = used
 
-            if modname and entry.type.is_extension_type:
-                entry.qualified_name = modname
-                entry.type.module_name = modname
-
-        dest_scope.merge_in(tree.scope, merge_unused=True)
+        original_scope = tree.scope
+        dest_scope.merge_in(original_scope, merge_unused=True)
         tree.scope = dest_scope
 
         for dep in self.requires:
             if dep.is_cython_utility:
                 dep.declare_in_scope(dest_scope)
+
+        return original_scope

Cython/Utility/Buffer.c

@@ -35,19 +35,9 @@ static void __Pyx_RaiseBufferFallbackError(void) {
      "Buffer acquisition failed on assignment; and then reacquiring the old buffer failed too!");
 }
 
-/////////////// BufferFormatCheck.proto ///////////////
-{{#
-
-    Buffer format string checking
-
-    Buffer type checking. Utility code for checking that acquired
-    buffers match our assumptions. We only need to check ndim and
-    the format string; the access mode/flags is checked by the
-    exporter.
-
-    The alignment code is copied from _struct.c in Python.
-}}
+/////////////// BufferFormatStructs.proto ///////////////
 
+#define IS_UNSIGNED(type) (((type) -1) > 0)
 
 /* Run-time type information about structs used with buffers */
 struct __Pyx_StructField_;
@@ -57,6 +47,7 @@ typedef struct {
   struct __Pyx_StructField_* fields;
   size_t size;     /* sizeof(type) */
   char typegroup; /* _R_eal, _C_omplex, Signed _I_nt, _U_nsigned int, _S_truct, _P_ointer, _O_bject */
+  char is_unsigned;
 } __Pyx_TypeInfo;
 
 typedef struct __Pyx_StructField_ {
@@ -82,6 +73,19 @@ typedef struct {
 } __Pyx_BufFmt_Context;
 
 
+/////////////// BufferFormatCheck.proto ///////////////
+{{#
+
+    Buffer format string checking
+
+    Buffer type checking. Utility code for checking that acquired
+    buffers match our assumptions. We only need to check ndim and
+    the format string; the access mode/flags is checked by the
+    exporter.
+
+    The alignment code is copied from _struct.c in Python.
+}}
+
 static CYTHON_INLINE int  __Pyx_GetBufferAndValidate(Py_buffer* buf, PyObject* obj,
     __Pyx_TypeInfo* dtype, int flags, int nd, int cast, __Pyx_BufFmt_StackElem* stack);
 static CYTHON_INLINE void __Pyx_SafeReleaseBuffer(Py_buffer* info);
@@ -246,6 +250,7 @@ static char __Pyx_BufFmt_TypeCharToGroup(char ch, int is_complex) {
   }
 }
 
+
 static void __Pyx_BufFmt_RaiseExpected(__Pyx_BufFmt_Context* ctx) {
   if (ctx->head == NULL || ctx->head->field == &ctx->root) {
     const char* expected;
@@ -519,3 +524,60 @@ static CYTHON_INLINE void __Pyx_SafeReleaseBuffer(Py_buffer* info) {
   if (info->suboffsets == __Pyx_minusones) info->suboffsets = NULL;
   __Pyx_ReleaseBuffer(info);
 }
+
+
+/////////////// TypeInfoToFormat.proto ///////////////
+struct __pyx_typeinfo_string {
+    char string[3];
+};
+static struct __pyx_typeinfo_string __Pyx_TypeInfoToFormat(__Pyx_TypeInfo *type);
+
+/////////////// TypeInfoToFormat ///////////////
+{{# See also MemoryView.pyx:BufferFormatFromTypeInfo }}
+
+static struct __pyx_typeinfo_string __Pyx_TypeInfoToFormat(__Pyx_TypeInfo *type) {
+    struct __pyx_typeinfo_string result = { {0} };
+    char *buf = (char *) result.string;
+    size_t size = type->size;
+
+    switch (type->typegroup) {
+        case 'I':
+        case 'U':
+            if (size == 1)
+                *buf = 'c';
+            else if (size == 2)
+                *buf = 'h';
+            else if (size == 4)
+                *buf = 'i';
+            else if (size == 8)
+                *buf = 'q';
+
+            if (type->is_unsigned)
+                *buf = toupper(*buf);
+            break;
+        case 'P':
+            *buf = 'P';
+            break;
+        case 'C':
+          {
+             __Pyx_TypeInfo complex_type = *type;
+            complex_type.typegroup = 'R';
+            complex_type.size /= 2;
+
+            *buf++ = 'Z';
+            /* Note: What about short/int/long complex? Probably not used? */
+            *buf = __Pyx_TypeInfoToFormat(&complex_type).string[0];
+            break;
+          }
+        case 'R':
+            if (size == 4)
+                *buf = 'f';
+            else if (size == 8)
+                *buf = 'd';
+            else
+                *buf = 'g';
+            break;
+    }
+
+    return result;
+}

Cython/Utility/MemoryView.pyx

@@ -12,6 +12,10 @@ cdef extern from "Python.h":
         PyBUF_ANY_CONTIGUOUS
         PyBUF_FORMAT
 
+
+    void Py_INCREF(object)
+    void Py_DECREF(object)
+
 cdef extern from *:
     object __pyx_memoryview_new(object obj, int flags)
 
@@ -52,6 +56,8 @@ cdef class array:
         self.strides = <Py_ssize_t *> malloc(sizeof(Py_ssize_t)*self.ndim)
 
         if not self.shape or not self.strides:
+            free(self.shape)
+            free(self.strides)
             raise MemoryError("unable to allocate shape or strides.")
 
         cdef int idx
@@ -93,7 +99,6 @@ cdef class array:
                 raise MemoryError("unable to allocate array data.")
 
     def __getbuffer__(self, Py_buffer *info, int flags):
-
         cdef int bufmode = -1
         if self.mode == b"c":
             bufmode = PyBUF_C_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS
@@ -114,27 +119,28 @@ cdef class array:
         else:
             info.format = NULL
 
-        # we do not need to call releasebuffer
-        info.obj = None
+        # info.obj = self
+        # Py_INCREF(self)
 
-    def __releasebuffer__(array self, Py_buffer* info):
-        # array.__releasebuffer__ should not be called,
-        # because the Py_buffer's 'obj' field is set to None.
-        raise NotImplementedError()
+    def __releasebuffer__(self, Py_buffer *info):
+        pass
 
     def __dealloc__(array self):
-        if self.data:
-            if self.callback_free_data != NULL:
-                self.callback_free_data(self.data)
-            else:
-                free(self.data)
-            self.data = NULL
+        if self.callback_free_data != NULL:
+            self.callback_free_data(self.data)
+        else:
+            free(self.data)
+
+        self.data = NULL
+
         if self.strides:
             free(self.strides)
             self.strides = NULL
+
         if self.shape:
             free(self.shape)
             self.shape = NULL
+
         self.format = NULL
         self.itemsize = 0
 
@@ -148,8 +154,15 @@ cdef class array:
 
 
 @cname("__pyx_array_new")
-cdef array array_cwrapper(tuple shape, Py_ssize_t itemsize, char *format, char *mode):
-    return array(shape, itemsize, format, mode.decode('ASCII'))
+cdef array array_cwrapper(tuple shape, Py_ssize_t itemsize, char *format, char *mode, char *buf):
+    cdef array result
+    if buf == NULL:
+        result = array(shape, itemsize, format, mode.decode('ASCII'))
+    else:
+        result = array(shape, itemsize, format, mode.decode('ASCII'), allocate_buffer=False)
+        result.data = buf
+
+    return result
 
 ########## View.MemoryView ##########
 
@@ -169,14 +182,7 @@ cdef extern from *:
     int __Pyx_GetBuffer(object, Py_buffer *, int) except -1
     void __Pyx_ReleaseBuffer(Py_buffer *)
 
-    ctypedef struct {{memviewslice_name}}:
-        char *data
-        Py_ssize_t shape[{{max_dims}}]
-        Py_ssize_t strides[{{max_dims}}]
-        Py_ssize_t suboffsets[{{max_dims}}]
-
-    void puts(char *)
-    void printf(char *, ...)
+    ctypedef struct PyObject
 
 
 @cname('__pyx_MemviewEnum')
@@ -206,14 +212,18 @@ cdef class memoryview(object):
 
     def __cinit__(memoryview self, object obj, int flags):
         self.obj = obj
-        __Pyx_GetBuffer(obj, &self.view, flags)
+
+        if type(self) is memoryview or obj is not None:
+            __Pyx_GetBuffer(obj, &self.view, flags)
 
         self.lock = PyThread_allocate_lock()
         if self.lock == NULL:
             raise MemoryError
 
     def __dealloc__(memoryview self):
-        __Pyx_ReleaseBuffer(&self.view)
+        if self.obj is not None:
+            __Pyx_ReleaseBuffer(&self.view)
+
         if self.lock != NULL:
             PyThread_free_lock(self.lock)
 
@@ -265,7 +275,11 @@ cdef class memoryview(object):
         cdef bytes bytesitem
         # Do a manual and complete check here instead of this easy hack
         bytesitem = itemp[:self.view.itemsize]
-        return struct.unpack(self.view.format, bytesitem)
+        result = struct.unpack(self.view.format, bytesitem)
+        if len(self.view.format) == 1:
+            return result[0]
+
+        return result
 
     cdef assign_item_from_object(self, char *itemp, object value):
         """Only used if instantiated manually by the user, or if Cython doesn't
@@ -283,11 +297,92 @@ cdef class memoryview(object):
         for i, c in enumerate(bytesvalue):
             itemp[i] = c
 
+    property _obj:
+        @cname('__pyx_memoryview__get__obj')
+        def __get__(self):
+            if (self.obj is None and <PyObject *> self.view.obj != NULL and
+                    self.view.obj is not None):
+                return <object> self.view.obj
+
+            return self.obj
+
     def __repr__(self):
-        return "<MemoryView of %r at 0x%x>" % (self.obj.__class__.__name__, id(self))
+        return "<MemoryView of %r at 0x%x>" % (self._obj.__class__.__name__, id(self))
 
     def __str__(self):
-        return "<MemoryView of %r object>" % (self.obj.__class__.__name__,)
+        return "<MemoryView of %r object>" % (self._obj.__class__.__name__,)
+
+
+@cname('__pyx_memoryview_new')
+cdef memoryview_cwrapper(object o, int flags):
+    return memoryview(o, flags)
+
+
+cdef _check_index(index):
+    if not PyIndex_Check(index):
+        raise TypeError("Cannot index with %s" % type(index))
+
+cdef tuple _unellipsify(tuple tup, int ndim):
+    if Ellipsis in tup:
+        result = []
+        for idx, item in enumerate(tup):
+            if item is Ellipsis:
+                result.extend([slice(None)] * (ndim - len(tup) + 1))
+                result.extend(tup[idx + 1:])
+                break
+
+            result.append(item)
+
+        return tuple(result)
+
+    return tup
+
+@cname('__pyx_pybuffer_index')
+cdef char *pybuffer_index(Py_buffer *view, char *bufp, Py_ssize_t index, int dim) except NULL:
+    cdef Py_ssize_t shape, stride, suboffset = -1
+    cdef Py_ssize_t itemsize = view.itemsize
+    cdef char *resultp
+
+    if view.ndim == 0:
+        shape = view.len / itemsize
+        stride = itemsize
+    else:
+        shape = view.shape[dim]
+        stride = view.strides[dim]
+        if view.suboffsets != NULL:
+            suboffset = view.suboffsets[dim]
+
+    if index < 0:
+        index += view.shape[dim]
+        if index < 0:
+            raise IndexError("Out of bounds on buffer access (axis %d)" % dim)
+
+    if index >= shape:
+        raise IndexError("Out of bounds on buffer access (axis %d)" % dim)
+
+    resultp = bufp + index * stride
+    if suboffset >= 0:
+        resultp = (<char **> resultp)[0] + suboffset
+
+    return resultp
+
+############### MemviewFromSlice ###############
+
+cdef extern from *:
+    cdef struct __pyx_memoryview:
+        Py_buffer view
+        PyObject *obj
+
+    ctypedef struct {{memviewslice_name}}:
+        __pyx_memoryview *memview
+        char *data
+        Py_ssize_t shape[{{max_dims}}]
+        Py_ssize_t strides[{{max_dims}}]
+        Py_ssize_t suboffsets[{{max_dims}}]
+
+
+    void __PYX_INC_MEMVIEW({{memviewslice_name}} *memslice, int have_gil)
+    void __PYX_XDEC_MEMVIEW({{memviewslice_name}} *memslice, int have_gil)
 
 
 @cname('__pyx_memoryviewslice')
@@ -296,17 +391,14 @@ cdef class _memoryviewslice(memoryview):
 
     # We need this to keep our shape/strides/suboffset pointers valid
     cdef {{memviewslice_name}} from_slice
-    # Restore the original Py_buffer before releasing
-    cdef Py_buffer orig_view
+    # We need this only to print it's classes name
+    cdef object from_object
 
     cdef object (*to_object_func)(char *)
     cdef int (*to_dtype_func)(char *, object) except 0
 
-    def __cinit__(self, object obj, int flags):
-        self.orig_view = self.view
-
     def __dealloc__(self):
-        self.view = self.orig_view
+        __PYX_XDEC_MEMVIEW(&self.from_slice, 1)
 
     cdef convert_item_to_object(self, char *itemp):
         if self.to_object_func != NULL:
@@ -320,21 +412,29 @@ cdef class _memoryviewslice(memoryview):
         else:
             memoryview.assign_item_from_object(self, itemp, value)
 
+    property _obj:
+        @cname('__pyx_memoryviewslice__get__obj')
+        def __get__(self):
+            return self.from_object
 
-@cname('__pyx_memoryview_new')
-cdef memoryview_cwrapper(object o, int flags):
-    return memoryview(o, flags)
 
 @cname('__pyx_memoryview_fromslice')
 cdef memoryview_from_memslice_cwrapper(
-            {{memviewslice_name}} *memviewslice, object orig_obj, int flags, int cur_ndim,
+            {{memviewslice_name}} *memviewslice, int cur_ndim,
              object (*to_object_func)(char *),
              int (*to_dtype_func)(char *, object) except 0):
-    cdef _memoryviewslice result = _memoryviewslice(orig_obj, flags)
-    cdef int new_ndim = result.view.ndim - cur_ndim
+
+    assert 0 < cur_ndim <= memviewslice.memview.view.ndim
+
+    cdef _memoryviewslice result = _memoryviewslice(None, 0)
+    cdef int new_ndim = memviewslice.memview.view.ndim - cur_ndim
 
     result.from_slice = memviewslice[0]
+    __PYX_INC_MEMVIEW(memviewslice, 1)
+
+    result.from_object = <object> memviewslice.memview.obj
 
+    result.view = memviewslice.memview.view
     result.view.shape = <Py_ssize_t *> (&result.from_slice.shape + new_ndim)
     result.view.strides = <Py_ssize_t *> (&result.from_slice.strides + new_ndim)
     result.view.suboffsets = <Py_ssize_t *> (&result.from_slice.suboffsets + new_ndim)
@@ -345,51 +445,55 @@ cdef memoryview_from_memslice_cwrapper(
 
     return result
 
-cdef _check_index(index):
-    if not PyIndex_Check(index):
-        raise TypeError("Cannot index with %s" % type(index))
+############### BufferFormatFromTypeInfo ###############
+cdef extern from *:
+    ctypedef struct __Pyx_StructField
 
-cdef tuple _unellipsify(tuple tup, int ndim):
-    if Ellipsis in tup:
-        result = []
-        for idx, item in enumerate(tup):
-            if item is Ellipsis:
-                result.extend([slice(None)] * (ndim - len(tup) + 1))
-                result.extend(tup[idx + 1:])
-                break
+    ctypedef struct __Pyx_TypeInfo:
+      char* name
+      __Pyx_StructField* fields
+      size_t size
+      char typegroup
+      char is_unsigned
 
-            result.append(item)
+    ctypedef struct __Pyx_StructField:
+      __Pyx_TypeInfo* type
+      char* name
+      size_t offset
 
-        return tuple(result)
+    ctypedef struct __Pyx_BufFmt_StackElem:
+      __Pyx_StructField* field
+      size_t parent_offset
 
-    return tup
+    #ctypedef struct __Pyx_BufFmt_Context:
+    #  __Pyx_StructField root
+      __Pyx_BufFmt_StackElem* head
 
-@cname('__pyx_pybuffer_index')
-cdef char *pybuffer_index(Py_buffer *view, char *bufp, Py_ssize_t index, int dim) except NULL:
-    cdef Py_ssize_t shape, stride, suboffset = -1
-    cdef Py_ssize_t itemsize = view.itemsize
-    cdef char *resultp
+    struct __pyx_typeinfo_string:
+        char string[3]
 
-    if view.ndim == 0:
-        shape = view.len / itemsize
-        stride = itemsize
-    else:
-        shape = view.shape[dim]
-        stride = view.strides[dim]
-        if view.suboffsets != NULL:
-            suboffset = view.suboffsets[dim]
+    __pyx_typeinfo_string __Pyx_TypeInfoToFormat(__Pyx_TypeInfo *)
 
-    if index < 0:
-        index += view.shape[dim]
-        if index < 0:
-            raise IndexError("Out of bounds on buffer access (axis %d)" % dim)
 
-    if index >= shape:
-        raise IndexError("Out of bounds on buffer access (axis %d)" % dim)
+@cname('__pyx_format_from_typeinfo')
+cdef format_from_typeinfo(__Pyx_TypeInfo *type):
+    cdef __Pyx_StructField *field
+    cdef __pyx_typeinfo_string fmt
 
-    resultp = bufp + index * stride
-    if suboffset >= 0:
-        resultp = (<char **> resultp)[0] + suboffset
+    if type.typegroup == 'S':
+        assert type.fields != NULL and type.fields.type != NULL
 
-    return resultp
+        parts = ["T{"]
+        field = type.fields
 
+        while field.type:
+            parts.append(format_from_typeinfo(field.type))
+            field += 1
+
+        parts.append("}")
+        result = "".join(parts)
+    else:
+        fmt = __Pyx_TypeInfoToFormat(type)
+        result = fmt.string
+
+    return result

Cython/Utility/MemoryView_C.c

@@ -56,9 +56,8 @@ static CYTHON_INLINE char *__pyx_memviewslice_index_full(const char *bufp, Py_ss
 {{#__Pyx_PyObject_to_MemoryviewSlice_<count>}}
 
 static CYTHON_INLINE {{memviewslice_name}} {{funcname}}(PyObject *obj) {
-    {{memviewslice_name}} result;
-    result.memview = NULL;
-    result.data = NULL;
+    {{memviewslice_name}} result = {0};
+
     struct __pyx_memoryview_obj *memview =  \
         (struct __pyx_memoryview_obj *) __pyx_memoryview_new(obj, {{buf_flag}});
     __Pyx_BufFmt_StackElem stack[{{struct_nesting_depth}}];
@@ -294,9 +293,9 @@ static CYTHON_INLINE void __Pyx_INC_MEMVIEW({{memviewslice_name}} *memslice,
         __pyx_fatalerror("Acquisition count is %d (line %d)",
                          memview->acquisition_count, lineno);
 
-    //PyThread_acquire_lock(memview->lock, 1);
+    PyThread_acquire_lock(memview->lock, 1);
     first_time = (memview->acquisition_count++ == 0);
-    //PyThread_release_lock(memview->lock);
+    PyThread_release_lock(memview->lock);
 
     if (first_time) {
         if (have_gil) {
@@ -321,9 +320,9 @@ static CYTHON_INLINE void __Pyx_XDEC_MEMVIEW({{memviewslice_name}} *memslice,
         __pyx_fatalerror("Acquisition count is %d (line %d)",
                          memview->acquisition_count, lineno);
 
-    //PyThread_acquire_lock(memview->lock, 1);
+    PyThread_acquire_lock(memview->lock, 1);
     last_time = (memview->acquisition_count-- == 1);
-    //PyThread_release_lock(memview->lock);
+    PyThread_release_lock(memview->lock);
 
     if (last_time) {
         if (have_gil) {
@@ -370,7 +369,7 @@ static __Pyx_memviewslice {{copy_name}}(const __Pyx_memviewslice from_mvs) {
         }
     }
 
-    array_obj = __pyx_array_new(shape_tuple, {{sizeof_dtype}}, buf->format, mode);
+    array_obj = __pyx_array_new(shape_tuple, {{sizeof_dtype}}, buf->format, mode, NULL);
     if (unlikely(!array_obj)) {
         goto fail;
     }
@@ -412,12 +411,15 @@ no_fail:
 
 /////////////// MemviewSliceIndex ///////////////
 
-static CYTHON_INLINE char *__pyx_memviewslice_index_full(const char *bufp, Py_ssize_t idx, Py_ssize_t stride, Py_ssize_t suboffset) {
+static CYTHON_INLINE char *
+__pyx_memviewslice_index_full(const char *bufp, Py_ssize_t idx,
+                              Py_ssize_t stride, Py_ssize_t suboffset)
+{
     bufp = bufp + idx * stride;
     if (suboffset >= 0) {
         bufp = *((char **) bufp) + suboffset;
     }
-    return bufp;
+    return (char *) bufp;
 }
 
 /////////////// MemviewDtypeToObject.proto ///////////////

tests/run/cythonarray.pyx

@@ -5,6 +5,8 @@ from __future__ import unicode_literals
 from cython cimport array
 cimport cython as cy
 
+from libc.stdlib cimport malloc, free
+
 def contiguity():
     '''
     >>> contiguity()
@@ -85,19 +87,79 @@ cdef create_array(shape, mode):
 
     return result
 
-def test_cython_array():
+def test_cython_array_getbuffer():
     """
-    >>> test_cython_array()
+    >>> test_cython_array_getbuffer()
     98
     61
     98
     61
     """
-    cdef int[:, ::1] carr = create_array((14, 10), 'c')
-    cdef int[::1, :] farr = create_array((14, 10), 'fortran')
+    cdef int[:, ::1] cslice = create_array((14, 10), 'c')
+    cdef int[::1, :] fslice = create_array((14, 10), 'fortran')
+
+    print cslice[9, 8]
+    print cslice[6, 1]
+
+    print fslice[9, 8]
+    print fslice[6, 1]
+
+def test_cython_array_index():
+    """
+    >>> test_cython_array_index()
+    98
+    61
+    98
+    61
+    """
+    c_array = create_array((14, 10), 'c')
+    f_array = create_array((14, 10), 'fortran')
+
+    print c_array[9, 8]
+    print c_array[6, 1]
+
+    print f_array[9, 8]
+    print f_array[6, 1]
+
+cdef int *getp(int dim1=10, int dim2=10) except NULL:
+    print "getp()"
+
+    cdef int *p = <int *> malloc(dim1 * dim2 * sizeof(int))
+
+    if p == NULL:
+        raise MemoryError
+
+    for i in range(dim1 * dim2):
+        p[i] = i
+
+    return p
+
+cdef void callback_free_data(char *p):
+    print 'callback free data called'
+    free(p)
+
+def test_array_from_pointer():
+    """
+    >>> test_array_from_pointer()
+    getp()
+    69
+    c
+    getp()
+    fortran
+    getp()
+    56
+    getp()
+    56
+    callback free data called
+    """
+    cdef int *p = getp()
+    cdef array c_arr = <int[:10, :10]> p
+    c_arr.callback_free_data = callback_free_data
+    print c_arr[6, 9]
+    print c_arr.mode
 
-    print carr[9, 8]
-    print carr[6, 1]
+    print (<int[:10:1, :10]> getp()).mode
 
-    print farr[9, 8]
-    print farr[6, 1]
+    cdef int[:, ::1] mslice = <int[:10, :10]> getp()
+    print mslice[5, 6]
+    print (<int[:12, :10]> getp(12, 10))[5, 6]

tests/run/memoryview.pyx

@@ -219,28 +219,20 @@ def get_int_2d(int[:, :] mslice, int i, int j):
     >>> C = IntMockBuffer("C", range(6), (2,3))
     >>> get_int_2d(C, 1, 1)
     acquired C
-    acquired C
-    released C
     released C
     4
 
     Check negative indexing:
     >>> get_int_2d(C, -1, 0)
     acquired C
-    acquired C
-    released C
     released C
     3
     >>> get_int_2d(C, -1, -2)
     acquired C
-    acquired C
-    released C
     released C
     4
     >>> get_int_2d(C, -2, -3)
     acquired C
-    acquired C
-    released C
     released C
     0
 
@@ -265,50 +257,34 @@ def set_int_2d(int[:, :] mslice, int i, int j, int value):
     >>> C = IntMockBuffer("C", range(6), (2,3))
     >>> set_int_2d(C, 1, 1, 10)
     acquired C
-    acquired C
-    released C
     released C
     >>> get_int_2d(C, 1, 1)
     acquired C
-    acquired C
-    released C
     released C
     10
 
     Check negative indexing:
     >>> set_int_2d(C, -1, 0, 3)
     acquired C
-    acquired C
-    released C
     released C
     >>> get_int_2d(C, -1, 0)
     acquired C
-    acquired C
-    released C
     released C
     3
 
     >>> set_int_2d(C, -1, -2, 8)
     acquired C
-    acquired C
-    released C
     released C
     >>> get_int_2d(C, -1, -2)
     acquired C
-    acquired C
-    released C
     released C
     8
 
     >>> set_int_2d(C, -2, -3, 9)
     acquired C
-    acquired C
-    released C
     released C
     >>> get_int_2d(C, -2, -3)
     acquired C
-    acquired C
-    released C
     released C
     9
 
@@ -336,8 +312,6 @@ def writable(unsigned short int[:, :, :] mslice):
     >>> R = UnsignedShortMockBuffer("R", range(27), shape=(3, 3, 3))
     >>> writable(R)
     acquired R
-    acquired R
-    released R
     released R
     >>> [str(x) for x in R.recieved_flags] # Py2/3
     ['FORMAT', 'ND', 'STRIDES', 'WRITABLE']
@@ -350,8 +324,6 @@ def strided(int[:] mslice):
     >>> A = IntMockBuffer("A", range(4))
     >>> strided(A)
     acquired A
-    acquired A
-    released A
     released A
     2
 
@@ -410,16 +382,12 @@ def generic(int[::view.generic, ::view.generic] mslice1,
     >>> generic(A, B)
     acquired A
     acquired B
-    acquired A
-    acquired B
     4
     4
     10
     11
     released A
     released B
-    released A
-    released B
     """
     buf1, buf2 = mslice1, mslice2
 
@@ -440,16 +408,12 @@ def generic_contig(int[::view.generic_contiguous, :] mslice1,
     >>> generic_contig(A, B)
     acquired A
     acquired B
-    acquired A
-    acquired B
     4
     4
     10
     11
     released A
     released B
-    released A
-    released B
     """
     buf1, buf2 = mslice1, mslice2