Merge pull request #61 from markflorisson88/_memview_rebase

Support for typed memoryviews, memoryview objects & cython.array

Cython/Compiler/Code.pxd

 
 cimport cython
 
-cdef class UtilityCode:
+cdef class UtilityCodeBase(object):
+    cdef public object name
+
+cdef class UtilityCode(UtilityCodeBase):
     cdef public object proto
     cdef public object impl
     cdef public object init
@@ -10,6 +13,7 @@ cdef class UtilityCode:
     cdef public dict _cache
     cdef public list specialize_list
     cdef public object proto_block
+    cdef public object file
 
     cpdef put_code(self, output)
 

Cython/Compiler/CythonScope.py

@@ -3,14 +3,19 @@ from PyrexTypes import *
 from UtilityCode import CythonUtilityCode
 from Errors import error
 from Scanning import StringSourceDescriptor
+import Options
+import Buffer
+import MemoryView
 
 class CythonScope(ModuleScope):
     is_cython_builtin = 1
 
-    def __init__(self):
+    def __init__(self, context):
         ModuleScope.__init__(self, u'cython', None, None)
         self.pxd_file_loaded = True
         self.populate_cython_scope()
+        # The Main.Context object
+        self.context = context
 
     def lookup_type(self, name):
         # This function should go away when types are all first-level objects.
@@ -66,28 +71,40 @@ class CythonScope(ModuleScope):
 #        self.test_cythonscope()
 
     def test_cythonscope(self):
-        # A special function just to make it easy to test the scope and
-        # utility code functionality in isolation. It is available to
-        # "end-users" but nobody will know it is there anyway...
-        cython_testscope_utility_code.declare_in_scope(self)
-        cython_test_extclass_utility_code.declare_in_scope(self)
+        """
+        Creates some entries for testing purposes and entries for
+        cython.array() and for cython.view.*.
+        """
+        cython_testscope_utility_code.declare_in_scope(
+                                self, cython_scope=self)
+        cython_test_extclass_utility_code.declare_in_scope(
+                                    self, cython_scope=self)
+
+        MemoryView.cython_array_utility_code.declare_in_scope(
+                                        self, cython_scope=self)
 
         #
         # The view sub-scope
         #
-        self.viewscope = viewscope = ModuleScope(u'cython.view', self, None)
-        self.declare_module('view', viewscope, None)
+        self.viewscope = viewscope = ModuleScope(u'view', self, None)
+        self.declare_module('view', viewscope, None).as_module = viewscope
         viewscope.is_cython_builtin = True
         viewscope.pxd_file_loaded = True
 
-        cythonview_testscope_utility_code.declare_in_scope(viewscope)
+        cythonview_testscope_utility_code.declare_in_scope(
+                                                viewscope, cython_scope=self)
+
+        view_utility_scope = MemoryView.view_utility_code.declare_in_scope(
+                                                viewscope, cython_scope=self)
+        # MemoryView.memview_fromslice_utility_code.from_scope = view_utility_scope
+        # MemoryView.memview_fromslice_utility_code.declare_in_scope(viewscope)
 
 
 def create_cython_scope(context, create_testscope):
     # One could in fact probably make it a singleton,
     # but not sure yet whether any code mutates it (which would kill reusing
     # it across different contexts)
-    scope = CythonScope()
+    scope = CythonScope(context)
 
     if create_testscope:
         scope.test_cythonscope()
@@ -95,80 +112,29 @@ def create_cython_scope(context, create_testscope):
     return scope
 
 
-cython_testscope_utility_code = CythonUtilityCode(u"""
-@cname('__pyx_testscope')
-cdef object _testscope(int value):
-    return "hello from cython scope, value=%d" % value
-""")
+# Load test utilities for the cython scope
+
+def load_testscope_utility(cy_util_name, **kwargs):
+    return CythonUtilityCode.load(cy_util_name, "TestCythonScope.pyx", **kwargs)
+
 
 undecorated_methods_protos = UtilityCode(proto=u"""
     /* These methods are undecorated and have therefore no prototype */
     static PyObject *__pyx_TestClass_cdef_method(
-            struct __pyx_TestClass *self, int value);
+            struct __pyx_TestClass_obj *self, int value);
     static PyObject *__pyx_TestClass_cpdef_method(
-            struct __pyx_TestClass *self, int value, int skip_dispatch);
+            struct __pyx_TestClass_obj *self, int value, int skip_dispatch);
     static PyObject *__pyx_TestClass_def_method(
             PyObject *self, PyObject *value);
 """)
 
-test_cython_utility_dep = CythonUtilityCode(u"""
-@cname('__pyx_test_dep')
-cdef test_dep(obj):
-    print 'test_dep', obj
-""")
-
-cython_test_extclass_utility_code = CythonUtilityCode(
-        name="TestClassUtilityCode",
-        prefix="__pyx_prefix_TestClass_",
-        requires=[undecorated_methods_protos, test_cython_utility_dep],
-        impl=u"""
-cdef extern from *:
-    cdef object __pyx_test_dep(object)
-
-@cname('__pyx_TestClass')
-cdef class TestClass(object):
-    cdef public int value
-
-    def __init__(self, int value):
-        self.value = value
-
-    def __str__(self):
-        return 'TestClass(%d)' % self.value
+cython_testscope_utility_code = load_testscope_utility("TestScope")
 
-    cdef cdef_method(self, int value):
-        print 'Hello from cdef_method', value
+test_cython_utility_dep = load_testscope_utility("TestDep")
 
-    cpdef cpdef_method(self, int value):
-        print 'Hello from cpdef_method', value
+cython_test_extclass_utility_code = \
+    load_testscope_utility("TestClass", name="TestClass",
+                           requires=[undecorated_methods_protos,
+                                     test_cython_utility_dep])
 
-    def def_method(self, int value):
-        print 'Hello from def_method', value
-
-    @cname('cdef_cname')
-    cdef cdef_cname_method(self, int value):
-        print "Hello from cdef_cname_method", value
-
-    @cname('cpdef_cname')
-    cpdef cpdef_cname_method(self, int value):
-        print "Hello from cpdef_cname_method", value
-
-    @cname('def_cname')
-    def def_cname_method(self, int value):
-        print "Hello from def_cname_method", value
-
-@cname('__pyx_test_call_other_cy_util')
-cdef test_call(obj):
-    print 'test_call'
-    __pyx_test_dep(obj)
-
-@cname('__pyx_TestClass_New')
-cdef _testclass_new(int value):
-    return TestClass(value)
-""")
-
-
-cythonview_testscope_utility_code = CythonUtilityCode(u"""
-@cname('__pyx_view_testscope')
-cdef object _testscope(int value):
-    return "hello from cython.view scope, value=%d" % value
-""")
+cythonview_testscope_utility_code = load_testscope_utility("View.TestScope")
\ No newline at end of file

Cython/Compiler/Main.py

@@ -430,7 +430,7 @@ def create_default_resultobj(compilation_source, options):
 
 def run_pipeline(source, options, full_module_name = None):
     import Pipeline
-    # Set up context
+
     context = options.create_context()
 
     # Set up source object
@@ -438,6 +438,7 @@ def run_pipeline(source, options, full_module_name = None):
     abs_path = os.path.abspath(source)
     source_ext = os.path.splitext(source)[1]
     full_module_name = full_module_name or context.extract_module_name(source, options)
+
     if options.relative_path_in_code_position_comments:
         rel_path = full_module_name.replace('.', os.sep) + source_ext
         if not abs_path.endswith(rel_path):
@@ -584,7 +585,8 @@ def compile_multiple(sources, options):
     a CompilationResultSet. Performs timestamp checking and/or recursion
     if these are specified in the options.
     """
-    context = options.create_context()
+    # run_pipeline creates the context
+    # context = options.create_context()
     sources = [os.path.abspath(source) for source in sources]
     processed = set()
     results = CompilationResultSet()

Cython/Compiler/ModuleNode.py

@@ -65,10 +65,25 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
             self.body.stats.extend(tree.stats)
         else:
             self.body.stats.append(tree)
-        selfscope = self.scope
-        selfscope.utility_code_list.extend(scope.utility_code_list)
+
+        self.scope.utility_code_list.extend(scope.utility_code_list)
+
+        def extend_if_not_in(L1, L2):
+            for x in L2:
+                if x not in L1:
+                    L1.append(x)
+
+        extend_if_not_in(self.scope.include_files, scope.include_files)
+        extend_if_not_in(self.scope.included_files, scope.included_files)
+        extend_if_not_in(self.scope.python_include_files,
+                         scope.python_include_files)
+
         if merge_scope:
-            selfscope.merge_in(scope)
+            # Ensure that we don't generate import code for these entries!
+            for entry in scope.c_class_entries:
+                entry.type.module_name = self.full_module_name
+
+            self.scope.merge_in(scope)
 
     def analyse_declarations(self, env):
         if Options.embed_pos_in_docstring:
@@ -126,7 +141,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         if (h_types or  h_vars or h_funcs or h_extension_types):
             result.h_file = replace_suffix(result.c_file, ".h")
             h_code = Code.CCodeWriter()
-            Code.GlobalState(h_code)
+            Code.GlobalState(h_code, self)
             if options.generate_pxi:
                 result.i_file = replace_suffix(result.c_file, ".pxi")
                 i_code = Code.PyrexCodeWriter(result.i_file)
@@ -195,7 +210,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         if api_vars or api_funcs or api_extension_types:
             result.api_file = replace_suffix(result.c_file, "_api.h")
             h_code = Code.CCodeWriter()
-            Code.GlobalState(h_code)
+            Code.GlobalState(h_code, self)
             api_guard = Naming.api_guard_prefix + self.api_name(env)
             h_code.put_h_guard(api_guard)
             h_code.putln('#include "Python.h"')
@@ -293,7 +308,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         else:
             emit_linenums = options.emit_linenums
             rootwriter = Code.CCodeWriter(emit_linenums=emit_linenums, c_line_in_traceback=options.c_line_in_traceback)
-        globalstate = Code.GlobalState(rootwriter, emit_linenums)
+        globalstate = Code.GlobalState(rootwriter, self, emit_linenums)
         globalstate.initialize_main_c_code()
         h_code = globalstate['h_code']
 
@@ -334,7 +349,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         self.generate_declarations_for_modules(env, modules, globalstate)
         h_code.write('\n')
 
-        for utilcode in env.utility_code_list:
+        for utilcode in env.utility_code_list[:]:
             globalstate.use_utility_code(utilcode)
         globalstate.finalize_main_c_code()
 
@@ -1185,10 +1200,13 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         type = scope.parent_type
         base_type = type.base_type
         py_attrs = []
+        memviewslice_attrs = []
         for entry in scope.var_entries:
             if entry.type.is_pyobject:
                 py_attrs.append(entry)
-        need_self_cast = type.vtabslot_cname or py_attrs
+            elif entry.type.is_memoryviewslice:
+                memviewslice_attrs.append(entry)
+        need_self_cast = type.vtabslot_cname or py_attrs or memviewslice_attrs
         code.putln("")
         code.putln(
             "static PyObject *%s(PyTypeObject *t, PyObject *a, PyObject *k) {"
@@ -1231,6 +1249,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                 code.putln("p->%s = 0;" % entry.cname)
             else:
                 code.put_init_var_to_py_none(entry, "p->%s", nanny=False)
+        for entry in memviewslice_attrs:
+            code.putln("p->%s.data = NULL;" % entry.cname)
+            code.putln("p->%s.memview = NULL;" % entry.cname)
         entry = scope.lookup_here("__new__")
         if entry and entry.is_special:
             if entry.trivial_signature:
@@ -2134,8 +2155,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         # variables to None.
         for entry in env.var_entries:
             if entry.visibility != 'extern':
-                if entry.type.is_pyobject and entry.used:
-                    code.put_init_var_to_py_none(entry, nanny=False)
+                if entry.used:
+                    entry.type.global_init_code(entry, code)
 
     def generate_c_variable_export_code(self, env, code):
         # Generate code to create PyCFunction wrappers for exported C functions.
@@ -2237,7 +2258,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         # Generate type import code for extern extension types
         # and type ready code for non-extern ones.
         for entry in env.c_class_entries:
-            if entry.visibility == 'extern':
+            if entry.visibility == 'extern' and not entry.utility_code_definition:
                 self.generate_type_import_code(env, entry.type, entry.pos, code)
             else:
                 self.generate_base_type_import_code(env, entry, code)
@@ -2247,8 +2268,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
 
     def generate_base_type_import_code(self, env, entry, code):
         base_type = entry.type.base_type
-        if base_type and base_type.module_name != env.qualified_name \
-               and not base_type.is_builtin_type:
+        if (base_type and base_type.module_name != env.qualified_name and not
+               base_type.is_builtin_type and not entry.utility_code_definition):
             self.generate_type_import_code(env, base_type, self.pos, code)
 
     def use_type_import_utility_code(self, env):
@@ -2419,8 +2440,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                     type.typeptr_cname, type.typeobj_cname))
 
 def generate_cfunction_declaration(entry, env, code, definition):
+    from_cy_utility = entry.used and entry.utility_code_definition
     if entry.inline_func_in_pxd or (not entry.in_cinclude and (definition
-            or entry.defined_in_pxd or entry.visibility == 'extern')):
+            or entry.defined_in_pxd or entry.visibility == 'extern' or from_cy_utility)):
         if entry.visibility == 'extern':
             storage_class = "%s " % Naming.extern_c_macro
             dll_linkage = "DL_IMPORT"

Cython/Compiler/Naming.py

@@ -38,10 +38,8 @@ typeobj_prefix    = pyrex_prefix + "type_"
 var_prefix        = pyrex_prefix + "v_"
 varptr_prefix     = pyrex_prefix + "vp_"
 wrapperbase_prefix= pyrex_prefix + "wrapperbase_"
-bufstruct_prefix  = pyrex_prefix + "bstruct_"
-bufstride_prefix  = pyrex_prefix + "bstride_"
-bufshape_prefix   = pyrex_prefix + "bshape_"
-bufsuboffset_prefix  = pyrex_prefix + "boffset_"
+pybuffernd_prefix   = pyrex_prefix + "pybuffernd_"
+pybufferstruct_prefix  = pyrex_prefix + "pybuffer_"
 vtable_prefix     = pyrex_prefix + "vtable_"
 vtabptr_prefix    = pyrex_prefix + "vtabptr_"
 vtabstruct_prefix = pyrex_prefix + "vtabstruct_"

Cython/Compiler/Options.py

@@ -67,11 +67,15 @@ old_style_globals = False
 cimport_from_pyx = False
 
 
+# max # of dims for buffers -- set to same value as max # of dims for numpy
+# arrays.
+buffer_max_dims = 32
 
 # Declare compiler directives
 directive_defaults = {
     'boundscheck' : True,
     'nonecheck' : False,
+    'initializedcheck' : True,
     'embedsignature' : False,
     'locals' : {},
     'auto_cpdef': False,

Cython/Compiler/ParseTreeTransforms.py

@@ -1037,7 +1037,8 @@ class ParallelRangeTransform(CythonTransform, SkipDeclarations):
             directive = directive.rstrip('.')
 
         cls = self.directive_to_node.get(directive)
-        if cls is None:
+        if cls is None and not (self.namenode_is_cython_module and
+                                self.parallel_directive[0] != 'parallel'):
             error(node.pos, "Invalid directive: %s" % directive)
 
         self.namenode_is_cython_module = False
@@ -1255,7 +1256,7 @@ class CnameDirectivesTransform(CythonTransform, SkipDeclarations):
     """
 
     def handle_function(self, node):
-        if not node.decorators:
+        if not getattr(node, 'decorators', None):
             return self.visit_Node(node)
 
         for i, decorator in enumerate(node.decorators):
@@ -1289,6 +1290,8 @@ class CnameDirectivesTransform(CythonTransform, SkipDeclarations):
 
     visit_FuncDefNode = handle_function
     visit_CClassDefNode = handle_function
+    visit_CEnumDefNode = handle_function
+    visit_CStructOrUnionDefNode = handle_function
 
 
 class ForwardDeclareTypes(CythonTransform):
@@ -1576,6 +1579,14 @@ if VALUE is not None:
         self.visitchildren(node)
         return None
 
+    def visit_CnameDecoratorNode(self, node):
+        self.visitchildren(node)
+
+        if not node.node:
+            return None
+
+        return node
+
     def create_Property(self, entry):
         if entry.visibility == 'public':
             if entry.type.is_pyobject:
@@ -2213,6 +2224,7 @@ class GilCheck(VisitorTransform):
         if self.env_stack and self.nogil and node.nogil_check:
             node.nogil_check(self.env_stack[-1])
         self.visitchildren(node)
+        node.in_nogil_context = self.nogil
         return node
 
 

Cython/Compiler/Parsing.pxd

@@ -7,8 +7,8 @@ ctypedef object (*p_sub_expr_func)(object)
 
 # entry points
 
-cpdef p_module(PyrexScanner s, pxd, full_module_name)
-cpdef p_code(PyrexScanner s, level= *)
+cpdef p_module(PyrexScanner s, pxd, full_module_name, ctx=*)
+cpdef p_code(PyrexScanner s, level= *, ctx=*)
 
 # internal parser states
 
@@ -131,6 +131,8 @@ cdef p_calling_convention(PyrexScanner s)
 cdef p_c_complex_base_type(PyrexScanner s)
 cpdef p_c_simple_base_type(PyrexScanner s, bint self_flag, bint nonempty, templates = *)
 cdef p_buffer_or_template(PyrexScanner s, base_type_node, templates)
+cdef is_memoryviewslice_access(PyrexScanner s)
+cdef p_memoryviewslice_access(PyrexScanner s, base_type_node)
 cdef bint looking_at_name(PyrexScanner s) except -2
 cdef bint looking_at_expr(PyrexScanner s) except -2
 cdef bint looking_at_base_type(PyrexScanner s) except -2

Cython/Compiler/Parsing.py

@@ -33,6 +33,7 @@ class Ctx(object):
     nogil = 0
     namespace = None
     templates = None
+    allow_struct_enum_decorator = False
 
     def __init__(self, **kwds):
         self.__dict__.update(kwds)
@@ -296,7 +297,8 @@ def p_typecast(s):
     pos = s.position()
     s.next()
     base_type = p_c_base_type(s)
-    if base_type.name is None:
+    is_memslice = isinstance(base_type, Nodes.MemoryViewSliceTypeNode)
+    if not is_memslice and base_type.name is None:
         s.error("Unknown type")
     declarator = p_c_declarator(s, empty = 1)
     if s.sy == '?':
@@ -306,6 +308,10 @@ def p_typecast(s):
         typecheck = 0
     s.expect(">")
     operand = p_factor(s)
+    if is_memslice:
+        return ExprNodes.CythonArrayNode(pos, base_type_node=base_type,
+                                         operand=operand)
+
     return ExprNodes.TypecastNode(pos,
         base_type = base_type,
         declarator = declarator,
@@ -1754,7 +1760,8 @@ def p_statement(s, ctx, first_statement = 0):
             s.error('decorator not allowed here')
         s.level = ctx.level
         decorators = p_decorators(s)
-        if s.sy not in ('def', 'cdef', 'cpdef', 'class'):
+        bad_toks =  'def', 'cdef', 'cpdef', 'class'
+        if not ctx.allow_struct_enum_decorator and s.sy not in bad_toks:
             s.error("Decorators can only be followed by functions or classes")
     elif s.sy == 'pass' and cdef_flag:
         # empty cdef block
@@ -1774,7 +1781,10 @@ def p_statement(s, ctx, first_statement = 0):
         s.level = ctx.level
         node = p_cdef_statement(s, ctx(overridable = overridable))
         if decorators is not None:
-            if not isinstance(node, (Nodes.CFuncDefNode, Nodes.CVarDefNode, Nodes.CClassDefNode)):
+            tup = Nodes.CFuncDefNode, Nodes.CVarDefNode, Nodes.CClassDefNode
+            if ctx.allow_struct_enum_decorator:
+                tup += Nodes.CStructOrUnionDefNode, Nodes.CEnumDefNode
+            if not isinstance(node, tup):
                 s.error("Decorators can only be followed by functions or classes")
             node.decorators = decorators
         return node
@@ -2002,7 +2012,11 @@ def p_c_simple_base_type(s, self_flag, nonempty, templates = None):
         complex = complex, longness = longness,
         is_self_arg = self_flag, templates = templates)
 
+    #    declarations here.
     if s.sy == '[':
+        if is_memoryviewslice_access(s):
+            type_node = p_memoryviewslice_access(s, type_node)
+        else:
             type_node = p_buffer_or_template(s, type_node, templates)
 
     if s.sy == '.':
@@ -2034,6 +2048,63 @@ def p_buffer_or_template(s, base_type_node, templates):
         base_type_node = base_type_node)
     return result
 
+def p_bracketed_base_type(s, base_type_node, nonempty, empty):
+    # s.sy == '['
+    if empty and not nonempty:
+        # sizeof-like thing.  Only anonymous C arrays allowed (int[SIZE]).
+        return base_type_node
+    elif not empty and nonempty:
+        # declaration of either memoryview slice or buffer.
+        if is_memoryviewslice_access(s):
+            return p_memoryviewslice_access(s, base_type_node)
+        else:
+            return p_buffer_or_template(s, base_type_node, None)
+            # return p_buffer_access(s, base_type_node)
+    elif not empty and not nonempty:
+        # only anonymous C arrays and memoryview slice arrays here.  We
+        # disallow buffer declarations for now, due to ambiguity with anonymous
+        # C arrays.
+        if is_memoryviewslice_access(s):
+            return p_memoryviewslice_access(s, base_type_node)
+        else:
+            return base_type_node
+
+def is_memoryviewslice_access(s):
+    # s.sy == '['
+    # a memoryview slice declaration is distinguishable from a buffer access
+    # declaration by the first entry in the bracketed list.  The buffer will
+    # not have an unnested colon in the first entry; the memoryview slice will.
+    saved = [(s.sy, s.systring)]
+    s.next()
+    retval = False
+    if s.systring == ':':
+        retval = True
+    elif s.sy == 'INT':
+        saved.append((s.sy, s.systring))
+        s.next()
+        if s.sy == ':':
+            retval = True
+
+    for sv in saved[::-1]:
+        s.put_back(*sv)
+
+    return retval
+
+def p_memoryviewslice_access(s, base_type_node):
+    # s.sy == '['
+    pos = s.position()
+    s.next()
+    subscripts = p_subscript_list(s)
+    # make sure each entry in subscripts is a slice
+    for subscript in subscripts:
+        if len(subscript) < 2:
+            s.error("An axis specification in memoryview declaration does not have a ':'.")
+    s.expect(']')
+    indexes = make_slice_nodes(pos, subscripts)
+    result = Nodes.MemoryViewSliceTypeNode(pos,
+            base_type_node = base_type_node,
+            axes = indexes)
+    return result
 
 def looking_at_name(s):
     return s.sy == 'IDENT' and not s.systring in calling_convention_words
@@ -2831,8 +2902,8 @@ def p_doc_string(s):
     else:
         return None
 
-def p_code(s, level=None):
-    body = p_statement_list(s, Ctx(level = level), first_statement = 1)
+def p_code(s, level=None, ctx=Ctx):
+    body = p_statement_list(s, ctx(level = level), first_statement = 1)
     if s.sy != 'EOF':
         s.error("Syntax error in statement [%s,%s]" % (
             repr(s.sy), repr(s.systring)))
@@ -2854,7 +2925,7 @@ def p_compiler_directive_comments(s):
         s.next()
     return result
 
-def p_module(s, pxd, full_module_name):
+def p_module(s, pxd, full_module_name, ctx=Ctx):
     pos = s.position()
 
     directive_comments = p_compiler_directive_comments(s)
@@ -2869,7 +2940,7 @@ def p_module(s, pxd, full_module_name):
     else:
         level = 'module'
 
-    body = p_statement_list(s, Ctx(level = level), first_statement = 1)
+    body = p_statement_list(s, ctx(level=level), first_statement = 1)
     if s.sy != 'EOF':
         s.error("Syntax error in statement [%s,%s]" % (
             repr(s.sy), repr(s.systring)))
@@ -2977,4 +3048,3 @@ def print_parse_tree(f, node, level, key = None):
             f.write("%s]\n" % ind)
             return
     f.write("%s%s\n" % (ind, node))
-

Cython/Compiler/Pipeline.py

@@ -4,6 +4,7 @@ from time import time
 import Errors
 import DebugFlags
 import Options
+from Visitor import CythonTransform
 from Errors import PyrexError, CompileError, InternalError, AbortError, error
 
 #
@@ -72,22 +73,41 @@ def use_utility_code_definitions(scope, target):
 
 def inject_utility_code_stage_factory(context):
     def inject_utility_code_stage(module_node):
-        # First, make sure any utility code pulled in by using symbols in the cython
-        # scope is included
-        use_utility_code_definitions(context.cython_scope, module_node.scope)
-
         added = []
         # Note: the list might be extended inside the loop (if some utility code
-        # pulls in other utility code)
+        # pulls in other utility code, explicitly or implicitly)
         for utilcode in module_node.scope.utility_code_list:
             if utilcode in added: continue
             added.append(utilcode)
+            if utilcode.requires:
+                for dep in utilcode.requires:
+                    if not dep in added and not dep in module_node.scope.utility_code_list:
+                        module_node.scope.utility_code_list.append(dep)
             tree = utilcode.get_tree()
             if tree:
                 module_node.merge_in(tree.body, tree.scope, merge_scope=True)
         return module_node
     return inject_utility_code_stage
 
+class UseUtilityCodeDefinitions(CythonTransform):
+    # Temporary hack to use any utility code in nodes' "utility_code_definitions".
+    # This should be moved to the code generation phase of the relevant nodes once
+    # it is safe to generate CythonUtilityCode at code generation time.
+    def __call__(self, node):
+        self.scope = node.scope
+        return super(UseUtilityCodeDefinitions, self).__call__(node)
+
+    def visit_AttributeNode(self, node):
+        if node.entry and node.entry.utility_code_definition:
+            self.scope.use_utility_code(node.entry.utility_code_definition)
+        return node
+
+    def visit_NameNode(self, node):
+        for e in (node.entry, node.type_entry):
+            if e and e.utility_code_definition:
+                self.scope.use_utility_code(e.utility_code_definition)
+        return node
+                     
 #
 # Pipeline factories
 #
@@ -113,6 +133,8 @@ def create_pipeline(context, mode, exclude_classes=()):
     from Optimize import DropRefcountingTransform
     from Buffer import IntroduceBufferAuxiliaryVars
     from ModuleNode import check_c_declarations, check_c_declarations_pxd
+    from ModuleNode import check_c_declarations
+
 
     if mode == 'pxd':
         _check_c_declarations = check_c_declarations_pxd
@@ -165,6 +187,7 @@ def create_pipeline(context, mode, exclude_classes=()):
         DropRefcountingTransform(),
         FinalOptimizePhase(context),
         GilCheck(),
+        UseUtilityCodeDefinitions(context),
         ]
     filtered_stages = []
     for s in stages:
@@ -265,7 +288,9 @@ def insert_into_pipeline(pipeline, transform, before=None, after=None):
 # Running a pipeline
 #
 
-def run_pipeline(pipeline, source):
+def run_pipeline(pipeline, source, printtree=True):
+    from Cython.Compiler.Visitor import PrintTree
+
     error = None
     data = source
     try:
@@ -275,6 +300,8 @@ def run_pipeline(pipeline, source):
                     if DebugFlags.debug_verbose_pipeline:
                         t = time()
                         print "Entering pipeline phase %r" % phase
+                    if not printtree and isinstance(phase, PrintTree):
+                        continue
                     data = phase(data)
                     if DebugFlags.debug_verbose_pipeline:
                         print "    %.3f seconds" % (time() - t)

Cython/Compiler/Symtab.py

@@ -39,12 +39,9 @@ def c_safe_identifier(cname):
 class BufferAux(object):
     writable_needed = False
 
-    def __init__(self, buffer_info_var, stridevars, shapevars,
-                 suboffsetvars):
-        self.buffer_info_var = buffer_info_var
-        self.stridevars = stridevars
-        self.shapevars = shapevars
-        self.suboffsetvars = suboffsetvars
+    def __init__(self, buflocal_nd_var, rcbuf_var):
+        self.buflocal_nd_var = buflocal_nd_var
+        self.rcbuf_var = rcbuf_var
 
     def __repr__(self):
         return "<BufferAux %r>" % self.__dict__
@@ -127,6 +124,8 @@ class Entry(object):
     #                             used uninitialized
     # cf_used          boolean    Entry is used
 
+    # TODO: utility_code and utility_code_definition serves the same purpose...
+
     inline_func_in_pxd = False
     borrowed = 0
     init = ""
@@ -292,8 +291,6 @@ class Scope(object):
         entries = [(name, entry)
                        for name, entry in other.entries.iteritems()
                            if entry.used or merge_unused]
-        # !@#$ py23
-        entries = dict(entries)
 
         self.entries.update(entries)
 
@@ -306,8 +303,9 @@ class Scope(object):
                      'cfunc_entries',
                      'c_class_entries'):
             self_entries = getattr(self, attr)
+            names = set([e.name for e in self_entries])
             for entry in getattr(other, attr):
-                if entry.used or merge_unused:
+                if (entry.used or merge_unused) and entry.name not in names:
                     self_entries.append(entry)
 
     def __str__(self):
@@ -383,6 +381,11 @@ class Scope(object):
 #                entries[name] = entry
             if not shadow:
                 entries[name] = entry
+
+        if type.is_memoryviewslice:
+            import MemoryView
+            entry.init = MemoryView.memslice_entry_init
+
         entry.scope = self
         entry.visibility = visibility
         return entry
@@ -1163,7 +1166,7 @@ class ModuleScope(Scope):
 
     def declare_c_class(self, name, pos, defining = 0, implementing = 0,
         module_name = None, base_type = None, objstruct_cname = None,
-        typeobj_cname = None, visibility = 'private', typedef_flag = 0, api = 0,
+        typeobj_cname = None, typeptr_cname = None, visibility = 'private', typedef_flag = 0, api = 0,
         buffer_defaults = None, shadow = 0):
         # If this is a non-extern typedef class, expose the typedef, but use
         # the non-typedef struct internally to avoid needing forward
@@ -1205,6 +1208,9 @@ class ModuleScope(Scope):
                 type.module_name = module_name
             else:
                 type.module_name = self.qualified_name
+            if typeptr_cname:
+                type.typeptr_cname = typeptr_cname
+            else:
                 type.typeptr_cname = self.mangle(Naming.typeptr_prefix, name)
             entry = self.declare_type(name, type, pos, visibility = visibility,
                 defining = 0, shadow = shadow)

Cython/Compiler/Tests/TestBuffer.py

@@ -98,3 +98,8 @@ class TestBufferOptions(CythonTest):
         self.assert_(stats[1].base_type.ndim == 3)
 
     # add exotic and impossible combinations as they come along...
+
+if __name__ == '__main__':
+    import unittest
+    unittest.main()
+

Cython/Compiler/Tests/TestMemView.py

+from Cython.TestUtils import CythonTest
+import Cython.Compiler.Errors as Errors
+from Cython.Compiler.Nodes import *
+from Cython.Compiler.ParseTreeTransforms import *
+from Cython.Compiler.Buffer import *
+
+
+class TestMemviewParsing(CythonTest):
+
+    def parse(self, s):
+        return self.should_not_fail(lambda: self.fragment(s)).root
+
+    def not_parseable(self, expected_error, s):
+        e = self.should_fail(lambda: self.fragment(s),  Errors.CompileError)
+        self.assertEqual(expected_error, e.message_only)
+    
+    def test_default_1dim(self):
+        self.parse(u"cdef int[:] x")
+        self.parse(u"cdef short int[:] x")
+
+    def test_default_ndim(self):
+        self.parse(u"cdef int[:,:,:,:,:] x")
+        self.parse(u"cdef unsigned long int[:,:,:,:,:] x")
+        self.parse(u"cdef unsigned int[:,:,:,:,:] x")
+
+    def test_zero_offset(self):
+        self.parse(u"cdef long double[0:] x")
+        self.parse(u"cdef int[0:] x")
+
+    def test_zero_offset_ndim(self):
+        self.parse(u"cdef int[0:,0:,0:,0:] x")
+
+    def test_def_arg(self):
+        self.parse(u"def foo(int[:,:] x): pass")
+
+    def test_cdef_arg(self):
+        self.parse(u"cdef foo(int[:,:] x): pass")
+
+    def test_general_slice(self):
+        self.parse(u'cdef float[::ptr, ::direct & contig, 0::full & strided] x')
+
+    def test_non_slice_memview(self):
+        self.not_parseable(u"An axis specification in memoryview declaration does not have a ':'.",
+                u"cdef double[:foo, bar] x")
+        self.not_parseable(u"An axis specification in memoryview declaration does not have a ':'.",
+                u"cdef double[0:foo, bar] x")
+
+    def test_basic(self):
+        t = self.parse(u"cdef int[:] x")
+        memv_node = t.stats[0].base_type
+        self.assert_(isinstance(memv_node, MemoryViewSliceTypeNode))
+
+    # we also test other similar declarations (buffers, anonymous C arrays)
+    # since the parsing has to distinguish between them.
+
+    def disable_test_no_buf_arg(self): # TODO
+        self.not_parseable(u"Expected ']'",
+                u"cdef extern foo(object[int, ndim=2])")
+
+    def disable_test_parse_sizeof(self): # TODO
+        self.parse(u"sizeof(int[NN])")
+        self.parse(u"sizeof(int[])")
+        self.parse(u"sizeof(int[][NN])")
+        self.not_parseable(u"Expected an identifier or literal",
+                u"sizeof(int[:NN])")
+        self.not_parseable(u"Expected ']'",
+                u"sizeof(foo[dtype=bar]")
+
+if __name__ == '__main__':
+    import unittest
+    unittest.main()

Cython/Compiler/Tests/TestUtilityLoad.py

+import unittest
+
+from Cython.Compiler import Code, UtilityCode
+
+
+def strip_2tup(tup):
+    return tup[0] and tup[0].strip(), tup[1] and tup[1].strip()
+
+class TestUtilityLoader(unittest.TestCase):
+    """
+    Test loading UtilityCodes
+    """
+
+    expected = "test {{loader}} prototype", "test {{loader}} impl"
+    expected_tempita = (expected[0].replace('{{loader}}', 'Loader'),
+                        expected[1].replace('{{loader}}', 'Loader'))
+
+    required = "I am a dependency proto", "I am a dependency impl"
+
+    context = dict(loader='Loader')
+
+    name = "TestUtilityLoader"
+    filename = "TestUtilityLoader.c"
+    cls = Code.UtilityCode
+
+    def test_load_as_string(self):
+        got = strip_2tup(self.cls.load_as_string(self.name))
+        self.assertEquals(got, self.expected)
+
+        got = strip_2tup(self.cls.load_as_string(self.name, self.filename))
+        self.assertEquals(got, self.expected)
+
+        got = strip_2tup(self.cls.load_as_string(self.name, context=self.context))
+        self.assertEquals(got, self.expected_tempita)
+
+    def test_load(self):
+        utility = self.cls.load(self.name)
+        got = strip_2tup((utility.proto, utility.impl))
+        self.assertEquals(got, self.expected)
+
+        # Not implemented yet
+        #required, = utility.requires
+        #self.assertEquals((required.proto, required.impl), self.required)
+
+        utility = self.cls.load(self.name, from_file=self.filename)
+        got = strip_2tup((utility.proto, utility.impl))
+        self.assertEquals(got, self.expected)
+
+
+class TestCythonUtilityLoader(TestUtilityLoader):
+    """
+    Test loading CythonUtilityCodes
+    """
+
+    # Just change the attributes and run the same tests
+    expected = None, "test {{cy_loader}} impl"
+    expected_tempita = None, "test CyLoader impl"
+
+    required = None, "I am a Cython dependency impl"
+
+    context = dict(cy_loader='CyLoader')
+
+    name = "TestCyUtilityLoader"
+    filename = "TestCyUtilityLoader.pyx"
+    cls = UtilityCode.CythonUtilityCode
+
+    # Small hack to pass our tests above
+    cls.proto = None
\ No newline at end of file

Cython/Compiler/TreeFragment.py

@@ -32,7 +32,7 @@ class StringParseContext(Main.Context):
         return ModuleScope(module_name, parent_module = None, context = self)
 
 def parse_from_strings(name, code, pxds={}, level=None, initial_pos=None,
-                       context=None):
+                       context=None, allow_struct_enum_decorator=False):
     """
     Utility method to parse a (unicode) string of code. This is mostly
     used for internal Cython compiler purposes (creating code snippets
@@ -66,12 +66,15 @@ def parse_from_strings(name, code, pxds={}, level=None, initial_pos=None,
 
     scanner = PyrexScanner(buf, code_source, source_encoding = encoding,
                      scope = scope, context = context, initial_pos = initial_pos)
+    ctx = Parsing.Ctx(allow_struct_enum_decorator=allow_struct_enum_decorator)
+
     if level is None:
-        tree = Parsing.p_module(scanner, 0, module_name)
+        tree = Parsing.p_module(scanner, 0, module_name, ctx=ctx)
         tree.scope = scope
         tree.is_pxd = False
     else:
-        tree = Parsing.p_code(scanner, level=level)
+        tree = Parsing.p_code(scanner, level=level, ctx=ctx)
+
     tree.scope = scope
     return tree
 

Cython/Compiler/UtilityCode.py

@@ -3,9 +3,15 @@ from Scanning import StringSourceDescriptor
 import Symtab
 import Naming
 from Cython.Compiler import Visitor
+import Code
 
 class NonManglingModuleScope(Symtab.ModuleScope):
 
+    def __init__(self, prefix, *args, **kw):
+        self.prefix = prefix
+        self.cython_scope = None
+        Symtab.ModuleScope.__init__(self, *args, **kw)
+
     def add_imported_entry(self, name, entry, pos):
         entry.used = True
         return super(NonManglingModuleScope, self).add_imported_entry(
@@ -13,32 +19,36 @@ class NonManglingModuleScope(Symtab.ModuleScope):
 
     def mangle(self, prefix, name=None):
         if name:
-            if prefix in (Naming.typeobj_prefix, Naming.func_prefix):
+            if prefix in (Naming.typeobj_prefix, Naming.func_prefix, Naming.var_prefix, Naming.pyfunc_prefix):
                 # Functions, classes etc. gets a manually defined prefix easily
                 # manually callable instead (the one passed to CythonUtilityCode)
                 prefix = self.prefix
             return "%s%s" % (prefix, name)
         else:
-            return self.base.name
+            return Symtab.ModuleScope.mangle(self, prefix)
 
 class CythonUtilityCodeContext(StringParseContext):
     scope = None
 
     def find_module(self, module_name, relative_to = None, pos = None,
                     need_pxd = 1):
+
         if module_name != self.module_name:
-            raise AssertionError("Not yet supporting any cimports/includes "
-                                 "from string code snippets")
+            if module_name not in self.modules:
+                raise AssertionError("Only the cython cimport is supported.")
+            else:
+                return self.modules[module_name]
 
         if self.scope is None:
-            self.scope = NonManglingModuleScope(
-                    module_name, parent_module=None, context=self)
-            self.scope.prefix = self.prefix
+            self.scope = NonManglingModuleScope(self.prefix,
+                                                module_name,
+                                                parent_module=None,
+                                                context=self)
 
         return self.scope
 
 
-class CythonUtilityCode(object):
+class CythonUtilityCode(Code.UtilityCodeBase):
     """
     Utility code written in the Cython language itself.
 
@@ -46,8 +56,8 @@ class CythonUtilityCode(object):
     Functions decorated with @cname('c_func_name') get the given cname.
 
     For cdef classes the rules are as follows:
-        obj struct      -> <cname>
-        obj type ptr    -> __pyx_ptype_<cname>
+        obj struct      -> <cname>_obj
+        obj type ptr    -> <cname>_type
         methods         -> <class_cname>_<method_cname>
 
     For methods the cname decorator is optional, but without the decorator the
@@ -57,19 +67,22 @@ class CythonUtilityCode(object):
 
     is_cython_utility = True
 
-    def __init__(self, impl, name="CythonUtilityCode", prefix="", requires=None):
+    def __init__(self, impl, name="__pyxutil", prefix="", requires=None,
+                 file=None, from_scope=None):
         # 1) We need to delay the parsing/processing, so that all modules can be
         #    imported without import loops
         # 2) The same utility code object can be used for multiple source files;
         #    while the generated node trees can be altered in the compilation of a
         #    single file.
         # Hence, delay any processing until later.
-        self.pyx = impl
+        self.impl = impl
         self.name = name
+        self.file = file
         self.prefix = prefix
         self.requires = requires or []
+        self.from_scope = from_scope
 
-    def get_tree(self):
+    def get_tree(self, entries_only=False, cython_scope=None):
         from AnalysedTreeTransforms import AutoTestDictTransform
         # The AutoTestDictTransform creates the statement "__test__ = {}",
         # which when copied into the main ModuleNode overwrites
@@ -79,10 +92,21 @@ class CythonUtilityCode(object):
         import Pipeline, ParseTreeTransforms
         context = CythonUtilityCodeContext(self.name)
         context.prefix = self.prefix
+        context.cython_scope = cython_scope
         #context = StringParseContext(self.name)
-        tree = parse_from_strings(self.name, self.pyx, context=context)
+        tree = parse_from_strings(self.name, self.impl, context=context,
+                                  allow_struct_enum_decorator=True)
         pipeline = Pipeline.create_pipeline(context, 'pyx', exclude_classes=excludes)
 
+        if entries_only:
+            p = []
+            for t in pipeline:
+                p.append(t)
+                if isinstance(p, ParseTreeTransforms.AnalyseDeclarationsTransform):
+                    break
+
+            pipeline = p
+
         transform = ParseTreeTransforms.CnameDirectivesTransform(context)
         # InterpretCompilerDirectives already does a cdef declarator check
         #before = ParseTreeTransforms.DecoratorTransform
@@ -90,21 +114,31 @@ class CythonUtilityCode(object):
         pipeline = Pipeline.insert_into_pipeline(pipeline, transform,
                                                  before=before)
 
-        (err, tree) = Pipeline.run_pipeline(pipeline, tree)
+        if self.from_scope:
+            def scope_transform(module_node):
+                module_node.scope.merge_in(self.from_scope)
+                return module_node
+
+            transform = ParseTreeTransforms.AnalyseDeclarationsTransform
+            pipeline = Pipeline.insert_into_pipeline(pipeline, scope_transform,
+                                                     before=transform)
+
+        (err, tree) = Pipeline.run_pipeline(pipeline, tree, printtree=False)
         assert not err, err
         return tree
 
     def put_code(self, output):
         pass
 
-    def declare_in_scope(self, dest_scope, used=False):
+    def declare_in_scope(self, dest_scope, used=False, cython_scope=None):
         """
         Declare all entries from the utility code in dest_scope. Code will only
-        be included for used entries.
+        be included for used entries. If module_name is given, declare the
+        type entries with that name.
         """
-        self.tree = self.get_tree()
+        tree = self.get_tree(entries_only=True, cython_scope=cython_scope)
 
-        entries = self.tree.scope.entries
+        entries = tree.scope.entries
         entries.pop('__name__')
         entries.pop('__file__')
         entries.pop('__builtins__')
@@ -114,9 +148,12 @@ class CythonUtilityCode(object):
             entry.utility_code_definition = self
             entry.used = used
 
-        dest_scope.merge_in(self.tree.scope, merge_unused=True)
-        self.tree.scope = dest_scope
+        original_scope = tree.scope
+        dest_scope.merge_in(original_scope, merge_unused=True)
+        tree.scope = dest_scope
 
         for dep in self.requires:
             if dep.is_cython_utility:
                 dep.declare_in_scope(dest_scope)
+
+        return original_scope

Cython/Utility/TestCyUtilityLoader.pyx

+########## TestCyUtilityLoader ##########
+test {{cy_loader}} impl