merge -- working fused types, broken cython testscope

.gitignore

 *.pyc
 *.pyo
 __pycache__
+*.so
 
 Cython/Compiler/*.c
 Cython/Plex/*.c

.hgignore

@@ -18,6 +18,7 @@ dist/
 *.rej
 *.dep
 *.swp
+*.so
 *~
 
 tags

Cython/Build/Dependencies.py

 from glob import glob
 import re, os, sys
-from cython import set
 
 
 from distutils.extension import Extension
@@ -8,7 +7,19 @@ from distutils.extension import Extension
 from Cython import Utils
 from Cython.Compiler.Main import Context, CompilationOptions, default_options
 
-# Unfortunately, Python 2.3 doesn't support decorators.
+def cached_function(f):
+    cache_name = '__%s_cache' % f.__name__
+    def wrapper(*args):
+        cache = getattr(f, cache_name, None)
+        if cache is None:
+            cache = {}
+            setattr(f, cache_name, cache)
+        if args in cache:
+            return cache[args]
+        res = cache[args] = f(*args)
+        return res
+    return wrapper
+
 def cached_method(f):
     cache_name = '__%s_cache' % f.__name__
     def wrapper(self, *args):
@@ -24,6 +35,16 @@ def cached_method(f):
 
 
 def parse_list(s):
+    """
+    >>> parse_list("a b c")
+    ['a', 'b', 'c']
+    >>> parse_list("[a, b, c]")
+    ['a', 'b', 'c']
+    >>> parse_list('a " " b')
+    ['a', ' ', 'b']
+    >>> parse_list('[a, ",a", "a,", ",", ]')
+    ['a', ',a', 'a,', ',']
+    """
     if s[0] == '[' and s[-1] == ']':
         s = s[1:-1]
         delimiter = ','
@@ -32,12 +53,11 @@ def parse_list(s):
     s, literals = strip_string_literals(s)
     def unquote(literal):
         literal = literal.strip()
-        if literal[0] == "'":
+        if literal[0] in "'\"":
             return literals[literal[1:-1]]
         else:
             return literal
-
-    return [unquote(item) for item in s.split(delimiter)]
+    return [unquote(item) for item in s.split(delimiter) if item.strip()]
 
 transitive_str = object()
 transitive_list = object()
@@ -254,12 +274,11 @@ class DependencyTree(object):
         self.context = context
         self._transitive_cache = {}
 
-    #@cached_method
+    @cached_method
     def parse_dependencies(self, source_filename):
         return parse_dependencies(source_filename)
-    parse_dependencies = cached_method(parse_dependencies)
     
-    #@cached_method
+    @cached_method
     def cimports_and_externs(self, filename):
         cimports, includes, externs = self.parse_dependencies(filename)[:3]
         cimports = set(cimports)
@@ -275,25 +294,22 @@ class DependencyTree(object):
             else:
                 print("Unable to locate '%s' referenced from '%s'" % (filename, include))
         return tuple(cimports), tuple(externs)
-    cimports_and_externs = cached_method(cimports_and_externs)
 
     def cimports(self, filename):
         return self.cimports_and_externs(filename)[0]
 
-    #@cached_method
+    @cached_method
     def package(self, filename):
-        dir = os.path.dirname(filename)
-        if os.path.exists(os.path.join(dir, '__init__.py')):
+        dir = os.path.dirname(os.path.abspath(filename))
+        if dir != filename and os.path.exists(os.path.join(dir, '__init__.py')):
             return self.package(dir) + (os.path.basename(dir),)
         else:
             return ()
-    package = cached_method(package)
 
-    #@cached_method
+    @cached_method
     def fully_qualifeid_name(self, filename):
         module = os.path.splitext(os.path.basename(filename))[0]
         return '.'.join(self.package(filename) + (module,))
-    fully_qualifeid_name = cached_method(fully_qualifeid_name)
 
     def find_pxd(self, module, filename=None):
         if module[0] == '.':
@@ -306,7 +322,7 @@ class DependencyTree(object):
         return self.context.find_pxd_file(module, None)
     find_pxd = cached_method(find_pxd)
 
-    #@cached_method
+    @cached_method
     def cimported_files(self, filename):
         if filename[-4:] == '.pyx' and os.path.exists(filename[:-4] + '.pxd'):
             self_pxd = [filename[:-4] + '.pxd']
@@ -319,7 +335,6 @@ class DependencyTree(object):
             print("\n\t".join(a))
             print("\n\t".join(b))
         return tuple(self_pxd + filter(None, [self.find_pxd(m, filename) for m in self.cimports(filename)]))
-    cimported_files = cached_method(cimported_files)
 
     def immediate_dependencies(self, filename):
         all = list(self.cimported_files(filename))
@@ -327,10 +342,9 @@ class DependencyTree(object):
             all.append(os.path.normpath(os.path.join(os.path.dirname(filename), extern)))
         return tuple(all)
 
-    #@cached_method
+    @cached_method
     def timestamp(self, filename):
         return os.path.getmtime(filename)
-    timestamp = cached_method(timestamp)
 
     def extract_timestamp(self, filename):
         # TODO: .h files from extern blocks
@@ -446,7 +460,7 @@ def create_extension_list(patterns, exclude=[], ctx=None, aliases=None):
     return module_list
 
 # This is the user-exposed entry point.
-def cythonize(module_list, exclude=[], nthreads=0, aliases=None, quiet=False, **options):
+def cythonize(module_list, exclude=[], nthreads=0, aliases=None, quiet=False, force=False, **options):
     if 'include_path' not in options:
         options['include_path'] = ['.']
     c_options = CompilationOptions(**options)
@@ -482,7 +496,7 @@ def cythonize(module_list, exclude=[], nthreads=0, aliases=None, quiet=False, **
                 else:
                     dep_timestamp, dep = deps.newest_dependency(source)
                     priority = 2 - (dep in deps.immediate_dependencies(source))
-                if c_timestamp < dep_timestamp:
+                if force or c_timestamp < dep_timestamp:
                     if not quiet:
                         if source == dep:
                             print("Compiling %s because it changed." % source)
@@ -505,14 +519,16 @@ def cythonize(module_list, exclude=[], nthreads=0, aliases=None, quiet=False, **
             nthreads = 0
     if not nthreads:
         for priority, pyx_file, c_file, options in to_compile:
-            cythonize_one(pyx_file, c_file, options)
+            cythonize_one(pyx_file, c_file, quiet, options)
     return module_list
 
 # TODO: Share context? Issue: pyx processing leaks into pxd module
-def cythonize_one(pyx_file, c_file, options=None):
+def cythonize_one(pyx_file, c_file, quiet, options=None):
     from Cython.Compiler.Main import compile, default_options
     from Cython.Compiler.Errors import CompileError, PyrexError
 
+    if not quiet:
+        print "Cythonizing %s" % pyx_file
     if options is None:
         options = CompilationOptions(default_options)
     options.output_file = c_file

Cython/Build/Inline.py

 import tempfile
 import sys, os, re, inspect
-from cython import set
 
 try:
     import hashlib
@@ -15,7 +14,9 @@ from Cython.Compiler.Main import Context, CompilationOptions, default_options
 
 from Cython.Compiler.ParseTreeTransforms import CythonTransform, SkipDeclarations, AnalyseDeclarationsTransform
 from Cython.Compiler.TreeFragment import parse_from_strings
-from Cython.Build.Dependencies import strip_string_literals, cythonize
+from Cython.Build.Dependencies import strip_string_literals, cythonize, cached_function
+from Cython.Compiler import Pipeline
+import cython as cython_module
 
 # A utility function to convert user-supplied ASCII strings to unicode.
 if sys.version_info[0] < 3:
@@ -37,13 +38,14 @@ class AllSymbols(CythonTransform, SkipDeclarations):
     def visit_NameNode(self, node):
         self.names.add(node.name)
 
+@cached_function
 def unbound_symbols(code, context=None):
     code = to_unicode(code)
     if context is None:
         context = Context([], default_options)
     from Cython.Compiler.ParseTreeTransforms import AnalyseDeclarationsTransform
     tree = parse_from_strings('(tree fragment)', code)
-    for phase in context.create_pipeline(pxd=False):
+    for phase in Pipeline.create_pipeline(context, 'pyx'):
         if phase is None:
             continue
         tree = phase(tree)
@@ -103,6 +105,7 @@ def cython_inline(code,
     if get_type is None:
         get_type = lambda x: 'object'
     code = to_unicode(code)
+    orig_code = code
     code, literals = strip_string_literals(code)
     code = strip_common_indent(code)
     ctx = Context(cython_include_dirs, default_options)
@@ -124,10 +127,15 @@ def cython_inline(code,
         if not quiet:
             # Parsing from strings not fully supported (e.g. cimports).
             print("Could not parse code as a string (to extract unbound symbols).")
+    cimports = []
+    for name, arg in kwds.items():
+        if arg is cython_module:
+            cimports.append('\ncimport cython as %s' % name)
+            del kwds[name]
     arg_names = kwds.keys()
     arg_names.sort()
     arg_sigs = tuple([(get_type(kwds[arg], ctx), arg) for arg in arg_names])
-    key = code, arg_sigs, sys.version_info, sys.executable, Cython.__version__
+    key = orig_code, arg_sigs, sys.version_info, sys.executable, Cython.__version__
     module_name = "_cython_inline_" + hashlib.md5(str(key).encode('utf-8')).hexdigest()
     try:
         if not os.path.exists(lib_dir):
@@ -141,7 +149,6 @@ def cython_inline(code,
     except ImportError:
         cflags = []
         c_include_dirs = []
-        cimports = []
         qualified = re.compile(r'([.\w]+)[.]')
         for type, _ in arg_sigs:
             m = qualified.match(type)

Cython/Build/Tests/TestInline.py

@@ -40,6 +40,15 @@ class TestInline(CythonTest):
     def test_globals(self):
         self.assertEquals(inline("return global_value + 1", **self.test_kwds), global_value + 1)
 
+    def test_pure(self):    
+        import cython as cy
+        b = inline("""
+        b = cy.declare(float, a)
+        c = cy.declare(cy.pointer(cy.float), &b)
+        return b
+        """, a=3)
+        self.assertEquals(type(b), float)
+
     if has_numpy:
 
         def test_numpy(self):

Cython/Compiler/Builtin.py

 #
-#   Pyrex - Builtin Definitions
+#   Builtin Definitions
 #
 
 from Symtab import BuiltinScope, StructOrUnionScope
@@ -23,6 +23,20 @@ proto = """
 """
 )
 
+abs_int_utility_code = UtilityCode(
+proto = '''
+#if HAVE_LONG_LONG
+#define __Pyx_abs_int(x) \
+    ((sizeof(x) <= sizeof(int)) ? ((unsigned int)abs(x)) : \
+     ((sizeof(x) <= sizeof(long)) ? ((unsigned long)labs(x)) : \
+      ((unsigned PY_LONG_LONG)llabs(x))))
+#else
+#define __Pyx_abs_int(x) \
+    ((sizeof(x) <= sizeof(int)) ? ((unsigned int)abs(x)) : ((unsigned long)labs(x)))
+#endif
+#define __Pyx_abs_long(x) __Pyx_abs_int(x)
+''')
+
 iter_next_utility_code = UtilityCode(
 proto = """
 #define __Pyx_PyIter_Next(obj) __Pyx_PyIter_Next2(obj, NULL);
@@ -127,14 +141,6 @@ bad:
 
 pyexec_utility_code = UtilityCode(
 proto = """
-#if PY_VERSION_HEX < 0x02040000
-#ifndef Py_COMPILE_H
-#include "compile.h"
-#endif
-#ifndef Py_EVAL_H
-#include "eval.h"
-#endif
-#endif
 static PyObject* __Pyx_PyRun(PyObject*, PyObject*, PyObject*);
 static CYTHON_INLINE PyObject* __Pyx_PyRun2(PyObject*, PyObject*);
 """,
@@ -234,12 +240,7 @@ static PyObject* __Pyx_Intern(PyObject* s) {
 }
 ''')
 
-def put_py23_set_init_utility_code(code, pos):
-    code.putln("#if PY_VERSION_HEX < 0x02040000")
-    code.putln(code.error_goto_if_neg("__Pyx_Py23SetsImport()", pos))
-    code.putln("#endif")
-
-py23_set_utility_code = UtilityCode(
+py_set_utility_code = UtilityCode(
 proto = """
 #if PY_VERSION_HEX < 0x02050000
 #ifndef PyAnySet_CheckExact
@@ -282,64 +283,13 @@ static CYTHON_INLINE int PySet_Add(PyObject *set, PyObject *key) {
 }
 
 #endif /* PyAnySet_CheckExact (<= Py2.4) */
-
-#if PY_VERSION_HEX < 0x02040000
-#ifndef Py_SETOBJECT_H
-#define Py_SETOBJECT_H
-
-static PyTypeObject *__Pyx_PySet_Type = NULL;
-static PyTypeObject *__Pyx_PyFrozenSet_Type = NULL;
-
-#define PySet_Type (*__Pyx_PySet_Type)
-#define PyFrozenSet_Type (*__Pyx_PyFrozenSet_Type)
-
-#define PyAnySet_Check(ob) \\
-    (PyAnySet_CheckExact(ob) || \\
-     PyType_IsSubtype((ob)->ob_type, &PySet_Type) || \\
-     PyType_IsSubtype((ob)->ob_type, &PyFrozenSet_Type))
-
-#define PyFrozenSet_CheckExact(ob) ((ob)->ob_type == &PyFrozenSet_Type)
-
-static int __Pyx_Py23SetsImport(void) {
-    PyObject *sets=0, *Set=0, *ImmutableSet=0;
-
-    sets = PyImport_ImportModule((char *)"sets");
-    if (!sets) goto bad;
-    Set = PyObject_GetAttrString(sets, (char *)"Set");
-    if (!Set) goto bad;
-    ImmutableSet = PyObject_GetAttrString(sets, (char *)"ImmutableSet");
-    if (!ImmutableSet) goto bad;
-    Py_DECREF(sets);
-
-    __Pyx_PySet_Type       = (PyTypeObject*) Set;
-    __Pyx_PyFrozenSet_Type = (PyTypeObject*) ImmutableSet;
-
-    return 0;
-
- bad:
-    Py_XDECREF(sets);
-    Py_XDECREF(Set);
-    Py_XDECREF(ImmutableSet);
-    return -1;
-}
-
-#else
-static int __Pyx_Py23SetsImport(void) { return 0; }
-#endif /* !Py_SETOBJECT_H */
-#endif /* < Py2.4  */
 #endif /* < Py2.5  */
 """,
-init = put_py23_set_init_utility_code,
-cleanup = """
-#if PY_VERSION_HEX < 0x02040000
-Py_XDECREF(__Pyx_PySet_Type); __Pyx_PySet_Type = NULL;
-Py_XDECREF(__Pyx_PyFrozenSet_Type); __Pyx_PyFrozenSet_Type = NULL;
-#endif /* < Py2.4  */
-""")
+)
 
 builtin_utility_code = {
-    'set'       : py23_set_utility_code,
-    'frozenset' : py23_set_utility_code,
+    'set'       : py_set_utility_code,
+    'frozenset' : py_set_utility_code,
 }
 
 
@@ -347,10 +297,12 @@ builtin_utility_code = {
 
 class _BuiltinOverride(object):
     def __init__(self, py_name, args, ret_type, cname, py_equiv = "*",
-                 utility_code = None, sig = None, func_type = None):
+                 utility_code = None, sig = None, func_type = None,
+                 is_strict_signature = False):
         self.py_name, self.cname, self.py_equiv = py_name, cname, py_equiv
         self.args, self.ret_type = args, ret_type
         self.func_type, self.sig = func_type, sig
+        self.is_strict_signature = is_strict_signature
         self.utility_code = utility_code
 
 class BuiltinAttribute(object):
@@ -376,6 +328,8 @@ class BuiltinFunction(_BuiltinOverride):
             if sig is None:
                 sig = Signature(self.args, self.ret_type)
             func_type = sig.function_type()
+            if self.is_strict_signature:
+                func_type.is_strict_signature = True
         scope.declare_builtin_cfunction(self.py_name, func_type, self.cname,
                                         self.py_equiv, self.utility_code)
 
@@ -388,13 +342,34 @@ class BuiltinMethod(_BuiltinOverride):
             # override 'self' type (first argument)
             self_arg = PyrexTypes.CFuncTypeArg("", self_type, None)
             self_arg.not_none = True
+            self_arg.accept_builtin_subtypes = True
             method_type = sig.function_type(self_arg)
+            if self.is_strict_signature:
+                method_type.is_strict_signature = True
         self_type.scope.declare_builtin_cfunction(
             self.py_name, method_type, self.cname, utility_code = self.utility_code)
 
 
 builtin_function_table = [
     # name,        args,   return,  C API func,           py equiv = "*"
+    BuiltinFunction('abs',        "d",    "d",     "fabs",
+                    is_strict_signature = True),
+    BuiltinFunction('abs',        "f",    "f",     "fabsf",
+                    is_strict_signature = True),
+    BuiltinFunction('abs',        None,    None,   "__Pyx_abs_int",
+                    utility_code = abs_int_utility_code,
+                    func_type = PyrexTypes.CFuncType(
+                        PyrexTypes.c_uint_type, [
+                            PyrexTypes.CFuncTypeArg("arg", PyrexTypes.c_int_type, None)
+                            ],
+                        is_strict_signature = True)),
+    BuiltinFunction('abs',        None,    None,   "__Pyx_abs_long",
+                    utility_code = abs_int_utility_code,
+                    func_type = PyrexTypes.CFuncType(
+                        PyrexTypes.c_ulong_type, [
+                            PyrexTypes.CFuncTypeArg("arg", PyrexTypes.c_long_type, None)
+                            ],
+                        is_strict_signature = True)),
     BuiltinFunction('abs',        "O",    "O",     "PyNumber_Absolute"),
     #('chr',       "",     "",      ""),
     #('cmp', "",   "",     "",      ""), # int PyObject_Cmp(PyObject *o1, PyObject *o2, int *result)
@@ -529,13 +504,13 @@ builtin_types_table = [
 #    ("file",    "PyFile_Type",     []),  # not in Py3
 
     ("set",       "PySet_Type",    [BuiltinMethod("clear",   "T",  "r", "PySet_Clear",
-                                                  utility_code = py23_set_utility_code),
+                                                  utility_code = py_set_utility_code),
                                     BuiltinMethod("discard", "TO", "r", "PySet_Discard",
-                                                  utility_code = py23_set_utility_code),
+                                                  utility_code = py_set_utility_code),
                                     BuiltinMethod("add",     "TO", "r", "PySet_Add",
-                                                  utility_code = py23_set_utility_code),
+                                                  utility_code = py_set_utility_code),
                                     BuiltinMethod("pop",     "T",  "O", "PySet_Pop",
-                                                  utility_code = py23_set_utility_code)]),
+                                                  utility_code = py_set_utility_code)]),
     ("frozenset", "PyFrozenSet_Type", []),
 ]
 

Cython/Compiler/Code.pxd

 
 cimport cython
 
-cdef class UtilityCode:
+cdef class UtilityCodeBase(object):
+    cdef public object name
+
+cdef class UtilityCode(UtilityCodeBase):
     cdef public object proto
     cdef public object impl
     cdef public object init
@@ -10,6 +13,7 @@ cdef class UtilityCode:
     cdef public dict _cache
     cdef public list specialize_list
     cdef public object proto_block
+    cdef public object file
 
     cpdef put_code(self, output)
 
@@ -33,6 +37,7 @@ cdef class FunctionState:
     cdef public dict temps_free
     cdef public dict temps_used_type
     cdef public size_t temp_counter
+    cdef public list collect_temps_stack
 
     cdef public object closure_temps
     cdef public bint should_declare_error_indicator
@@ -43,6 +48,8 @@ cdef class FunctionState:
     cpdef set_loop_labels(self, labels)
     cpdef tuple get_all_labels(self)
     cpdef set_all_labels(self, labels)
+    cpdef start_collecting_temps(self)
+    cpdef stop_collecting_temps(self)
 
     cpdef list temps_in_use(self)
 

Cython/Compiler/CythonScope.py

 from Symtab import ModuleScope
 from PyrexTypes import *
-
-shape_func_type = CFuncType(
-    c_ptr_type(c_py_ssize_t_type),
-    [CFuncTypeArg("buffer", py_object_type, None)])
+from UtilityCode import CythonUtilityCode
+from Errors import error
+from Scanning import StringSourceDescriptor
+import Options
+import Buffer
+import MemoryView
 
 class CythonScope(ModuleScope):
+    is_cython_builtin = 1
+
     def __init__(self, context):
-        ModuleScope.__init__(self, u'cython', None, context)
+        ModuleScope.__init__(self, u'cython', None, None)
         self.pxd_file_loaded = True
-
-        self.shape_entry = self.declare_cfunction('shape',
-                                                  shape_func_type,
-                                                  pos=None,
-                                                  defining = 1,
-                                                  cname='<error>')
+        self.populate_cython_scope()
+        # The Main.Context object
+        self.context = context
 
         for fused_type in (cy_integral_type, cy_floating_type, cy_numeric_type):
             entry = self.declare_typedef(fused_type.name,
@@ -31,28 +32,118 @@ class CythonScope(ModuleScope):
 
         return super(CythonScope, self).lookup_type(name)
 
-def create_cython_scope(context):
-    create_utility_scope(context)
-    return CythonScope(context)
+    def find_module(self, module_name, pos):
+        error("cython.%s is not available" % module_name, pos)
 
+    def find_submodule(self, module_name):
+        entry = self.entries.get(module_name, None)
+        if entry and entry.as_module:
+            return entry.as_module
+        else:
+            # TODO: fix find_submodule control flow so that we're not
+            # expected to create a submodule here (to protect CythonScope's
+            # possible immutability). Hack ourselves out of the situation
+            # for now.
+            raise error((StringSourceDescriptor(u"cython", u""), 0, 0),
+                  "cython.%s is not available" % module_name)
 
-def create_utility_scope(context):
-    global utility_scope
-    utility_scope = ModuleScope(u'utility', None, context)
+    def lookup_qualified_name(self, qname):
+        # ExprNode.as_cython_attribute generates qnames and we untangle it here...
+        name_path = qname.split(u'.')
+        scope = self
+        while len(name_path) > 1:
+            scope = scope.lookup_here(name_path[0]).as_module
+            del name_path[0]
+            if scope is None:
+                return None
+        else:
+            return scope.lookup_here(name_path[0])
 
+    def populate_cython_scope(self):
         # These are used to optimize isinstance in FinalOptimizePhase
-    type_object = utility_scope.declare_typedef('PyTypeObject',
+        type_object = self.declare_typedef(
+            'PyTypeObject',
             base_type = c_void_type,
             pos = None,
             cname = 'PyTypeObject')
         type_object.is_void = True
+        type_object_type = type_object.type
 
-    utility_scope.declare_cfunction(
+        self.declare_cfunction(
             'PyObject_TypeCheck',
             CFuncType(c_bint_type, [CFuncTypeArg("o", py_object_type, None),
-                                        CFuncTypeArg("t", c_ptr_type(type_object), None)]),
+                                    CFuncTypeArg("t", c_ptr_type(type_object_type), None)]),
             pos = None,
             defining = 1,
             cname = 'PyObject_TypeCheck')
 
-    return utility_scope
+#        self.test_cythonscope()
+
+    def test_cythonscope(self):
+        """
+        Creates some entries for testing purposes and entries for
+        cython.array() and for cython.view.*.
+        """
+        cython_testscope_utility_code.declare_in_scope(
+                                self, cython_scope=self)
+        cython_test_extclass_utility_code.declare_in_scope(
+                                    self, cython_scope=self)
+
+        MemoryView.cython_array_utility_code.declare_in_scope(
+                                        self, cython_scope=self)
+
+        #
+        # The view sub-scope
+        #
+        self.viewscope = viewscope = ModuleScope(u'view', self, None)
+        self.declare_module('view', viewscope, None).as_module = viewscope
+        viewscope.is_cython_builtin = True
+        viewscope.pxd_file_loaded = True
+
+        cythonview_testscope_utility_code.declare_in_scope(
+                                                viewscope, cython_scope=self)
+
+        view_utility_scope = MemoryView.view_utility_code.declare_in_scope(
+                                                viewscope, cython_scope=self)
+        # MemoryView.memview_fromslice_utility_code.from_scope = view_utility_scope
+        # MemoryView.memview_fromslice_utility_code.declare_in_scope(viewscope)
+
+
+def create_cython_scope(context, create_testscope):
+    # One could in fact probably make it a singleton,
+    # but not sure yet whether any code mutates it (which would kill reusing
+    # it across different contexts)
+    scope = CythonScope(context)
+
+    if create_testscope:
+        scope.test_cythonscope()
+
+    return scope
+
+
+# Load test utilities for the cython scope
+
+def load_testscope_utility(cy_util_name, **kwargs):
+    return CythonUtilityCode.load(cy_util_name, "TestCythonScope.pyx", **kwargs)
+
+
+undecorated_methods_protos = UtilityCode(proto=u"""
+    /* These methods are undecorated and have therefore no prototype */
+    static PyObject *__pyx_TestClass_cdef_method(
+            struct __pyx_TestClass_obj *self, int value);
+    static PyObject *__pyx_TestClass_cpdef_method(
+            struct __pyx_TestClass_obj *self, int value, int skip_dispatch);
+    static PyObject *__pyx_TestClass_def_method(
+            PyObject *self, PyObject *value);
+""")
+
+cython_testscope_utility_code = load_testscope_utility("TestScope")
+
+test_cython_utility_dep = load_testscope_utility("TestDep")
+
+cython_test_extclass_utility_code = \
+    load_testscope_utility("TestClass", name="TestClass",
+                           requires=[undecorated_methods_protos,
+                                     test_cython_utility_dep])
+
+cythonview_testscope_utility_code = load_testscope_utility("View.TestScope")

Cython/Compiler/Errors.py

 #
-#   Pyrex - Errors
+#   Errors
 #
 
 import sys
@@ -32,7 +32,7 @@ def context(position):
 
 def format_position(position):
     if position:
-        return u"%s:%d:%d: " % (position[0].get_description(),
+        return u"%s:%d:%d: " % (position[0].get_error_description(),
                                 position[1], position[2])
     return u''
 

Cython/Compiler/FlowControl.pxd

@@ -54,7 +54,7 @@ cdef class ControlFlow:
      cpdef nextblock(self, parent=*)
      cpdef bint is_tracked(self, entry)
      cpdef mark_position(self, node)
-     cpdef mark_assignment(self, lhs, rhs, entry=*)
+     cpdef mark_assignment(self, lhs, rhs, entry)
      cpdef mark_argument(self, lhs, rhs, entry)
      cpdef mark_deletion(self, node, entry)
      cpdef mark_reference(self, node, entry)

Cython/Compiler/Naming.py

 #
-#   Pyrex - C naming conventions
+#   C naming conventions
 #
 #
 #   Prefixes for generating C names.
@@ -38,10 +38,8 @@ typeobj_prefix    = pyrex_prefix + "type_"
 var_prefix        = pyrex_prefix + "v_"
 varptr_prefix     = pyrex_prefix + "vp_"
 wrapperbase_prefix= pyrex_prefix + "wrapperbase_"
-bufstruct_prefix  = pyrex_prefix + "bstruct_"
-bufstride_prefix  = pyrex_prefix + "bstride_"
-bufshape_prefix   = pyrex_prefix + "bshape_"
-bufsuboffset_prefix  = pyrex_prefix + "boffset_"
+pybuffernd_prefix   = pyrex_prefix + "pybuffernd_"
+pybufferstruct_prefix  = pyrex_prefix + "pybuffer_"
 vtable_prefix     = pyrex_prefix + "vtable_"
 vtabptr_prefix    = pyrex_prefix + "vtabptr_"
 vtabstruct_prefix = pyrex_prefix + "vtabstruct_"
@@ -109,6 +107,14 @@ exc_value_name  = pyrex_prefix + "exc_value"
 exc_tb_name     = pyrex_prefix + "exc_tb"
 exc_lineno_name = pyrex_prefix + "exc_lineno"
 
+parallel_exc_type = pyrex_prefix + "parallel_exc_type"
+parallel_exc_value = pyrex_prefix + "parallel_exc_value"
+parallel_exc_tb = pyrex_prefix + "parallel_exc_tb"
+parallel_filename = pyrex_prefix + "parallel_filename"
+parallel_lineno = pyrex_prefix + "parallel_lineno"
+parallel_clineno = pyrex_prefix + "parallel_clineno"
+parallel_why = pyrex_prefix + "parallel_why"
+
 exc_vars = (exc_type_name, exc_value_name, exc_tb_name)
 
 api_name        = pyrex_prefix + "capi__"

Cython/Compiler/Options.py

@@ -67,11 +67,15 @@ old_style_globals = False
 cimport_from_pyx = False
 
 
+# max # of dims for buffers -- set to same value as max # of dims for numpy
+# arrays.
+buffer_max_dims = 32
 
 # Declare compiler directives
 directive_defaults = {
     'boundscheck' : True,
     'nonecheck' : False,
+    'initializedcheck' : True,
     'embedsignature' : False,
     'locals' : {},
     'auto_cpdef': False,
@@ -141,7 +145,7 @@ for key, val in directive_defaults.items():
 
 directive_scopes = { # defaults to available everywhere
     # 'module', 'function', 'class', 'with statement'
-    'final' : ('cclass',),   # add 'method' in the future
+    'final' : ('cclass', 'function'),
     'internal' : ('cclass',),
     'autotestdict' : ('module',),
     'autotestdict.all' : ('module',),

Cython/Compiler/Parsing.pxd

@@ -7,8 +7,8 @@ ctypedef object (*p_sub_expr_func)(object)
 
 # entry points
 
-cpdef p_module(PyrexScanner s, pxd, full_module_name)
-cpdef p_code(PyrexScanner s, level= *)
+cpdef p_module(PyrexScanner s, pxd, full_module_name, ctx=*)
+cpdef p_code(PyrexScanner s, level= *, ctx=*)
 
 # internal parser states
 
@@ -62,7 +62,7 @@ cdef p_name(PyrexScanner s, name)
 cdef p_cat_string_literal(PyrexScanner s)
 cdef p_opt_string_literal(PyrexScanner s, required_type=*)
 cdef bint check_for_non_ascii_characters(unicode string)
-@cython.locals(systr=unicode, is_python3_source=bint)
+@cython.locals(systr=unicode, is_python3_source=bint, is_raw=bint)
 cdef p_string_literal(PyrexScanner s, kind_override=*)
 cdef p_list_maker(PyrexScanner s)
 cdef p_comp_iter(PyrexScanner s, body)
@@ -131,6 +131,8 @@ cdef p_calling_convention(PyrexScanner s)
 cdef p_c_complex_base_type(PyrexScanner s)
 cpdef p_c_simple_base_type(PyrexScanner s, bint self_flag, bint nonempty, templates = *)
 cdef p_buffer_or_template(PyrexScanner s, base_type_node, templates)
+cdef is_memoryviewslice_access(PyrexScanner s)
+cdef p_memoryviewslice_access(PyrexScanner s, base_type_node)
 cdef bint looking_at_name(PyrexScanner s) except -2
 cdef object looking_at_expr(PyrexScanner s)# except -2
 cdef bint looking_at_base_type(PyrexScanner s) except -2

Cython/Compiler/Scanning.py

@@ -204,6 +204,13 @@ class FileSourceDescriptor(SourceDescriptor):
     def get_description(self):
         return self.path_description
 
+    def get_error_description(self):
+        path = self.filename
+        cwd = Utils.decode_filename(os.getcwd() + os.path.sep)
+        if path.startswith(cwd):
+            return path[len(cwd):]
+        return path
+
     def get_filenametable_entry(self):
         return self.filename
 
@@ -239,11 +246,16 @@ class StringSourceDescriptor(SourceDescriptor):
     def get_description(self):
         return self.name
 
+    get_error_description = get_description
+
     def get_filenametable_entry(self):
         return "stringsource"
 
     def __hash__(self):
-        return hash(self.name)
+        return id(self)
+        # Do not hash on the name, an identical string source should be the
+        # same object (name is often defaulted in other places)
+        # return hash(self.name)
 
     def __eq__(self, other):
         return isinstance(other, StringSourceDescriptor) and self.name == other.name
@@ -279,10 +291,10 @@ class PyrexScanner(Scanner):
         self.source_encoding = source_encoding
         if filename.is_python_file():
             self.in_python_file = True
-            self.keywords = cython.set(py_reserved_words)
+            self.keywords = set(py_reserved_words)
         else:
             self.in_python_file = False
-            self.keywords = cython.set(pyx_reserved_words)
+            self.keywords = set(pyx_reserved_words)
         self.trace = trace_scanner
         self.indentation_stack = [0]
         self.indentation_char = None

Cython/Compiler/Tests/TestBuffer.py

@@ -98,3 +98,8 @@ class TestBufferOptions(CythonTest):
         self.assert_(stats[1].base_type.ndim == 3)
 
     # add exotic and impossible combinations as they come along...
+
+if __name__ == '__main__':
+    import unittest
+    unittest.main()
+

Cython/Compiler/Tests/TestDecorators.py

-import unittest
-from Cython.TestUtils import TransformTest
-from Cython.Compiler.ParseTreeTransforms import DecoratorTransform
-
-class TestDecorator(TransformTest):
-
-    def test_decorator(self):
-        t = self.run_pipeline([DecoratorTransform(None)], u"""
-        def decorator(fun):
-            return fun
-        @decorator
-        def decorated():
-            pass
-        """)
-
-        self.assertCode(u"""
-        def decorator(fun):
-            return fun
-        def decorated():
-            pass
-        decorated = decorator(decorated)
-        """, t)
-
-if __name__ == '__main__':
-    unittest.main()

Cython/Compiler/Tests/TestMemView.py

+from Cython.TestUtils import CythonTest
+import Cython.Compiler.Errors as Errors
+from Cython.Compiler.Nodes import *
+from Cython.Compiler.ParseTreeTransforms import *
+from Cython.Compiler.Buffer import *
+
+
+class TestMemviewParsing(CythonTest):
+
+    def parse(self, s):
+        return self.should_not_fail(lambda: self.fragment(s)).root
+
+    def not_parseable(self, expected_error, s):
+        e = self.should_fail(lambda: self.fragment(s),  Errors.CompileError)
+        self.assertEqual(expected_error, e.message_only)
+    
+    def test_default_1dim(self):
+        self.parse(u"cdef int[:] x")
+        self.parse(u"cdef short int[:] x")
+
+    def test_default_ndim(self):
+        self.parse(u"cdef int[:,:,:,:,:] x")
+        self.parse(u"cdef unsigned long int[:,:,:,:,:] x")
+        self.parse(u"cdef unsigned int[:,:,:,:,:] x")
+
+    def test_zero_offset(self):
+        self.parse(u"cdef long double[0:] x")
+        self.parse(u"cdef int[0:] x")
+
+    def test_zero_offset_ndim(self):
+        self.parse(u"cdef int[0:,0:,0:,0:] x")
+
+    def test_def_arg(self):
+        self.parse(u"def foo(int[:,:] x): pass")
+
+    def test_cdef_arg(self):
+        self.parse(u"cdef foo(int[:,:] x): pass")
+
+    def test_general_slice(self):
+        self.parse(u'cdef float[::ptr, ::direct & contig, 0::full & strided] x')
+
+    def test_non_slice_memview(self):
+        self.not_parseable(u"An axis specification in memoryview declaration does not have a ':'.",
+                u"cdef double[:foo, bar] x")
+        self.not_parseable(u"An axis specification in memoryview declaration does not have a ':'.",
+                u"cdef double[0:foo, bar] x")
+
+    def test_basic(self):
+        t = self.parse(u"cdef int[:] x")
+        memv_node = t.stats[0].base_type
+        self.assert_(isinstance(memv_node, MemoryViewSliceTypeNode))
+
+    # we also test other similar declarations (buffers, anonymous C arrays)
+    # since the parsing has to distinguish between them.
+
+    def disable_test_no_buf_arg(self): # TODO
+        self.not_parseable(u"Expected ']'",
+                u"cdef extern foo(object[int, ndim=2])")
+
+    def disable_test_parse_sizeof(self): # TODO
+        self.parse(u"sizeof(int[NN])")
+        self.parse(u"sizeof(int[])")
+        self.parse(u"sizeof(int[][NN])")
+        self.not_parseable(u"Expected an identifier or literal",
+                u"sizeof(int[:NN])")
+        self.not_parseable(u"Expected ']'",
+                u"sizeof(foo[dtype=bar]")
+
+if __name__ == '__main__':
+    import unittest
+    unittest.main()

Cython/Includes/posix/ioctl.pxd

+cdef extern from "sys/ioctl.h" nogil:
+    enum: FIONBIO
+
+    int ioctl(int fd, int request, ...)

Cython/Utility/TestCyUtilityLoader.pyx

+########## TestCyUtilityLoader ##########
+test {{cy_loader}} impl

Cython/Utility/TestUtilityLoader.c

+////////// TestUtilityLoader.proto //////////
+test {{loader}} prototype
+
+////////// TestUtilityLoader //////////
+test {{loader}} impl