Merge branch 'master' into mypy

.gitignore

@@ -11,14 +11,15 @@ Cython/Compiler/*.c
 Cython/Plex/*.c
 Cython/Runtime/refnanny.c
 Cython/Tempita/*.c
+Cython/*.c
 
 Tools/*.elc
 
-TEST_TMP/
-build/
-wheelhouse*/
+/TEST_TMP/
+/build/
+/wheelhouse*/
 !tests/build/
-dist/
+/dist/
 .gitrev
 .coverage
 *.orig

.travis.yml

 os: linux
 dist: trusty
-sudo: false
+# 'sudo' is enabled automatically by the 'apt' addon below.
+#sudo: false
 
 addons:
   apt:
+    sources:
+      - sourceline: 'ppa:ubuntu-toolchain-r/test'
+    update: true
     packages:
       - gdb
       - python-dbg
       - python3-dbg
+      - libzmq-dev  # needed by IPython/Tornado
+      - gcc-8
+      - g++-8
 
 cache:
   pip: true
@@ -19,11 +26,8 @@ python:
   - 2.7
   - 3.6
   - 2.6
-  - 3.3
   - 3.4
   - 3.5
-  - 3.6-dev
-  - 3.7-dev
   - pypy
   - pypy3
 
@@ -32,14 +36,30 @@ env:
     - USE_CCACHE=1
     - CCACHE_SLOPPINESS=pch_defines,time_macros
     - CCACHE_COMPRESS=1
-    - CCACHE_MAXSIZE=100M
-    - PATH="/usr/lib/ccache:$PATH"
+    - CCACHE_MAXSIZE=150M
+    - PATH="/usr/lib/ccache:$HOME/gcclinks:$HOME/miniconda/bin:$PATH"
   matrix:
     - BACKEND=c
     - BACKEND=cpp
 
 matrix:
   include:
+    - python: 3.7
+      dist: xenial    # Required for Python 3.7
+      sudo: required  # travis-ci/travis-ci#9069
+      env: BACKEND=c
+    - python: 3.7
+      dist: xenial    # Required for Python 3.7
+      sudo: required  # travis-ci/travis-ci#9069
+      env: BACKEND=cpp
+    - python: 3.8-dev
+      dist: xenial    # Required for Python 3.7
+      sudo: required  # travis-ci/travis-ci#9069
+      env: BACKEND=c
+    - python: 3.8-dev
+      dist: xenial    # Required for Python 3.7
+      sudo: required  # travis-ci/travis-ci#9069
+      env: BACKEND=cpp
     - os: osx
       osx_image: xcode6.4
       env: BACKEND=c PY=2
@@ -68,10 +88,16 @@ matrix:
       language: cpp
       compiler: clang
       cache: false
+    - env: STACKLESS=true BACKEND=c PY=2
+      python: 2.7
+    - env: STACKLESS=true BACKEND=c PY=3
+      python: 3.6
   allow_failures:
     - python: pypy
     - python: pypy3
-    - python: 3.7-dev
+    - python: 3.8-dev
+    - env: STACKLESS=true BACKEND=c PY=2
+    - env: STACKLESS=true BACKEND=c PY=3
   exclude:
     - python: pypy
       env: BACKEND=cpp
@@ -85,16 +111,34 @@ branches:
 
 before_install:
   - |
-    if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then # Install Miniconda
-      curl -s -o miniconda.sh https://repo.continuum.io/miniconda/Miniconda$PY-latest-MacOSX-x86_64.sh;
-      bash miniconda.sh -b -p $HOME/miniconda && rm miniconda.sh;
-      export PATH="$HOME/miniconda/bin:$PATH"; hash -r;
-      #conda install --quiet --yes nomkl --file=test-requirements.txt --file=test-requirements-cpython.txt;
+    if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
+      mkdir -p $HOME/gcclinks
+      ln -s /usr/bin/gcc-8 $HOME/gcclinks/gcc
+      ln -s /usr/bin/g++-8 $HOME/gcclinks/g++
+      export  CC=gcc-8  CXX=g++-8
+      $CC --version
+      $CXX --version
+    fi
+
+  - |
+    if [[ "$TRAVIS_OS_NAME" == "osx" ]] || [[ "$STACKLESS" == "true" ]]; then # Install Miniconda
+      if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then CONDA_PLATFORM=MacOSX; else CONDA_PLATFORM=Linux; fi
+      curl -s -o miniconda.sh https://repo.continuum.io/miniconda/Miniconda$PY-latest-${CONDA_PLATFORM}-x86_64.sh
+      bash miniconda.sh -b -p $HOME/miniconda && rm miniconda.sh
+      $HOME/miniconda/bin/conda install --quiet --yes nomkl  clang  clang++
+      #conda install --quiet --yes nomkl --file=test-requirements.txt --file=test-requirements-cpython.txt
+      if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then which clang && clang --version && export CC=clang|| true; fi
+      if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then which clang++ && clang++ --version && export CXX=clang++ || true; fi
+    fi
+
+  - if [[ "$STACKLESS" == "true" ]]; then
+      conda config --add channels stackless;
+      conda install --quiet --yes stackless;
     fi
 
 install:
   - python -c 'import sys; print("Python %s" % (sys.version,))'
-  - if [ -n "${TRAVIS_PYTHON_VERSION##*-dev}" -a -n "${TRAVIS_PYTHON_VERSION##2.6*}" ]; then pip install -r test-requirements.txt $( [ -z "${TRAVIS_PYTHON_VERSION##pypy*}" ] || echo " -r test-requirements-cpython.txt" ) ; fi
+  - if [ -n "${TRAVIS_PYTHON_VERSION##*-dev}" -a -n "${TRAVIS_PYTHON_VERSION##2.6*}" ]; then pip install -r test-requirements.txt $( [ -z "${TRAVIS_PYTHON_VERSION##pypy*}" -o -z "${TRAVIS_PYTHON_VERSION##3.7*}" ] || echo " -r test-requirements-cpython.txt" ) ; fi
   - CFLAGS="-O2 -ggdb -Wall -Wextra $(python -c 'import sys; print("-fno-strict-aliasing" if sys.version_info[0] == 2 else "")')" python setup.py build
 
 before_script: ccache -s || true

Cython/Build/Cythonize.py

@@ -25,9 +25,14 @@ class _FakePool(object):
         for _ in imap(func, args):
             pass
 
-    def close(self): pass
-    def terminate(self): pass
-    def join(self): pass
+    def close(self):
+        pass
+
+    def terminate(self):
+        pass
+
+    def join(self):
+        pass
 
 
 def parse_directives(option, name, value, parser):
@@ -52,6 +57,13 @@ def parse_options(option, name, value, parser):
     setattr(parser.values, dest, options)
 
 
+def parse_compile_time_env(option, name, value, parser):
+    dest = option.dest
+    old_env = dict(getattr(parser.values, dest, {}))
+    new_env = Options.parse_compile_time_env(value, current_settings=old_env)
+    setattr(parser.values, dest, new_env)
+
+
 def find_package_base(path):
     base_dir, package_path = os.path.split(path)
     while os.path.isfile(os.path.join(base_dir, '__init__.py')):
@@ -85,6 +97,7 @@ def cython_compile(path_pattern, options):
                 exclude_failures=options.keep_going,
                 exclude=options.excludes,
                 compiler_directives=options.directives,
+                compile_time_env=options.compile_time_env,
                 force=options.force,
                 quiet=options.quiet,
                 **options.options)
@@ -136,11 +149,17 @@ def parse_args(args):
     from optparse import OptionParser
     parser = OptionParser(usage='%prog [options] [sources and packages]+')
 
-    parser.add_option('-X', '--directive', metavar='NAME=VALUE,...', dest='directives',
-                      type=str, action='callback', callback=parse_directives, default={},
+    parser.add_option('-X', '--directive', metavar='NAME=VALUE,...',
+                      dest='directives', default={}, type="str",
+                      action='callback', callback=parse_directives,
                       help='set a compiler directive')
-    parser.add_option('-s', '--option', metavar='NAME=VALUE', dest='options',
-                      type=str, action='callback', callback=parse_options, default={},
+    parser.add_option('-E', '--compile-time-env', metavar='NAME=VALUE,...',
+                      dest='compile_time_env', default={}, type="str",
+                      action='callback', callback=parse_compile_time_env,
+                      help='set a compile time environment variable')
+    parser.add_option('-s', '--option', metavar='NAME=VALUE',
+                      dest='options', default={}, type="str",
+                      action='callback', callback=parse_options,
                       help='set a cythonize option')
     parser.add_option('-3', dest='python3_mode', action='store_true',
                       help='use Python 3 syntax mode by default')

Cython/Build/IpythonMagic.py

@@ -14,7 +14,7 @@ Magic command interface for interactive work with Cython
 Usage
 =====
 
-To enable the magics below, execute ``%load_ext cythonmagic``.
+To enable the magics below, execute ``%load_ext cython``.
 
 ``%%cython``
 
@@ -166,11 +166,15 @@ class CythonMagics(Magics):
             f.write(cell)
         if 'pyximport' not in sys.modules or not self._pyximport_installed:
             import pyximport
-            pyximport.install(reload_support=True)
+            pyximport.install()
             self._pyximport_installed = True
         if module_name in self._reloads:
             module = self._reloads[module_name]
-            reload(module)
+            # Note: reloading extension modules is not actually supported
+            # (requires PEP-489 reinitialisation support).
+            # Don't know why this should ever have worked as it reads here.
+            # All we really need to do is to update the globals below.
+            #reload(module)
         else:
             __import__(module_name)
             module = sys.modules[module_name]

Cython/Build/Tests/TestCyCache.py

+import difflib
+import glob
+import gzip
+import os
+import tempfile
+
+import Cython.Build.Dependencies
+import Cython.Utils
+from Cython.TestUtils import CythonTest
+
+
+class TestCyCache(CythonTest):
+
+    def setUp(self):
+        CythonTest.setUp(self)
+        self.temp_dir = tempfile.mkdtemp(
+            prefix='cycache-test',
+            dir='TEST_TMP' if os.path.isdir('TEST_TMP') else None)
+        self.src_dir = tempfile.mkdtemp(prefix='src', dir=self.temp_dir)
+        self.cache_dir = tempfile.mkdtemp(prefix='cache', dir=self.temp_dir)
+
+    def cache_files(self, file_glob):
+        return glob.glob(os.path.join(self.cache_dir, file_glob))
+
+    def fresh_cythonize(self, *args, **kwargs):
+        Cython.Utils.clear_function_caches()
+        Cython.Build.Dependencies._dep_tree = None  # discard method caches
+        Cython.Build.Dependencies.cythonize(*args, **kwargs)
+
+    def test_cycache_switch(self):
+        content1 = 'value = 1\n'
+        content2 = 'value = 2\n'
+        a_pyx = os.path.join(self.src_dir, 'a.pyx')
+        a_c = a_pyx[:-4] + '.c'
+
+        open(a_pyx, 'w').write(content1)
+        self.fresh_cythonize(a_pyx, cache=self.cache_dir)
+        self.fresh_cythonize(a_pyx, cache=self.cache_dir)
+        self.assertEqual(1, len(self.cache_files('a.c*')))
+        a_contents1 = open(a_c).read()
+        os.unlink(a_c)
+
+        open(a_pyx, 'w').write(content2)
+        self.fresh_cythonize(a_pyx, cache=self.cache_dir)
+        a_contents2 = open(a_c).read()
+        os.unlink(a_c)
+
+        self.assertNotEqual(a_contents1, a_contents2, 'C file not changed!')
+        self.assertEqual(2, len(self.cache_files('a.c*')))
+
+        open(a_pyx, 'w').write(content1)
+        self.fresh_cythonize(a_pyx, cache=self.cache_dir)
+        self.assertEqual(2, len(self.cache_files('a.c*')))
+        a_contents = open(a_c).read()
+        self.assertEqual(
+            a_contents, a_contents1,
+            msg='\n'.join(list(difflib.unified_diff(
+                a_contents.split('\n'), a_contents1.split('\n')))[:10]))
+
+    def test_cycache_uses_cache(self):
+        a_pyx = os.path.join(self.src_dir, 'a.pyx')
+        a_c = a_pyx[:-4] + '.c'
+        open(a_pyx, 'w').write('pass')
+        self.fresh_cythonize(a_pyx, cache=self.cache_dir)
+        a_cache = os.path.join(self.cache_dir, os.listdir(self.cache_dir)[0])
+        gzip.GzipFile(a_cache, 'wb').write('fake stuff'.encode('ascii'))
+        os.unlink(a_c)
+        self.fresh_cythonize(a_pyx, cache=self.cache_dir)
+        a_contents = open(a_c).read()
+        self.assertEqual(a_contents, 'fake stuff',
+                         'Unexpected contents: %s...' % a_contents[:100])
+
+    def test_multi_file_output(self):
+        a_pyx = os.path.join(self.src_dir, 'a.pyx')
+        a_c = a_pyx[:-4] + '.c'
+        a_h = a_pyx[:-4] + '.h'
+        a_api_h = a_pyx[:-4] + '_api.h'
+        open(a_pyx, 'w').write('cdef public api int foo(int x): return x\n')
+        self.fresh_cythonize(a_pyx, cache=self.cache_dir)
+        expected = [a_c, a_h, a_api_h]
+        for output in expected:
+            self.assertTrue(os.path.exists(output), output)
+            os.unlink(output)
+        self.fresh_cythonize(a_pyx, cache=self.cache_dir)
+        for output in expected:
+            self.assertTrue(os.path.exists(output), output)
+
+    def test_options_invalidation(self):
+        hash_pyx = os.path.join(self.src_dir, 'options.pyx')
+        hash_c = hash_pyx[:-len('.pyx')] + '.c'
+
+        open(hash_pyx, 'w').write('pass')
+        self.fresh_cythonize(hash_pyx, cache=self.cache_dir, cplus=False)
+        self.assertEqual(1, len(self.cache_files('options.c*')))
+
+        os.unlink(hash_c)
+        self.fresh_cythonize(hash_pyx, cache=self.cache_dir, cplus=True)
+        self.assertEqual(2, len(self.cache_files('options.c*')))
+
+        os.unlink(hash_c)
+        self.fresh_cythonize(hash_pyx, cache=self.cache_dir, cplus=False, show_version=False)
+        self.assertEqual(2, len(self.cache_files('options.c*')))
+
+        os.unlink(hash_c)
+        self.fresh_cythonize(hash_pyx, cache=self.cache_dir, cplus=False, show_version=True)
+        self.assertEqual(2, len(self.cache_files('options.c*')))

Cython/Compiler/Annotate.py

@@ -79,14 +79,6 @@ class AnnotationCCodeWriter(CCodeWriter):
             css.append(HtmlFormatter().get_style_defs('.cython'))
         return '\n'.join(css)
 
-    _js = """
-    function toggleDiv(id) {
-        theDiv = id.nextElementSibling
-        if (theDiv.style.display != 'block') theDiv.style.display = 'block';
-        else theDiv.style.display = 'none';
-    }
-    """.strip()
-
     _css_template = textwrap.dedent("""
         body.cython { font-family: courier; font-size: 12; }
 
@@ -114,6 +106,14 @@ class AnnotationCCodeWriter(CCodeWriter):
         .cython.code .c_call  { color: #0000FF; }
     """)
 
+    # on-click toggle function to show/hide C source code
+    _onclick_attr = ' onclick="{0}"'.format((
+        "(function(s){"
+        "    s.display =  s.display === 'block' ? 'none' : 'block'"
+        "})(this.nextElementSibling.style)"
+        ).replace(' ', '')  # poor dev's JS minification
+    )
+
     def save_annotation(self, source_filename, target_filename, coverage_xml=None):
         with Utils.open_source_file(source_filename) as f:
             code = f.read()
@@ -141,9 +141,6 @@ class AnnotationCCodeWriter(CCodeWriter):
                 <style type="text/css">
                 {css}
                 </style>
-                <script>
-                {js}
-                </script>
             </head>
             <body class="cython">
             <p><span style="border-bottom: solid 1px grey;">Generated by Cython {watermark}</span>{more_info}</p>
@@ -151,7 +148,7 @@ class AnnotationCCodeWriter(CCodeWriter):
                 <span style="background-color: #FFFF00">Yellow lines</span> hint at Python interaction.<br />
                 Click on a line that starts with a "<code>+</code>" to see the C code that Cython generated for it.
             </p>
-            ''').format(css=self._css(), js=self._js, watermark=Version.watermark,
+            ''').format(css=self._css(), watermark=Version.watermark,
                         filename=os.path.basename(source_filename) if source_filename else '',
                         more_info=coverage_info)
         ]
@@ -253,7 +250,7 @@ class AnnotationCCodeWriter(CCodeWriter):
                      calls['py_macro_api'] + calls['pyx_macro_api'])
 
             if c_code:
-                onclick = " onclick='toggleDiv(this)'"
+                onclick = self._onclick_attr
                 expandsymbol = '+'
             else:
                 onclick = ''

Cython/Compiler/Buffer.py

@@ -370,7 +370,7 @@ def put_assign_to_buffer(lhs_cname, rhs_cname, buf_entry,
     pybuffernd_struct = buffer_aux.buflocal_nd_var.cname
     flags = get_flags(buffer_aux, buffer_type)
 
-    code.putln("{")  # Set up necesarry stack for getbuffer
+    code.putln("{")  # Set up necessary stack for getbuffer
     code.putln("__Pyx_BufFmt_StackElem __pyx_stack[%d];" % buffer_type.dtype.struct_nesting_depth())
 
     getbuffer = get_getbuffer_call(code, "%s", buffer_aux, buffer_type) # fill in object below

Cython/Compiler/Builtin.py

@@ -124,7 +124,8 @@ builtin_function_table = [
                                   PyrexTypes.c_double_complex_type,
                                   PyrexTypes.c_longdouble_complex_type)
                         ) + [
-    BuiltinFunction('abs',        "O",    "O",     "PyNumber_Absolute"),
+    BuiltinFunction('abs',        "O",    "O",     "__Pyx_PyNumber_Absolute",
+                    utility_code=UtilityCode.load("py_abs", "Builtins.c")),
     #('all',       "",     "",      ""),
     #('any',       "",     "",      ""),
     #('ascii',     "",     "",      ""),
@@ -328,7 +329,10 @@ builtin_types_table = [
     ("set",       "PySet_Type",    [BuiltinMethod("__contains__",  "TO",   "b", "PySequence_Contains"),
                                     BuiltinMethod("clear",   "T",  "r", "PySet_Clear"),
                                     # discard() and remove() have a special treatment for unhashable values
-#                                    BuiltinMethod("discard", "TO", "r", "PySet_Discard"),
+                                    BuiltinMethod("discard", "TO", "r", "__Pyx_PySet_Discard",
+                                                  utility_code=UtilityCode.load("py_set_discard", "Optimize.c")),
+                                    BuiltinMethod("remove",  "TO", "r", "__Pyx_PySet_Remove",
+                                                  utility_code=UtilityCode.load("py_set_remove", "Optimize.c")),
                                     # update is actually variadic (see Github issue #1645)
 #                                    BuiltinMethod("update",     "TO", "r", "__Pyx_PySet_Update",
 #                                                  utility_code=UtilityCode.load_cached("PySet_Update", "Builtins.c")),
@@ -388,6 +392,8 @@ def init_builtin_types():
         utility = builtin_utility_code.get(name)
         if name == 'frozenset':
             objstruct_cname = 'PySetObject'
+        elif name == 'bytearray':
+            objstruct_cname = 'PyByteArrayObject'
         elif name == 'bool':
             objstruct_cname = None
         elif name == 'Exception':

Cython/Compiler/CmdLine.py

@@ -47,10 +47,11 @@ Options:
   --warning-errors, -Werror      Make all warnings into errors
   --warning-extra, -Wextra       Enable extra warnings
   -X, --directive <name>=<value>[,<name=value,...] Overrides a compiler directive
+  -E, --compile-time-env name=value[,<name=value,...] Provides compile time env like DEF would do.
 """
 
 
-#The following experimental options are supported only on MacOSX:
+# The following experimental options are supported only on MacOSX:
 #  -C, --compile    Compile generated .c file to .o file
 #  --link           Link .o file to produce extension module (implies -C)
 #  -+, --cplus      Use C++ compiler for compiling and linking
@@ -154,6 +155,8 @@ def parse_command_line(args):
                 options.capi_reexport_cincludes = True
             elif option == "--fast-fail":
                 Options.fast_fail = True
+            elif option == "--cimport-from-pyx":
+                Options.cimport_from_pyx = True
             elif option in ('-Werror', '--warning-errors'):
                 Options.warning_errors = True
             elif option in ('-Wextra', '--warning-extra'):
@@ -172,6 +175,17 @@ def parse_command_line(args):
                 except ValueError as e:
                     sys.stderr.write("Error in compiler directive: %s\n" % e.args[0])
                     sys.exit(1)
+            elif option == "--compile-time-env" or option.startswith('-E'):
+                if option.startswith('-E') and option[2:].strip():
+                    x_args = option[2:]
+                else:
+                    x_args = pop_value()
+                try:
+                    options.compile_time_env = Options.parse_compile_time_env(
+                        x_args, current_settings=options.compile_time_env)
+                except ValueError as e:
+                    sys.stderr.write("Error in compile-time-env: %s\n" % e.args[0])
+                    sys.exit(1)
             elif option.startswith('--debug'):
                 option = option[2:].replace('-', '_')
                 from . import DebugFlags

Cython/Compiler/Code.pxd

@@ -2,6 +2,7 @@
 from __future__ import absolute_import
 
 cimport cython
+from ..StringIOTree cimport StringIOTree
 
 
 cdef class UtilityCodeBase(object):
@@ -95,7 +96,27 @@ cdef class StringConst:
 
 #def funccontext_property(name):
 
-#class CCodeWriter(object):
+cdef class CCodeWriter(object):
+    cdef readonly StringIOTree buffer
+    cdef readonly list pyclass_stack
+    cdef readonly object globalstate
+    cdef readonly object funcstate
+    cdef object code_config
+    cdef object last_pos
+    cdef object last_marked_pos
+    cdef Py_ssize_t level
+    cdef public Py_ssize_t call_level  # debug-only, see Nodes.py
+    cdef bint bol
+
+    cpdef write(self, s)
+    cpdef put(self, code)
+    cpdef put_safe(self, code)
+    cpdef putln(self, code=*, bint safe=*)
+    @cython.final
+    cdef increase_indent(self)
+    @cython.final
+    cdef decrease_indent(self)
+
 
 cdef class PyrexCodeWriter:
     cdef public object f

Cython/Compiler/CythonScope.py

@@ -26,6 +26,10 @@ class CythonScope(ModuleScope):
                                          cname='<error>')
             entry.in_cinclude = True
 
+    def is_cpp(self):
+        # Allow C++ utility code in C++ contexts.
+        return self.context.cpp
+
     def lookup_type(self, name):
         # This function should go away when types are all first-level objects.
         type = parse_basic_type(name)

Cython/Compiler/FusedNode.py

@@ -276,7 +276,7 @@ class FusedCFuncDefNode(StatListNode):
 
     def _fused_instance_checks(self, normal_types, pyx_code, env):
         """
-        Genereate Cython code for instance checks, matching an object to
+        Generate Cython code for instance checks, matching an object to
         specialized types.
         """
         for specialized_type in normal_types:
@@ -390,7 +390,7 @@ class FusedCFuncDefNode(StatListNode):
             coerce_from_py_func=memslice_type.from_py_function,
             dtype=dtype)
         decl_code.putln(
-            "{{memviewslice_cname}} {{coerce_from_py_func}}(object)")
+            "{{memviewslice_cname}} {{coerce_from_py_func}}(object, int)")
 
         pyx_code.context.update(
             specialized_type_name=specialized_type.specialization_string,
@@ -400,7 +400,7 @@ class FusedCFuncDefNode(StatListNode):
             u"""
                 # try {{dtype}}
                 if itemsize == -1 or itemsize == {{sizeof_dtype}}:
-                    memslice = {{coerce_from_py_func}}(arg)
+                    memslice = {{coerce_from_py_func}}(arg, 0)
                     if memslice.memview:
                         __PYX_XDEC_MEMVIEW(&memslice, 1)
                         # print 'found a match for the buffer through format parsing'
@@ -421,10 +421,11 @@ class FusedCFuncDefNode(StatListNode):
         # The first thing to find a match in this loop breaks out of the loop
         pyx_code.put_chunk(
             u"""
+                """ + (u"arg_is_pythran_compatible = False" if pythran_types else u"") + u"""
                 if ndarray is not None:
                     if isinstance(arg, ndarray):
                         dtype = arg.dtype
-                        arg_is_pythran_compatible = True
+                        """ + (u"arg_is_pythran_compatible = True" if pythran_types else u"") + u"""
                     elif __pyx_memoryview_check(arg):
                         arg_base = arg.base
                         if isinstance(arg_base, ndarray):
@@ -438,24 +439,30 @@ class FusedCFuncDefNode(StatListNode):
                     if dtype is not None:
                         itemsize = dtype.itemsize
                         kind = ord(dtype.kind)
-                        # We only support the endianness of the current compiler
+                        dtype_signed = kind == 'i'
+            """)
+        pyx_code.indent(2)
+        if pythran_types:
+            pyx_code.put_chunk(
+                u"""
+                        # Pythran only supports the endianness of the current compiler
                         byteorder = dtype.byteorder
                         if byteorder == "<" and not __Pyx_Is_Little_Endian():
                             arg_is_pythran_compatible = False
-                        if byteorder == ">" and __Pyx_Is_Little_Endian():
+                        elif byteorder == ">" and __Pyx_Is_Little_Endian():
                             arg_is_pythran_compatible = False
-                        dtype_signed = kind == 'i'
                         if arg_is_pythran_compatible:
                             cur_stride = itemsize
-                            for dim,stride in zip(reversed(arg.shape),reversed(arg.strides)):
-                                if stride != cur_stride:
+                            shape = arg.shape
+                            strides = arg.strides
+                            for i in range(arg.ndim-1, -1, -1):
+                                if (<Py_ssize_t>strides[i]) != cur_stride:
                                     arg_is_pythran_compatible = False
                                     break
-                                cur_stride *= dim
+                                cur_stride *= <Py_ssize_t> shape[i]
                             else:
-                                arg_is_pythran_compatible = not (arg.flags.f_contiguous and arg.ndim > 1)
-            """)
-        pyx_code.indent(2)
+                                arg_is_pythran_compatible = not (arg.flags.f_contiguous and (<Py_ssize_t>arg.ndim) > 1)
+                """)
         pyx_code.named_insertion_point("numpy_dtype_checks")
         self._buffer_check_numpy_dtype(pyx_code, buffer_types, pythran_types)
         pyx_code.dedent(2)
@@ -464,7 +471,7 @@ class FusedCFuncDefNode(StatListNode):
             self._buffer_parse_format_string_check(
                     pyx_code, decl_code, specialized_type, env)
 
-    def _buffer_declarations(self, pyx_code, decl_code, all_buffer_types):
+    def _buffer_declarations(self, pyx_code, decl_code, all_buffer_types, pythran_types):
         """
         If we have any buffer specializations, write out some variable
         declarations and imports.
@@ -484,10 +491,14 @@ class FusedCFuncDefNode(StatListNode):
                 cdef Py_ssize_t itemsize
                 cdef bint dtype_signed
                 cdef char kind
-                cdef bint arg_is_pythran_compatible
 
                 itemsize = -1
-                arg_is_pythran_compatible = False
+            """)
+
+        if pythran_types:
+            pyx_code.local_variable_declarations.put_chunk(u"""
+                cdef bint arg_is_pythran_compatible
+                cdef Py_ssize_t cur_stride
             """)
 
         pyx_code.imports.put_chunk(
@@ -514,7 +525,7 @@ class FusedCFuncDefNode(StatListNode):
                     pyx_code.local_variable_declarations.put_chunk(
                         u"""
                             cdef bint {{dtype_name}}_is_signed
-                            {{dtype_name}}_is_signed = <{{dtype_type}}> -1 < 0
+                            {{dtype_name}}_is_signed = not (<{{dtype_type}}> -1 > 0)
                         """)
 
     def _split_fused_types(self, arg):
@@ -670,7 +681,7 @@ class FusedCFuncDefNode(StatListNode):
                 default_idx += 1
 
         if all_buffer_types:
-            self._buffer_declarations(pyx_code, decl_code, all_buffer_types)
+            self._buffer_declarations(pyx_code, decl_code, all_buffer_types, pythran_types)
             env.use_utility_code(Code.UtilityCode.load_cached("Import", "ImportExport.c"))
             env.use_utility_code(Code.UtilityCode.load_cached("ImportNumPyArray", "ImportExport.c"))
 

Cython/Compiler/Lexicon.py

@@ -3,7 +3,7 @@
 #   Cython Scanner - Lexical Definitions
 #
 
-from __future__ import absolute_import
+from __future__ import absolute_import, unicode_literals
 
 raw_prefixes = "rR"
 bytes_prefixes = "bB"

Cython/Compiler/MemoryView.py

@@ -28,12 +28,12 @@ def concat_flags(*flags):
 
 format_flag = "PyBUF_FORMAT"
 
-memview_c_contiguous = "(PyBUF_C_CONTIGUOUS | PyBUF_FORMAT | PyBUF_WRITABLE)"
-memview_f_contiguous = "(PyBUF_F_CONTIGUOUS | PyBUF_FORMAT | PyBUF_WRITABLE)"
-memview_any_contiguous = "(PyBUF_ANY_CONTIGUOUS | PyBUF_FORMAT | PyBUF_WRITABLE)"
-memview_full_access = "PyBUF_FULL"
-#memview_strided_access = "PyBUF_STRIDED"
-memview_strided_access = "PyBUF_RECORDS"
+memview_c_contiguous = "(PyBUF_C_CONTIGUOUS | PyBUF_FORMAT)"
+memview_f_contiguous = "(PyBUF_F_CONTIGUOUS | PyBUF_FORMAT)"
+memview_any_contiguous = "(PyBUF_ANY_CONTIGUOUS | PyBUF_FORMAT)"
+memview_full_access = "PyBUF_FULL_RO"
+#memview_strided_access = "PyBUF_STRIDED_RO"
+memview_strided_access = "PyBUF_RECORDS_RO"
 
 MEMVIEW_DIRECT = '__Pyx_MEMVIEW_DIRECT'
 MEMVIEW_PTR    = '__Pyx_MEMVIEW_PTR'
@@ -484,18 +484,23 @@ def copy_c_or_fortran_cname(memview):
     return "__pyx_memoryview_copy_slice_%s_%s" % (
             memview.specialization_suffix(), c_or_f)
 
+
 def get_copy_new_utility(pos, from_memview, to_memview):
-    if from_memview.dtype != to_memview.dtype:
-        return error(pos, "dtypes must be the same!")
+    if (from_memview.dtype != to_memview.dtype and
+            not (from_memview.dtype.is_const and from_memview.dtype.const_base_type == to_memview.dtype)):
+        error(pos, "dtypes must be the same!")
+        return
     if len(from_memview.axes) != len(to_memview.axes):
-        return error(pos, "number of dimensions must be same")
+        error(pos, "number of dimensions must be same")
+        return
     if not (to_memview.is_c_contig or to_memview.is_f_contig):
-        return error(pos, "to_memview must be c or f contiguous.")
+        error(pos, "to_memview must be c or f contiguous.")
+        return
 
     for (access, packing) in from_memview.axes:
         if access != 'direct':
-            return error(
-                    pos, "cannot handle 'full' or 'ptr' access at this time.")
+            error(pos, "cannot handle 'full' or 'ptr' access at this time.")
+            return
 
     if to_memview.is_c_contig:
         mode = 'c'
@@ -516,6 +521,7 @@ def get_copy_new_utility(pos, from_memview, to_memview):
             dtype_is_object=int(to_memview.dtype.is_pyobject)),
         requires=[copy_contents_new_utility])
 
+
 def get_axes_specs(env, axes):
     '''
     get_axes_specs(env, axes) -> list of (access, packing) specs for each axis.

Cython/Compiler/Naming.py

@@ -28,6 +28,7 @@ const_prefix      = pyrex_prefix + "k_"
 py_const_prefix   = pyrex_prefix + "kp_"
 label_prefix      = pyrex_prefix + "L"
 pymethdef_prefix  = pyrex_prefix + "mdef_"
+method_wrapper_prefix = pyrex_prefix + "specialmethod_"
 methtab_prefix    = pyrex_prefix + "methods_"
 memtab_prefix     = pyrex_prefix + "members_"
 objstruct_prefix  = pyrex_prefix + "obj_"

Cython/Compiler/ParseTreeTransforms.pxd

@@ -7,9 +7,6 @@ from .Visitor cimport (
     CythonTransform, VisitorTransform, TreeVisitor,
     ScopeTrackingTransform, EnvTransform)
 
-cdef class NameNodeCollector(TreeVisitor):
-    cdef list name_nodes
-
 cdef class SkipDeclarations: # (object):
     pass
 
@@ -73,6 +70,9 @@ cdef class CreateClosureClasses(CythonTransform):
     cdef create_class_from_scope(self, node, target_module_scope, inner_node=*)
     cdef find_entries_used_in_closures(self, node)
 
+#cdef class InjectGilHandling(VisitorTransform, SkipDeclarations):
+#    cdef bint nogil
+
 cdef class GilCheck(VisitorTransform):
     cdef list env_stack
     cdef bint nogil

Cython/Compiler/Parsing.pxd

@@ -191,7 +191,7 @@ cdef p_c_class_options(PyrexScanner s)
 cdef p_property_decl(PyrexScanner s)
 cdef p_doc_string(PyrexScanner s)
 cdef p_ignorable_statement(PyrexScanner s)
-cdef p_compiler_directive_comments(PyrexScanner s)
+cdef dict p_compiler_directive_comments(PyrexScanner s)
 cdef p_template_definition(PyrexScanner s)
 cdef p_cpp_class_definition(PyrexScanner s, pos, ctx)
 cdef p_cpp_class_attribute(PyrexScanner s, ctx)

Cython/Compiler/Pipeline.py

@@ -141,7 +141,7 @@ def create_pipeline(context, mode, exclude_classes=()):
     assert mode in ('pyx', 'py', 'pxd')
     from .Visitor import PrintTree
     from .ParseTreeTransforms import WithTransform, NormalizeTree, PostParse, PxdPostParse
-    from .ParseTreeTransforms import ForwardDeclareTypes, AnalyseDeclarationsTransform
+    from .ParseTreeTransforms import ForwardDeclareTypes, InjectGilHandling, AnalyseDeclarationsTransform
     from .ParseTreeTransforms import AnalyseExpressionsTransform, FindInvalidUseOfFusedTypes
     from .ParseTreeTransforms import CreateClosureClasses, MarkClosureVisitor, DecoratorTransform
     from .ParseTreeTransforms import TrackNumpyAttributes, InterpretCompilerDirectives, TransformBuiltinMethods
@@ -194,6 +194,7 @@ def create_pipeline(context, mode, exclude_classes=()):
         FlattenInListTransform(),
         DecoratorTransform(context),
         ForwardDeclareTypes(context),
+        InjectGilHandling(),
         AnalyseDeclarationsTransform(context),
         AutoTestDictTransform(context),
         EmbedSignature(context),

Cython/Compiler/Pythran.py

@@ -6,16 +6,20 @@ from .PyrexTypes import CType, CTypedefType, CStructOrUnionType
 
 import cython
 
+try:
+    import pythran
+    _pythran_available = True
+except ImportError:
+    _pythran_available = False
+
 
 # Pythran/Numpy specific operations
 
 def has_np_pythran(env):
-    while env is not None:
-        directives = getattr(env, 'directives', None)
-        if directives and env.directives.get('np_pythran', False):
-            return True
-        env = env.outer_scope
-
+    if env is None:
+        return False
+    directives = getattr(env, 'directives', None)
+    return (directives and directives.get('np_pythran', False))
 
 @cython.ccall
 def is_pythran_supported_dtype(type_):
@@ -111,10 +115,32 @@ def pythran_indexing_type(type_, indices):
 def pythran_indexing_code(indices):
     return _index_access(_index_code, indices)
 
+def np_func_to_list(func):
+    if not func.is_numpy_attribute:
+        return []
+    return np_func_to_list(func.obj) + [func.attribute]
+
+if _pythran_available:
+    def pythran_is_numpy_func_supported(func):
+        CurF = pythran.tables.MODULES['numpy']
+        FL = np_func_to_list(func)
+        for F in FL:
+            CurF = CurF.get(F, None)
+            if CurF is None:
+                return False
+        return True
+else:
+    def pythran_is_numpy_func_supported(name):
+        return False
+
+def pythran_functor(func):
+    func = np_func_to_list(func)
+    submodules = "::".join(func[:-1] + ["functor"])
+    return "pythonic::numpy::%s::%s" % (submodules, func[-1])
 
 def pythran_func_type(func, args):
     args = ",".join(("std::declval<%s>()" % pythran_type(a.type) for a in args))
-    return "decltype(pythonic::numpy::functor::%s{}(%s))" % (func, args)
+    return "decltype(%s{}(%s))" % (pythran_functor(func), args)
 
 
 @cython.ccall
@@ -168,6 +194,9 @@ def is_pythran_buffer(type_):
     return (type_.is_numpy_buffer and is_pythran_supported_dtype(type_.dtype) and
             type_.mode in ("c", "strided") and not type_.cast)
 
+def pythran_get_func_include_file(func):
+    func = np_func_to_list(func)
+    return "pythonic/include/numpy/%s.hpp" % "/".join(func)
 
 def include_pythran_generic(env):
     # Generic files

Cython/Compiler/StringEncoding.py

@@ -191,6 +191,14 @@ def bytes_literal(s, encoding):
     return s
 
 
+def encoded_string(s, encoding):
+    assert isinstance(s, (_unicode, bytes))
+    s = EncodedString(s)
+    if encoding is not None:
+        s.encoding = encoding
+    return s
+
+
 char_from_escape_sequence = {
     r'\a' : u'\a',
     r'\b' : u'\b',

Cython/Compiler/Tests/TestTreeFragment.py

@@ -45,7 +45,7 @@ class TestTreeFragments(CythonTest):
         T = F.substitute({"v" : NameNode(pos=None, name="a")})
         v = F.root.stats[1].rhs.operand2.operand1
         a = T.stats[1].rhs.operand2.operand1
-        self.assertEquals(v.pos, a.pos)
+        self.assertEqual(v.pos, a.pos)
 
     def test_temps(self):
         TemplateTransform.temp_name_counter = 0

Cython/Includes/cpython/long.pxd

@@ -146,4 +146,4 @@ cdef extern from "Python.h":
     # pointer. If pylong cannot be converted, an OverflowError will be
     # raised. This is only assured to produce a usable void pointer
     # for values created with PyLong_FromVoidPtr(). For values outside
-    # 0..LONG_MAX, both signed and unsigned integers are acccepted.
+    # 0..LONG_MAX, both signed and unsigned integers are accepted.