Merge branch 'master' into readonly_buffers

.travis.yml

-language: python
+os: linux
 dist: trusty
 sudo: false
 
+addons:
+  apt:
+    sources:
+      - ubuntu-toolchain-r-test
+    packages:
+      - gdb
+      - python-dbg
+      - python3-dbg
+      - gcc-6
+      - g++-6
+      # GCC-7 currently takes 5-7 *minutes* to download on travis
+      #- gcc-7
+      #- g++-7
+
 cache:
   pip: true
   directories:
     - $HOME/.ccache
 
+language: python
 python:
   - 2.7
   - 3.6
@@ -25,45 +40,86 @@ env:
     - CCACHE_SLOPPINESS=pch_defines,time_macros
     - CCACHE_COMPRESS=1
     - CCACHE_MAXSIZE=100M
-    - PATH="/usr/lib/ccache:$PATH"
+    - PATH="/usr/lib/ccache:$HOME/gcc-symlinks:$PATH"
   matrix:
     - BACKEND=c
     - BACKEND=cpp
 
+matrix:
+  include:
+    #- python: 3.7-dev
+    #  env: BACKEND=c PY=3 CC=gcc-7
+    - os: osx
+      osx_image: xcode6.4
+      env: BACKEND=c PY=2
+      python: 2
+      language: c
+      compiler: clang
+      cache: false
+    - os: osx
+      osx_image: xcode6.4
+      env: BACKEND=cpp PY=2
+      python: 2
+      language: cpp
+      compiler: clang
+      cache: false
+    - os: osx
+      osx_image: xcode6.4
+      env: BACKEND=c PY=3
+      python: 3
+      language: c
+      compiler: clang
+      cache: false
+    - os: osx
+      osx_image: xcode6.4
+      env: BACKEND=cpp PY=3
+      python: 3
+      language: cpp
+      compiler: clang
+      cache: false
+  allow_failures:
+    - python: pypy
+    - python: pypy3
+    - python: 3.7-dev
+  exclude:
+    - python: pypy
+      env: BACKEND=cpp
+    - python: pypy3
+      env: BACKEND=cpp
+
 branches:
   only:
     - master
     - release
 
+before_install:
+  - |
+    if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
+      mkdir "$HOME/gcc-symlinks"
+      ln -s /usr/bin/gcc-6 $HOME/gcc-symlinks/gcc 
+      ln -s /usr/bin/g++-6 $HOME/gcc-symlinks/g++ 
+      
+      if [ -n "$CC" ]; then "$CC" --version; else gcc --version; fi
+    fi
+
+  - |
+    if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then # Install Miniconda
+      curl -s -o miniconda.sh https://repo.continuum.io/miniconda/Miniconda$PY-latest-MacOSX-x86_64.sh;
+      bash miniconda.sh -b -p $HOME/miniconda && rm miniconda.sh;
+      export PATH="$HOME/miniconda/bin:$PATH"; hash -r;
+      #conda install --quiet --yes nomkl --file=test-requirements.txt --file=test-requirements-cpython.txt;
+    fi
+
 install:
   - python -c 'import sys; print("Python %s" % (sys.version,))'
   - if [ -n "${TRAVIS_PYTHON_VERSION##*-dev}" -a -n "${TRAVIS_PYTHON_VERSION##2.6*}" ]; then pip install -r test-requirements.txt $( [ -z "${TRAVIS_PYTHON_VERSION##pypy*}" ] || echo " -r test-requirements-cpython.txt" ) ; fi
   - CFLAGS="-O2 -ggdb -Wall -Wextra $(python -c 'import sys; print("-fno-strict-aliasing" if sys.version_info[0] == 2 else "")')" python setup.py build
 
-before_script: ccache -s
+before_script: ccache -s || true
 
 script:
   - PYTHON_DBG="python$( python -c 'import sys; print("%d.%d" % sys.version_info[:2])' )-dbg"
   - if $PYTHON_DBG -V >&2; then CFLAGS="-O0 -ggdb" $PYTHON_DBG runtests.py -vv Debugger --backends=$BACKEND; fi
-  - if [ false && "$BACKEND" = "cpp" ]; then pip install pythran; fi  # disabled: needs Pythran > 0.8.1
+  - if [ "$BACKEND" = "cpp" -a -n "${TRAVIS_PYTHON_VERSION##2.6*}" ]; then pip install pythran; fi
   - CFLAGS="-O2 -ggdb -Wall -Wextra $(python -c 'import sys; print("-fno-strict-aliasing" if sys.version_info[0] == 2 else "")')" python setup.py build_ext -i
-  - CFLAGS="-O0 -ggdb -Wall -Wextra" python runtests.py -vv -x Debugger --backends=$BACKEND -j4
-
-matrix:
-  allow_failures:
-    - python: pypy
-    - python: pypy3
-    - python: 3.6-dev
-    - python: 3.7-dev
-  exclude:
-    - python: pypy
-      env: BACKEND=cpp
-    - python: pypy3
-      env: BACKEND=cpp
-
-addons:
-  apt:
-    packages:
-      - gdb
-      - python-dbg
-      - python3-dbg
+  - CFLAGS="-O0 -ggdb -Wall -Wextra" python runtests.py -vv -x Debugger --backends=$BACKEND -j7

Cython/Build/Dependencies.py

@@ -3,9 +3,17 @@ from __future__ import absolute_import, print_function
 import cython
 from .. import __version__
 
+import os
+import shutil
+import hashlib
+import subprocess
 import collections
-import re, os, sys, time
+import re, sys, time
 from glob import iglob
+from io import open as io_open
+from os.path import relpath as _relpath
+from distutils.extension import Extension
+from distutils.util import strtobool
 
 try:
     import gzip
@@ -14,34 +22,6 @@ try:
 except ImportError:
     gzip_open = open
     gzip_ext = ''
-import shutil
-import subprocess
-import os
-
-try:
-    import hashlib
-except ImportError:
-    import md5 as hashlib
-
-try:
-    from io import open as io_open
-except ImportError:
-    from codecs import open as io_open
-
-try:
-    from os.path import relpath as _relpath
-except ImportError:
-    # Py<2.6
-    def _relpath(path, start=os.path.curdir):
-        if not path:
-            raise ValueError("no path specified")
-        start_list = os.path.abspath(start).split(os.path.sep)
-        path_list = os.path.abspath(path).split(os.path.sep)
-        i = len(os.path.commonprefix([start_list, path_list]))
-        rel_list = [os.path.pardir] * (len(start_list)-i) + path_list[i:]
-        if not rel_list:
-            return os.path.curdir
-        return os.path.join(*rel_list)
 
 try:
     import pythran
@@ -50,9 +30,6 @@ try:
 except:
     PythranAvailable = False
 
-from distutils.extension import Extension
-from distutils.util import strtobool
-
 from .. import Utils
 from ..Utils import (cached_function, cached_method, path_exists,
     safe_makedirs, copy_file_to_dir_if_newer, is_package_dir)
@@ -800,7 +777,6 @@ def create_extension_list(patterns, exclude=None, ctx=None, aliases=None, quiet=
         for file in nonempty(sorted(extended_iglob(filepattern)), "'%s' doesn't match any files" % filepattern):
             if os.path.abspath(file) in to_exclude:
                 continue
-            pkg = deps.package(file)
             module_name = deps.fully_qualified_name(file)
             if '*' in name:
                 if module_name in explicit_modules:
@@ -810,6 +786,9 @@ def create_extension_list(patterns, exclude=None, ctx=None, aliases=None, quiet=
                     name, module_name, file))
                 module_name = name
                 
+            if module_name == 'cython':
+                raise ValueError('cython is a special module, cannot be used as a module name')
+
             if module_name not in seen:
                 try:
                     kwds = deps.distutils_info(file, aliases, base).values
@@ -850,8 +829,13 @@ def create_extension_list(patterns, exclude=None, ctx=None, aliases=None, quiet=
                 module_metadata[module_name] = metadata
 
                 if file not in m.sources:
-                    # Old setuptools unconditionally replaces .pyx with .c
-                    m.sources.remove(file.rsplit('.')[0] + '.c')
+                    # Old setuptools unconditionally replaces .pyx with .c/.cpp
+                    target_file = os.path.splitext(file)[0] + ('.cpp' if m.language == 'c++' else '.c')
+                    try:
+                        m.sources.remove(target_file)
+                    except ValueError:
+                        # never seen this in the wild, but probably better to warn about this unexpected case
+                        print("Warning: Cython source file not found in sources list, adding %s" % file)
                     m.sources.insert(0, file)
                 seen.add(name)
     return module_list, module_metadata
@@ -916,22 +900,33 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False,
     deps = create_dependency_tree(ctx, quiet=quiet)
     build_dir = getattr(options, 'build_dir', None)
 
-    modules_by_cfile = {}
-    to_compile = []
-    for m in module_list:
-        if build_dir:
-            root = os.getcwd()  # distutil extension depends are relative to cwd
-            def copy_to_build_dir(filepath, root=root):
+    def copy_to_build_dir(filepath, root=os.getcwd()):
         filepath_abs = os.path.abspath(filepath)
         if os.path.isabs(filepath):
             filepath = filepath_abs
         if filepath_abs.startswith(root):
+            # distutil extension depends are relative to cwd
             mod_dir = join_path(build_dir,
                                 os.path.dirname(_relpath(filepath, root)))
             copy_once_if_newer(filepath_abs, mod_dir)
+
+    modules_by_cfile = collections.defaultdict(list)
+    to_compile = []
+    for m in module_list:
+        if build_dir:
             for dep in m.depends:
                 copy_to_build_dir(dep)
 
+        cy_sources = [
+            source for source in m.sources
+            if os.path.splitext(source)[1] in ('.pyx', '.py')]
+        if len(cy_sources) == 1:
+            # normal "special" case: believe the Extension module name to allow user overrides
+            full_module_name = m.name
+        else:
+            # infer FQMN from source files
+            full_module_name = None
+
         new_sources = []
         for source in m.sources:
             base, ext = os.path.splitext(source)
@@ -976,12 +971,11 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False,
                         fingerprint = deps.transitive_fingerprint(source, extra)
                     else:
                         fingerprint = None
-                    to_compile.append((priority, source, c_file, fingerprint, quiet,
-                                       options, not exclude_failures, module_metadata.get(m.name)))
+                    to_compile.append((
+                        priority, source, c_file, fingerprint, quiet,
+                        options, not exclude_failures, module_metadata.get(m.name),
+                        full_module_name))
                 new_sources.append(c_file)
-                if c_file not in modules_by_cfile:
-                    modules_by_cfile[c_file] = [m]
-                else:
                 modules_by_cfile[c_file].append(m)
             else:
                 new_sources.append(source)
@@ -1098,17 +1092,15 @@ else:
 
 # TODO: Share context? Issue: pyx processing leaks into pxd module
 @record_results
-def cythonize_one(pyx_file, c_file, fingerprint, quiet, options=None, raise_on_failure=True, embedded_metadata=None, progress=""):
-    from ..Compiler.Main import compile, default_options
+def cythonize_one(pyx_file, c_file, fingerprint, quiet, options=None,
+                  raise_on_failure=True, embedded_metadata=None, full_module_name=None,
+                  progress=""):
+    from ..Compiler.Main import compile_single, default_options
     from ..Compiler.Errors import CompileError, PyrexError
 
     if fingerprint:
         if not os.path.exists(options.cache):
-            try:
-                os.mkdir(options.cache)
-            except:
-                if not os.path.exists(options.cache):
-                    raise
+            safe_makedirs(options.cache)
         # Cython-generated c files are highly compressible.
         # (E.g. a compression ratio of about 10 for Sage).
         fingerprint_file = join_path(
@@ -1136,7 +1128,7 @@ def cythonize_one(pyx_file, c_file, fingerprint, quiet, options=None, raise_on_f
 
     any_failures = 0
     try:
-        result = compile([pyx_file], options)
+        result = compile_single(pyx_file, options, full_module_name=full_module_name)
         if result.num_errors > 0:
             any_failures = 1
     except (EnvironmentError, PyrexError) as e:

Cython/Build/IpythonMagic.py

@@ -53,8 +53,10 @@ import re
 import sys
 import time
 import copy
+import distutils.log
 import textwrap
 
+
 try:
     reload
 except NameError:   # Python 3
@@ -236,6 +238,11 @@ class CythonMagics(Magics):
         help=("Enable profile guided optimisation in the C compiler. "
               "Compiles the cell twice and executes it in between to generate a runtime profile.")
     )
+    @magic_arguments.argument(
+        '--verbose', dest='quiet', action='store_false', default=True,
+        help=("Print debug information like generated .c/.cpp file location "
+              "and exact gcc/g++ command invoked.")
+    )
     @cell_magic
     def cython(self, line, cell):
         """Compile and import everything from a Cython code cell.
@@ -282,7 +289,6 @@ class CythonMagics(Magics):
         args = magic_arguments.parse_argstring(self.cython, line)
         code = cell if cell.endswith('\n') else cell + '\n'
         lib_dir = os.path.join(get_ipython_cache_dir(), 'cython')
-        quiet = True
         key = (code, line, sys.version_info, sys.executable, cython_version)
 
         if not os.path.exists(lib_dir):
@@ -311,7 +317,7 @@ class CythonMagics(Magics):
 
         extension = None
         if need_cythonize:
-            extensions = self._cythonize(module_name, code, lib_dir, args, quiet=quiet)
+            extensions = self._cythonize(module_name, code, lib_dir, args, quiet=args.quiet)
             assert len(extensions) == 1
             extension = extensions[0]
             self._code_cache[key] = module_name
@@ -319,7 +325,8 @@ class CythonMagics(Magics):
             if args.pgo:
                 self._profile_pgo_wrapper(extension, lib_dir)
 
-        self._build_extension(extension, lib_dir, pgo_step_name='use' if args.pgo else None)
+        self._build_extension(extension, lib_dir, pgo_step_name='use' if args.pgo else None,
+                              quiet=args.quiet)
 
         module = imp.load_dynamic(module_name, module_path)
         self._import_all(module)
@@ -386,7 +393,7 @@ class CythonMagics(Magics):
         so_module_path = os.path.join(lib_dir, pgo_module_name + self.so_ext)
         imp.load_dynamic(pgo_module_name, so_module_path)
 
-    def _cythonize(self, module_name, code, lib_dir, args, quiet=False):
+    def _cythonize(self, module_name, code, lib_dir, args, quiet=True):
         pyx_file = os.path.join(lib_dir, module_name + '.pyx')
         pyx_file = py3compat.cast_bytes_py2(pyx_file, encoding=sys.getfilesystemencoding())
 
@@ -422,10 +429,17 @@ class CythonMagics(Magics):
         except CompileError:
             return None
 
-    def _build_extension(self, extension, lib_dir, temp_dir=None, pgo_step_name=None):
+    def _build_extension(self, extension, lib_dir, temp_dir=None, pgo_step_name=None, quiet=True):
         build_extension = self._get_build_extension(
             extension, lib_dir=lib_dir, temp_dir=temp_dir, pgo_step_name=pgo_step_name)
+        old_threshold = None
+        try:
+            if not quiet:
+                old_threshold = distutils.log.set_threshold(distutils.log.DEBUG)
             build_extension.run()
+        finally:
+            if not quiet and old_threshold is not None:
+                distutils.log.set_threshold(old_threshold)
 
     def _add_pgo_flags(self, build_extension, step_name, temp_dir):
         compiler_type = build_extension.compiler.compiler_type

Cython/Build/Tests/TestIpythonMagic.py

@@ -3,25 +3,38 @@
 
 """Tests for the Cython magics extension."""
 
+from __future__ import absolute_import
+
 import os
 import sys
+from contextlib import contextmanager
+from Cython.Build import IpythonMagic
+from Cython.TestUtils import CythonTest
 
 try:
-    from IPython.testing.globalipapp import get_ipython
+    import IPython.testing.globalipapp
     from IPython.utils import py3compat
-except:
-    __test__ = False
+except ImportError:
+    # Disable tests and fake helpers for initialisation below.
+    class _py3compat(object):
+        def str_to_unicode(self, s):
+            return s
+
+    py3compat = _py3compat()
+
+    def skip_if_not_installed(_):
+        return None
+else:
+    def skip_if_not_installed(c):
+        return c
 
 try:
-    # disable IPython history thread to avoid having to clean it up
+    # disable IPython history thread before it gets started to avoid having to clean it up
     from IPython.core.history import HistoryManager
     HistoryManager.enabled = False
 except ImportError:
     pass
 
-from Cython.TestUtils import CythonTest
-
-ip = get_ipython()
 code = py3compat.str_to_unicode("""\
 def f(x):
     return 2*x
@@ -61,19 +74,27 @@ else:
         return _skip_win32
 
 
+@skip_if_not_installed
 class TestIPythonMagic(CythonTest):
 
+    @classmethod
+    def setUpClass(cls):
+        CythonTest.setUpClass()
+        cls._ip = IPython.testing.globalipapp.get_ipython()
+
     def setUp(self):
         CythonTest.setUp(self)
-        ip.extension_manager.load_extension('cython')
+        self._ip.extension_manager.load_extension('cython')
 
     def test_cython_inline(self):
+        ip = self._ip
         ip.ex('a=10; b=20')
         result = ip.run_cell_magic('cython_inline', '', 'return a+b')
         self.assertEqual(result, 30)
 
     @skip_win32('Skip on Windows')
     def test_cython_pyximport(self):
+        ip = self._ip
         module_name = '_test_cython_pyximport'
         ip.run_cell_magic('cython_pyximport', module_name, code)
         ip.ex('g = f(10)')
@@ -87,12 +108,14 @@ class TestIPythonMagic(CythonTest):
             pass
 
     def test_cython(self):
+        ip = self._ip
         ip.run_cell_magic('cython', '', code)
         ip.ex('g = f(10)')
         self.assertEqual(ip.user_ns['g'], 20.0)
 
     def test_cython_name(self):
         # The Cython module named 'mymodule' defines the function f.
+        ip = self._ip
         ip.run_cell_magic('cython', '--name=mymodule', code)
         # This module can now be imported in the interactive namespace.
         ip.ex('import mymodule; g = mymodule.f(10)')
@@ -100,6 +123,7 @@ class TestIPythonMagic(CythonTest):
 
     def test_cython_language_level(self):
         # The Cython cell defines the functions f() and call().
+        ip = self._ip
         ip.run_cell_magic('cython', '', cython3_code)
         ip.ex('g = f(10); h = call(10)')
         if sys.version_info[0] < 3:
@@ -111,6 +135,7 @@ class TestIPythonMagic(CythonTest):
 
     def test_cython3(self):
         # The Cython cell defines the functions f() and call().
+        ip = self._ip
         ip.run_cell_magic('cython', '-3', cython3_code)
         ip.ex('g = f(10); h = call(10)')
         self.assertEqual(ip.user_ns['g'], 2.0 / 10.0)
@@ -118,13 +143,16 @@ class TestIPythonMagic(CythonTest):
 
     def test_cython2(self):
         # The Cython cell defines the functions f() and call().
+        ip = self._ip
         ip.run_cell_magic('cython', '-2', cython3_code)
         ip.ex('g = f(10); h = call(10)')
         self.assertEqual(ip.user_ns['g'], 2 // 10)
         self.assertEqual(ip.user_ns['h'], 2 // 10)
 
+    @skip_win32('Skip on Windows')
     def test_cython3_pgo(self):
         # The Cython cell defines the functions f() and call().
+        ip = self._ip
         ip.run_cell_magic('cython', '-3 --pgo', pgo_cython3_code)
         ip.ex('g = f(10); h = call(10); main()')
         self.assertEqual(ip.user_ns['g'], 2.0 / 10.0)
@@ -132,6 +160,7 @@ class TestIPythonMagic(CythonTest):
 
     @skip_win32('Skip on Windows')
     def test_extlibs(self):
+        ip = self._ip
         code = py3compat.str_to_unicode("""
 from libc.math cimport sin
 x = sin(0.0)
@@ -139,3 +168,45 @@ x = sin(0.0)
         ip.user_ns['x'] = 1
         ip.run_cell_magic('cython', '-l m', code)
         self.assertEqual(ip.user_ns['x'], 0)
+
+
+    def test_cython_verbose(self):
+        ip = self._ip
+        ip.run_cell_magic('cython', '--verbose', code)
+        ip.ex('g = f(10)')
+        self.assertEqual(ip.user_ns['g'], 20.0)
+
+    def test_cython_verbose_thresholds(self):
+        @contextmanager
+        def mock_distutils():
+            class MockLog:
+                DEBUG = 1
+                INFO = 2
+                thresholds = [INFO]
+
+                def set_threshold(self, val):
+                    self.thresholds.append(val)
+                    return self.thresholds[-2]
+
+
+            new_log = MockLog()
+            old_log = IpythonMagic.distutils.log
+            try:
+                IpythonMagic.distutils.log = new_log
+                yield new_log
+            finally:
+                IpythonMagic.distutils.log = old_log
+
+        ip = self._ip
+        with mock_distutils() as verbose_log:
+            ip.run_cell_magic('cython', '--verbose', code)
+            ip.ex('g = f(10)')
+        self.assertEqual(ip.user_ns['g'], 20.0)
+        self.assertEquals([verbose_log.INFO, verbose_log.DEBUG, verbose_log.INFO],
+                          verbose_log.thresholds)
+
+        with mock_distutils() as normal_log:
+            ip.run_cell_magic('cython', '', code)
+            ip.ex('g = f(10)')
+        self.assertEqual(ip.user_ns['g'], 20.0)
+        self.assertEquals([normal_log.INFO], normal_log.thresholds)

Cython/CodeWriter.py

@@ -363,7 +363,7 @@ class CodeWriter(DeclarationWriter):
             self.dedent()
 
     def visit_IfStatNode(self, node):
-        # The IfClauseNode is handled directly without a seperate match
+        # The IfClauseNode is handled directly without a separate match
         # for clariy.
         self.startline(u"if ")
         self.visit(node.if_clauses[0].condition)

Cython/Compiler/Buffer.py

@@ -326,7 +326,7 @@ def put_acquire_arg_buffer(entry, code, pos):
     code.putln("__Pyx_BufFmt_StackElem __pyx_stack[%d];" % entry.type.dtype.struct_nesting_depth())
     code.putln(code.error_goto_if("%s == -1" % getbuffer, pos))
     code.putln("}")
-    # An exception raised in arg parsing cannot be catched, so no
+    # An exception raised in arg parsing cannot be caught, so no
     # need to care about the buffer then.
     put_unpack_buffer_aux_into_scope(entry, code)
 
@@ -617,7 +617,7 @@ class GetAndReleaseBufferUtilityCode(object):
 
 
 def mangle_dtype_name(dtype):
-    # Use prefixes to seperate user defined types from builtins
+    # Use prefixes to separate user defined types from builtins
     # (consider "typedef float unsigned_int")
     if dtype.is_pyobject:
         return "object"
@@ -636,7 +636,7 @@ def get_type_information_cname(code, dtype, maxdepth=None):
     and return the name of the type info struct.
 
     Structs with two floats of the same size are encoded as complex numbers.
-    One can seperate between complex numbers declared as struct or with native
+    One can separate between complex numbers declared as struct or with native
     encoding by inspecting to see if the fields field of the type is
     filled in.
     """

Cython/Compiler/Builtin.py

@@ -328,7 +328,10 @@ builtin_types_table = [
     ("set",       "PySet_Type",    [BuiltinMethod("__contains__",  "TO",   "b", "PySequence_Contains"),
                                     BuiltinMethod("clear",   "T",  "r", "PySet_Clear"),
                                     # discard() and remove() have a special treatment for unhashable values
-#                                    BuiltinMethod("discard", "TO", "r", "PySet_Discard"),
+                                    BuiltinMethod("discard", "TO", "r", "__Pyx_PySet_Discard",
+                                                  utility_code=UtilityCode.load("py_set_discard", "Optimize.c")),
+                                    BuiltinMethod("remove",  "TO", "r", "__Pyx_PySet_Remove",
+                                                  utility_code=UtilityCode.load("py_set_remove", "Optimize.c")),
                                     # update is actually variadic (see Github issue #1645)
 #                                    BuiltinMethod("update",     "TO", "r", "__Pyx_PySet_Update",
 #                                                  utility_code=UtilityCode.load_cached("PySet_Update", "Builtins.c")),

Cython/Compiler/CodeGeneration.py

@@ -12,7 +12,7 @@ class ExtractPxdCode(VisitorTransform):
     The result is a tuple (StatListNode, ModuleScope), i.e.
     everything that is needed from the pxd after it is processed.
 
-    A purer approach would be to seperately compile the pxd code,
+    A purer approach would be to separately compile the pxd code,
     but the result would have to be slightly more sophisticated
     than pure strings (functions + wanted interned strings +
     wanted utility code + wanted cached objects) so for now this

Cython/Compiler/FlowControl.py

@@ -341,14 +341,6 @@ class NameAssignment(object):
             return self.entry.type
         return self.inferred_type
 
-    def __getstate__(self):
-        return (self.lhs, self.rhs, self.entry, self.pos,
-                self.refs, self.is_arg, self.is_deletion, self.inferred_type)
-
-    def __setstate__(self, state):
-        (self.lhs, self.rhs, self.entry, self.pos,
-         self.refs, self.is_arg, self.is_deletion, self.inferred_type) = state
-
 
 class StaticAssignment(NameAssignment):
     """Initialised at declaration time, e.g. stack allocation."""

Cython/Compiler/FusedNode.py

@@ -127,9 +127,6 @@ class FusedCFuncDefNode(StatListNode):
         #                                            len(permutations))
         # import pprint; pprint.pprint([d for cname, d in permutations])
 
-        if self.node.entry in env.cfunc_entries:
-            env.cfunc_entries.remove(self.node.entry)
-
         # Prevent copying of the python function
         self.orig_py_func = orig_py_func = self.node.py_func
         self.node.py_func = None
@@ -139,12 +136,26 @@ class FusedCFuncDefNode(StatListNode):
         fused_types = self.node.type.get_fused_types()
         self.fused_compound_types = fused_types
 
+        new_cfunc_entries = []
         for cname, fused_to_specific in permutations:
             copied_node = copy.deepcopy(self.node)
 
-            # Make the types in our CFuncType specific
+            # Make the types in our CFuncType specific.
             type = copied_node.type.specialize(fused_to_specific)
             entry = copied_node.entry
+            type.specialize_entry(entry, cname)
+
+            # Reuse existing Entries (e.g. from .pxd files).
+            for i, orig_entry in enumerate(env.cfunc_entries):
+                if entry.cname == orig_entry.cname and type.same_as_resolved_type(orig_entry.type):
+                    copied_node.entry = env.cfunc_entries[i]
+                    if not copied_node.entry.func_cname:
+                        copied_node.entry.func_cname = entry.func_cname
+                    entry = copied_node.entry
+                    type = entry.type
+                    break
+            else:
+                new_cfunc_entries.append(entry)
 
             copied_node.type = type
             entry.type, type.entry = type, entry
@@ -165,9 +176,6 @@ class FusedCFuncDefNode(StatListNode):
             self._specialize_function_args(copied_node.cfunc_declarator.args,
                                            fused_to_specific)
 
-            type.specialize_entry(entry, cname)
-            env.cfunc_entries.append(entry)
-
             # If a cpdef, declare all specialized cpdefs (this
             # also calls analyse_declarations)
             copied_node.declare_cpdef_wrapper(env)
@@ -181,6 +189,14 @@ class FusedCFuncDefNode(StatListNode):
             if not self.replace_fused_typechecks(copied_node):
                 break
 
+        # replace old entry with new entries
+        try:
+            cindex = env.cfunc_entries.index(self.node.entry)
+        except ValueError:
+            env.cfunc_entries.extend(new_cfunc_entries)
+        else:
+            env.cfunc_entries[cindex:cindex+1] = new_cfunc_entries
+
         if orig_py_func:
             self.py_func = self.make_fused_cpdef(orig_py_func, env,
                                                  is_def=False)
@@ -209,7 +225,7 @@ class FusedCFuncDefNode(StatListNode):
         """
         Create a new local scope for the copied node and append it to
         self.nodes. A new local scope is needed because the arguments with the
-        fused types are aready in the local scope, and we need the specialized
+        fused types are already in the local scope, and we need the specialized
         entries created after analyse_declarations on each specialized version
         of the (CFunc)DefNode.
         f2s is a dict mapping each fused type to its specialized version
@@ -422,7 +438,7 @@ class FusedCFuncDefNode(StatListNode):
                     if dtype is not None:
                         itemsize = dtype.itemsize
                         kind = ord(dtype.kind)
-                        # We only support the endianess of the current compiler
+                        # We only support the endianness of the current compiler
                         byteorder = dtype.byteorder
                         if byteorder == "<" and not __Pyx_Is_Little_Endian():
                             arg_is_pythran_compatible = False

Cython/Compiler/Main.py

@@ -18,12 +18,12 @@ try:
 except ImportError:
     basestring = str
 
-from . import Errors
 # Do not import Parsing here, import it when needed, because Parsing imports
 # Nodes, which globally needs debug command line options initialized to set a
 # conditional metaclass. These options are processed by CmdLine called from
 # main() in this file.
 # import Parsing
+from . import Errors
 from .StringEncoding import EncodedString
 from .Scanning import PyrexScanner, FileSourceDescriptor
 from .Errors import PyrexError, CompileError, error, warning
@@ -38,6 +38,7 @@ module_name_pattern = re.compile(r"[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_
 
 verbose = 0
 
+
 class CompilationData(object):
     #  Bundles the information that is passed from transform to transform.
     #  (For now, this is only)
@@ -52,6 +53,7 @@ class CompilationData(object):
     #  result                CompilationResult
     pass
 
+
 class Context(object):
     #  This class encapsulates the context needed for compiling
     #  one or more Cython implementation files along with their
@@ -356,7 +358,7 @@ class Context(object):
                         from ..Parser import ConcreteSyntaxTree
                     except ImportError:
                         raise RuntimeError(
-                            "Formal grammer can only be used with compiled Cython with an available pgen.")
+                            "Formal grammar can only be used with compiled Cython with an available pgen.")
                     ConcreteSyntaxTree.p_module(source_filename)
         except UnicodeDecodeError as e:
             #import traceback
@@ -426,6 +428,7 @@ class Context(object):
                 pass
             result.c_file = None
 
+
 def get_output_filename(source_filename, cwd, options):
     if options.cplus:
         c_suffix = ".cpp"
@@ -441,6 +444,7 @@ def get_output_filename(source_filename, cwd, options):
     else:
         return suggested_file_name
 
+
 def create_default_resultobj(compilation_source, options):
     result = CompilationResult()
     result.main_source_file = compilation_source.source_desc.filename
@@ -451,6 +455,7 @@ def create_default_resultobj(compilation_source, options):
     result.embedded_metadata = options.embedded_metadata
     return result
 
+
 def run_pipeline(source, options, full_module_name=None, context=None):
     from . import Pipeline
 
@@ -496,11 +501,11 @@ def run_pipeline(source, options, full_module_name=None, context=None):
     return result
 
 
-#------------------------------------------------------------------------
+# ------------------------------------------------------------------------
 #
 #  Main Python entry points
 #
-#------------------------------------------------------------------------
+# ------------------------------------------------------------------------
 
 class CompilationSource(object):
     """
@@ -512,6 +517,7 @@ class CompilationSource(object):
         self.full_module_name = full_module_name
         self.cwd = cwd
 
+
 class CompilationOptions(object):
     """
     Options to the Cython compiler:
@@ -678,13 +684,14 @@ def compile_multiple(sources, options):
             processed.add(source)
     return results
 
+
 def compile(source, options = None, full_module_name = None, **kwds):
     """
     compile(source [, options], [, <option> = <value>]...)
 
     Compile one or more Pyrex implementation files, with optional timestamp
-    checking and recursing on dependecies. The source argument may be a string
-    or a sequence of strings If it is a string and no recursion or timestamp
+    checking and recursing on dependencies.  The source argument may be a string
+    or a sequence of strings.  If it is a string and no recursion or timestamp
     checking is requested, a CompilationResult is returned, otherwise a
     CompilationResultSet is returned.
     """
@@ -694,14 +701,17 @@ def compile(source, options = None, full_module_name = None, **kwds):
     else:
         return compile_multiple(source, options)
 
-#------------------------------------------------------------------------
+
+# ------------------------------------------------------------------------
 #
 #  Main command-line entry point
 #
-#------------------------------------------------------------------------
+# ------------------------------------------------------------------------
+
 def setuptools_main():
     return main(command_line = 1)
 
+
 def main(command_line = 0):
     args = sys.argv[1:]
     any_failures = 0
@@ -727,12 +737,11 @@ def main(command_line = 0):
         sys.exit(1)
 
 
-
-#------------------------------------------------------------------------
+# ------------------------------------------------------------------------
 #
 #  Set the default options depending on the platform
 #
-#------------------------------------------------------------------------
+# ------------------------------------------------------------------------
 
 default_options = dict(
     show_version = 0,

Cython/Compiler/MemoryView.py

@@ -390,19 +390,15 @@ def get_memoryview_flag(access, packing):
         return 'contiguous'
 
 
-def get_is_contig_func_name(c_or_f, ndim):
-    return "__pyx_memviewslice_is_%s_contig%d" % (c_or_f, ndim)
+def get_is_contig_func_name(contig_type, ndim):
+    assert contig_type in ('C', 'F')
+    return "__pyx_memviewslice_is_contig_%s%d" % (contig_type, ndim)
 
 
-def get_is_contig_utility(c_contig, ndim):
-    C = dict(context, ndim=ndim)
-    if c_contig:
-        utility = load_memview_c_utility("MemviewSliceIsCContig", C,
-                                         requires=[is_contig_utility])
-    else:
-        utility = load_memview_c_utility("MemviewSliceIsFContig", C,
-                                         requires=[is_contig_utility])
-
+def get_is_contig_utility(contig_type, ndim):
+    assert contig_type in ('C', 'F')
+    C = dict(context, ndim=ndim, contig_type=contig_type)
+    utility = load_memview_c_utility("MemviewSliceCheckContig", C, requires=[is_contig_utility])
     return utility
 
 

Cython/Compiler/ParseTreeTransforms.py

@@ -15,6 +15,7 @@ from . import ExprNodes
 from . import Nodes
 from . import Options
 from . import Builtin
+from . import Errors
 
 from .Visitor import VisitorTransform, TreeVisitor
 from .Visitor import CythonTransform, EnvTransform, ScopeTrackingTransform
@@ -632,7 +633,7 @@ class TrackNumpyAttributes(VisitorTransform, SkipDeclarations):
     visit_Node = VisitorTransform.recurse_to_children
 
 
-class InterpretCompilerDirectives(CythonTransform, SkipDeclarations):
+class InterpretCompilerDirectives(CythonTransform):
     """
     After parsing, directives can be stored in a number of places:
     - #cython-comments at the top of the file (stored in ModuleNode)
@@ -857,6 +858,11 @@ class InterpretCompilerDirectives(CythonTransform, SkipDeclarations):
                 node.cython_attribute = directive
         return node
 
+    def visit_NewExprNode(self, node):
+        self.visit(node.cppclass)
+        self.visitchildren(node)
+        return node
+
     def try_to_parse_directives(self, node):
         # If node is the contents of an directive (in a with statement or
         # decorator), returns a list of (directivename, value) pairs.
@@ -987,7 +993,7 @@ class InterpretCompilerDirectives(CythonTransform, SkipDeclarations):
     def visit_CVarDefNode(self, node):
         directives = self._extract_directives(node, 'function')
         if not directives:
-            return node
+            return self.visit_Node(node)
         for name, value in directives.items():
             if name == 'locals':
                 node.directive_locals = value
@@ -1027,7 +1033,8 @@ class InterpretCompilerDirectives(CythonTransform, SkipDeclarations):
         directives = []
         realdecs = []
         both = []
-        for dec in node.decorators:
+        # Decorators coming first take precedence.
+        for dec in node.decorators[::-1]:
             new_directives = self.try_to_parse_directives(dec.decorator)
             if new_directives is not None:
                 for directive in new_directives:
@@ -1037,15 +1044,17 @@ class InterpretCompilerDirectives(CythonTransform, SkipDeclarations):
                             directives.append(directive)
                         if directive[0] == 'staticmethod':
                             both.append(dec)
+                    # Adapt scope type based on decorators that change it.
+                    if directive[0] == 'cclass' and scope_name == 'class':
+                        scope_name = 'cclass'
             else:
                 realdecs.append(dec)
-        if realdecs and isinstance(node, (Nodes.CFuncDefNode, Nodes.CClassDefNode, Nodes.CVarDefNode)):
+        if realdecs and (scope_name == 'cclass' or
+                         isinstance(node, (Nodes.CFuncDefNode, Nodes.CClassDefNode, Nodes.CVarDefNode))):
             raise PostParseError(realdecs[0].pos, "Cdef functions/classes cannot take arbitrary decorators.")
-        else:
-            node.decorators = realdecs + both
+        node.decorators = realdecs[::-1] + both[::-1]
         # merge or override repeated directives
         optdict = {}
-        directives.reverse() # Decorators coming first take precedence
         for directive in directives:
             name, value = directive
             if name in optdict:
@@ -1292,7 +1301,7 @@ class WithTransform(CythonTransform, SkipDeclarations):
                                 pos, with_stat=node,
                                 test_if_run=False,
                                 args=excinfo_target,
-                                await=ExprNodes.AwaitExprNode(pos, arg=None) if is_async else None)),
+                                await_expr=ExprNodes.AwaitExprNode(pos, arg=None) if is_async else None)),
                         body=Nodes.ReraiseStatNode(pos),
                     ),
                 ],
@@ -1314,7 +1323,7 @@ class WithTransform(CythonTransform, SkipDeclarations):
                     test_if_run=True,
                     args=ExprNodes.TupleNode(
                         pos, args=[ExprNodes.NoneNode(pos) for _ in range(3)]),
-                    await=ExprNodes.AwaitExprNode(pos, arg=None) if is_async else None)),
+                    await_expr=ExprNodes.AwaitExprNode(pos, arg=None) if is_async else None)),
             handle_error_case=False,
         )
         return node
@@ -1385,9 +1394,14 @@ class DecoratorTransform(ScopeTrackingTransform, SkipDeclarations):
             elif decorator.is_attribute and decorator.obj.name in properties:
                 handler_name = self._map_property_attribute(decorator.attribute)
                 if handler_name:
-                    assert decorator.obj.name == node.name
-                    if len(node.decorators) > 1:
+                    if decorator.obj.name != node.name:
+                        # CPython does not generate an error or warning, but not something useful either.
+                        error(decorator_node.pos,
+                              "Mismatching property names, expected '%s', got '%s'" % (
+                                  decorator.obj.name, node.name))
+                    elif len(node.decorators) > 1:
                         return self._reject_decorated_property(node, decorator_node)
+                    else:
                         return self._add_to_property(properties, node, handler_name, decorator_node)
 
         # we clear node.decorators, so we need to set the
@@ -1535,6 +1549,13 @@ class ForwardDeclareTypes(CythonTransform):
     def visit_CClassDefNode(self, node):
         if node.class_name not in self.module_scope.entries:
             node.declare(self.module_scope)
+        # Expand fused methods of .pxd declared types to construct the final vtable order.
+        type = self.module_scope.entries[node.class_name].type
+        if type is not None and type.is_extension_type and not type.is_builtin_type and type.scope:
+            scope = type.scope
+            for entry in scope.cfunc_entries:
+                if entry.type and entry.type.is_fused:
+                    entry.type.get_all_specialized_function_types()
         return node
 
 
@@ -1859,7 +1880,7 @@ if VALUE is not None:
 
     def visit_FuncDefNode(self, node):
         """
-        Analyse a function and its body, as that hasn't happend yet.  Also
+        Analyse a function and its body, as that hasn't happened yet.  Also
         analyse the directive_locals set by @cython.locals().
 
         Then, if we are a function with fused arguments, replace the function
@@ -1922,6 +1943,8 @@ if VALUE is not None:
             binding = self.current_directives.get('binding')
             rhs = ExprNodes.PyCFunctionNode.from_defnode(node, binding)
             node.code_object = rhs.code_object
+            if node.is_generator:
+                node.gbody.code_object = node.code_object
 
         if env.is_py_class_scope:
             rhs.binding = True
@@ -2048,7 +2071,7 @@ if VALUE is not None:
 
     # Some nodes are no longer needed after declaration
     # analysis and can be dropped. The analysis was performed
-    # on these nodes in a seperate recursive process from the
+    # on these nodes in a separate recursive process from the
     # enclosing function or module, so we can simply drop them.
     def visit_CDeclaratorNode(self, node):
         # necessary to ensure that all CNameDeclaratorNodes are visited.
@@ -2573,10 +2596,27 @@ class MarkClosureVisitor(CythonTransform):
         collector.visitchildren(node)
 
         if node.is_async_def:
-            coroutine_type = Nodes.AsyncGenNode if collector.has_yield else Nodes.AsyncDefNode
+            coroutine_type = Nodes.AsyncDefNode
             if collector.has_yield:
+                coroutine_type = Nodes.AsyncGenNode
                 for yield_expr in collector.yields + collector.returns:
                     yield_expr.in_async_gen = True
+            elif node.decorators:
+                # evaluate @asyncio.coroutine() decorator at compile time if it's the inner-most one
+                # TODO: better decorator validation: should come from imported module
+                decorator = node.decorators[-1].decorator
+                if decorator.is_name and decorator.name == 'coroutine':
+                    pass
+                elif decorator.is_attribute and decorator.attribute == 'coroutine':
+                    if decorator.obj.is_name and decorator.obj.name in ('types', 'asyncio'):
+                        pass
+                    else:
+                        decorator = None
+                else:
+                    decorator = None
+                if decorator is not None:
+                    node.decorators.pop()
+                    coroutine_type = Nodes.IterableAsyncDefNode
         elif collector.has_await:
             found = next(y for y in collector.yields if y.is_await)
             error(found.pos, "'await' not allowed in generators (use 'yield')")
@@ -3140,6 +3180,7 @@ class ReplaceFusedTypeChecks(VisitorTransform):
         return self.transform(node)
 
     def visit_PrimaryCmpNode(self, node):
+        with Errors.local_errors(ignore=True):
           type1 = node.operand1.analyse_as_type(self.local_scope)
           type2 = node.operand2.analyse_as_type(self.local_scope)
 

Cython/Compiler/Parsing.py

@@ -2713,6 +2713,7 @@ special_basic_c_types = cython.declare(dict, {
     "ssize_t"    : (2, 0),
     "size_t"     : (0, 0),
     "ptrdiff_t"  : (2, 0),
+    "Py_tss_t"   : (1, 0),
 })
 
 sign_and_longness_words = cython.declare(
@@ -3084,9 +3085,13 @@ def p_cdef_extern_block(s, pos, ctx):
         ctx.namespace = p_string_literal(s, 'u')[2]
     if p_nogil(s):
         ctx.nogil = 1
-    body = p_suite(s, ctx)
+
+    # Use "docstring" as verbatim string to include
+    verbatim_include, body = p_suite_with_docstring(s, ctx, True)
+
     return Nodes.CDefExternNode(pos,
         include_file = include_file,
+        verbatim_include = verbatim_include,
         body = body,
         namespace = ctx.namespace)
 
@@ -3438,19 +3443,15 @@ def p_c_class_definition(s, pos,  ctx):
         as_name = class_name
     objstruct_name = None
     typeobj_name = None
-    base_class_module = None
-    base_class_name = None
+    bases = None
     if s.sy == '(':
-        s.next()
-        base_class_path = [p_ident(s)]
-        while s.sy == '.':
-            s.next()
-            base_class_path.append(p_ident(s))
-        if s.sy == ',':
-            s.error("C class may only have one base class", fatal=False)
-        s.expect(')')
-        base_class_module = ".".join(base_class_path[:-1])
-        base_class_name = base_class_path[-1]
+        positional_args, keyword_args = p_call_parse_args(s, allow_genexp=False)
+        if keyword_args:
+            s.error("C classes cannot take keyword bases.")
+        bases, _ = p_call_build_packed_args(pos, positional_args, keyword_args)
+    if bases is None:
+        bases = ExprNodes.TupleNode(pos, args=[])
+
     if s.sy == '[':
         if ctx.visibility not in ('public', 'extern') and not ctx.api:
             error(s.position(), "Name options only allowed for 'public', 'api', or 'extern' C class")
@@ -3490,8 +3491,7 @@ def p_c_class_definition(s, pos,  ctx):
         module_name = ".".join(module_path),
         class_name = class_name,
         as_name = as_name,
-        base_class_module = base_class_module,
-        base_class_name = base_class_name,
+        bases = bases,
         objstruct_name = objstruct_name,
         typeobj_name = typeobj_name,
         in_pxd = ctx.level == 'module_pxd',

Cython/Compiler/Pythran.py

+# cython: language_level=3
+
 from __future__ import absolute_import
 
 from .PyrexTypes import CType, CTypedefType, CStructOrUnionType
@@ -58,8 +60,14 @@ def pythran_unaryop_type(op, type_):
         op, pythran_type(type_))
 
 
-def pythran_indexing_type(type_, indices):
-    def index_code(idx):
+@cython.cfunc
+def _index_access(index_code, indices):
+    indexing = ",".join([index_code(idx) for idx in indices])
+    return ('[%s]' if len(indices) == 1 else '(%s)') % indexing
+
+
+def _index_type_code(index_with_type):
+    idx, index_type = index_with_type
     if idx.is_slice:
         if idx.step.is_none:
             func = "contiguous_slice"
@@ -69,18 +77,14 @@ def pythran_indexing_type(type_, indices):
             n = 3
         return "pythonic::types::%s(%s)" % (
             func, ",".join(["0"]*n))
-        elif idx.type.is_int:
-            return "std::declval<%s>()" % idx.type.sign_and_name()
-        elif idx.type.is_pythran_expr:
-            return "std::declval<%s>()" % idx.type.pythran_type
-        raise ValueError("unsupported indexing type %s!" % idx.type)
+    elif index_type.is_int:
+        return "std::declval<%s>()" % index_type.sign_and_name()
+    elif index_type.is_pythran_expr:
+        return "std::declval<%s>()" % index_type.pythran_type
+    raise ValueError("unsupported indexing type %s!" % index_type)
 
-    indexing = ",".join(index_code(idx) for idx in indices)
-    return type_remove_ref("decltype(std::declval<%s>()[%s])" % (pythran_type(type_), indexing))
 
-
-def pythran_indexing_code(indices):
-    def index_code(idx):
+def _index_code(idx):
     if idx.is_slice:
         values = idx.start, idx.stop, idx.step
         if idx.step.is_none:
@@ -96,7 +100,16 @@ def pythran_indexing_code(indices):
         return idx.pythran_result()
     raise ValueError("unsupported indexing type %s" % idx.type)
 
-    return ",".join(index_code(idx) for idx in indices)
+
+def pythran_indexing_type(type_, indices):
+    return type_remove_ref("decltype(std::declval<%s>()%s)" % (
+        pythran_type(type_),
+        _index_access(_index_type_code, indices),
+    ))
+
+
+def pythran_indexing_code(indices):
+    return _index_access(_index_code, indices)
 
 
 def pythran_func_type(func, args):

Cython/Compiler/Scanning.py

-# cython: infer_types=True, language_level=3, py2_import=True
+# cython: infer_types=True, language_level=3, py2_import=True, auto_pickle=False
 #
 #   Cython Scanner
 #
@@ -63,6 +63,12 @@ class Method(object):
         # self.kwargs is almost always unused => avoid call overhead
         return method(text, **self.kwargs) if self.kwargs is not None else method(text)
 
+    def __copy__(self):
+        return self  # immutable, no need to copy
+
+    def __deepcopy__(self, memo):
+        return self  # immutable, no need to copy
+
 
 #------------------------------------------------------------------
 
@@ -162,7 +168,7 @@ class SourceDescriptor(object):
         if self._escaped_description is None:
             esc_desc = \
                 self.get_description().encode('ASCII', 'replace').decode("ASCII")
-            # Use foreward slashes on Windows since these paths
+            # Use forward slashes on Windows since these paths
             # will be used in the #line directives in the C/C++ files.
             self._escaped_description = esc_desc.replace('\\', '/')
         return self._escaped_description
@@ -188,6 +194,12 @@ class SourceDescriptor(object):
         except AttributeError:
             return False
 
+    def __copy__(self):
+        return self  # immutable, no need to copy
+
+    def __deepcopy__(self, memo):
+        return self  # immutable, no need to copy
+
 
 class FileSourceDescriptor(SourceDescriptor):
     """

Cython/Compiler/StringEncoding.py

@@ -191,6 +191,14 @@ def bytes_literal(s, encoding):
     return s
 
 
+def encoded_string(s, encoding):
+    assert isinstance(s, (_unicode, bytes))
+    s = EncodedString(s)
+    if encoding is not None:
+        s.encoding = encoding
+    return s
+
+
 char_from_escape_sequence = {
     r'\a' : u'\a',
     r'\b' : u'\b',

Cython/Compiler/TypeInference.py

@@ -415,6 +415,24 @@ class SimpleAssignmentTypeInferer(object):
             entry = node.entry
             return spanning_type(types, entry.might_overflow, entry.pos, scope)
 
+        def inferred_types(entry):
+            has_none = False
+            has_pyobjects = False
+            types = []
+            for assmt in entry.cf_assignments:
+                if assmt.rhs.is_none:
+                    has_none = True
+                else:
+                    rhs_type = assmt.inferred_type
+                    if rhs_type and rhs_type.is_pyobject:
+                        has_pyobjects = True
+                    types.append(rhs_type)
+            # Ignore None assignments as long as there are concrete Python type assignments.
+            # but include them if None is the only assigned Python object.
+            if has_none and not has_pyobjects:
+                types.append(py_object_type)
+            return types
+
         def resolve_assignments(assignments):
             resolved = set()
             for assmt in assignments:
@@ -467,7 +485,7 @@ class SimpleAssignmentTypeInferer(object):
                 continue
             entry_type = py_object_type
             if assmts_resolved.issuperset(entry.cf_assignments):
-                types = [assmt.inferred_type for assmt in entry.cf_assignments]
+                types = inferred_types(entry)
                 if types and all(types):
                     entry_type = spanning_type(
                         types, entry.might_overflow, entry.pos, scope)
@@ -477,8 +495,9 @@ class SimpleAssignmentTypeInferer(object):
         def reinfer():
             dirty = False
             for entry in inferred:
-                types = [assmt.infer_type()
-                         for assmt in entry.cf_assignments]
+                for assmt in entry.cf_assignments:
+                    assmt.infer_type()
+                types = inferred_types(entry)
                 new_type = spanning_type(types, entry.might_overflow, entry.pos, scope)
                 if new_type != entry.type:
                     self.set_entry_type(entry, new_type)
@@ -544,6 +563,8 @@ def safe_spanning_type(types, might_overflow, pos, scope):
         # find_spanning_type() only returns 'bint' for clean boolean
         # operations without other int types, so this is safe, too
         return result_type
+    elif result_type.is_pythran_expr:
+        return result_type
     elif result_type.is_ptr:
         # Any pointer except (signed|unsigned|) char* can't implicitly
         # become a PyObject, and inferring char* is now accepted, too.

Cython/Compiler/TypeSlots.py

@@ -576,7 +576,7 @@ def get_special_method_signature(name):
     if slot:
         return slot.signature
     elif name in richcmp_special_methods:
-        return binaryfunc
+        return ibinaryfunc
     else:
         return None
 
@@ -612,6 +612,20 @@ def get_slot_function(scope, slot):
             return slot_code
     return None
 
+
+def get_slot_by_name(slot_name):
+    # For now, only search the type struct, no referenced sub-structs.
+    for slot in slot_table:
+        if slot.slot_name == slot_name:
+            return slot
+    assert False, "Slot not found: %s" % slot_name
+
+
+def get_slot_code_by_name(scope, slot_name):
+    slot = get_slot_by_name(slot_name)
+    return slot.slot_code(scope)
+
+
 #------------------------------------------------------------------------------------------
 #
 #  Signatures for generic Python functions and methods.
@@ -678,8 +692,7 @@ delattrofunc = Signature("TO", 'r')
 cmpfunc = Signature("TO", "i")             # typedef int (*cmpfunc)(PyObject *, PyObject *);
 reprfunc = Signature("T", "O")             # typedef PyObject *(*reprfunc)(PyObject *);
 hashfunc = Signature("T", "h")             # typedef Py_hash_t (*hashfunc)(PyObject *);
-                                           # typedef PyObject *(*richcmpfunc) (PyObject *, PyObject *, int);
-richcmpfunc = Signature("OOi", "O")        # typedef PyObject *(*richcmpfunc) (PyObject *, PyObject *, int);
+richcmpfunc = Signature("TOi", "O")        # typedef PyObject *(*richcmpfunc) (PyObject *, PyObject *, int);
 getiterfunc = Signature("T", "O")          # typedef PyObject *(*getiterfunc) (PyObject *);
 iternextfunc = Signature("T", "O")         # typedef PyObject *(*iternextfunc) (PyObject *);
 descrgetfunc = Signature("TOO", "O")       # typedef PyObject *(*descrgetfunc) (PyObject *, PyObject *, PyObject *);

Cython/Compiler/UtilNodes.py

 #
 # Nodes used as utilities and support for transforms etc.
 # These often make up sets including both Nodes and ExprNodes
-# so it is convenient to have them in a seperate module.
+# so it is convenient to have them in a separate module.
 #
 
 from __future__ import absolute_import
@@ -267,6 +267,9 @@ class EvalWithTempExprNode(ExprNodes.ExprNode, LetNodeMixin):
     def infer_type(self, env):
         return self.subexpression.infer_type(env)
 
+    def may_be_none(self):
+        return self.subexpression.may_be_none()
+
     def result(self):
         return self.subexpression.result()
 

Cython/Compiler/Visitor.py

@@ -581,15 +581,23 @@ class MethodDispatcherTransform(EnvTransform):
             # into a C function call (defined in the builtin scope)
             if not function.entry:
                 return node
+            entry = function.entry
             is_builtin = (
-                function.entry.is_builtin or
-                function.entry is self.current_env().builtin_scope().lookup_here(function.name))
+                entry.is_builtin or
+                entry is self.current_env().builtin_scope().lookup_here(function.name))
             if not is_builtin:
                 if function.cf_state and function.cf_state.is_single:
                     # we know the value of the variable
                     # => see if it's usable instead
                     return self._delegate_to_assigned_value(
                         node, function, arg_list, kwargs)
+                if arg_list and entry.is_cmethod and entry.scope and entry.scope.parent_type.is_builtin_type:
+                    if entry.scope.parent_type is arg_list[0].type:
+                        # Optimised (unbound) method of a builtin type => try to "de-optimise".
+                        return self._dispatch_to_method_handler(
+                            entry.name, self_arg=None, is_unbound_method=True,
+                            type_name=entry.scope.parent_type.name,
+                            node=node, function=function, arg_list=arg_list, kwargs=kwargs)
                 return node
             function_handler = self._find_handler(
                 "function_%s" % function.name, kwargs)
@@ -615,8 +623,7 @@ class MethodDispatcherTransform(EnvTransform):
             obj_type = self_arg.type
             is_unbound_method = False
             if obj_type.is_builtin_type:
-                if (obj_type is Builtin.type_type and self_arg.is_name and
-                        arg_list and arg_list[0].type.is_pyobject):
+                if obj_type is Builtin.type_type and self_arg.is_name and arg_list and arg_list[0].type.is_pyobject:
                     # calling an unbound method like 'list.append(L,x)'
                     # (ignoring 'type.mro()' here ...)
                     type_name = self_arg.name

Cython/Distutils/build_ext.py

@@ -14,9 +14,11 @@ else:
 class new_build_ext(_build_ext, object):
     def finalize_options(self):
         if self.distribution.ext_modules:
+            nthreads = getattr(self, 'parallel', None)  # -j option in Py3.5+
+            nthreads = int(nthreads) if nthreads else None
             from Cython.Build.Dependencies import cythonize
             self.distribution.ext_modules[:] = cythonize(
-                self.distribution.ext_modules)
+                self.distribution.ext_modules, nthreads=nthreads, force=self.force)
         super(new_build_ext, self).finalize_options()
 
 # This will become new_build_ext in the future.

Cython/Distutils/old_build_ext.py

 """Cython.Distutils.old_build_ext
 
 Implements a version of the Distutils 'build_ext' command, for
-building Cython extension modules."""
+building Cython extension modules.
 
-# This module should be kept compatible with Python 2.3.
+Note that this module is deprecated.  Use cythonize() instead.
+"""
 
 __revision__ = "$Id:$"