Add LIMITED_API support and remove static state (GH-3223)

Blacklists failing test for now

.travis.yml

@@ -51,6 +51,10 @@ matrix:
       dist: xenial    # Required for Python 3.7
       sudo: required  # travis-ci/travis-ci#9069
       env: TEST_CODE_STYLE=1
+    - python: 3.7
+      dist: xenial    # Required for Python 3.7
+      sudo: required  # travis-ci/travis-ci#9069
+      env: LIMITED_API=--limited-api EXCLUDE=--no-file
     - python: 3.4
       env: BACKEND=c
     - python: 3.4
@@ -63,6 +67,9 @@ matrix:
       env: BACKEND=c
     - python: 3.6
       env: BACKEND=cpp
+    - python: 3.6
+      sudo: required  # travis-ci/travis-ci#9069
+      env: LIMITED_API=--limited-api EXCLUDE=--no-file
     - python: 3.8
       dist: xenial    # Required for Python 3.7
       sudo: required  # travis-ci/travis-ci#9069
@@ -71,6 +78,10 @@ matrix:
       dist: xenial    # Required for Python 3.7
       sudo: required  # travis-ci/travis-ci#9069
       env: BACKEND=cpp
+    - python: 3.8-dev
+      dist: xenial    # Required for Python 3.8
+      sudo: required  # travis-ci/travis-ci#9069
+      env: LIMITED_API=--limited-api EXCLUDE=--no-file
     - os: osx
       osx_image: xcode6.4
       env: PY=2
@@ -154,5 +165,5 @@ script:
 # Need to clear the ccache? Try something like this:
 #  - if [ -n "${BACKEND##*cpp*}" -a -z "${TRAVIS_PYTHON_VERSION##*3.4}" ]; then ccache -C || true; fi
   - if [ "$COVERAGE" != "1" ]; then CFLAGS="-O2 -ggdb -Wall -Wextra $(python -c 'import sys; print("-fno-strict-aliasing" if sys.version_info[0] == 2 else "")')" python setup.py build_ext -i; fi
-  - CFLAGS="-O0 -ggdb -Wall -Wextra" python runtests.py -vv $STYLE_ARGS -x Debugger --backends=$BACKEND $(if [ "$COVERAGE" == "1" ]; then echo " --coverage"; fi) $(if [ -z "$TEST_CODE_STYLE" ]; then echo " -j7 "; fi)
+  - CFLAGS="-O0 -ggdb -Wall -Wextra" python runtests.py -vv $STYLE_ARGS -x Debugger --backends=$BACKEND $LIMITED_API $EXCLUDE $(if [ "$COVERAGE" == "1" ]; then echo " --coverage"; fi) $(if [ -z "$TEST_CODE_STYLE" ]; then echo " -j7 "; fi)
   - ccache -s || true

Cython/Compiler/Code.py

@@ -1080,6 +1080,10 @@ class GlobalState(object):
         'h_code',
         'filename_table',
         'utility_code_proto_before_types',
+        'module_state',
+        'module_state_clear',
+        'module_state_traverse',
+        'module_state_defines',
         'numeric_typedefs',          # Let these detailed individual parts stay!,
         'complex_type_declarations', # as the proper solution is to make a full DAG...
         'type_declarations',         # More coarse-grained blocks would simply hide
@@ -1420,9 +1424,18 @@ class GlobalState(object):
                   for c in self.py_constants]
         consts.sort()
         decls_writer = self.parts['decls']
+        decls_writer.putln("#if !CYTHON_COMPILING_IN_LIMITED_API")
         for _, cname, c in consts:
+            self.parts['module_state'].putln("%s;" % c.type.declaration_code(cname))
+            self.parts['module_state_defines'].putln(
+                "#define %s %s->%s" % (cname, Naming.modulestateglobal_cname, cname))
+            self.parts['module_state_clear'].putln(
+                "Py_CLEAR(clear_module_state->%s);" % cname)
+            self.parts['module_state_traverse'].putln(
+                "Py_VISIT(traverse_module_state->%s);" % cname)
             decls_writer.putln(
                 "static %s;" % c.type.declaration_code(cname))
+        decls_writer.putln("#endif")
 
     def generate_cached_methods_decls(self):
         if not self.cached_cmethods:
@@ -1476,13 +1489,15 @@ class GlobalState(object):
                 decls_writer.putln("static Py_UNICODE %s[] = { %s };" % (cname, utf16_array))
                 decls_writer.putln("#endif")
 
+        init_globals = self.parts['init_globals']
         if py_strings:
             self.use_utility_code(UtilityCode.load_cached("InitStrings", "StringTools.c"))
             py_strings.sort()
             w = self.parts['pystring_table']
             w.putln("")
             w.putln("static __Pyx_StringTabEntry %s[] = {" % Naming.stringtab_cname)
-            for c_cname, _, py_string in py_strings:
+            for idx, py_string_args in enumerate(py_strings):
+                c_cname, _, py_string = py_string_args
                 if not py_string.is_str or not py_string.encoding or \
                         py_string.encoding in ('ASCII', 'USASCII', 'US-ASCII',
                                                'UTF8', 'UTF-8'):
@@ -1490,8 +1505,19 @@ class GlobalState(object):
                 else:
                     encoding = '"%s"' % py_string.encoding.lower()
 
+                self.parts['module_state'].putln("PyObject *%s;" % py_string.cname)
+                self.parts['module_state_defines'].putln("#define %s %s->%s" % (
+                    py_string.cname,
+                    Naming.modulestateglobal_cname,
+                    py_string.cname))
+                self.parts['module_state_clear'].putln("Py_CLEAR(clear_module_state->%s);" %
+                    py_string.cname)
+                self.parts['module_state_traverse'].putln("Py_VISIT(traverse_module_state->%s);" %
+                    py_string.cname)
+                decls_writer.putln("#if !CYTHON_COMPILING_IN_LIMITED_API")
                 decls_writer.putln(
                     "static PyObject *%s;" % py_string.cname)
+                decls_writer.putln("#endif")
                 if py_string.py3str_cstring:
                     w.putln("#if PY_MAJOR_VERSION >= 3")
                     w.putln("{&%s, %s, sizeof(%s), %s, %d, %d, %d}," % (
@@ -1502,6 +1528,17 @@ class GlobalState(object):
                         py_string.intern
                         ))
                     w.putln("#else")
+
+                w.putln("#if CYTHON_COMPILING_IN_LIMITED_API")
+                w.putln("{0, %s, sizeof(%s), %s, %d, %d, %d}," % (
+                    c_cname,
+                    c_cname,
+                    encoding,
+                    py_string.is_unicode,
+                    py_string.is_str,
+                    py_string.intern
+                    ))
+                w.putln("#else")
                 w.putln("{&%s, %s, sizeof(%s), %s, %d, %d, %d}," % (
                     py_string.cname,
                     c_cname,
@@ -1511,25 +1548,42 @@ class GlobalState(object):
                     py_string.is_str,
                     py_string.intern
                     ))
+                w.putln("#endif")
+                init_globals.putln("#if CYTHON_COMPILING_IN_LIMITED_API")
+                init_globals.putln("if (__Pyx_InitString(%s[%d], &%s) < 0) %s;" % (
+                    Naming.stringtab_cname,
+                    idx,
+                    py_string.cname,
+                    init_globals.error_goto(self.module_pos)))
+                init_globals.putln("#endif")
                 if py_string.py3str_cstring:
                     w.putln("#endif")
             w.putln("{0, 0, 0, 0, 0, 0, 0}")
             w.putln("};")
 
-            init_globals = self.parts['init_globals']
+            init_globals.putln("#if !CYTHON_COMPILING_IN_LIMITED_API")
             init_globals.putln(
                 "if (__Pyx_InitStrings(%s) < 0) %s;" % (
                     Naming.stringtab_cname,
                     init_globals.error_goto(self.module_pos)))
+            init_globals.putln("#endif")
 
     def generate_num_constants(self):
         consts = [(c.py_type, c.value[0] == '-', len(c.value), c.value, c.value_code, c)
                   for c in self.num_const_index.values()]
         consts.sort()
         decls_writer = self.parts['decls']
+        decls_writer.putln("#if !CYTHON_COMPILING_IN_LIMITED_API")
         init_globals = self.parts['init_globals']
         for py_type, _, _, value, value_code, c in consts:
             cname = c.cname
+            self.parts['module_state'].putln("PyObject *%s;" % cname)
+            self.parts['module_state_defines'].putln("#define %s %s->%s" % (
+                cname, Naming.modulestateglobal_cname, cname))
+            self.parts['module_state_clear'].putln(
+                "Py_CLEAR(clear_module_state->%s);" % cname)
+            self.parts['module_state_traverse'].putln(
+                "Py_VISIT(traverse_module_state->%s);" % cname)
             decls_writer.putln("static PyObject *%s;" % cname)
             if py_type == 'float':
                 function = 'PyFloat_FromDouble(%s)'
@@ -1544,6 +1598,7 @@ class GlobalState(object):
             init_globals.putln('%s = %s; %s' % (
                 cname, function % value_code,
                 init_globals.error_goto_if_null(cname, self.module_pos)))
+        decls_writer.putln("#endif")
 
     # The functions below are there in a transition phase only
     # and will be deprecated. They are called from Nodes.BlockNode.

Cython/Compiler/ExprNodes.py

@@ -2301,6 +2301,14 @@ class NameNode(AtomicExprNode):
                 setter = '__Pyx_' + n
             else:
                 assert False, repr(entry)
+            code.putln("#if CYTHON_COMPILING_IN_LIMITED_API")
+            code.put_incref(rhs.py_result(), py_object_type)
+            code.put_error_if_neg(self.pos, '%s(%s, %s, %s)' % (
+              "PyModule_AddObject",
+              Naming.module_cname,
+              code.get_string_const(self.entry.name),
+              rhs.py_result()))
+            code.putln("#else")
             code.put_error_if_neg(
                 self.pos,
                 '%s(%s, %s, %s)' % (
@@ -2308,6 +2316,7 @@ class NameNode(AtomicExprNode):
                     namespace,
                     interned_cname,
                     rhs.py_result()))
+            code.putln("#endif")
             if debug_disposal_code:
                 print("NameNode.generate_assignment_code:")
                 print("...generating disposal code for %s" % rhs)
@@ -9370,6 +9379,27 @@ class PyCFunctionNode(ExprNode, ModuleNameMixin):
         else:
             flags = '0'
 
+        code.putln('#if CYTHON_COMPILING_IN_LIMITED_API')
+        dict_temp = code.funcstate.allocate_temp(py_object_type, manage_ref=True)
+        code.putln('%s = PyDict_New(); %s' % (
+            dict_temp,
+            code.error_goto_if_null(dict_temp, self.pos)))
+        code.put_gotref(dict_temp)
+        code.putln(
+            '%s = %s(&%s, %s, %s, %s, %s, %s, %s); %s' % (
+                self.result(),
+                constructor,
+                self.pymethdef_cname,
+                flags,
+                self.get_py_qualified_name(code),
+                self.closure_result_code(),
+                self.get_py_mod_name(code),
+                dict_temp,
+                code_object_result,
+                code.error_goto_if_null(self.result(), self.pos)))
+        code.put_decref_clear(dict_temp, type=py_object_type)
+        code.funcstate.release_temp(dict_temp)
+        code.putln('#else')
         code.putln(
             '%s = %s(&%s, %s, %s, %s, %s, %s, %s); %s' % (
                 self.result(),
@@ -9382,6 +9412,7 @@ class PyCFunctionNode(ExprNode, ModuleNameMixin):
                 Naming.moddict_cname,
                 code_object_result,
                 code.error_goto_if_null(self.result(), self.pos)))
+        code.putln('#endif')
 
         code.put_gotref(self.py_result())
 

Cython/Compiler/Naming.py

@@ -96,6 +96,8 @@ clineno_cname    = pyrex_prefix + "clineno"
 cfilenm_cname    = pyrex_prefix + "cfilenm"
 local_tstate_cname = pyrex_prefix + "tstate"
 module_cname     = pyrex_prefix + "m"
+modulestate_cname = pyrex_prefix + "mstate"
+modulestateglobal_cname = pyrex_prefix + "mstate_global"
 moddoc_cname     = pyrex_prefix + "mdoc"
 methtable_cname  = pyrex_prefix + "methods"
 retval_cname     = pyrex_prefix + "r"

Cython/Compiler/Nodes.py

@@ -3715,10 +3715,17 @@ class DefNodeWrapper(FuncDefNode):
         code.putln('{')
         all_args = tuple(positional_args) + tuple(kw_only_args)
         non_posonly_args = [arg for arg in all_args if not arg.pos_only]
+        non_pos_args_id = ','.join(
+            ['&%s' % code.intern_identifier(arg.name) for arg in non_posonly_args] + ['0'])
+        code.putln("#if CYTHON_COMPILING_IN_LIMITED_API")
+        code.putln("PyObject **%s[] = {%s};" % (
+            Naming.pykwdlist_cname,
+            non_pos_args_id))
+        code.putln("#else")
         code.putln("static PyObject **%s[] = {%s};" % (
             Naming.pykwdlist_cname,
-            ','.join(['&%s' % code.intern_identifier(arg.name)
-                      for arg in non_posonly_args] + ['0'])))
+            non_pos_args_id))
+        code.putln("#endif")
 
         # Before being converted and assigned to the target variables,
         # borrowed references to all unpacked argument values are
@@ -5070,6 +5077,13 @@ class CClassDefNode(ClassDefNode):
         if not scope:  # could be None if there was an error
             return
         if entry.visibility != 'extern':
+            code.putln("#if CYTHON_COMPILING_IN_LIMITED_API")
+            code.putln(
+                "%s = PyType_FromSpec(&%s_spec); %s" % (
+                    typeobj_cname,
+                    typeobj_cname,
+                    code.error_goto_if_null(typeobj_cname, entry.pos)))
+            code.putln("#else")
             for slot in TypeSlots.slot_table:
                 slot.generate_dynamic_init_code(scope, code)
             if heap_type_bases:
@@ -5108,6 +5122,7 @@ class CClassDefNode(ClassDefNode):
                 code.putln("%s.tp_getattro = %s;" % (
                     typeobj_cname, py_cfunc))
                 code.putln("}")
+            code.putln("#endif")
 
             # Fix special method docstrings. This is a bit of a hack, but
             # unless we let PyType_Ready create the slot wrappers we have
@@ -5145,11 +5160,19 @@ class CClassDefNode(ClassDefNode):
             if type.vtable_cname:
                 code.globalstate.use_utility_code(
                     UtilityCode.load_cached('SetVTable', 'ImportExport.c'))
+                code.putln("#if CYTHON_COMPILING_IN_LIMITED_API")
+                code.putln(
+                    "if (__Pyx_SetVtable(%s, %s) < 0) %s" % (
+                        typeobj_cname,
+                        type.vtabptr_cname,
+                        code.error_goto(entry.pos)))
+                code.putln("#else")
                 code.putln(
                     "if (__Pyx_SetVtable(%s.tp_dict, %s) < 0) %s" % (
                         typeobj_cname,
                         type.vtabptr_cname,
                         code.error_goto(entry.pos)))
+                code.putln("#endif")
                 if heap_type_bases:
                     code.globalstate.use_utility_code(
                         UtilityCode.load_cached('MergeVTables', 'ImportExport.c'))
@@ -5160,12 +5183,21 @@ class CClassDefNode(ClassDefNode):
                 # scope.is_internal is set for types defined by
                 # Cython (such as closures), the 'internal'
                 # directive is set by users
+                code.putln("#if CYTHON_COMPILING_IN_LIMITED_API")
+                code.putln(
+                    'if (PyObject_SetAttr(%s, %s, %s) < 0) %s' % (
+                        Naming.module_cname,
+                        code.intern_identifier(scope.class_name),
+                        typeobj_cname,
+                        code.error_goto(entry.pos)))
+                code.putln("#else")
                 code.putln(
                     'if (PyObject_SetAttr(%s, %s, (PyObject *)&%s) < 0) %s' % (
                         Naming.module_cname,
                         code.intern_identifier(scope.class_name),
                         typeobj_cname,
                         code.error_goto(entry.pos)))
+                code.putln("#endif")
             weakref_entry = scope.lookup_here("__weakref__") if not scope.is_closure_class_scope else None
             if weakref_entry:
                 if weakref_entry.type is py_object_type:
@@ -5187,15 +5219,19 @@ class CClassDefNode(ClassDefNode):
                 # do so at runtime.
                 code.globalstate.use_utility_code(
                     UtilityCode.load_cached('SetupReduce', 'ExtensionTypes.c'))
+                code.putln("#if !CYTHON_COMPILING_IN_LIMITED_API")
                 code.putln('if (__Pyx_setup_reduce((PyObject*)&%s) < 0) %s' % (
                               typeobj_cname,
                               code.error_goto(entry.pos)))
+                code.putln("#endif")
         # Generate code to initialise the typeptr of an extension
         # type defined in this module to point to its type object.
         if type.typeobj_cname:
+            code.putln("#if !CYTHON_COMPILING_IN_LIMITED_API")
             code.putln(
                 "%s = &%s;" % (
                     type.typeptr_cname, type.typeobj_cname))
+            code.putln("#endif")
 
     def annotate(self, code):
         if self.type_init_args:

Cython/Compiler/PyrexTypes.py

@@ -1310,7 +1310,7 @@ class BuiltinObjectType(PyObjectType):
             name = '"%s"' % self.name
             # avoid wasting too much space but limit number of different format strings
             space_for_name = (len(self.name) // 16 + 1) * 16
-        error = '(PyErr_Format(PyExc_TypeError, "Expected %%.%ds, got %%.200s", %s, Py_TYPE(%s)->tp_name), 0)' % (
+        error = '(PyErr_Format(PyExc_TypeError, "Expected %%.%ds, got %%.200s", %s, __Pyx_PyType_Name(Py_TYPE(%s))), 0)' % (
             space_for_name, name, arg)
         return check + '||' + error
 

Cython/Compiler/TypeSlots.py

@@ -242,7 +242,15 @@ class SlotDescriptor(object):
             guard = ("#if PY_MAJOR_VERSION >= 3")
         return guard
 
-    def generate(self, scope, code):
+    def spec_slot_value(self, scope):
+        if self.is_initialised_dynamically:
+            return None
+        result = self.slot_code(scope)
+        if result == "0":
+            return None
+        return result
+
+    def generate(self, scope, code, spec=False):
         preprocessor_guard = self.preprocessor_guard_code()
         if preprocessor_guard:
             code.putln(preprocessor_guard)
@@ -271,6 +279,10 @@ class SlotDescriptor(object):
                     code.putln("#else")
                     end_pypy_guard = True
 
+        if spec:
+            if value != "0":
+                code.putln("{Py_%s, (void *)%s}," % (self.slot_name, value))
+        else:
             code.putln("%s, /*%s*/" % (value, self.slot_name))
 
         if end_pypy_guard:
@@ -555,6 +567,11 @@ class SuiteSlot(SlotDescriptor):
             if self.ifdef:
                 code.putln("#endif")
 
+    def generate_substructure_spec(self, scope, code):
+        if not self.is_empty(scope):
+            for slot in self.sub_slots:
+                slot.generate(scope, code, spec=True)
+
 substructures = []   # List of all SuiteSlot instances
 
 class MethodTableSlot(SlotDescriptor):

Cython/Utility/Exceptions.c

@@ -284,7 +284,7 @@ static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject
     PyErr_SetObject(type, value);
 
     if (tb) {
-#if CYTHON_COMPILING_IN_PYPY
+#if CYTHON_COMPILING_IN_PYPY ||  CYTHON_COMPILING_IN_LIMITED_API
         PyObject *tmp_type, *tmp_value, *tmp_tb;
         PyErr_Fetch(&tmp_type, &tmp_value, &tmp_tb);
         Py_INCREF(tb);
@@ -705,6 +705,15 @@ static void __Pyx_AddTraceback(const char *funcname, int c_line,
 #include "frameobject.h"
 #include "traceback.h"
 
+#if CYTHON_COMPILING_IN_LIMITED_API
+static void __Pyx_AddTraceback(const char *funcname, int c_line,
+                               int py_line, const char *filename) {
+    if (c_line) {
+        c_line = __Pyx_CLineForTraceback(__Pyx_PyThreadState_Current, c_line);
+    }
+    _PyTraceback_Add(funcname, filename, c_line ? -c_line : py_line);
+}
+#else
 static PyCodeObject* __Pyx_CreateCodeObjectForTraceback(
             const char *funcname, int c_line,
             int py_line, const char *filename) {
@@ -791,3 +800,4 @@ bad:
     Py_XDECREF(py_code);
     Py_XDECREF(py_frame);
 }
+#endif

Cython/Utility/ExtensionTypes.c

@@ -172,13 +172,16 @@ static void __Pyx_call_next_tp_clear(PyObject* obj, inquiry current_tp_clear) {
 
 /////////////// SetupReduce.proto ///////////////
 
+#if !CYTHON_COMPILING_IN_LIMITED_API
 static int __Pyx_setup_reduce(PyObject* type_obj);
+#endif
 
 /////////////// SetupReduce ///////////////
 //@requires: ObjectHandling.c::PyObjectGetAttrStrNoError
 //@requires: ObjectHandling.c::PyObjectGetAttrStr
 //@substitute: naming
 
+#if !CYTHON_COMPILING_IN_LIMITED_API
 static int __Pyx_setup_reduce_is_named(PyObject* meth, PyObject* name) {
   int ret;
   PyObject *name_attr;
@@ -250,7 +253,7 @@ static int __Pyx_setup_reduce(PyObject* type_obj) {
 
 BAD:
     if (!PyErr_Occurred())
-        PyErr_Format(PyExc_RuntimeError, "Unable to initialize pickling for %s", ((PyTypeObject*)type_obj)->tp_name);
+        PyErr_Format(PyExc_RuntimeError, "Unable to initialize pickling for %s", __Pyx_PyType_Name(type_obj));
     ret = -1;
 GOOD:
 #if !CYTHON_USE_PYTYPE_LOOKUP
@@ -264,3 +267,4 @@ GOOD:
     Py_XDECREF(setstate_cython);
     return ret;
 }
+#endif

Cython/Utility/ImportExport.c

@@ -182,8 +182,13 @@ static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level) {
         if (level == -1) {
             if (strchr(__Pyx_MODULE_NAME, '.')) {
                 /* try package relative import first */
+                #if CYTHON_COMPILING_IN_LIMITED_API
+                module = PyImport_ImportModuleLevelObject(
+                    name, empty_dict, empty_dict, from_list, 1);
+                #else
                 module = PyImport_ImportModuleLevelObject(
                     name, $moddict_cname, empty_dict, from_list, 1);
+                #endif
                 if (unlikely(!module)) {
                     if (unlikely(!PyErr_ExceptionMatches(PyExc_ImportError)))
                         goto bad;
@@ -202,9 +207,14 @@ static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level) {
                 name, $moddict_cname, empty_dict, from_list, py_level, (PyObject *)NULL);
             Py_DECREF(py_level);
             #else
+            #if CYTHON_COMPILING_IN_LIMITED_API
+            module = PyImport_ImportModuleLevelObject(
+                name, empty_dict, empty_dict, from_list, level);
+            #else
             module = PyImport_ImportModuleLevelObject(
                 name, $moddict_cname, empty_dict, from_list, level);
             #endif
+            #endif
         }
     }
 bad:
@@ -686,11 +696,19 @@ static int __Pyx_SetVtable(PyObject *dict, void *vtable); /*proto*/
 
 /////////////// SetVTable ///////////////
 
+#if CYTHON_COMPILING_IN_LIMITED_API
+static int __Pyx_SetVtable(PyObject *type, void *vtable) {
+#else
 static int __Pyx_SetVtable(PyObject *dict, void *vtable) {
+#endif
     PyObject *ob = PyCapsule_New(vtable, 0, 0);
     if (!ob)
         goto bad;
+#if CYTHON_COMPILING_IN_LIMITED_API
+    if (PyObject_SetAttr(type, PYIDENT("__pyx_vtable__"), ob) < 0)
+#else
     if (PyDict_SetItem(dict, PYIDENT("__pyx_vtable__"), ob) < 0)
+#endif
         goto bad;
     Py_DECREF(ob);
     return 0;

Cython/Utility/ModuleSetupCode.c

@@ -44,6 +44,7 @@
   #define CYTHON_COMPILING_IN_PYPY 1
   #define CYTHON_COMPILING_IN_PYSTON 0
   #define CYTHON_COMPILING_IN_CPYTHON 0
+  #define CYTHON_COMPILING_IN_LIMITED_API 0
 
   #undef CYTHON_USE_TYPE_SLOTS
   #define CYTHON_USE_TYPE_SLOTS 0
@@ -90,6 +91,7 @@
   #define CYTHON_COMPILING_IN_PYPY 0
   #define CYTHON_COMPILING_IN_PYSTON 1
   #define CYTHON_COMPILING_IN_CPYTHON 0
+  #define CYTHON_COMPILING_IN_LIMITED_API 0
 
   #ifndef CYTHON_USE_TYPE_SLOTS
     #define CYTHON_USE_TYPE_SLOTS 1
@@ -133,10 +135,53 @@
   #undef CYTHON_USE_EXC_INFO_STACK
   #define CYTHON_USE_EXC_INFO_STACK 0
 
+#elif defined(CYTHON_LIMITED_API)
+  #define CYTHON_COMPILING_IN_PYPY 0
+  #define CYTHON_COMPILING_IN_PYSTON 0
+  #define CYTHON_COMPILING_IN_CPYTHON 0
+  #define CYTHON_COMPILING_IN_LIMITED_API 1
+
+  #undef CYTHON_USE_TYPE_SLOTS
+  #define CYTHON_USE_TYPE_SLOTS 0
+  #undef CYTHON_USE_PYTYPE_LOOKUP
+  #define CYTHON_USE_PYTYPE_LOOKUP 0
+  #undef CYTHON_USE_PYLIST_INTERNALS
+  #define CYTHON_USE_ASYNC_SLOTS 0
+  #define CYTHON_USE_PYLIST_INTERNALS 0
+  #undef CYTHON_USE_UNICODE_INTERNALS
+  #define CYTHON_USE_UNICODE_INTERNALS 0
+  #undef CYTHON_USE_UNICODE_WRITER
+  #define CYTHON_USE_UNICODE_WRITER 1
+  #undef CYTHON_USE_PYLONG_INTERNALS
+  #define CYTHON_USE_PYLONG_INTERNALS 0
+  #undef CYTHON_AVOID_BORROWED_REFS
+  #define CYTHON_AVOID_BORROWED_REFS 0
+  #undef CYTHON_ASSUME_SAFE_MACROS
+  #define CYTHON_ASSUME_SAFE_MACROS 0
+  #undef CYTHON_UNPACK_METHODS
+  #define CYTHON_UNPACK_METHODS 0
+  #undef CYTHON_FAST_THREAD_STATE
+  #define CYTHON_FAST_THREAD_STATE 0
+  #undef CYTHON_FAST_GIL
+  #define CYTHON_FAST_GIL 0
+  #undef CYTHON_METH_FASTCALL
+  #define CYTHON_METH_FASTCALL 0
+  #undef CYTHON_FAST_PYCALL
+  #define CYTHON_FAST_PYCALL 0
+  #undef CYTHON_PEP489_MULTI_PHASE_INIT
+  #define CYTHON_PEP489_MULTI_PHASE_INIT 0
+  #undef CYTHON_USE_TP_FINALIZE
+  #define CYTHON_USE_TP_FINALIZE 1
+  #undef CYTHON_USE_DICT_VERSIONS
+  #define CYTHON_USE_DICT_VERSIONS 0
+  #undef CYTHON_USE_EXC_INFO_STACK
+  #define CYTHON_USE_EXC_INFO_STACK 0
+
 #else
   #define CYTHON_COMPILING_IN_PYPY 0
   #define CYTHON_COMPILING_IN_PYSTON 0
   #define CYTHON_COMPILING_IN_CPYTHON 1
+  #define CYTHON_COMPILING_IN_LIMITED_API 0
 
   #ifndef CYTHON_USE_TYPE_SLOTS
     #define CYTHON_USE_TYPE_SLOTS 1
@@ -495,12 +540,17 @@ class __Pyx_FakeReference {
   // special C-API functions only in Pyston
   #define __Pyx_PyCode_HasFreeVars(co)  PyCode_HasFreeVars(co)
   #define __Pyx_PyFrame_SetLineNumber(frame, lineno) PyFrame_SetLineNumber(frame, lineno)
+#elif CYTHON_COMPILING_IN_LIMITED_API
+  #define __Pyx_PyCode_HasFreeVars(co)  (PyCode_GetNumFree(co) > 0)
+  #define __Pyx_PyFrame_SetLineNumber(frame, lineno)
 #else
   #define __Pyx_PyCode_HasFreeVars(co)  (PyCode_GetNumFree(co) > 0)
   #define __Pyx_PyFrame_SetLineNumber(frame, lineno)  (frame)->f_lineno = (lineno)
 #endif
 
-#if !CYTHON_FAST_THREAD_STATE
+#if CYTHON_COMPILING_IN_LIMITED_API
+  #define __Pyx_PyThreadState_Current PyThreadState_Get()
+#elif !CYTHON_FAST_THREAD_STATE
   #define __Pyx_PyThreadState_Current PyThreadState_GET()
 #elif PY_VERSION_HEX >= 0x03060000
   //#elif PY_VERSION_HEX >= 0x03050200
@@ -512,6 +562,18 @@ class __Pyx_FakeReference {
   #define __Pyx_PyThreadState_Current _PyThreadState_Current
 #endif
 
+#if CYTHON_COMPILING_IN_LIMITED_API
+static inline void *__Pyx_PyModule_GetState(PyObject *op)
+{
+    void *result;
+
+    result = PyModule_GetState(op);
+    if (!result)
+        Py_FatalError("Couldn't find the module state");
+    return result;
+}
+#endif
+
 // TSS (Thread Specific Storage) API
 #if PY_VERSION_HEX < 0x030700A2 && !defined(PyThread_tss_create) && !defined(Py_tss_NEEDS_INIT)
 #include "pythread.h"
@@ -582,8 +644,39 @@ static CYTHON_INLINE PyObject * __Pyx_PyDict_GetItemStrWithError(PyObject *dict,
 #define __Pyx_PyDict_GetItemStr           PyDict_GetItem
 #endif
 
-/* new Py3.3 unicode type (PEP 393) */
-#if PY_VERSION_HEX > 0x03030000 && defined(PyUnicode_KIND)
+/* Type slots */
+#if CYTHON_COMPILING_IN_LIMITED_API
+  #if defined(_PyType_Name)
+    #define __Pyx_PyType_Name(tp)     (_PyType_Name((PyTypeObject *)tp))
+  #else
+    #define __Pyx_PyType_Name(tp)     (((PyTypeObject *)tp)->tp_name)
+  #endif
+  #define __Pyx_PyType_GetFlags(tp)   (PyType_GetFlags((PyTypeObject *)tp))
+#else
+  #define __Pyx_PyType_Name(tp)       (((PyTypeObject *)tp)->tp_name)
+  #define __Pyx_PyType_GetFlags(tp)   (((PyTypeObject *)tp)->tp_flags)
+#endif
+
+#if CYTHON_COMPILING_IN_LIMITED_API
+  #if !defined(PyUnicode_GET_SIZE)
+    #define PyUnicode_GET_SIZE(u)          PyUnicode_GetSize(u)
+  #endif
+  #define CYTHON_PEP393_ENABLED 1
+  #define PyUnicode_1BYTE_KIND  1
+  #define PyUnicode_2BYTE_KIND  2
+  #define PyUnicode_4BYTE_KIND  4
+  #define __Pyx_PyUnicode_READY(op)       (0)
+  #define __Pyx_PyUnicode_GET_LENGTH(u)   PyUnicode_GetSize(u)
+  #define __Pyx_PyUnicode_READ_CHAR(u, i) ((Py_UCS4)(PyUnicode_AsUnicode(u)[i]))
+  #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u)   ((sizeof(wchar_t) == 2) ? 65535 : 1114111)
+  #define __Pyx_PyUnicode_KIND(u)         (sizeof(wchar_t))
+  #define __Pyx_PyUnicode_DATA(u)         ((void*)PyUnicode_AsUnicode(u))
+  /* (void)(k) => avoid unused variable warning due to macro: */
+  #define __Pyx_PyUnicode_READ(k, d, i)   ((void)(k), (Py_UCS4)(((wchar_t*)d)[i]))
+  #define __Pyx_PyUnicode_WRITE(k, d, i, ch)  (((void)(k)), ((wchar_t*)d)[i] = ch)
+  #define __Pyx_PyUnicode_IS_TRUE(u)      (0 != PyUnicode_GetSize(u))
+#elif PY_VERSION_HEX > 0x03030000 && defined(PyUnicode_KIND)
+  /* new Py3.3 unicode type (PEP 393) */
   #define CYTHON_PEP393_ENABLED 1
   #define __Pyx_PyUnicode_READY(op)       (likely(PyUnicode_IS_READY(op)) ? \
                                               0 : _PyUnicode_Ready((PyObject *)(op)))
@@ -987,12 +1080,20 @@ static CYTHON_SMALL_CODE int __Pyx_check_single_interpreter(void) {
     return 0;
 }
 
+#if CYTHON_COMPILING_IN_LIMITED_API
+static CYTHON_SMALL_CODE int __Pyx_copy_spec_to_module(PyObject *spec, PyObject *module, const char* from_name, const char* to_name, int allow_none) {
+#else
 static CYTHON_SMALL_CODE int __Pyx_copy_spec_to_module(PyObject *spec, PyObject *moddict, const char* from_name, const char* to_name, int allow_none) {
+#endif
     PyObject *value = PyObject_GetAttrString(spec, from_name);
     int result = 0;
     if (likely(value)) {
         if (allow_none || value != Py_None) {
+#if CYTHON_COMPILING_IN_LIMITED_API
+            result = PyModule_AddObject(module, to_name, value);
+#else
             result = PyDict_SetItemString(moddict, to_name, value);
+#endif
         }
         Py_DECREF(value);
     } else if (PyErr_ExceptionMatches(PyExc_AttributeError)) {
@@ -1019,9 +1120,13 @@ static CYTHON_SMALL_CODE PyObject* ${pymodule_create_func_cname}(PyObject *spec,
     Py_DECREF(modname);
     if (unlikely(!module)) goto bad;
 
+#if CYTHON_COMPILING_IN_LIMITED_API
+    moddict = module;
+#else
     moddict = PyModule_GetDict(module);
     if (unlikely(!moddict)) goto bad;
     // moddict is a borrowed reference
+#endif
 
     if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "loader", "__loader__", 1) < 0)) goto bad;
     if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "origin", "__file__", 1) < 0)) goto bad;
@@ -1038,6 +1143,7 @@ bad:
 
 /////////////// CodeObjectCache.proto ///////////////
 
+#if !CYTHON_COMPILING_IN_LIMITED_API
 typedef struct {
     PyCodeObject* code_object;
     int code_line;
@@ -1054,11 +1160,13 @@ static struct __Pyx_CodeObjectCache __pyx_code_cache = {0,0,NULL};
 static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line);
 static PyCodeObject *__pyx_find_code_object(int code_line);
 static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object);
+#endif
 
 /////////////// CodeObjectCache ///////////////
 // Note that errors are simply ignored in the code below.
 // This is just a cache, if a lookup or insertion fails - so what?
 
+#if !CYTHON_COMPILING_IN_LIMITED_API
 static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line) {
     int start = 0, mid = 0, end = count - 1;
     if (end >= 0 && code_line > entries[end].code_line) {
@@ -1139,9 +1247,11 @@ static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object) {
     __pyx_code_cache.count++;
     Py_INCREF(code_object);
 }
+#endif
 
 /////////////// CodeObjectCache.cleanup ///////////////
 
+  #if !CYTHON_COMPILING_IN_LIMITED_API
   if (__pyx_code_cache.entries) {
       __Pyx_CodeObjectCacheEntry* entries = __pyx_code_cache.entries;
       int i, count = __pyx_code_cache.count;
@@ -1153,6 +1263,7 @@ static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object) {
       }
       PyMem_Free(entries);
   }
+  #endif
 
 /////////////// CheckBinaryVersion.proto ///////////////
 

Cython/Utility/ObjectHandling.c

@@ -1289,6 +1289,14 @@ static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name)
     } else if (unlikely(PyErr_Occurred())) {
         return NULL;
     }
+#elif CYTHON_COMPILING_IN_LIMITED_API
+    if (unlikely(!$module_cname)) {
+        return NULL;
+    }
+    result = PyObject_GetItem($module_cname, name);
+    if (likely(result)) {
+        return result;
+    }
 #else
     result = PyDict_GetItem($moddict_cname, name);
     __PYX_UPDATE_DICT_CACHE($moddict_cname, result, *dict_cached_value, *dict_version)

Cython/Utility/Optimize.c

@@ -776,7 +776,11 @@ static CYTHON_INLINE {{c_ret_type}} __Pyx_PyInt_{{'' if ret_type.is_pyobject els
 
     if (PyFloat_CheckExact({{pyval}})) {
         const long {{'a' if order == 'CObj' else 'b'}} = intval;
+#if CYTHON_COMPILING_IN_LIMITED_API
+        double {{ival}} = __pyx_PyFloat_AsDouble({{pyval}});
+#else
         double {{ival}} = PyFloat_AS_DOUBLE({{pyval}});
+#endif
         {{return_compare('(double)a', '(double)b', c_op)}}
     }
 
@@ -1064,7 +1068,11 @@ static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, CYTHON_UNUSED
     {{if c_op in '+-*' or op in ('TrueDivide', 'Eq', 'Ne')}}
     if (PyFloat_CheckExact({{pyval}})) {
         const long {{'a' if order == 'CObj' else 'b'}} = intval;
+#if CYTHON_COMPILING_IN_LIMITED_API
+        double {{ival}} = __pyx_PyFloat_AsDouble({{pyval}});
+#else
         double {{ival}} = PyFloat_AS_DOUBLE({{pyval}});
+#endif
         {{if op in ('Eq', 'Ne')}}
             if ((double)a {{c_op}} (double)b) {
                 {{return_true}};
@@ -1143,7 +1151,11 @@ static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, double floatv
     {{endif}}
 
     if (likely(PyFloat_CheckExact({{pyval}}))) {
+#if CYTHON_COMPILING_IN_LIMITED_API
+        {{fval}} = __pyx_PyFloat_AsDouble({{pyval}});
+#else
         {{fval}} = PyFloat_AS_DOUBLE({{pyval}});
+#endif
         {{zerodiv_check(fval)}}
     } else
 

Cython/Utility/StringTools.c

@@ -43,13 +43,42 @@ static CYTHON_INLINE Py_ssize_t __Pyx_Py_UNICODE_ssize_strlen(const Py_UNICODE *
 
 //////////////////// InitStrings.proto ////////////////////
 
+#if CYTHON_COMPILING_IN_LIMITED_API
+static int __Pyx_InitString(__Pyx_StringTabEntry t, PyObject **str); /*proto*/
+#else
 static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); /*proto*/
+#endif
 
 //////////////////// InitStrings ////////////////////
 
+#if PY_MAJOR_VERSION >= 3
+static int __Pyx_InitString(__Pyx_StringTabEntry t, PyObject **str) {
+    if (t.is_unicode | t.is_str) {
+        if (t.intern) {
+            *str = PyUnicode_InternFromString(t.s);
+        } else if (t.encoding) {
+            *str = PyUnicode_Decode(t.s, t.n - 1, t.encoding, NULL);
+        } else {
+            *str = PyUnicode_FromStringAndSize(t.s, t.n - 1);
+        }
+    } else {
+        *str = PyBytes_FromStringAndSize(t.s, t.n - 1);
+    }
+    if (!*str)
+        return -1;
+    // initialise cached hash value
+    if (PyObject_Hash(*str) == -1)
+        return -1;
+    return 0;
+}
+#endif
+
+#if !CYTHON_COMPILING_IN_LIMITED_API
 static int __Pyx_InitStrings(__Pyx_StringTabEntry *t) {
     while (t->p) {
-        #if PY_MAJOR_VERSION < 3
+        #if PY_MAJOR_VERSION >= 3  /* Python 3+ has unicode identifiers */
+        __Pyx_InitString(*t, t->p);
+        #else
         if (t->is_unicode) {
             *t->p = PyUnicode_DecodeUTF8(t->s, t->n - 1, NULL);
         } else if (t->intern) {
@@ -57,28 +86,17 @@ static int __Pyx_InitStrings(__Pyx_StringTabEntry *t) {
         } else {
             *t->p = PyString_FromStringAndSize(t->s, t->n - 1);
         }
-        #else  /* Python 3+ has unicode identifiers */
-        if (t->is_unicode | t->is_str) {
-            if (t->intern) {
-                *t->p = PyUnicode_InternFromString(t->s);
-            } else if (t->encoding) {
-                *t->p = PyUnicode_Decode(t->s, t->n - 1, t->encoding, NULL);
-            } else {
-                *t->p = PyUnicode_FromStringAndSize(t->s, t->n - 1);
-            }
-        } else {
-            *t->p = PyBytes_FromStringAndSize(t->s, t->n - 1);
-        }
-        #endif
         if (!*t->p)
             return -1;
         // initialise cached hash value
         if (PyObject_Hash(*t->p) == -1)
             return -1;
+        #endif
         ++t;
     }
     return 0;
 }
+#endif
 
 //////////////////// BytesContains.proto ////////////////////
 
@@ -189,7 +207,7 @@ static CYTHON_INLINE int __Pyx_PyUnicode_Equals(PyObject* s1, PyObject* s2, int
 //@requires: BytesEquals
 
 static CYTHON_INLINE int __Pyx_PyUnicode_Equals(PyObject* s1, PyObject* s2, int equals) {
-#if CYTHON_COMPILING_IN_PYPY
+#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API
     return PyObject_RichCompareBool(s1, s2, equals);
 #else
 #if PY_MAJOR_VERSION < 3
@@ -300,7 +318,7 @@ static CYTHON_INLINE int __Pyx_PyBytes_Equals(PyObject* s1, PyObject* s2, int eq
 //@requires: IncludeStringH
 
 static CYTHON_INLINE int __Pyx_PyBytes_Equals(PyObject* s1, PyObject* s2, int equals) {
-#if CYTHON_COMPILING_IN_PYPY
+#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API
     return PyObject_RichCompareBool(s1, s2, equals);
 #else
     if (s1 == s2) {

Cython/Utility/TypeConversion.c

@@ -80,11 +80,21 @@ static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char*);
 #define __Pyx_PyUnicode_FromCString(s) __Pyx_PyUnicode_FromString((const char*)s)
 
 // There used to be a Py_UNICODE_strlen() in CPython 3.x, but it is deprecated since Py3.3.
-static CYTHON_INLINE size_t __Pyx_Py_UNICODE_strlen(const Py_UNICODE *u) {
+#if CYTHON_COMPILING_IN_LIMITED_API
+static CYTHON_INLINE size_t __Pyx_Py_UNICODE_strlen(const wchar_t *u)
+{
+    const wchar_t *u_end = u;
+    while (*u_end++) ;
+    return (size_t)(u_end - u - 1);
+}
+#else
+static CYTHON_INLINE size_t __Pyx_Py_UNICODE_strlen(const Py_UNICODE *u)
+{
     const Py_UNICODE *u_end = u;
     while (*u_end++) ;
     return (size_t)(u_end - u - 1);
 }
+#endif
 
 #define __Pyx_PyUnicode_FromUnicode(u)       PyUnicode_FromUnicode(u, __Pyx_Py_UNICODE_strlen(u))
 #define __Pyx_PyUnicode_FromUnicodeAndLength PyUnicode_FromUnicode
@@ -271,7 +281,7 @@ static CYTHON_INLINE const char* __Pyx_PyObject_AsStringAndSize(PyObject* o, Py_
     } else
 #endif /* __PYX_DEFAULT_STRING_ENCODING_IS_ASCII  || __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT */
 
-#if (!CYTHON_COMPILING_IN_PYPY) || (defined(PyByteArray_AS_STRING) && defined(PyByteArray_GET_SIZE))
+#if (!CYTHON_COMPILING_IN_PYPY && !CYTHON_COMPILING_IN_LIMITED_API) || (defined(PyByteArray_AS_STRING) && defined(PyByteArray_GET_SIZE))
     if (PyByteArray_Check(o)) {
         *length = PyByteArray_GET_SIZE(o);
         return PyByteArray_AS_STRING(o);
@@ -311,7 +321,7 @@ static PyObject* __Pyx_PyNumber_IntOrLongWrongResultType(PyObject* result, const
                 "__int__ returned non-int (type %.200s).  "
                 "The ability to return an instance of a strict subclass of int "
                 "is deprecated, and may be removed in a future version of Python.",
-                Py_TYPE(result)->tp_name)) {
+                __Pyx_PyType_Name(Py_TYPE(result)))) {
             Py_DECREF(result);
             return NULL;
         }
@@ -320,7 +330,7 @@ static PyObject* __Pyx_PyNumber_IntOrLongWrongResultType(PyObject* result, const
 #endif
     PyErr_Format(PyExc_TypeError,
                  "__%.4s__ returned non-%.4s (type %.200s)",
-                 type_name, type_name, Py_TYPE(result)->tp_name);
+                 type_name, type_name, __Pyx_PyType_Name(Py_TYPE(result)));
     Py_DECREF(result);
     return NULL;
 }
@@ -947,9 +957,9 @@ static CYTHON_INLINE {{TYPE}} {{FROM_PY_FUNCTION}}(PyObject *x) {
             }
         }
         {
-#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray)
+#if (CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) && !defined(_PyLong_AsByteArray)
             PyErr_SetString(PyExc_RuntimeError,
-                            "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers");
+                            "_PyLong_AsByteArray() not available, cannot convert large numbers");
 #else
             {{TYPE}} val;
             PyObject *v = __Pyx_PyNumber_IntOrLong(x);

runtests.py

@@ -2149,6 +2149,8 @@ def main():
                       help="specify Pythran include directory. This will run the C++ tests using Pythran backend for Numpy")
     parser.add_option("--no-capture", dest="capture", default=True, action="store_false",
                       help="do not capture stdout, stderr in srctree tests. Makes pdb.set_trace interactive")
+    parser.add_option("--limited-api", dest="limited_api", default=False, action="store_true",
+                      help="Compiles Cython using CPython's LIMITED_API")
 
     options, cmd_args = parser.parse_args(args)
 
@@ -2368,6 +2370,10 @@ def runtests(options, cmd_args, coverage=None):
         sys.path.insert(0, os.path.split(libpath)[0])
         CFLAGS.append("-DCYTHON_REFNANNY=1")
 
+    if options.limited_api:
+        CFLAGS.append("-DCYTHON_LIMITED_API=1")
+
+
     if xml_output_dir and options.fork:
         # doesn't currently work together
         sys.stderr.write("Disabling forked testing to support XML test output\n")
@@ -2426,6 +2432,7 @@ def runtests(options, cmd_args, coverage=None):
         bug_files = [
             ('bugs.txt', True),
             ('pypy_bugs.txt', IS_PYPY),
+            ('limited_api_bugs.txt', options.limited_api),
             ('windows_bugs.txt', sys.platform == 'win32'),
             ('cygwin_bugs.txt', sys.platform == 'cygwin')
         ]

tests/limited_api_bugs.txt

+# This file contains tests corresponding to unresolved bugs using CPython's
+# Limited API which will be skipped in the normal testing run.
+
+calc_pi_2
+calc_pi_3
+calc_pi_4
+clone
+convolve2
+cpdef_sin
+dict_animal
+integrate
+landscaping
+mymodule
+not_none
+overhead
+profile
+profile_2
+queue3
+resize
+safe_usage
+shrubbery_2
+sin_of_square
+test_queue
+unsafe_usage
+wave_function