Merge branch 'master' into 0.20.x

CHANGES.rst

@@ -23,6 +23,9 @@ Features added
 
 * Constant Python float values are cached.
 
+* String equality comparisons can use faster type specific code in
+  more cases than before.
+
 * String/Unicode formatting using the '%' operator uses a faster
   C-API call.
 
@@ -73,9 +76,18 @@ Features added
 Bugs fixed
 ----------
 
+* Abstract Python classes that subtyped a Cython extension type
+  failed to raise an exception on instantiation, and thus ended
+  up being instantiated.
+
+* ``set.add(a_tuple)`` and ``set.discard(a_tuple)`` failed with a
+  TypeError in Py2.4.
+
 * The PEP 3155 ``__qualname__`` was incorrect for nested classes and
   inner classes/functions declared as ``global``.
 
+* Several corner cases in the try-finally statement were fixed.
+
 * The metaclass of a Python class was not inherited from its parent
   class(es).  It is now extracted from the list of base classes if not
   provided explicitly using the Py3 ``metaclass`` keyword argument.

Cython/Compiler/Builtin.py

@@ -18,52 +18,7 @@ pyexec_utility_code = UtilityCode.load("PyExec", "Builtins.c")
 pyexec_globals_utility_code = UtilityCode.load("PyExecGlobals", "Builtins.c")
 globals_utility_code = UtilityCode.load("Globals", "Builtins.c")
 
-py_set_utility_code = UtilityCode(
-proto = """
-#if PY_VERSION_HEX < 0x02050000
-#ifndef PyAnySet_CheckExact
-
-#define PyAnySet_CheckExact(ob) \\
-    ((ob)->ob_type == &PySet_Type || \\
-     (ob)->ob_type == &PyFrozenSet_Type)
-
-#define PySet_New(iterable) \\
-    PyObject_CallFunctionObjArgs((PyObject *)&PySet_Type, (iterable), NULL)
-
-#define Pyx_PyFrozenSet_New(iterable) \\
-    PyObject_CallFunctionObjArgs((PyObject *)&PyFrozenSet_Type, (iterable), NULL)
-
-#define PySet_Size(anyset) \\
-    PyObject_Size((anyset))
-
-#define PySet_Contains(anyset, key) \\
-    PySequence_Contains((anyset), (key))
-
-#define PySet_Pop(set) \\
-    PyObject_CallMethod(set, (char *)"pop", NULL)
-
-static CYTHON_INLINE int PySet_Clear(PyObject *set) {
-    PyObject *ret = PyObject_CallMethod(set, (char *)"clear", NULL);
-    if (!ret) return -1;
-    Py_DECREF(ret); return 0;
-}
-
-static CYTHON_INLINE int PySet_Discard(PyObject *set, PyObject *key) {
-    PyObject *ret = PyObject_CallMethod(set, (char *)"discard", (char *)"O", key);
-    if (!ret) return -1;
-    Py_DECREF(ret); return 0;
-}
-
-static CYTHON_INLINE int PySet_Add(PyObject *set, PyObject *key) {
-    PyObject *ret = PyObject_CallMethod(set, (char *)"add", (char *)"O", key);
-    if (!ret) return -1;
-    Py_DECREF(ret); return 0;
-}
-
-#endif /* PyAnySet_CheckExact (<= Py2.4) */
-#endif /* < Py2.5  */
-""",
-)
+py_set_utility_code = UtilityCode.load("pyset_compat", "Builtins.c")
 
 builtin_utility_code = {
     'set'       : py_set_utility_code,

Cython/Compiler/ExprNodes.py

@@ -9930,11 +9930,11 @@ class CmpNode(object):
             return (container_type.is_ptr or container_type.is_array) \
                 and not container_type.is_string
 
-    def find_special_bool_compare_function(self, env, operand1):
+    def find_special_bool_compare_function(self, env, operand1, result_is_bool=False):
         # note: currently operand1 must get coerced to a Python object if we succeed here!
         if self.operator in ('==', '!='):
             type1, type2 = operand1.type, self.operand2.type
-            if type1.is_builtin_type and type2.is_builtin_type:
+            if result_is_bool or (type1.is_builtin_type and type2.is_builtin_type):
                 if type1 is Builtin.unicode_type or type2 is Builtin.unicode_type:
                     self.special_bool_cmp_utility_code = UtilityCode.load_cached("UnicodeEquals", "StringTools.c")
                     self.special_bool_cmp_function = "__Pyx_PyUnicode_Equals"
@@ -9943,6 +9943,10 @@ class CmpNode(object):
                     self.special_bool_cmp_utility_code = UtilityCode.load_cached("BytesEquals", "StringTools.c")
                     self.special_bool_cmp_function = "__Pyx_PyBytes_Equals"
                     return True
+                elif type1 is Builtin.basestring_type or type2 is Builtin.basestring_type:
+                    self.special_bool_cmp_utility_code = UtilityCode.load_cached("UnicodeEquals", "StringTools.c")
+                    self.special_bool_cmp_function = "__Pyx_PyUnicode_Equals"
+                    return True
                 elif type1 is Builtin.str_type or type2 is Builtin.str_type:
                     self.special_bool_cmp_utility_code = UtilityCode.load_cached("StrEquals", "StringTools.c")
                     self.special_bool_cmp_function = "__Pyx_PyString_Equals"
@@ -10180,6 +10184,7 @@ class PrimaryCmpNode(ExprNode, CmpNode):
         else:
             self.operand1 = self.operand1.coerce_to(func_type.args[0].type, env)
             self.operand2 = self.operand2.coerce_to(func_type.args[1].type, env)
+        self.is_pycmp = False
         self.type = func_type.return_type
 
     def analyse_memoryviewslice_comparison(self, env):
@@ -10195,6 +10200,23 @@ class PrimaryCmpNode(ExprNode, CmpNode):
 
         return False
 
+    def coerce_to_boolean(self, env):
+        if self.is_pycmp:
+            # coercing to bool => may allow for more efficient comparison code
+            if self.find_special_bool_compare_function(
+                    env, self.operand1, result_is_bool=True):
+                self.is_pycmp = False
+                self.type = PyrexTypes.c_bint_type
+                self.is_temp = 1
+                if self.cascade:
+                    operand2 = self.cascade.optimise_comparison(
+                        self.operand2, env, result_is_bool=True)
+                    if operand2 is not self.operand2:
+                        self.coerced_operand2 = operand2
+                return self
+        # TODO: check if we can optimise parts of the cascade here
+        return ExprNode.coerce_to_boolean(self, env)
+
     def has_python_operands(self):
         return (self.operand1.type.is_pyobject
             or self.operand2.type.is_pyobject)
@@ -10316,12 +10338,14 @@ class CascadedCmpNode(Node, CmpNode):
     def has_python_operands(self):
         return self.operand2.type.is_pyobject
 
-    def optimise_comparison(self, operand1, env):
-        if self.find_special_bool_compare_function(env, operand1):
+    def optimise_comparison(self, operand1, env, result_is_bool=False):
+        if self.find_special_bool_compare_function(env, operand1, result_is_bool):
+            self.is_pycmp = False
+            self.type = PyrexTypes.c_bint_type
             if not operand1.type.is_pyobject:
                 operand1 = operand1.coerce_to_pyobject(env)
         if self.cascade:
-            operand2 = self.cascade.optimise_comparison(self.operand2, env)
+            operand2 = self.cascade.optimise_comparison(self.operand2, env, result_is_bool)
             if operand2 is not self.operand2:
                 self.coerced_operand2 = operand2
         return operand1

Cython/Compiler/ModuleNode.py

@@ -1125,7 +1125,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                 code.globalstate.use_utility_code(
                     UtilityCode.load_cached("IncludeStringH", "StringTools.c"))
                 obj_struct = type.declaration_code("", deref=True)
-                code.putln("if (likely((%s > 0) & (t->tp_basicsize == sizeof(%s)))) {" % (
+                code.putln("if (likely((%s > 0) & (t->tp_basicsize == sizeof(%s)) & ((t->tp_flags & Py_TPFLAGS_IS_ABSTRACT) == 0))) {" % (
                     freecount_name, obj_struct))
                 code.putln("o = (PyObject*)%s[--%s];" % (
                     freelist_name, freecount_name))
@@ -1134,7 +1134,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                 if scope.needs_gc():
                     code.putln("PyObject_GC_Track(o);")
                 code.putln("} else {")
-            code.putln("o = (*t->tp_alloc)(t, 0);")
+            code.putln("o = (PyObject *) PyBaseObject_Type.tp_new(t, %s, 0);" % Naming.empty_tuple)
         code.putln("if (unlikely(!o)) return 0;")
         if freelist_size and not base_type:
             code.putln('}')

Cython/Compiler/Nodes.py

@@ -5275,8 +5275,10 @@ class ReraiseStatNode(StatNode):
         vars = code.funcstate.exc_vars
         if vars:
             code.globalstate.use_utility_code(restore_exception_utility_code)
-            for varname in vars:
-                code.put_giveref(varname)
+            code.put_giveref(vars[0])
+            code.put_giveref(vars[1])
+            # fresh exceptions may not have a traceback yet (-> finally!)
+            code.put_xgiveref(vars[2])
             code.putln("__Pyx_ErrRestore(%s, %s, %s);" % tuple(vars))
             for varname in vars:
                 code.put("%s = 0; " % varname)
@@ -6506,7 +6508,10 @@ class TryFinallyStatNode(StatNode):
             finally_old_labels = code.all_new_labels()
 
             code.putln('{')
+            old_exc_vars = code.funcstate.exc_vars
+            code.funcstate.exc_vars = exc_vars[:3]
             fresh_finally_clause().generate_execution_code(code)
+            code.funcstate.exc_vars = old_exc_vars
             code.putln('}')
 
             if needs_success_cleanup:

Cython/Utility/Builtins.c

@@ -369,3 +369,48 @@ static CYTHON_INLINE PyObject* __Pyx_PyDict_ViewItems(PyObject* d); /*proto*/
 static CYTHON_INLINE PyObject* __Pyx_PyDict_ViewItems(PyObject* d) {
     return __Pyx_PyObject_CallMethod0(d, (PY_MAJOR_VERSION >= 3) ? PYIDENT("items") : PYIDENT("viewitems"));
 }
+
+//////////////////// pyset_compat.proto ////////////////////
+
+#if PY_VERSION_HEX < 0x02050000
+#ifndef PyAnySet_CheckExact
+
+#define PyAnySet_CheckExact(ob) \
+    ((ob)->ob_type == &PySet_Type || \
+     (ob)->ob_type == &PyFrozenSet_Type)
+
+#define PySet_New(iterable) \
+    PyObject_CallFunctionObjArgs((PyObject *)&PySet_Type, (iterable), NULL)
+
+#define Pyx_PyFrozenSet_New(iterable) \
+    PyObject_CallFunctionObjArgs((PyObject *)&PyFrozenSet_Type, (iterable), NULL)
+
+#define PySet_Size(anyset) \
+    PyObject_Size((anyset))
+
+#define PySet_Contains(anyset, key) \
+    PySequence_Contains((anyset), (key))
+
+#define PySet_Pop(set) \
+    PyObject_CallMethod((set), (char*)"pop", NULL)
+
+static CYTHON_INLINE int PySet_Clear(PyObject *set) {
+    PyObject *ret = PyObject_CallMethod(set, (char*)"clear", NULL);
+    if (!ret) return -1;
+    Py_DECREF(ret); return 0;
+}
+
+static CYTHON_INLINE int PySet_Discard(PyObject *set, PyObject *key) {
+    PyObject *ret = PyObject_CallMethod(set, (char*)"discard", (char*)"(O)", key);
+    if (!ret) return -1;
+    Py_DECREF(ret); return 0;
+}
+
+static CYTHON_INLINE int PySet_Add(PyObject *set, PyObject *key) {
+    PyObject *ret = PyObject_CallMethod(set, (char*)"add", (char*)"(O)", key);
+    if (!ret) return -1;
+    Py_DECREF(ret); return 0;
+}
+
+#endif /* PyAnySet_CheckExact (<= Py2.4) */
+#endif /* < Py2.5  */

Cython/Utility/Exceptions.c

@@ -430,6 +430,9 @@ static void __Pyx_WriteUnraisable(const char *name, CYTHON_UNUSED int clineno,
     PyObject *ctx;
     __Pyx_ErrFetch(&old_exc, &old_val, &old_tb);
     if (full_traceback) {
+        Py_XINCREF(old_exc);
+        Py_XINCREF(old_val);
+        Py_XINCREF(old_tb);
         __Pyx_ErrRestore(old_exc, old_val, old_tb);
         PyErr_PrintEx(1);
     }

Cython/Utility/ModuleSetupCode.c

@@ -133,7 +133,9 @@
 #if PY_VERSION_HEX < 0x02060000
   #define Py_TPFLAGS_HAVE_VERSION_TAG 0
 #endif
-
+#if PY_VERSION_HEX < 0x02060000 && !defined(Py_TPFLAGS_IS_ABSTRACT)
+  #define Py_TPFLAGS_IS_ABSTRACT 0
+#endif
 #if PY_VERSION_HEX < 0x030400a1 && !defined(Py_TPFLAGS_HAVE_FINALIZE)
   #define Py_TPFLAGS_HAVE_FINALIZE 0
 #endif

Cython/Utility/StringTools.c

@@ -133,15 +133,40 @@ static CYTHON_INLINE int __Pyx_PyUnicode_Contains(PyObject* substring, PyObject*
 static CYTHON_INLINE int __Pyx_PyUnicode_Equals(PyObject* s1, PyObject* s2, int equals); /*proto*/
 
 //////////////////// UnicodeEquals ////////////////////
+//@requires: BytesEquals
 
 static CYTHON_INLINE int __Pyx_PyUnicode_Equals(PyObject* s1, PyObject* s2, int equals) {
 #if CYTHON_COMPILING_IN_PYPY
     return PyObject_RichCompareBool(s1, s2, equals);
 #else
+#if PY_MAJOR_VERSION < 3
+    PyObject* owned_ref = NULL;
+#endif
+    int s1_is_unicode, s2_is_unicode;
     if (s1 == s2) {
         /* as done by PyObject_RichCompareBool(); also catches the (interned) empty string */
-        return (equals == Py_EQ);
-    } else if (PyUnicode_CheckExact(s1) & PyUnicode_CheckExact(s2)) {
+        goto return_eq;
+    }
+    s1_is_unicode = PyUnicode_CheckExact(s1);
+    s2_is_unicode = PyUnicode_CheckExact(s2);
+#if PY_MAJOR_VERSION < 3
+    if ((s1_is_unicode & (!s2_is_unicode)) && PyString_CheckExact(s2)) {
+        owned_ref = PyUnicode_FromObject(s2);
+        if (unlikely(!owned_ref))
+            return -1;
+        s2 = owned_ref;
+        s2_is_unicode = 1;
+    } else if ((s2_is_unicode & (!s1_is_unicode)) && PyString_CheckExact(s1)) {
+        owned_ref = PyUnicode_FromObject(s1);
+        if (unlikely(!owned_ref))
+            return -1;
+        s1 = owned_ref;
+        s1_is_unicode = 1;
+    } else if (((!s2_is_unicode) & (!s1_is_unicode))) {
+        return __Pyx_PyBytes_Equals(s1, s2, equals);
+    }
+#endif
+    if (s1_is_unicode & s2_is_unicode) {
         Py_ssize_t length;
         int kind;
         void *data1, *data2;
@@ -150,26 +175,31 @@ static CYTHON_INLINE int __Pyx_PyUnicode_Equals(PyObject* s1, PyObject* s2, int
             return -1;
         #endif
         length = __Pyx_PyUnicode_GET_LENGTH(s1);
-        if (length != __Pyx_PyUnicode_GET_LENGTH(s2))
-            return (equals == Py_NE);
+        if (length != __Pyx_PyUnicode_GET_LENGTH(s2)) {
+            goto return_ne;
+        }
         // len(s1) == len(s2) >= 1  (empty string is interned, and "s1 is not s2")
         kind = __Pyx_PyUnicode_KIND(s1);
-        if (kind != __Pyx_PyUnicode_KIND(s2))
-            return (equals == Py_NE);
+        if (kind != __Pyx_PyUnicode_KIND(s2)) {
+            goto return_ne;
+        }
         data1 = __Pyx_PyUnicode_DATA(s1);
         data2 = __Pyx_PyUnicode_DATA(s2);
         if (__Pyx_PyUnicode_READ(kind, data1, 0) != __Pyx_PyUnicode_READ(kind, data2, 0)) {
-            return (equals == Py_NE);
+            goto return_ne;
         } else if (length == 1) {
-            return (equals == Py_EQ);
+            goto return_eq;
         } else {
             int result = memcmp(data1, data2, length * kind);
+            #if PY_MAJOR_VERSION < 3
+            Py_XDECREF(owned_ref);
+            #endif
             return (equals == Py_EQ) ? (result == 0) : (result != 0);
         }
-    } else if ((s1 == Py_None) & PyUnicode_CheckExact(s2)) {
-        return (equals == Py_NE);
-    } else if ((s2 == Py_None) & PyUnicode_CheckExact(s1)) {
-        return (equals == Py_NE);
+    } else if ((s1 == Py_None) & s2_is_unicode) {
+        goto return_ne;
+    } else if ((s2 == Py_None) & s1_is_unicode) {
+        goto return_ne;
     } else {
         int result;
         PyObject* py_result = PyObject_RichCompare(s1, s2, equals);
@@ -179,6 +209,16 @@ static CYTHON_INLINE int __Pyx_PyUnicode_Equals(PyObject* s1, PyObject* s2, int
         Py_DECREF(py_result);
         return result;
     }
+return_eq:
+    #if PY_MAJOR_VERSION < 3
+    Py_XDECREF(owned_ref);
+    #endif
+    return (equals == Py_EQ);
+return_ne:
+    #if PY_MAJOR_VERSION < 3
+    Py_XDECREF(owned_ref);
+    #endif
+    return (equals == Py_NE);
 #endif
 }
 

tests/run/richcmp_str_equals.py

@@ -11,12 +11,16 @@ class testobj(object):
     def __eq__(self, other):
         return plop()
 
-def test_equals():
+def test_equals(x):
     """
-    >>> result = test_equals()
+    >>> x = testobj()
+    >>> result = test_equals(x)
     >>> isinstance(result, plop)
     True
+    >>> test_equals('hihi')
+    False
+    >>> test_equals('coucou')
+    True
     """
-    blah = testobj()
-    eq = blah == 'coucou'  # not every str equals returns a bool ...
+    eq = x == 'coucou'  # not every str equals returns a bool ...
     return eq

tests/run/set.pyx

@@ -35,13 +35,14 @@ def test_set_add():
     >>> type(test_set_add()) is _set
     True
     >>> sorted(test_set_add())
-    ['a', 1]
+    ['a', 1, (1, 2)]
     """
     cdef set s1
-    s1 = set([1])
+    s1 = set([1, (1, 2)])
     s1.add(1)
     s1.add('a')
     s1.add(1)
+    s1.add((1,2))
     return s1
 
 def test_set_clear():
@@ -159,14 +160,18 @@ def test_set_of_tuple():
     return set((1, 2, 3))
 
 def sorted(it):
-    # Py3 can't compare strings to ints
+    # Py3 can't compare different types
     chars = []
     nums = []
+    tuples = []
     for item in it:
         if type(item) is int:
             nums.append(item)
+        elif type(item) is tuple:
+            tuples.append(item)
         else:
             chars.append(item)
     nums.sort()
     chars.sort()
-    return chars+nums
+    tuples.sort()
+    return chars+nums+tuples

tests/run/tryfinally.pyx

@@ -73,6 +73,31 @@ def except_finally_reraise():
         raise
 
 
+def except_finally_reraise_new():
+    """
+    >>> def py_check():
+    ...     try: raise ValueError
+    ...     except ValueError:
+    ...         try: raise TypeError
+    ...         finally:
+    ...             raise
+    >>> try: py_check()
+    ... except ValueError: assert not IS_PY3
+    ... except TypeError: assert IS_PY3
+    ... else: assert False
+    >>> try: except_finally_reraise_new()
+    ... except TypeError: pass  # currently only Py3 semantics implemented
+    ... else: assert False
+    """
+    try:
+        raise ValueError
+    except ValueError:
+        try:
+            raise TypeError
+        finally:
+            raise
+
+
 def finally_exception_check_return():
     """
     >>> if not IS_PY3: