Concatenate strings in place if possible (GH-3451)

Optimized inplace operations for str/unicode so that they're
genuinely done in place if no-one else needs the object. This
is what CPython tries to do (and string concatenation was
a point where it significantly beat Cython at times).

This only works if the types are known at compile time, so with
unknown types CPython will still be faster in some cases.

Note: Uses slightly odd-macro trickery - does modify the input argument too
(although only in places where it shouldn't matter).

Cython/Compiler/ExprNodes.py

@@ -11376,7 +11376,7 @@ class AddNode(NumBinopNode):
 
     def py_operation_function(self, code):
         type1, type2 = self.operand1.type, self.operand2.type
-
+        func = None
         if type1 is unicode_type or type2 is unicode_type:
             if type1 in (unicode_type, str_type) and type2 in (unicode_type, str_type):
                 is_unicode_concat = True
@@ -11388,10 +11388,22 @@ class AddNode(NumBinopNode):
                 is_unicode_concat = False
 
             if is_unicode_concat:
+                if self.inplace or self.operand1.is_temp:
+                    code.globalstate.use_utility_code(
+                        UtilityCode.load_cached("UnicodeConcatInPlace", "ObjectHandling.c"))
+                func = '__Pyx_PyUnicode_Concat'
+        elif type1 is str_type and type2 is str_type:
+            code.globalstate.use_utility_code(
+                    UtilityCode.load_cached("StrConcatInPlace", "ObjectHandling.c"))
+            func = '__Pyx_PyStr_Concat'
+
+        if func:
+            # any necessary utility code will be got by "NumberAdd" in generate_evaluation_code
+            if self.inplace or self.operand1.is_temp:
+                func += 'InPlace'  # upper case to indicate unintuitive macro
             if self.operand1.may_be_none() or self.operand2.may_be_none():
-                    return '__Pyx_PyUnicode_ConcatSafe'
-                else:
-                    return '__Pyx_PyUnicode_Concat'
+                func += 'Safe'
+            return func
 
         return super(AddNode, self).py_operation_function(code)
 

Cython/Utility/ObjectHandling.c

@@ -2553,3 +2553,115 @@ static PyObject *__Pyx_PyMethod_New(PyObject *func, PyObject *self, CYTHON_UNUSE
 #else
     #define __Pyx_PyMethod_New PyMethod_New
 #endif
+
+/////////////// UnicodeConcatInPlace.proto ////////////////
+
+# if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3
+// __Pyx_PyUnicode_ConcatInPlace may modify the first argument 'left'
+// However, unlike `PyUnicode_Append` it will never NULL it.
+// It behaves like a regular function - returns a new reference and NULL on error
+    #if CYTHON_REFNANNY
+        #define __Pyx_PyUnicode_ConcatInPlace(left, right) __Pyx_PyUnicode_ConcatInPlaceImpl(&left, right, __pyx_refnanny)
+    #else
+        #define __Pyx_PyUnicode_ConcatInPlace(left, right) __Pyx_PyUnicode_ConcatInPlaceImpl(&left, right, NULL)
+    #endif
+    // __Pyx_PyUnicode_ConcatInPlace is slightly odd because it has the potential to modify the input
+    // argument (but only in cases where no user should notice). Therefore, it needs to keep Cython's
+    // refnanny informed.
+    static CYTHON_INLINE PyObject *__Pyx_PyUnicode_ConcatInPlaceImpl(PyObject **p_left, PyObject *right, void* __pyx_refnanny); /* proto */
+#else
+#define __Pyx_PyUnicode_ConcatInPlace __Pyx_PyUnicode_Concat
+#endif
+#define __Pyx_PyUnicode_ConcatInPlaceSafe(left, right) ((unlikely((left) == Py_None) || unlikely((right) == Py_None)) ? \
+    PyNumber_InPlaceAdd(left, right) : __Pyx_PyUnicode_ConcatInPlace(left, right))
+
+/////////////// UnicodeConcatInPlace ////////////////
+//@substitute: naming
+
+# if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3
+// copied directly from unicode_object.c "unicode_modifiable
+// removing _PyUnicode_HASH since it's a macro we don't have
+//  - this is OK because trying PyUnicode_Resize on a non-modifyable
+//  object will still work, it just won't happen in place
+static int
+__Pyx_unicode_modifiable(PyObject *unicode)
+{
+    if (Py_REFCNT(unicode) != 1)
+        return 0;
+    if (!PyUnicode_CheckExact(unicode))
+        return 0;
+    if (PyUnicode_CHECK_INTERNED(unicode))
+        return 0;
+    return 1;
+}
+
+static CYTHON_INLINE PyObject *__Pyx_PyUnicode_ConcatInPlaceImpl(PyObject **p_left, PyObject *right, void* __pyx_refnanny) {
+    // heavily based on PyUnicode_Append
+    PyObject *left = *p_left;
+    Py_ssize_t left_len, right_len, new_len;
+
+    if (unlikely(PyUnicode_READY(left) == -1))
+        return NULL;
+    if (unlikely(PyUnicode_READY(right) == -1))
+        return NULL;
+
+    // Shortcuts
+    left_len = PyUnicode_GET_LENGTH(left);
+    if (left_len == 0) {
+        Py_INCREF(right);
+        return right;
+    }
+    right_len = PyUnicode_GET_LENGTH(right);
+    if (right_len == 0) {
+        Py_INCREF(left);
+        return left;
+    }
+    if (unlikely(left_len > PY_SSIZE_T_MAX - right_len)) {
+        PyErr_SetString(PyExc_OverflowError,
+                        "strings are too large to concat");
+        return NULL;
+    }
+    new_len = left_len + right_len;
+
+    if (__Pyx_unicode_modifiable(left)
+            && PyUnicode_CheckExact(right)
+            && PyUnicode_KIND(right) <= PyUnicode_KIND(left)
+            // Don't resize for ascii += latin1. Convert ascii to latin1 requires
+            //   to change the structure size, but characters are stored just after
+            //   the structure, and so it requires to move all characters which is
+            //   not so different than duplicating the string.
+            && !(PyUnicode_IS_ASCII(left) && !PyUnicode_IS_ASCII(right))) {
+
+        __Pyx_GIVEREF(*p_left);
+        if (unlikely(PyUnicode_Resize(p_left, new_len) != 0)) {
+            // on failure PyUnicode_Resize does not deallocate the the input
+            // so left will remain unchanged - simply undo the giveref
+            __Pyx_GOTREF(*p_left);
+            return NULL;
+        }
+        __Pyx_INCREF(*p_left);
+
+        // copy 'right' into the newly allocated area of 'left'
+        _PyUnicode_FastCopyCharacters(*p_left, left_len, right, 0, right_len);
+        return *p_left;
+    } else {
+        return __Pyx_PyUnicode_Concat(left, right);
+    }
+  }
+#endif
+
+////////////// StrConcatInPlace.proto ///////////////////////
+//@requires: UnicodeConcatInPlace
+
+#if PY_MAJOR_VERSION >= 3
+    // allow access to the more efficient versions where we know str_type is unicode
+    #define __Pyx_PyStr_Concat __Pyx_PyUnicode_Concat
+    #define __Pyx_PyStr_ConcatInPlace __Pyx_PyUnicode_ConcatInPlace
+#else
+    #define __Pyx_PyStr_Concat PyNumber_Add
+    #define __Pyx_PyStr_ConcatInPlace PyNumber_InPlaceAdd
+#endif
+#define __Pyx_PyStr_ConcatSafe(a, b) ((unlikely((a) == Py_None) || unlikely((b) == Py_None)) ? \
+    PyNumber_Add(a, b) : __Pyx_PyStr_Concat(a, b))
+#define __Pyx_PyStr_ConcatInPlaceSafe(a, b) ((unlikely((a) == Py_None) || unlikely((b) == Py_None)) ? \
+    PyNumber_InPlaceAdd(a, b) : __Pyx_PyStr_ConcatInPlace(a, b))