Merge pull request #713 from kmod/perf3

optimize regex handling

from_cpython/Modules/_sre.c

@@ -1768,6 +1768,9 @@ state_init(SRE_STATE* state, PatternObject* pattern, PyObject* string,
     if (!ptr)
         return NULL;
 
+    // Pyston change:
+    assert(charsize == 1 || charsize == sizeof(Py_UNICODE));
+
     /* adjust boundaries */
     if (start < 0)
         start = 0;
@@ -1813,8 +1816,15 @@ state_fini(SRE_STATE* state)
 }
 
 /* calculate offset from start of string */
+// Pyston change: this division is expensive
+/*
 #define STATE_OFFSET(state, member)\
     (((char*)(member) - (char*)(state)->beginning) / (state)->charsize)
+*/
+#define STATE_OFFSET(state, member) \
+    ((state)->charsize == 1 ? \
+     (((char*)(member) - (char*)(state)->beginning)) : \
+     (((char*)(member) - (char*)(state)->beginning) / sizeof(Py_UNICODE)))
 
 LOCAL(PyObject*)
 state_getslice(SRE_STATE* state, Py_ssize_t index, PyObject* string, int empty)

src/capi/abstract.cpp

@@ -1441,14 +1441,55 @@ extern "C" PyObject* PySequence_GetItem(PyObject* o, Py_ssize_t i) noexcept {
     }
 }
 
-extern "C" PyObject* PySequence_GetSlice(PyObject* o, Py_ssize_t i1, Py_ssize_t i2) noexcept {
-    try {
-        // Not sure if this is really the same:
-        return getitem(o, createSlice(boxInt(i1), boxInt(i2), None));
-    } catch (ExcInfo e) {
-        fatalOrError(PyExc_NotImplementedError, "unimplemented");
-        return nullptr;
+PyObject* _PySlice_FromIndices(Py_ssize_t istart, Py_ssize_t istop) {
+    PyObject* start, *end, *slice;
+    start = PyInt_FromSsize_t(istart);
+    if (!start)
+        return NULL;
+    end = PyInt_FromSsize_t(istop);
+    if (!end) {
+        Py_DECREF(start);
+        return NULL;
     }
+
+    slice = PySlice_New(start, end, NULL);
+    Py_DECREF(start);
+    Py_DECREF(end);
+    return slice;
+}
+
+extern "C" PyObject* PySequence_GetSlice(PyObject* s, Py_ssize_t i1, Py_ssize_t i2) noexcept {
+    PySequenceMethods* m;
+    PyMappingMethods* mp;
+
+    if (!s)
+        return null_error();
+
+    m = s->cls->tp_as_sequence;
+    if (m && m->sq_slice) {
+        if (i1 < 0 || i2 < 0) {
+            if (m->sq_length) {
+                Py_ssize_t l = (*m->sq_length)(s);
+                if (l < 0)
+                    return NULL;
+                if (i1 < 0)
+                    i1 += l;
+                if (i2 < 0)
+                    i2 += l;
+            }
+        }
+        return m->sq_slice(s, i1, i2);
+    } else if ((mp = s->cls->tp_as_mapping) && mp->mp_subscript) {
+        PyObject* res;
+        PyObject* slice = _PySlice_FromIndices(i1, i2);
+        if (!slice)
+            return NULL;
+        res = mp->mp_subscript(s, slice);
+        Py_DECREF(slice);
+        return res;
+    }
+
+    return type_error("'%.200s' object is unsliceable", s);
 }
 
 extern "C" int PySequence_SetItem(PyObject* o, Py_ssize_t i, PyObject* v) noexcept {

src/capi/typeobject.cpp

@@ -1112,12 +1112,14 @@ static int slot_sq_ass_slice(PyObject* self, Py_ssize_t i, Py_ssize_t j, PyObjec
 // Copied from CPython:
 #define SLOT0(FUNCNAME, OPSTR)                                                                                         \
     static PyObject* FUNCNAME(PyObject* self) noexcept {                                                               \
+        STAT_TIMER(t0, "us_timer_" #FUNCNAME, SLOT_AVOIDABILITY(self));                                                \
         static PyObject* cache_str;                                                                                    \
         return call_method(self, OPSTR, &cache_str, "()");                                                             \
     }
 
 #define SLOT1(FUNCNAME, OPSTR, ARG1TYPE, ARGCODES)                                                                     \
     /* Pyston change: static */ PyObject* FUNCNAME(PyObject* self, ARG1TYPE arg1) noexcept {                           \
+        STAT_TIMER(t0, "us_timer_" #FUNCNAME, SLOT_AVOIDABILITY(self));                                                \
         static PyObject* cache_str;                                                                                    \
         return call_method(self, OPSTR, &cache_str, "(" ARGCODES ")", arg1);                                           \
     }

src/codegen/ast_interpreter.cpp

@@ -38,6 +38,7 @@
 #include "runtime/generator.h"
 #include "runtime/import.h"
 #include "runtime/inline/boxing.h"
+#include "runtime/inline/list.h"
 #include "runtime/long.h"
 #include "runtime/objmodel.h"
 #include "runtime/set.h"

src/codegen/baseline_jit.cpp

@@ -21,6 +21,7 @@
 #include "codegen/memmgr.h"
 #include "codegen/type_recording.h"
 #include "core/cfg.h"
+#include "runtime/inline/list.h"
 #include "runtime/objmodel.h"
 #include "runtime/set.h"
 #include "runtime/types.h"

src/codegen/runtime_hooks.cpp

@@ -40,6 +40,7 @@
 #include "runtime/generator.h"
 #include "runtime/import.h"
 #include "runtime/inline/boxing.h"
+#include "runtime/inline/list.h"
 #include "runtime/int.h"
 #include "runtime/long.h"
 #include "runtime/objmodel.h"

src/gc/heap.cpp

@@ -685,7 +685,7 @@ GCAllocation* LargeArena::allocationFrom(void* ptr) {
 
 void LargeArena::prepareForCollection() {
     for (LargeObj* lo = head; lo; lo = lo->next) {
-        lookup.push_back(ObjLookupCache(lo, &lo->data[0], lo->size));
+        lookup.push_back(ObjLookupCache(&lo->data[0], lo->size));
     }
     std::sort(lookup.begin(), lookup.end(),
               [](const ObjLookupCache& lo1, const ObjLookupCache& lo2) { return lo1.data < lo2.data; });
@@ -896,7 +896,7 @@ GCAllocation* HugeArena::allocationFrom(void* ptr) {
 
 void HugeArena::prepareForCollection() {
     for (HugeObj* lo = head; lo; lo = lo->next) {
-        lookup.push_back(ObjLookupCache(lo, &lo->data[0], lo->size));
+        lookup.push_back(ObjLookupCache(&lo->data[0], lo->size));
     }
     std::sort(lookup.begin(), lookup.end(),
               [](const ObjLookupCache& lo1, const ObjLookupCache& lo2) { return lo1.data < lo2.data; });

src/gc/heap.h

@@ -365,11 +365,10 @@ private:
 };
 
 struct ObjLookupCache {
-    void* objptr;
     void* data;
     size_t size;
 
-    ObjLookupCache(void* objptr, void* data, size_t size) : objptr(objptr), data(data), size(size) {}
+    ObjLookupCache(void* data, size_t size) : data(data), size(size) {}
 };
 
 

src/runtime/builtin_modules/builtins.cpp

@@ -31,6 +31,7 @@
 #include "runtime/file.h"
 #include "runtime/ics.h"
 #include "runtime/import.h"
+#include "runtime/inline/list.h"
 #include "runtime/inline/xrange.h"
 #include "runtime/iterobject.h"
 #include "runtime/list.h"

src/runtime/builtin_modules/sys.cpp

@@ -26,6 +26,7 @@
 #include "gc/collector.h"
 #include "runtime/file.h"
 #include "runtime/inline/boxing.h"
+#include "runtime/inline/list.h"
 #include "runtime/int.h"
 #include "runtime/types.h"
 #include "runtime/util.h"

src/runtime/dict.cpp

@@ -20,6 +20,7 @@
 #include "core/stats.h"
 #include "core/types.h"
 #include "gc/collector.h"
+#include "runtime/inline/list.h"
 #include "runtime/objmodel.h"
 #include "runtime/types.h"
 #include "runtime/util.h"

src/runtime/import.cpp

@@ -24,6 +24,7 @@
 #include "codegen/parser.h"
 #include "codegen/unwinding.h"
 #include "core/ast.h"
+#include "runtime/inline/list.h"
 #include "runtime/objmodel.h"
 
 namespace pyston {

src/runtime/inline/link_forcer.cpp

@@ -25,6 +25,7 @@
 #include "runtime/generator.h"
 #include "runtime/import.h"
 #include "runtime/inline/boxing.h"
+#include "runtime/inline/list.h"
 #include "runtime/int.h"
 #include "runtime/list.h"
 #include "runtime/long.h"

src/runtime/inline/list.cpp

@@ -13,6 +13,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include "runtime/inline/list.h"
+
 #include <cstring>
 
 #include "runtime/list.h"
@@ -111,38 +113,6 @@ void BoxedList::shrink() {
     }
 }
 
-// TODO the inliner doesn't want to inline these; is there any point to having them in the inline section?
-void BoxedList::ensure(int space) {
-    if (size + space > capacity) {
-        if (capacity == 0) {
-            const int INITIAL_CAPACITY = 8;
-            int initial = std::max(INITIAL_CAPACITY, space);
-            elts = new (initial) GCdArray();
-            capacity = initial;
-        } else {
-            int new_capacity = std::max(capacity * 2, size + space);
-            elts = GCdArray::realloc(elts, new_capacity);
-            capacity = new_capacity;
-        }
-    }
-    assert(capacity >= size + space);
-}
-
-// TODO the inliner doesn't want to inline these; is there any point to having them in the inline section?
-extern "C" void listAppendInternal(Box* s, Box* v) {
-    // Lock must be held!
-
-    assert(isSubclass(s->cls, list_cls));
-    BoxedList* self = static_cast<BoxedList*>(s);
-
-    assert(self->size <= self->capacity);
-    self->ensure(1);
-
-    assert(self->size < self->capacity);
-    self->elts->elts[self->size] = v;
-    self->size++;
-}
-
 
 extern "C" void listAppendArrayInternal(Box* s, Box** v, int nelts) {
     // Lock must be held!

src/runtime/inline/list.h

+// Copyright (c) 2014-2015 Dropbox, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef PYSTON_RUNTIME_INLINE_LIST_H
+#define PYSTON_RUNTIME_INLINE_LIST_H
+
+#include "runtime/list.h"
+#include "runtime/objmodel.h"
+
+namespace pyston {
+
+// TODO the inliner doesn't want to inline these; is there any point to having them in the inline section?
+inline void BoxedList::grow(int min_free) {
+    if (capacity == 0) {
+        const int INITIAL_CAPACITY = 8;
+        int initial = std::max(INITIAL_CAPACITY, min_free);
+        elts = new (initial) GCdArray();
+        capacity = initial;
+    } else {
+        int new_capacity = std::max(capacity * 2, size + min_free);
+        elts = GCdArray::realloc(elts, new_capacity);
+        capacity = new_capacity;
+    }
+}
+
+inline void BoxedList::ensure(int min_free) {
+    if (unlikely(size + min_free > capacity)) {
+        grow(min_free);
+    }
+    assert(capacity >= size + min_free);
+}
+
+// TODO the inliner doesn't want to inline these; is there any point to having them in the inline section?
+extern "C" inline void listAppendInternal(Box* s, Box* v) {
+    // Lock must be held!
+
+    assert(isSubclass(s->cls, list_cls));
+    BoxedList* self = static_cast<BoxedList*>(s);
+
+    assert(self->size <= self->capacity);
+    self->ensure(1);
+
+    assert(self->size < self->capacity);
+    self->elts->elts[self->size] = v;
+    self->size++;
+}
+}
+
+#endif

src/runtime/list.cpp

@@ -26,6 +26,7 @@
 #include "core/types.h"
 #include "gc/collector.h"
 #include "gc/roots.h"
+#include "runtime/inline/list.h"
 #include "runtime/objmodel.h"
 #include "runtime/types.h"
 #include "runtime/util.h"
@@ -33,8 +34,9 @@
 namespace pyston {
 
 extern "C" int PyList_Append(PyObject* op, PyObject* newitem) noexcept {
+    RELEASE_ASSERT(PyList_Check(op), "");
     try {
-        listAppend(op, newitem);
+        listAppendInternal(op, newitem);
     } catch (ExcInfo e) {
         abort();
     }
@@ -131,6 +133,10 @@ extern "C" Box* listLen(BoxedList* self) {
     return boxInt(self->size);
 }
 
+static Py_ssize_t list_length(Box* self) noexcept {
+    return static_cast<BoxedList*>(self)->size;
+}
+
 Box* _listSlice(BoxedList* self, i64 start, i64 stop, i64 step, i64 length) {
     // printf("%ld %ld %ld\n", start, stop, step);
     assert(step != 0);
@@ -151,6 +157,38 @@ Box* _listSlice(BoxedList* self, i64 start, i64 stop, i64 step, i64 length) {
     return rtn;
 }
 
+static Box* list_slice(Box* o, Py_ssize_t ilow, Py_ssize_t ihigh) noexcept {
+    BoxedList* a = static_cast<BoxedList*>(o);
+
+    PyObject** src, **dest;
+    Py_ssize_t i, len;
+    if (ilow < 0)
+        ilow = 0;
+    else if (ilow > Py_SIZE(a))
+        ilow = Py_SIZE(a);
+    if (ihigh < ilow)
+        ihigh = ilow;
+    else if (ihigh > Py_SIZE(a))
+        ihigh = Py_SIZE(a);
+    len = ihigh - ilow;
+
+    BoxedList* np = new BoxedList();
+
+    np->ensure(len);
+    if (len) {
+        src = a->elts->elts + ilow;
+        dest = np->elts->elts;
+        for (i = 0; i < len; i++) {
+            PyObject* v = src[i];
+            Py_INCREF(v);
+            dest[i] = v;
+        }
+    }
+    np->size = len;
+
+    return (PyObject*)np;
+}
+
 extern "C" Box* listGetitemUnboxed(BoxedList* self, int64_t n) {
     LOCK_REGION(self->lock.asRead());
 
@@ -1141,6 +1179,9 @@ void setupList() {
     list_cls->giveAttr("__hash__", None);
     list_cls->freeze();
 
+    list_cls->tp_as_sequence->sq_slice = list_slice;
+    list_cls->tp_as_sequence->sq_length = list_length;
+
     CLFunction* hasnext = boxRTFunction((void*)listiterHasnextUnboxed, BOOL, 1);
     addRTFunction(hasnext, (void*)listiterHasnext, BOXED_BOOL);
     list_iterator_cls->giveAttr("__hasnext__", new BoxedFunction(hasnext));

src/runtime/str.cpp

@@ -57,13 +57,10 @@ BoxedString* EmptyString;
 BoxedString* characters[UCHAR_MAX + 1];
 
 BoxedString::BoxedString(const char* s, size_t n) : interned_state(SSTATE_NOT_INTERNED) {
+    assert(s);
     RELEASE_ASSERT(n != llvm::StringRef::npos, "");
-    if (s) {
     memmove(data(), s, n);
     data()[n] = 0;
-    } else {
-        memset(data(), 0, n + 1);
-    }
 }
 
 BoxedString::BoxedString(llvm::StringRef lhs, llvm::StringRef rhs) : interned_state(SSTATE_NOT_INTERNED) {
@@ -85,6 +82,13 @@ BoxedString::BoxedString(size_t n, char c) : interned_state(SSTATE_NOT_INTERNED)
     data()[n] = 0;
 }
 
+BoxedString::BoxedString(size_t n) : interned_state(SSTATE_NOT_INTERNED) {
+    RELEASE_ASSERT(n != llvm::StringRef::npos, "");
+    // Note: no memset.  add the null-terminator for good measure though
+    // (CPython does the same thing).
+    data()[n] = 0;
+}
+
 extern "C" char PyString_GetItem(PyObject* op, ssize_t n) noexcept {
     RELEASE_ASSERT(PyString_Check(op), "");
     return static_cast<const BoxedString*>(op)->s()[n];
@@ -1577,15 +1581,38 @@ Box* _strSlice(BoxedString* self, i64 start, i64 stop, i64 step, i64 length) {
         assert(start < s.size());
         assert(-1 <= stop);
     }
+    assert(length >= 0);
 
     if (length == 0)
         return EmptyString;
 
-    BoxedString* bs = new (length) BoxedString(nullptr, length);
+    BoxedString* bs = BoxedString::createUninitializedString(length);
     copySlice(bs->data(), s.data(), start, step, length);
     return bs;
 }
 
+static Box* str_slice(Box* o, Py_ssize_t i, Py_ssize_t j) {
+    BoxedString* a = static_cast<BoxedString*>(o);
+    if (i < 0)
+        i = 0;
+    if (j < 0)
+        j = 0; /* Avoid signed/unsigned bug in next line */
+    if (j > Py_SIZE(a))
+        j = Py_SIZE(a);
+    if (i == 0 && j == Py_SIZE(a) && PyString_CheckExact(a)) {
+        /* It's the same as a */
+        Py_INCREF(a);
+        return (PyObject*)a;
+    }
+    if (j < i)
+        j = i;
+    return PyString_FromStringAndSize(a->data() + i, j - i);
+}
+
+static Py_ssize_t str_length(Box* a) {
+    return Py_SIZE(a);
+}
+
 Box* strIsAlpha(BoxedString* self) {
     assert(PyString_Check(self));
 
@@ -2307,17 +2334,9 @@ extern "C" int PyString_AsStringAndSize(register PyObject* obj, register char**
     return 0;
 }
 
-BoxedString* createUninitializedString(ssize_t n) {
-    return new (n) BoxedString(n, 0);
-}
-
-char* getWriteableStringContents(BoxedString* s) {
-    return s->data();
-}
-
 extern "C" PyObject* PyString_FromStringAndSize(const char* s, ssize_t n) noexcept {
     if (s == NULL)
-        return createUninitializedString(n);
+        return BoxedString::createUninitializedString(n);
     return boxString(llvm::StringRef(s, n));
 }
 
@@ -2334,7 +2353,7 @@ extern "C" char* PyString_AsString(PyObject* o) noexcept {
         return string_getbuffer(o);
 
     BoxedString* s = static_cast<BoxedString*>(o);
-    return getWriteableStringContents(s);
+    return s->getWriteableStringContents();
 }
 
 extern "C" Py_ssize_t PyString_Size(PyObject* op) noexcept {
@@ -2748,6 +2767,9 @@ void setupStr() {
     add_operators(str_cls);
     str_cls->freeze();
 
+    str_cls->tp_as_sequence->sq_slice = str_slice;
+    str_cls->tp_as_sequence->sq_length = str_length;
+
     basestring_cls->giveAttr("__doc__",
                              boxString("Type basestring cannot be instantiated; it is the base for str and unicode."));
     basestring_cls->giveAttr("__new__",

src/runtime/traceback.cpp

@@ -24,6 +24,7 @@
 #include "core/stats.h"
 #include "core/types.h"
 #include "gc/collector.h"
+#include "runtime/inline/list.h"
 #include "runtime/list.h"
 #include "runtime/objmodel.h"
 #include "runtime/types.h"

src/runtime/types.cpp

@@ -1674,6 +1674,8 @@ extern "C" int PySlice_GetIndicesEx(PySliceObject* _r, Py_ssize_t length, Py_ssi
 
     if ((*step < 0 && *stop >= *start) || (*step > 0 && *start >= *stop)) {
         *slicelength = 0;
+    } else if (*step == 1) { // Pyston change: added this branch to make the common step==1 case avoid the div:
+        *slicelength = (*stop - *start - 1) + 1;
     } else if (*step < 0) {
         *slicelength = (*stop - *start + 1) / (*step) + 1;
     } else {

src/runtime/types.h

@@ -120,14 +120,7 @@ BoxedString* boxStringTwine(const llvm::Twine& s);
 
 extern "C" Box* decodeUTF8StringPtr(llvm::StringRef s);
 
-// creates an uninitialized string of length n; useful for directly constructing into the string and avoiding copies:
-BoxedString* createUninitializedString(ssize_t n);
-// Gets a writeable pointer to the contents of a string.
-// Is only meant to be used with something just created from createUninitializedString(), though
-// in theory it might work in more cases.
-char* getWriteableStringContents(BoxedString* s);
-
-extern "C" void listAppendInternal(Box* self, Box* v);
+extern "C" inline void listAppendInternal(Box* self, Box* v) __attribute__((visibility("default")));
 extern "C" void listAppendArrayInternal(Box* self, Box** v, int nelts);
 extern "C" Box* boxCLFunction(CLFunction* f, BoxedClosure* closure, Box* globals,
                               std::initializer_list<Box*> defaults) noexcept;
@@ -480,9 +473,20 @@ public:
     explicit BoxedString(llvm::StringRef s) __attribute__((visibility("default")));
     explicit BoxedString(llvm::StringRef lhs, llvm::StringRef rhs) __attribute__((visibility("default")));
 
+    // creates an uninitialized string of length n; useful for directly constructing into the string and avoiding
+    // copies:
+    static BoxedString* createUninitializedString(ssize_t n) { return new (n) BoxedString(n); }
+
+    // Gets a writeable pointer to the contents of a string.
+    // Is only meant to be used with something just created from createUninitializedString(), though
+    // in theory it might work in more cases.
+    char* getWriteableStringContents() { return s_data; }
+
 private:
     void* operator new(size_t size) = delete;
 
+    BoxedString(size_t n); // non-initializing constructor
+
     char s_data[0];
 
     friend void setupRuntime();
@@ -549,6 +553,9 @@ public:
 };
 
 class BoxedList : public Box {
+private:
+    void grow(int min_free);
+
 public:
     int64_t size, capacity;
     GCdArray* elts;
@@ -557,7 +564,7 @@ public:
 
     BoxedList() __attribute__((visibility("default"))) : size(0), capacity(0) {}
 
-    void ensure(int space);
+    void ensure(int min_free);
     void shrink();
     static const int INITIAL_CAPACITY;