Merge pull request #761 from kmod/perf2

optimize some misc runtime functions

from_cpython/Include/modsupport.h

@@ -16,6 +16,7 @@ extern "C" {
 #define PyArg_Parse			_PyArg_Parse_SizeT
 #define PyArg_ParseTuple		_PyArg_ParseTuple_SizeT
 #define PyArg_ParseTupleAndKeywords	_PyArg_ParseTupleAndKeywords_SizeT
+#define PyArg_ParseSingle		_PyArg_ParseSingle_SizeT
 #define PyArg_VaParse			_PyArg_VaParse_SizeT
 #define PyArg_VaParseTupleAndKeywords	_PyArg_VaParseTupleAndKeywords_SizeT
 #define Py_BuildValue			_Py_BuildValue_SizeT
@@ -28,6 +29,8 @@ PyAPI_FUNC(int) PyArg_Parse(PyObject *, const char *, ...) PYSTON_NOEXCEPT;
 PyAPI_FUNC(int) PyArg_ParseTuple(PyObject *, const char *, ...) Py_FORMAT_PARSETUPLE(PyArg_ParseTuple, 2, 3) PYSTON_NOEXCEPT;
 PyAPI_FUNC(int) PyArg_ParseTupleAndKeywords(PyObject *, PyObject *,
                                                   const char *, char **, ...) PYSTON_NOEXCEPT;
+// Pyston addition:
+PyAPI_FUNC(int) PyArg_ParseSingle(PyObject* obj, int arg_idx, const char* fname, const char* format, ...) PYSTON_NOEXCEPT;
 PyAPI_FUNC(int) PyArg_UnpackTuple(PyObject *, const char *, Py_ssize_t, Py_ssize_t, ...) PYSTON_NOEXCEPT;
 PyAPI_FUNC(PyObject *) Py_BuildValue(const char *, ...) PYSTON_NOEXCEPT;
 PyAPI_FUNC(PyObject *) _Py_BuildValue_SizeT(const char *, ...) PYSTON_NOEXCEPT;

from_cpython/Include/unicodeobject.h

@@ -1408,6 +1408,9 @@ PyAPI_FUNC(int) _PyUnicode_IsAlpha(
     Py_UNICODE ch       /* Unicode character */
     ) PYSTON_NOEXCEPT;
 
+// Pyston addition:
+PyAPI_FUNC(PyObject*) unicode_new_inner(PyObject* x, char* encoding, char* errors) PYSTON_NOEXCEPT;
+
 #ifdef __cplusplus
 }
 #endif

from_cpython/Objects/unicodeobject.c

@@ -8745,6 +8745,15 @@ static PyBufferProcs unicode_as_buffer = {
 static PyObject *
 unicode_subtype_new(PyTypeObject *type, PyObject *args, PyObject *kwds);
 
+PyObject* unicode_new_inner(PyObject* x, char* encoding, char* errors) {
+    if (x == NULL)
+        return (PyObject *)_PyUnicode_New(0);
+    if (encoding == NULL && errors == NULL)
+        return PyObject_Unicode(x);
+    else
+        return PyUnicode_FromEncodedObject(x, encoding, errors);
+}
+
 static PyObject *
 unicode_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
 {
@@ -8758,12 +8767,7 @@ unicode_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
     if (!PyArg_ParseTupleAndKeywords(args, kwds, "|Oss:unicode",
                                      kwlist, &x, &encoding, &errors))
         return NULL;
-    if (x == NULL)
-        return (PyObject *)_PyUnicode_New(0);
-    if (encoding == NULL && errors == NULL)
-        return PyObject_Unicode(x);
-    else
-        return PyUnicode_FromEncodedObject(x, encoding, errors);
+    return unicode_new_inner(x, encoding, errors);
 }
 
 static PyObject *

from_cpython/Python/getargs.c

@@ -569,6 +569,35 @@ float_argument_error(PyObject *arg)
         return 0;
 }
 
+int _PyArg_ParseSingle_SizeT(PyObject* obj, int arg_idx, const char* fname, const char* format, ...) {
+    va_list va;
+    char* msg;
+    char msgbuf[256];
+
+    assert(format[0] != '\0');
+    assert(format[0] != '(');
+    assert(format[0] != '|');
+    assert(format[0] != '|');
+
+    assert(format[1] != '*'); // would need to pass a non-null freelist
+    assert(format[0] != 'e'); // would need to pass a non-null freelist
+
+    va_start(va, format);
+    msg = convertsimple(obj, &format, &va, FLAG_SIZE_T, msgbuf, sizeof(msgbuf), NULL);
+    va_end(va);
+
+    if (msg) {
+        int levels[1];
+        levels[0] = 0;
+        seterror(arg_idx + 1, msg, levels, fname, NULL);
+        return 0;
+    }
+
+    // Should have consumed the entire format string:
+    assert(format[0] == '\0');
+    return 1;
+}
+
 /* Convert a non-tuple argument.  Return NULL if conversion went OK,
    or a string with a message describing the failure.  The message is
    formatted as "must be <desired type>, not <actual type>".

microbenchmarks/django_tiny.py

+import os
+import sys
+sys.path.append(os.path.join(os.path.dirname(__file__), "../test/integration/django"))
+
+from django.conf import settings
+settings.configure()
+from django.template import Context, Template, Node
+
+import time
+
+DJANGO_TMPL = Template("""
+{{ col|escape }}
+""".strip())
+
+def test_django():
+    table = [xrange(50) for _ in xrange(50)]
+    context = Context({"table": table, 'col': 1})
+
+    times = []
+
+    node = DJANGO_TMPL.nodelist[0]
+    context.push()
+    for _ in xrange(100000):
+        node.render(context)
+    context.pop()
+    return times
+
+test_django()

src/capi/abstract.cpp

@@ -749,15 +749,6 @@ extern "C" PyObject* PyObject_GetIter(PyObject* o) noexcept {
     }
 }
 
-extern "C" PyObject* PyObject_Repr(PyObject* obj) noexcept {
-    try {
-        return repr(obj);
-    } catch (ExcInfo e) {
-        setCAPIException(e);
-        return NULL;
-    }
-}
-
 static int recursive_issubclass(PyObject* derived, PyObject* cls) noexcept {
     int retval;
 
@@ -1454,14 +1445,26 @@ extern "C" PyObject* PySequence_InPlaceRepeat(PyObject* o, Py_ssize_t count) noe
     return nullptr;
 }
 
-extern "C" PyObject* PySequence_GetItem(PyObject* o, Py_ssize_t i) noexcept {
-    try {
-        // Not sure if this is really the same:
-        return getitem(o, boxInt(i));
-    } catch (ExcInfo e) {
-        setCAPIException(e);
-        return NULL;
+extern "C" PyObject* PySequence_GetItem(PyObject* s, Py_ssize_t i) noexcept {
+    PySequenceMethods* m;
+
+    if (s == NULL)
+        return null_error();
+
+    m = s->cls->tp_as_sequence;
+    if (m && m->sq_item) {
+        if (i < 0) {
+            if (m->sq_length) {
+                Py_ssize_t l = (*m->sq_length)(s);
+                if (l < 0)
+                    return NULL;
+                i += l;
+            }
+        }
+        return m->sq_item(s, i);
     }
+
+    return type_error("'%.200s' object does not support indexing", s);
 }
 
 PyObject* _PySlice_FromIndices(Py_ssize_t istart, Py_ssize_t istop) {

src/capi/object.cpp

@@ -379,19 +379,80 @@ extern "C" PyObject* PyObject_Unicode(PyObject* v) noexcept {
     return res;
 }
 
-extern "C" PyObject* _PyObject_Str(PyObject* v) noexcept {
+extern "C" PyObject* PyObject_Repr(PyObject* v) noexcept {
+    if (PyErr_CheckSignals())
+        return NULL;
+#ifdef USE_STACKCHECK
+    if (PyOS_CheckStack()) {
+        PyErr_SetString(PyExc_MemoryError, "stack overflow");
+        return NULL;
+    }
+#endif
     if (v == NULL)
-        return boxString("<NULL>");
+        return PyString_FromString("<NULL>");
+    else if (Py_TYPE(v)->tp_repr == NULL)
+        return PyString_FromFormat("<%s object at %p>", Py_TYPE(v)->tp_name, v);
+    else {
+        PyObject* res;
+        res = (*Py_TYPE(v)->tp_repr)(v);
+        if (res == NULL)
+            return NULL;
+#ifdef Py_USING_UNICODE
+        if (PyUnicode_Check(res)) {
+            PyObject* str;
+            str = PyUnicode_AsEncodedString(res, NULL, NULL);
+            Py_DECREF(res);
+            if (str)
+                res = str;
+            else
+                return NULL;
+        }
+#endif
+        if (!PyString_Check(res)) {
+            PyErr_Format(PyExc_TypeError, "__repr__ returned non-string (type %.200s)", Py_TYPE(res)->tp_name);
+            Py_DECREF(res);
+            return NULL;
+        }
+        return res;
+    }
+}
 
-    if (v->cls == str_cls)
+extern "C" PyObject* _PyObject_Str(PyObject* v) noexcept {
+    PyObject* res;
+    int type_ok;
+    if (v == NULL)
+        return PyString_FromString("<NULL>");
+    if (PyString_CheckExact(v)) {
+        Py_INCREF(v);
+        return v;
+    }
+#ifdef Py_USING_UNICODE
+    if (PyUnicode_CheckExact(v)) {
+        Py_INCREF(v);
         return v;
+    }
+#endif
+    if (Py_TYPE(v)->tp_str == NULL)
+        return PyObject_Repr(v);
 
-    try {
-        return str(v);
-    } catch (ExcInfo e) {
-        setCAPIException(e);
+    /* It is possible for a type to have a tp_str representation that loops
+       infinitely. */
+    if (Py_EnterRecursiveCall(" while getting the str of an object"))
+        return NULL;
+    res = (*Py_TYPE(v)->tp_str)(v);
+    Py_LeaveRecursiveCall();
+    if (res == NULL)
+        return NULL;
+    type_ok = PyString_Check(res);
+#ifdef Py_USING_UNICODE
+    type_ok = type_ok || PyUnicode_Check(res);
+#endif
+    if (!type_ok) {
+        PyErr_Format(PyExc_TypeError, "__str__ returned non-string (type %.200s)", Py_TYPE(res)->tp_name);
+        Py_DECREF(res);
         return NULL;
     }
+    return res;
 }
 
 extern "C" PyObject* PyObject_Str(PyObject* v) noexcept {
@@ -473,7 +534,10 @@ extern "C" int PyObject_SetAttrString(PyObject* v, const char* name, PyObject* w
 
 extern "C" PyObject* PyObject_GetAttrString(PyObject* o, const char* attr) noexcept {
     try {
-        return getattr(o, internStringMortal(attr));
+        Box* r = getattrInternal(o, internStringMortal(attr), NULL);
+        if (!r)
+            PyErr_Format(PyExc_AttributeError, "'%.50s' object has no attribute '%.400s'", o->cls->tp_name, attr);
+        return r;
     } catch (ExcInfo e) {
         setCAPIException(e);
         return NULL;

src/codegen/unwinding.cpp

@@ -604,6 +604,9 @@ static const LineInfo lineInfoForFrame(PythonFrameIteratorImpl* frame_it) {
 }
 
 void exceptionCaughtInInterpreter(LineInfo line_info, ExcInfo* exc_info) {
+    static StatCounter frames_unwound("num_frames_unwound_python");
+    frames_unwound.log();
+
     // basically the same as PythonUnwindSession::addTraceback, but needs to
     // be callable after an PythonUnwindSession has ended.  The interpreter
     // will call this from catch blocks if it needs to ensure that a
@@ -632,6 +635,9 @@ void unwindingThroughFrame(PythonUnwindSession* unwind_session, unw_cursor_t* cu
         assert(!unwind_session->shouldSkipFrame());
         unwind_session->setShouldSkipNextFrame(true);
     } else if (frameIsPythonFrame(ip, bp, cursor, &frame_iter)) {
+        static StatCounter frames_unwound("num_frames_unwound_python");
+        frames_unwound.log();
+
         if (!unwind_session->shouldSkipFrame())
             unwind_session->addTraceback(lineInfoForFrame(&frame_iter));
 

src/runtime/builtin_modules/builtins.cpp

@@ -854,9 +854,7 @@ Box* zip(BoxedTuple* containers) {
 }
 
 static Box* callable(Box* obj) {
-    Box* r = PyBool_FromLong((long)PyCallable_Check(obj));
-    checkAndThrowCAPIException();
-    return r;
+    return PyBool_FromLong((long)PyCallable_Check(obj));
 }
 
 BoxedClass* notimplemented_cls;

src/runtime/capi.cpp

@@ -638,9 +638,19 @@ extern "C" int PyObject_Print(PyObject* obj, FILE* fp, int flags) noexcept {
 extern "C" int PyCallable_Check(PyObject* x) noexcept {
     if (x == NULL)
         return 0;
-
-    static BoxedString* call_attr = internStringImmortal("__call__");
-    return typeLookup(x->cls, call_attr, NULL) != NULL;
+    if (PyInstance_Check(x)) {
+        PyObject* call = PyObject_GetAttrString(x, "__call__");
+        if (call == NULL) {
+            PyErr_Clear();
+            return 0;
+        }
+        /* Could test recursively but don't, for fear of endless
+           recursion if some joker sets self.__call__ = self */
+        Py_DECREF(call);
+        return 1;
+    } else {
+        return x->cls->tp_call != NULL;
+    }
 }
 
 extern "C" int Py_FlushLine(void) noexcept {

src/runtime/cxx_unwind.cpp

@@ -496,6 +496,9 @@ static inline void unwind_loop(ExcInfo* exc_data) {
     while (unw_step(&cursor) > 0) {
         unw_proc_info_t pip;
 
+        static StatCounter frames_unwound("num_frames_unwound_cxx");
+        frames_unwound.log();
+
         // NB. unw_get_proc_info is slow; a significant chunk of all time spent unwinding is spent here.
         check(unw_get_proc_info(&cursor, &pip));
 
@@ -668,6 +671,9 @@ static uint64_t* unwinding_stattimer = pyston::Stats::getStatCounter("us_timer_u
 #endif
 
 extern "C" void __cxa_throw(void* exc_obj, std::type_info* tinfo, void (*dtor)(void*)) {
+    static pyston::StatCounter num_cxa_throw("num_cxa_throw");
+    num_cxa_throw.log();
+
     assert(!pyston::in_cleanup_code);
     assert(exc_obj);
     RELEASE_ASSERT(tinfo == &EXCINFO_TYPE_INFO, "can't throw a non-ExcInfo value! type info: %p", tinfo);

src/runtime/int.cpp

@@ -1142,6 +1142,8 @@ void setupInt() {
 
     add_operators(int_cls);
     int_cls->freeze();
+
+    int_cls->tp_repr = (reprfunc)int_to_decimal_string;
 }
 
 void teardownInt() {

src/runtime/str.cpp

@@ -1557,15 +1557,19 @@ extern "C" Box* strNonzero(BoxedString* self) {
 extern "C" Box* strNew(BoxedClass* cls, Box* obj) {
     assert(isSubclass(cls, str_cls));
 
-    Box* rtn = str(obj);
-    assert(PyString_Check(rtn));
+    if (cls != str_cls) {
+        Box* tmp = strNew(str_cls, obj);
+        assert(isSubclass(tmp->cls, str_cls));
+        BoxedString* tmp_s = static_cast<BoxedString*>(tmp);
 
-    if (cls == str_cls)
-        return rtn;
-
-    BoxedString* _rtn = static_cast<BoxedString*>(rtn);
+        return new (cls, tmp_s->size()) BoxedString(tmp_s->s());
+    }
 
-    return new (cls, _rtn->size()) BoxedString(_rtn->s());
+    Box* r = PyObject_Str(obj);
+    if (!r)
+        throwCAPIException();
+    assert(PyString_Check(r));
+    return r;
 }
 
 extern "C" Box* basestringNew(BoxedClass* cls, Box* args, Box* kwargs) {

src/runtime/tuple.cpp

@@ -517,6 +517,19 @@ static PyObject* tupleslice(PyTupleObject* a, Py_ssize_t ilow, Py_ssize_t ihigh)
     return (PyObject*)np;
 }
 
+static PyObject* tupleitem(register PyTupleObject* a, register Py_ssize_t i) {
+    if (i < 0 || i >= Py_SIZE(a)) {
+        PyErr_SetString(PyExc_IndexError, "tuple index out of range");
+        return NULL;
+    }
+    Py_INCREF(a->ob_item[i]);
+    return a->ob_item[i];
+}
+
+static Py_ssize_t tuplelength(PyTupleObject* a) {
+    return Py_SIZE(a);
+}
+
 void setupTuple() {
     tuple_iterator_cls = BoxedHeapClass::create(type_cls, object_cls, &tupleIteratorGCHandler, 0, 0,
                                                 sizeof(BoxedTupleIterator), false, "tuple");
@@ -528,8 +541,6 @@ void setupTuple() {
     addRTFunction(getitem, (void*)tupleGetitem, UNKNOWN, std::vector<ConcreteCompilerType*>{ UNKNOWN, UNKNOWN });
     tuple_cls->giveAttr("__getitem__", new BoxedFunction(getitem));
 
-    tuple_cls->tp_as_sequence->sq_slice = (ssizessizeargfunc)&tupleslice;
-
     tuple_cls->giveAttr("__contains__", new BoxedFunction(boxRTFunction((void*)tupleContains, BOXED_BOOL, 2)));
     tuple_cls->giveAttr("index", new BoxedFunction(boxRTFunction((void*)tupleIndex, BOXED_INT, 4, 2, false, false),
                                                    { boxInt(0), boxInt(std::numeric_limits<Py_ssize_t>::max()) }));
@@ -550,10 +561,14 @@ void setupTuple() {
     tuple_cls->giveAttr("__rmul__", new BoxedFunction(boxRTFunction((void*)tupleMul, BOXED_TUPLE, 2)));
 
     tuple_cls->tp_hash = (hashfunc)tuple_hash;
+    tuple_cls->tp_as_sequence->sq_slice = (ssizessizeargfunc)&tupleslice;
     add_operators(tuple_cls);
 
     tuple_cls->freeze();
 
+    tuple_cls->tp_as_sequence->sq_item = (ssizeargfunc)tupleitem;
+    tuple_cls->tp_as_sequence->sq_length = (lenfunc)tuplelength;
+
     CLFunction* hasnext = boxRTFunction((void*)tupleiterHasnextUnboxed, BOOL, 1);
     addRTFunction(hasnext, (void*)tupleiterHasnext, BOXED_BOOL);
     tuple_iterator_cls->giveAttr("__hasnext__", new BoxedFunction(hasnext));

test/tests/encoding_test.py

+import codecs
+
+### Codec APIs
+
+class MyUnicode(unicode):
+    def __new__(*args):
+        print "MyUnicode.__new__", map(type, args)
+        return unicode.__new__(*args)
+
+    def __init__(*args):
+        print "MyUnicode.__init__", map(type, args)
+
+def encode(input, errors='strict'):
+    raise Exception()
+
+def decode(input, errors='strict'):
+    return (MyUnicode(u"."), 1)
+
+class IncrementalEncoder(codecs.IncrementalEncoder):
+    def encode(self, input, final=False):
+        return codecs.utf_8_encode(input, self.errors)[0]
+
+class IncrementalDecoder(codecs.BufferedIncrementalDecoder):
+    _buffer_decode = codecs.utf_8_decode
+
+class StreamWriter(codecs.StreamWriter):
+    encode = codecs.utf_8_encode
+
+class StreamReader(codecs.StreamReader):
+    decode = codecs.utf_8_decode
+
+codec = codecs.CodecInfo(
+    name='myunicode',
+    encode=encode,
+    decode=decode,
+    incrementalencoder=IncrementalEncoder,
+    incrementaldecoder=IncrementalDecoder,
+    streamreader=StreamReader,
+    streamwriter=StreamWriter,
+)
+
+def search(name):
+    if name == "myunicode":
+        return codec
+codecs.register(search)
+
+
+u = unicode("hello world", "myunicode", "strict")
+print type(u)

test/tests/intmethods.py

@@ -132,4 +132,9 @@ print
 
 # These return longs:
 print int("12938719238719827398172938712983791827938712987312")
+print int(u"12938719238719827398172938712983791827938712987312")
+print int("12938719238719827398172938712983791827938712987312", 16)
+print int(u"12938719238719827398172938712983791827938712987312", 16)
 print int(1e100)
+print int(*[1e100])
+print int(x=1e100)

test/tests/object_new_arguments.py

@@ -21,3 +21,8 @@ except TypeError as e:
 # are being passed, but really they are not.
 type.__call__(*[C2])
 type.__call__(C2, **{})
+
+try:
+    type.__call__(*[])
+except TypeError as e:
+    print "caught typeerror"

test/tests/str_subclassing.py

@@ -10,3 +10,15 @@ print
 
 print repr("hello" + MyStr("world"))
 print int(MyStr("2"))
+
+class MyStr(str):
+    def __init__(*args):
+        print "MyStr.__init__", map(type, args)
+
+class C(object):
+    def __str__(self):
+        return MyStr("hello world")
+
+print type(str(C()))
+m = MyStr(C())
+print type(m), repr(m)

test/tests/unicode_test.py

@@ -155,3 +155,13 @@ print "".join([u"\xB2", u"\xB3"])
 
 import sys
 print type(sys.maxunicode)
+
+class MyUnicode(unicode):
+    def __init__(*args):
+        print "MyUnicode.__init__", map(type, args)
+
+class C(object):
+    def __unicode__(self):
+        return MyUnicode("hello world")
+
+print type(unicode(C()))