Merge pull request #761 from kmod/perf2

optimize some misc runtime functions

from_cpython/Include/modsupport.h

@@ -16,6 +16,7 @@ extern "C" {
 #define PyArg_Parse			_PyArg_Parse_SizeT
 #define PyArg_ParseTuple		_PyArg_ParseTuple_SizeT
 #define PyArg_ParseTupleAndKeywords	_PyArg_ParseTupleAndKeywords_SizeT
+#define PyArg_ParseSingle		_PyArg_ParseSingle_SizeT
 #define PyArg_VaParse			_PyArg_VaParse_SizeT
 #define PyArg_VaParseTupleAndKeywords	_PyArg_VaParseTupleAndKeywords_SizeT
 #define Py_BuildValue			_Py_BuildValue_SizeT
@@ -28,6 +29,8 @@ PyAPI_FUNC(int) PyArg_Parse(PyObject *, const char *, ...) PYSTON_NOEXCEPT;
 PyAPI_FUNC(int) PyArg_ParseTuple(PyObject *, const char *, ...) Py_FORMAT_PARSETUPLE(PyArg_ParseTuple, 2, 3) PYSTON_NOEXCEPT;
 PyAPI_FUNC(int) PyArg_ParseTupleAndKeywords(PyObject *, PyObject *,
                                                   const char *, char **, ...) PYSTON_NOEXCEPT;
+// Pyston addition:
+PyAPI_FUNC(int) PyArg_ParseSingle(PyObject* obj, int arg_idx, const char* fname, const char* format, ...) PYSTON_NOEXCEPT;
 PyAPI_FUNC(int) PyArg_UnpackTuple(PyObject *, const char *, Py_ssize_t, Py_ssize_t, ...) PYSTON_NOEXCEPT;
 PyAPI_FUNC(PyObject *) Py_BuildValue(const char *, ...) PYSTON_NOEXCEPT;
 PyAPI_FUNC(PyObject *) _Py_BuildValue_SizeT(const char *, ...) PYSTON_NOEXCEPT;

from_cpython/Include/unicodeobject.h

@@ -1408,6 +1408,9 @@ PyAPI_FUNC(int) _PyUnicode_IsAlpha(
     Py_UNICODE ch       /* Unicode character */
     ) PYSTON_NOEXCEPT;
 
+// Pyston addition:
+PyAPI_FUNC(PyObject*) unicode_new_inner(PyObject* x, char* encoding, char* errors) PYSTON_NOEXCEPT;
+
 #ifdef __cplusplus
 }
 #endif

from_cpython/Objects/unicodeobject.c

@@ -8745,6 +8745,15 @@ static PyBufferProcs unicode_as_buffer = {
 static PyObject *
 unicode_subtype_new(PyTypeObject *type, PyObject *args, PyObject *kwds);
 
+PyObject* unicode_new_inner(PyObject* x, char* encoding, char* errors) {
+    if (x == NULL)
+        return (PyObject *)_PyUnicode_New(0);
+    if (encoding == NULL && errors == NULL)
+        return PyObject_Unicode(x);
+    else
+        return PyUnicode_FromEncodedObject(x, encoding, errors);
+}
+
 static PyObject *
 unicode_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
 {
@@ -8758,12 +8767,7 @@ unicode_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
     if (!PyArg_ParseTupleAndKeywords(args, kwds, "|Oss:unicode",
                                      kwlist, &x, &encoding, &errors))
         return NULL;
-    if (x == NULL)
-        return (PyObject *)_PyUnicode_New(0);
-    if (encoding == NULL && errors == NULL)
-        return PyObject_Unicode(x);
-    else
-        return PyUnicode_FromEncodedObject(x, encoding, errors);
+    return unicode_new_inner(x, encoding, errors);
 }
 
 static PyObject *

from_cpython/Python/getargs.c

@@ -569,6 +569,35 @@ float_argument_error(PyObject *arg)
         return 0;
 }
 
+int _PyArg_ParseSingle_SizeT(PyObject* obj, int arg_idx, const char* fname, const char* format, ...) {
+    va_list va;
+    char* msg;
+    char msgbuf[256];
+
+    assert(format[0] != '\0');
+    assert(format[0] != '(');
+    assert(format[0] != '|');
+    assert(format[0] != '|');
+
+    assert(format[1] != '*'); // would need to pass a non-null freelist
+    assert(format[0] != 'e'); // would need to pass a non-null freelist
+
+    va_start(va, format);
+    msg = convertsimple(obj, &format, &va, FLAG_SIZE_T, msgbuf, sizeof(msgbuf), NULL);
+    va_end(va);
+
+    if (msg) {
+        int levels[1];
+        levels[0] = 0;
+        seterror(arg_idx + 1, msg, levels, fname, NULL);
+        return 0;
+    }
+
+    // Should have consumed the entire format string:
+    assert(format[0] == '\0');
+    return 1;
+}
+
 /* Convert a non-tuple argument.  Return NULL if conversion went OK,
    or a string with a message describing the failure.  The message is
    formatted as "must be <desired type>, not <actual type>".

microbenchmarks/django_tiny.py

+import os
+import sys
+sys.path.append(os.path.join(os.path.dirname(__file__), "../test/integration/django"))
+
+from django.conf import settings
+settings.configure()
+from django.template import Context, Template, Node
+
+import time
+
+DJANGO_TMPL = Template("""
+{{ col|escape }}
+""".strip())
+
+def test_django():
+    table = [xrange(50) for _ in xrange(50)]
+    context = Context({"table": table, 'col': 1})
+
+    times = []
+
+    node = DJANGO_TMPL.nodelist[0]
+    context.push()
+    for _ in xrange(100000):
+        node.render(context)
+    context.pop()
+    return times
+
+test_django()

src/capi/abstract.cpp

@@ -749,15 +749,6 @@ extern "C" PyObject* PyObject_GetIter(PyObject* o) noexcept {
     }
 }
 
-extern "C" PyObject* PyObject_Repr(PyObject* obj) noexcept {
-    try {
-        return repr(obj);
-    } catch (ExcInfo e) {
-        setCAPIException(e);
-        return NULL;
-    }
-}
-
 static int recursive_issubclass(PyObject* derived, PyObject* cls) noexcept {
     int retval;
 
@@ -1454,14 +1445,26 @@ extern "C" PyObject* PySequence_InPlaceRepeat(PyObject* o, Py_ssize_t count) noe
     return nullptr;
 }
 
-extern "C" PyObject* PySequence_GetItem(PyObject* o, Py_ssize_t i) noexcept {
-    try {
-        // Not sure if this is really the same:
-        return getitem(o, boxInt(i));
-    } catch (ExcInfo e) {
-        setCAPIException(e);
-        return NULL;
+extern "C" PyObject* PySequence_GetItem(PyObject* s, Py_ssize_t i) noexcept {
+    PySequenceMethods* m;
+
+    if (s == NULL)
+        return null_error();
+
+    m = s->cls->tp_as_sequence;
+    if (m && m->sq_item) {
+        if (i < 0) {
+            if (m->sq_length) {
+                Py_ssize_t l = (*m->sq_length)(s);
+                if (l < 0)
+                    return NULL;
+                i += l;
+            }
+        }
+        return m->sq_item(s, i);
     }
+
+    return type_error("'%.200s' object does not support indexing", s);
 }
 
 PyObject* _PySlice_FromIndices(Py_ssize_t istart, Py_ssize_t istop) {

src/capi/object.cpp

@@ -379,19 +379,80 @@ extern "C" PyObject* PyObject_Unicode(PyObject* v) noexcept {
     return res;
 }
 
-extern "C" PyObject* _PyObject_Str(PyObject* v) noexcept {
+extern "C" PyObject* PyObject_Repr(PyObject* v) noexcept {
+    if (PyErr_CheckSignals())
+        return NULL;
+#ifdef USE_STACKCHECK
+    if (PyOS_CheckStack()) {
+        PyErr_SetString(PyExc_MemoryError, "stack overflow");
+        return NULL;
+    }
+#endif
     if (v == NULL)
-        return boxString("<NULL>");
+        return PyString_FromString("<NULL>");
+    else if (Py_TYPE(v)->tp_repr == NULL)
+        return PyString_FromFormat("<%s object at %p>", Py_TYPE(v)->tp_name, v);
+    else {
+        PyObject* res;
+        res = (*Py_TYPE(v)->tp_repr)(v);
+        if (res == NULL)
+            return NULL;
+#ifdef Py_USING_UNICODE
+        if (PyUnicode_Check(res)) {
+            PyObject* str;
+            str = PyUnicode_AsEncodedString(res, NULL, NULL);
+            Py_DECREF(res);
+            if (str)
+                res = str;
+            else
+                return NULL;
+        }
+#endif
+        if (!PyString_Check(res)) {
+            PyErr_Format(PyExc_TypeError, "__repr__ returned non-string (type %.200s)", Py_TYPE(res)->tp_name);
+            Py_DECREF(res);
+            return NULL;
+        }
+        return res;
+    }
+}
 
-    if (v->cls == str_cls)
+extern "C" PyObject* _PyObject_Str(PyObject* v) noexcept {
+    PyObject* res;
+    int type_ok;
+    if (v == NULL)
+        return PyString_FromString("<NULL>");
+    if (PyString_CheckExact(v)) {
+        Py_INCREF(v);
+        return v;
+    }
+#ifdef Py_USING_UNICODE
+    if (PyUnicode_CheckExact(v)) {
+        Py_INCREF(v);
         return v;
+    }
+#endif
+    if (Py_TYPE(v)->tp_str == NULL)
+        return PyObject_Repr(v);
 
-    try {
-        return str(v);
-    } catch (ExcInfo e) {
-        setCAPIException(e);
+    /* It is possible for a type to have a tp_str representation that loops
+       infinitely. */
+    if (Py_EnterRecursiveCall(" while getting the str of an object"))
+        return NULL;
+    res = (*Py_TYPE(v)->tp_str)(v);
+    Py_LeaveRecursiveCall();
+    if (res == NULL)
+        return NULL;
+    type_ok = PyString_Check(res);
+#ifdef Py_USING_UNICODE
+    type_ok = type_ok || PyUnicode_Check(res);
+#endif
+    if (!type_ok) {
+        PyErr_Format(PyExc_TypeError, "__str__ returned non-string (type %.200s)", Py_TYPE(res)->tp_name);
+        Py_DECREF(res);
         return NULL;
     }
+    return res;
 }
 
 extern "C" PyObject* PyObject_Str(PyObject* v) noexcept {
@@ -473,7 +534,10 @@ extern "C" int PyObject_SetAttrString(PyObject* v, const char* name, PyObject* w
 
 extern "C" PyObject* PyObject_GetAttrString(PyObject* o, const char* attr) noexcept {
     try {
-        return getattr(o, internStringMortal(attr));
+        Box* r = getattrInternal(o, internStringMortal(attr), NULL);
+        if (!r)
+            PyErr_Format(PyExc_AttributeError, "'%.50s' object has no attribute '%.400s'", o->cls->tp_name, attr);
+        return r;
     } catch (ExcInfo e) {
         setCAPIException(e);
         return NULL;

src/codegen/unwinding.cpp

@@ -604,6 +604,9 @@ static const LineInfo lineInfoForFrame(PythonFrameIteratorImpl* frame_it) {
 }
 
 void exceptionCaughtInInterpreter(LineInfo line_info, ExcInfo* exc_info) {
+    static StatCounter frames_unwound("num_frames_unwound_python");
+    frames_unwound.log();
+
     // basically the same as PythonUnwindSession::addTraceback, but needs to
     // be callable after an PythonUnwindSession has ended.  The interpreter
     // will call this from catch blocks if it needs to ensure that a
@@ -632,6 +635,9 @@ void unwindingThroughFrame(PythonUnwindSession* unwind_session, unw_cursor_t* cu
         assert(!unwind_session->shouldSkipFrame());
         unwind_session->setShouldSkipNextFrame(true);
     } else if (frameIsPythonFrame(ip, bp, cursor, &frame_iter)) {
+        static StatCounter frames_unwound("num_frames_unwound_python");
+        frames_unwound.log();
+
         if (!unwind_session->shouldSkipFrame())
             unwind_session->addTraceback(lineInfoForFrame(&frame_iter));
 

src/runtime/builtin_modules/builtins.cpp

@@ -854,9 +854,7 @@ Box* zip(BoxedTuple* containers) {
 }
 
 static Box* callable(Box* obj) {
-    Box* r = PyBool_FromLong((long)PyCallable_Check(obj));
-    checkAndThrowCAPIException();
-    return r;
+    return PyBool_FromLong((long)PyCallable_Check(obj));
 }
 
 BoxedClass* notimplemented_cls;

src/runtime/capi.cpp

@@ -638,9 +638,19 @@ extern "C" int PyObject_Print(PyObject* obj, FILE* fp, int flags) noexcept {
 extern "C" int PyCallable_Check(PyObject* x) noexcept {
     if (x == NULL)
         return 0;
-
-    static BoxedString* call_attr = internStringImmortal("__call__");
-    return typeLookup(x->cls, call_attr, NULL) != NULL;
+    if (PyInstance_Check(x)) {
+        PyObject* call = PyObject_GetAttrString(x, "__call__");
+        if (call == NULL) {
+            PyErr_Clear();
+            return 0;
+        }
+        /* Could test recursively but don't, for fear of endless
+           recursion if some joker sets self.__call__ = self */
+        Py_DECREF(call);
+        return 1;
+    } else {
+        return x->cls->tp_call != NULL;
+    }
 }
 
 extern "C" int Py_FlushLine(void) noexcept {

src/runtime/cxx_unwind.cpp

@@ -496,6 +496,9 @@ static inline void unwind_loop(ExcInfo* exc_data) {
     while (unw_step(&cursor) > 0) {
         unw_proc_info_t pip;
 
+        static StatCounter frames_unwound("num_frames_unwound_cxx");
+        frames_unwound.log();
+
         // NB. unw_get_proc_info is slow; a significant chunk of all time spent unwinding is spent here.
         check(unw_get_proc_info(&cursor, &pip));
 
@@ -668,6 +671,9 @@ static uint64_t* unwinding_stattimer = pyston::Stats::getStatCounter("us_timer_u
 #endif
 
 extern "C" void __cxa_throw(void* exc_obj, std::type_info* tinfo, void (*dtor)(void*)) {
+    static pyston::StatCounter num_cxa_throw("num_cxa_throw");
+    num_cxa_throw.log();
+
     assert(!pyston::in_cleanup_code);
     assert(exc_obj);
     RELEASE_ASSERT(tinfo == &EXCINFO_TYPE_INFO, "can't throw a non-ExcInfo value! type info: %p", tinfo);

src/runtime/int.cpp

@@ -1142,6 +1142,8 @@ void setupInt() {
 
     add_operators(int_cls);
     int_cls->freeze();
+
+    int_cls->tp_repr = (reprfunc)int_to_decimal_string;
 }
 
 void teardownInt() {

src/runtime/str.cpp

@@ -1557,15 +1557,19 @@ extern "C" Box* strNonzero(BoxedString* self) {
 extern "C" Box* strNew(BoxedClass* cls, Box* obj) {
     assert(isSubclass(cls, str_cls));
 
-    Box* rtn = str(obj);
-    assert(PyString_Check(rtn));
+    if (cls != str_cls) {
+        Box* tmp = strNew(str_cls, obj);
+        assert(isSubclass(tmp->cls, str_cls));
+        BoxedString* tmp_s = static_cast<BoxedString*>(tmp);
 
-    if (cls == str_cls)
-        return rtn;
-
-    BoxedString* _rtn = static_cast<BoxedString*>(rtn);
+        return new (cls, tmp_s->size()) BoxedString(tmp_s->s());
+    }
 
-    return new (cls, _rtn->size()) BoxedString(_rtn->s());
+    Box* r = PyObject_Str(obj);
+    if (!r)
+        throwCAPIException();
+    assert(PyString_Check(r));
+    return r;
 }
 
 extern "C" Box* basestringNew(BoxedClass* cls, Box* args, Box* kwargs) {

src/runtime/tuple.cpp

@@ -517,6 +517,19 @@ static PyObject* tupleslice(PyTupleObject* a, Py_ssize_t ilow, Py_ssize_t ihigh)
     return (PyObject*)np;
 }
 
+static PyObject* tupleitem(register PyTupleObject* a, register Py_ssize_t i) {
+    if (i < 0 || i >= Py_SIZE(a)) {
+        PyErr_SetString(PyExc_IndexError, "tuple index out of range");
+        return NULL;
+    }
+    Py_INCREF(a->ob_item[i]);
+    return a->ob_item[i];
+}
+
+static Py_ssize_t tuplelength(PyTupleObject* a) {
+    return Py_SIZE(a);
+}
+
 void setupTuple() {
     tuple_iterator_cls = BoxedHeapClass::create(type_cls, object_cls, &tupleIteratorGCHandler, 0, 0,
                                                 sizeof(BoxedTupleIterator), false, "tuple");
@@ -528,8 +541,6 @@ void setupTuple() {
     addRTFunction(getitem, (void*)tupleGetitem, UNKNOWN, std::vector<ConcreteCompilerType*>{ UNKNOWN, UNKNOWN });
     tuple_cls->giveAttr("__getitem__", new BoxedFunction(getitem));
 
-    tuple_cls->tp_as_sequence->sq_slice = (ssizessizeargfunc)&tupleslice;
-
     tuple_cls->giveAttr("__contains__", new BoxedFunction(boxRTFunction((void*)tupleContains, BOXED_BOOL, 2)));
     tuple_cls->giveAttr("index", new BoxedFunction(boxRTFunction((void*)tupleIndex, BOXED_INT, 4, 2, false, false),
                                                    { boxInt(0), boxInt(std::numeric_limits<Py_ssize_t>::max()) }));
@@ -550,10 +561,14 @@ void setupTuple() {
     tuple_cls->giveAttr("__rmul__", new BoxedFunction(boxRTFunction((void*)tupleMul, BOXED_TUPLE, 2)));
 
     tuple_cls->tp_hash = (hashfunc)tuple_hash;
+    tuple_cls->tp_as_sequence->sq_slice = (ssizessizeargfunc)&tupleslice;
     add_operators(tuple_cls);
 
     tuple_cls->freeze();
 
+    tuple_cls->tp_as_sequence->sq_item = (ssizeargfunc)tupleitem;
+    tuple_cls->tp_as_sequence->sq_length = (lenfunc)tuplelength;
+
     CLFunction* hasnext = boxRTFunction((void*)tupleiterHasnextUnboxed, BOOL, 1);
     addRTFunction(hasnext, (void*)tupleiterHasnext, BOXED_BOOL);
     tuple_iterator_cls->giveAttr("__hasnext__", new BoxedFunction(hasnext));

src/runtime/types.cpp

@@ -11,6 +11,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#define PY_SSIZE_T_CLEAN
+
 #include "runtime/types.h"
 
 #include <cassert>
@@ -603,7 +605,7 @@ static Box* typeTppCall(Box* self, CallRewriteArgs* rewrite_args, ArgPassSpec ar
                         Box** args, const std::vector<BoxedString*>* keyword_names) {
     int npassed_args = argspec.totalPassed();
 
-    if (argspec.has_starargs) {
+    if (argspec.has_starargs || argspec.has_kwargs) {
         // This would fail in typeCallInner
         rewrite_args = NULL;
     }
@@ -633,8 +635,11 @@ static Box* typeCallInternal(BoxedFunctionBase* f, CallRewriteArgs* rewrite_args
     static StatCounter slowpath_typecall("slowpath_typecall");
     slowpath_typecall.log();
 
-    if (argspec.has_starargs)
+    if (argspec.has_starargs || argspec.num_args == 0) {
+        // Get callFunc to expand the arguments.
+        // TODO: update this to use rearrangeArguments instead.
         return callFunc(f, rewrite_args, argspec, arg1, arg2, arg3, args, keyword_names);
+    }
 
     return typeCallInner(rewrite_args, argspec, arg1, arg2, arg3, args, keyword_names);
 }
@@ -682,10 +687,33 @@ static PyObject* cpythonTypeCall(BoxedClass* type, PyObject* args, PyObject* kwd
     return r;
 }
 
+static Box* unicodeNewHelper(BoxedClass* type, Box* string, Box* encoding_obj, Box** _args) {
+    Box* errors_obj = _args[0];
+
+    assert(type == unicode_cls);
+
+    char* encoding = NULL;
+    char* errors = NULL;
+    if (encoding_obj)
+        if (!PyArg_ParseSingle(encoding_obj, 1, "unicode", "s", &encoding))
+            throwCAPIException();
+    if (errors_obj)
+        if (!PyArg_ParseSingle(errors_obj, 1, "unicode", "s", &errors))
+            throwCAPIException();
+
+    Box* r = unicode_new_inner(string, encoding, errors);
+    if (!r)
+        throwCAPIException();
+    assert(r->cls == unicode_cls); // otherwise we'd need to call this object's init
+    return r;
+}
+
 static Box* typeCallInner(CallRewriteArgs* rewrite_args, ArgPassSpec argspec, Box* arg1, Box* arg2, Box* arg3,
                           Box** args, const std::vector<BoxedString*>* keyword_names) {
     int npassed_args = argspec.totalPassed();
+    int npositional = argspec.num_args;
 
+    // We need to know what the class is.  We could potentially call rearrangeArguments here
     assert(argspec.num_args >= 1);
     Box* _cls = arg1;
 
@@ -696,6 +724,42 @@ static Box* typeCallInner(CallRewriteArgs* rewrite_args, ArgPassSpec argspec, Bo
 
     BoxedClass* cls = static_cast<BoxedClass*>(_cls);
 
+    if (cls == unicode_cls && !argspec.has_kwargs && !argspec.has_starargs
+        && (argspec.num_args == 1 || (argspec.num_args == 2 && arg2->cls == str_cls))) {
+        // unicode() takes an "encoding" parameter which can cause the constructor to return unicode subclasses.
+
+        if (rewrite_args) {
+            rewrite_args->arg1->addGuard((intptr_t)cls);
+            if (argspec.num_args >= 2)
+                rewrite_args->arg2->addGuard((intptr_t)arg2->cls);
+        }
+
+        // Special-case unicode for now, maybe there's something about this that can eventually be generalized:
+        ParamReceiveSpec paramspec(4, 3, false, false);
+        bool rewrite_success = false;
+        Box* oarg1, *oarg2, *oarg3;
+        static ParamNames param_names({ "string", "encoding", "errors" }, "", "");
+        static Box* defaults[3] = { NULL, NULL, NULL };
+        Box* oargs[1];
+
+        rearrangeArguments(paramspec, &param_names, "unicode", defaults, rewrite_args, rewrite_success, argspec, arg1,
+                           arg2, arg3, args, keyword_names, oarg1, oarg2, oarg3, oargs);
+        assert(oarg1 == cls);
+
+        if (!rewrite_success)
+            rewrite_args = NULL;
+
+        if (rewrite_args) {
+            rewrite_args->out_rtn
+                = rewrite_args->rewriter->call(true, (void*)unicodeNewHelper, rewrite_args->arg1, rewrite_args->arg2,
+                                               rewrite_args->arg3, rewrite_args->args);
+            rewrite_args->out_success = true;
+        }
+
+        // TODO other encodings could return non-unicode?
+        return unicodeNewHelper(cls, oarg2, oarg3, oargs);
+    }
+
     if (cls->tp_new != object_cls->tp_new && cls->tp_new != slot_tp_new) {
         // Looks like we're calling an extension class and we're not going to be able to
         // separately rewrite the new + init calls.  But we can rewrite the fact that we
@@ -719,12 +783,16 @@ static Box* typeCallInner(CallRewriteArgs* rewrite_args, ArgPassSpec argspec, Bo
         return cpythonTypeCall(cls, oarg2, oarg3);
     }
 
+    if (argspec.has_starargs || argspec.has_kwargs)
+        rewrite_args = NULL;
+
     RewriterVar* r_ccls = NULL;
     RewriterVar* r_new = NULL;
     RewriterVar* r_init = NULL;
     Box* new_attr, *init_attr;
     if (rewrite_args) {
         assert(!argspec.has_starargs);
+        assert(!argspec.has_kwargs);
         assert(argspec.num_args > 0);
 
         r_ccls = rewrite_args->arg1;
@@ -782,20 +850,35 @@ static Box* typeCallInner(CallRewriteArgs* rewrite_args, ArgPassSpec argspec, Bo
 
     // typeCall is tricky to rewrite since it has complicated behavior: we are supposed to
     // call the __init__ method of the *result of the __new__ call*, not of the original
-    // class.  (And only if the result is an instance of the original class, but that's not
-    // even the tricky part here.)
+    // class.  (And only if the result is an instance of the original class (or a subclass),
+    // but that's not even the tricky part here.)
     //
     // By the time we know the type of the result of __new__(), it's too late to add traditional
     // guards.  So, instead of doing that, we're going to add a guard that makes sure that __new__
-    // has the property that __new__(kls) always returns an instance of kls.
-    //
-    // Whitelist a set of __new__ methods that we know work like this.  Most importantly: object.__new__.
+    // has the property that it will always return an instance where we know what __init__ has to be
+    // called on it.  There are a couple cases:
+    // - Some __new__ functions, such as object.__new__, always return an instance of the requested class.
+    //   We can whitelist these __new__ functions.
+    // - There are cls+arg pairs such that cls(arg) always returns an instance of cls.  For example,
+    //   str() of an int is always a str, but str of arbitrary types does not necessarily return a str
+    //   (could return a subtype of str)
+    // - There are cls+arg pairs where we know that we don't have to call an __init__, despite the return
+    //   value having variable type.  For instance, int(float) can return a long on overflow, but in either
+    //   case no __init__ should be called.
+    // - There's a final special case that type(obj) does not call __init__ even if type.__new__(type, obj)
+    //   happens to return a subclass of type.  This is a special case in cpython's code that we have as well.
     //
-    // Most builtin classes behave this way, but not all!
-    // Notably, "type" itself does not.  For instance, assuming M is a subclass of
-    // type, type.__new__(M, 1) will return the int class, which is not an instance of M.
 
-    // this is ok with not using StlCompatAllocator since we will manually register these objects with the GC
+    // For debugging, keep track of why we think we can rewrite this:
+    enum { NOT_ALLOWED, VERIFIED, NO_INIT, TYPE_NEW_SPECIAL_CASE, } why_rewrite_allowed = NOT_ALLOWED;
+
+    // These are __new__ functions that have the property that __new__(kls) always returns an instance of kls.
+    // These are ok to call regardless of what type was requested.
+    //
+    // TODO what if an extension type defines a tp_alloc that returns something that's not an instance of that
+    // type?  then object.__new__ would not be able to be here:
+    //
+    // this array is ok with not using StlCompatAllocator since we will manually register these objects with the GC
     static std::vector<Box*> allowable_news;
     if (allowable_news.empty()) {
         for (BoxedClass* allowed_cls : { object_cls, enumerate_cls, xrange_cls, tuple_cls, list_cls, dict_cls }) {
@@ -805,9 +888,6 @@ static Box* typeCallInner(CallRewriteArgs* rewrite_args, ArgPassSpec argspec, Bo
         }
     }
 
-    // For debugging, keep track of why we think we can rewrite this:
-    enum { NOT_ALLOWED, VERIFIED, NO_INIT, TYPE_NEW_SPECIAL_CASE, } why_rewrite_allowed = NOT_ALLOWED;
-
     if (rewrite_args) {
         for (auto b : allowable_news) {
             if (b == new_attr) {
@@ -816,13 +896,44 @@ static Box* typeCallInner(CallRewriteArgs* rewrite_args, ArgPassSpec argspec, Bo
             }
         }
 
-        if (cls == int_cls || cls == float_cls || cls == long_cls) {
-            if (npassed_args == 1) {
+        bool know_first_arg = !argspec.has_starargs && !argspec.has_kwargs && argspec.num_keywords == 0;
+
+        if (know_first_arg) {
+            if (argspec.num_args == 1
+                && (cls == int_cls || cls == float_cls || cls == long_cls || cls == str_cls || cls == unicode_cls))
                 why_rewrite_allowed = VERIFIED;
-            } else if (npassed_args == 2 && (arg2->cls == int_cls || arg2->cls == str_cls || arg2->cls == float_cls)) {
+
+            if (argspec.num_args == 2 && (cls == int_cls || cls == float_cls || cls == long_cls)
+                && (arg2->cls == int_cls || arg2->cls == str_cls || arg2->cls == float_cls
+                    || arg2->cls == unicode_cls)) {
                 why_rewrite_allowed = NO_INIT;
                 rewrite_args->arg2->addAttrGuard(offsetof(Box, cls), (intptr_t)arg2->cls);
             }
+
+            // str(obj) can return str-subtypes, but for builtin types it won't:
+            if (argspec.num_args == 2 && cls == str_cls && (arg2->cls == int_cls || arg2->cls == float_cls)) {
+                why_rewrite_allowed = VERIFIED;
+                rewrite_args->arg2->addAttrGuard(offsetof(Box, cls), (intptr_t)arg2->cls);
+            }
+
+            // int(str, base) can only return int/long
+            if (argspec.num_args == 3 && cls == int_cls) {
+                why_rewrite_allowed = NO_INIT;
+            }
+
+#if 0
+            if (why_rewrite_allowed == NOT_ALLOWED) {
+                std::string per_name_stat_name = "zzz_norewrite_" + std::string(cls->tp_name);
+                if (argspec.num_args == 1)
+                    per_name_stat_name += "_1arg";
+                else if (argspec.num_args == 2)
+                    per_name_stat_name += "_" + std::string(arg2->cls->tp_name);
+                else
+                    per_name_stat_name += "_narg";
+                uint64_t* counter = Stats::getStatCounter(per_name_stat_name);
+                Stats::log(counter);
+            }
+#endif
         }
 
         if (cls == type_cls && argspec == ArgPassSpec(2))

test/tests/encoding_test.py

+import codecs
+
+### Codec APIs
+
+class MyUnicode(unicode):
+    def __new__(*args):
+        print "MyUnicode.__new__", map(type, args)
+        return unicode.__new__(*args)
+
+    def __init__(*args):
+        print "MyUnicode.__init__", map(type, args)
+
+def encode(input, errors='strict'):
+    raise Exception()
+
+def decode(input, errors='strict'):
+    return (MyUnicode(u"."), 1)
+
+class IncrementalEncoder(codecs.IncrementalEncoder):
+    def encode(self, input, final=False):
+        return codecs.utf_8_encode(input, self.errors)[0]
+
+class IncrementalDecoder(codecs.BufferedIncrementalDecoder):
+    _buffer_decode = codecs.utf_8_decode
+
+class StreamWriter(codecs.StreamWriter):
+    encode = codecs.utf_8_encode
+
+class StreamReader(codecs.StreamReader):
+    decode = codecs.utf_8_decode
+
+codec = codecs.CodecInfo(
+    name='myunicode',
+    encode=encode,
+    decode=decode,
+    incrementalencoder=IncrementalEncoder,
+    incrementaldecoder=IncrementalDecoder,
+    streamreader=StreamReader,
+    streamwriter=StreamWriter,
+)
+
+def search(name):
+    if name == "myunicode":
+        return codec
+codecs.register(search)
+
+
+u = unicode("hello world", "myunicode", "strict")
+print type(u)

test/tests/intmethods.py

@@ -132,4 +132,9 @@ print
 
 # These return longs:
 print int("12938719238719827398172938712983791827938712987312")
+print int(u"12938719238719827398172938712983791827938712987312")
+print int("12938719238719827398172938712983791827938712987312", 16)
+print int(u"12938719238719827398172938712983791827938712987312", 16)
 print int(1e100)
+print int(*[1e100])
+print int(x=1e100)

test/tests/object_new_arguments.py

@@ -21,3 +21,8 @@ except TypeError as e:
 # are being passed, but really they are not.
 type.__call__(*[C2])
 type.__call__(C2, **{})
+
+try:
+    type.__call__(*[])
+except TypeError as e:
+    print "caught typeerror"

test/tests/str_subclassing.py

@@ -10,3 +10,15 @@ print
 
 print repr("hello" + MyStr("world"))
 print int(MyStr("2"))
+
+class MyStr(str):
+    def __init__(*args):
+        print "MyStr.__init__", map(type, args)
+
+class C(object):
+    def __str__(self):
+        return MyStr("hello world")
+
+print type(str(C()))
+m = MyStr(C())
+print type(m), repr(m)

test/tests/unicode_test.py

@@ -155,3 +155,13 @@ print "".join([u"\xB2", u"\xB3"])
 
 import sys
 print type(sys.maxunicode)
+
+class MyUnicode(unicode):
+    def __init__(*args):
+        print "MyUnicode.__init__", map(type, args)
+
+class C(object):
+    def __unicode__(self):
+        return MyUnicode("hello world")
+
+print type(unicode(C()))