Optimize calling str() and unicode()

They are tricky since these are types, which means they invoke
the relatively-complicated constructor logic.  ie str() doesn't
just call __str__ on the argument: if the result is a subclass
of str, it calls result.__init__().  Similarly for unicode, except
unicode is even trickier since it takes some more arguments, one
of which is "encoding" which will have non-type-based dynamic
behavior.

I didn't realize that at first and optimized unicode() by exposing an
inner version of it that takes its arguments in registers, which we
can take advantage of using our jit-arg-rearrangement capability.
This means we have to do parts of PyArg_ParseTuple ourselves, so I
added a PyArg_ParseSingle that runs a single object through the
arg-conversion code.  PyArg_ParseSingle could be further optimized if
we want to.  Or rather, if we have functions of the form
PyArg_ParseSingle_s (which corresponds to the "s" format code) we
could skip some more of the overhead.

I had to disable most of that once I realized the encoding issue, but
I left it in since hopefully we will be able to use it again once
we have some "do some guards after mutations if we know how to resume
after a failed guard" rewriter support.

from_cpython/Include/modsupport.h

@@ -16,6 +16,7 @@ extern "C" {
 #define PyArg_Parse			_PyArg_Parse_SizeT
 #define PyArg_ParseTuple		_PyArg_ParseTuple_SizeT
 #define PyArg_ParseTupleAndKeywords	_PyArg_ParseTupleAndKeywords_SizeT
+#define PyArg_ParseSingle		_PyArg_ParseSingle_SizeT
 #define PyArg_VaParse			_PyArg_VaParse_SizeT
 #define PyArg_VaParseTupleAndKeywords	_PyArg_VaParseTupleAndKeywords_SizeT
 #define Py_BuildValue			_Py_BuildValue_SizeT
@@ -28,6 +29,8 @@ PyAPI_FUNC(int) PyArg_Parse(PyObject *, const char *, ...) PYSTON_NOEXCEPT;
 PyAPI_FUNC(int) PyArg_ParseTuple(PyObject *, const char *, ...) Py_FORMAT_PARSETUPLE(PyArg_ParseTuple, 2, 3) PYSTON_NOEXCEPT;
 PyAPI_FUNC(int) PyArg_ParseTupleAndKeywords(PyObject *, PyObject *,
                                                   const char *, char **, ...) PYSTON_NOEXCEPT;
+// Pyston addition:
+PyAPI_FUNC(int) PyArg_ParseSingle(PyObject* obj, int arg_idx, const char* fname, const char* format, ...) PYSTON_NOEXCEPT;
 PyAPI_FUNC(int) PyArg_UnpackTuple(PyObject *, const char *, Py_ssize_t, Py_ssize_t, ...) PYSTON_NOEXCEPT;
 PyAPI_FUNC(PyObject *) Py_BuildValue(const char *, ...) PYSTON_NOEXCEPT;
 PyAPI_FUNC(PyObject *) _Py_BuildValue_SizeT(const char *, ...) PYSTON_NOEXCEPT;

from_cpython/Include/unicodeobject.h

@@ -1408,6 +1408,9 @@ PyAPI_FUNC(int) _PyUnicode_IsAlpha(
     Py_UNICODE ch       /* Unicode character */
     ) PYSTON_NOEXCEPT;
 
+// Pyston addition:
+PyAPI_FUNC(PyObject*) unicode_new_inner(PyObject* x, char* encoding, char* errors) PYSTON_NOEXCEPT;
+
 #ifdef __cplusplus
 }
 #endif

from_cpython/Objects/unicodeobject.c

@@ -8745,6 +8745,15 @@ static PyBufferProcs unicode_as_buffer = {
 static PyObject *
 unicode_subtype_new(PyTypeObject *type, PyObject *args, PyObject *kwds);
 
+PyObject* unicode_new_inner(PyObject* x, char* encoding, char* errors) {
+    if (x == NULL)
+        return (PyObject *)_PyUnicode_New(0);
+    if (encoding == NULL && errors == NULL)
+        return PyObject_Unicode(x);
+    else
+        return PyUnicode_FromEncodedObject(x, encoding, errors);
+}
+
 static PyObject *
 unicode_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
 {
@@ -8758,12 +8767,7 @@ unicode_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
     if (!PyArg_ParseTupleAndKeywords(args, kwds, "|Oss:unicode",
                                      kwlist, &x, &encoding, &errors))
         return NULL;
-    if (x == NULL)
-        return (PyObject *)_PyUnicode_New(0);
-    if (encoding == NULL && errors == NULL)
-        return PyObject_Unicode(x);
-    else
-        return PyUnicode_FromEncodedObject(x, encoding, errors);
+    return unicode_new_inner(x, encoding, errors);
 }
 
 static PyObject *

from_cpython/Python/getargs.c

@@ -569,6 +569,35 @@ float_argument_error(PyObject *arg)
         return 0;
 }
 
+int _PyArg_ParseSingle_SizeT(PyObject* obj, int arg_idx, const char* fname, const char* format, ...) {
+    va_list va;
+    char* msg;
+    char msgbuf[256];
+
+    assert(format[0] != '\0');
+    assert(format[0] != '(');
+    assert(format[0] != '|');
+    assert(format[0] != '|');
+
+    assert(format[1] != '*'); // would need to pass a non-null freelist
+    assert(format[0] != 'e'); // would need to pass a non-null freelist
+
+    va_start(va, format);
+    msg = convertsimple(obj, &format, &va, FLAG_SIZE_T, msgbuf, sizeof(msgbuf), NULL);
+    va_end(va);
+
+    if (msg) {
+        int levels[1];
+        levels[0] = 0;
+        seterror(arg_idx + 1, msg, levels, fname, NULL);
+        return 0;
+    }
+
+    // Should have consumed the entire format string:
+    assert(format[0] == '\0');
+    return 1;
+}
+
 /* Convert a non-tuple argument.  Return NULL if conversion went OK,
    or a string with a message describing the failure.  The message is
    formatted as "must be <desired type>, not <actual type>".

src/runtime/types.cpp

@@ -11,6 +11,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#define PY_SSIZE_T_CLEAN
+
 #include "runtime/types.h"
 
 #include <cassert>
@@ -603,7 +605,7 @@ static Box* typeTppCall(Box* self, CallRewriteArgs* rewrite_args, ArgPassSpec ar
                         Box** args, const std::vector<BoxedString*>* keyword_names) {
     int npassed_args = argspec.totalPassed();
 
-    if (argspec.has_starargs) {
+    if (argspec.has_starargs || argspec.has_kwargs) {
         // This would fail in typeCallInner
         rewrite_args = NULL;
     }
@@ -633,8 +635,11 @@ static Box* typeCallInternal(BoxedFunctionBase* f, CallRewriteArgs* rewrite_args
     static StatCounter slowpath_typecall("slowpath_typecall");
     slowpath_typecall.log();
 
-    if (argspec.has_starargs)
+    if (argspec.has_starargs || argspec.num_args == 0) {
+        // Get callFunc to expand the arguments.
+        // TODO: update this to use rearrangeArguments instead.
         return callFunc(f, rewrite_args, argspec, arg1, arg2, arg3, args, keyword_names);
+    }
 
     return typeCallInner(rewrite_args, argspec, arg1, arg2, arg3, args, keyword_names);
 }
@@ -682,10 +687,33 @@ static PyObject* cpythonTypeCall(BoxedClass* type, PyObject* args, PyObject* kwd
     return r;
 }
 
+static Box* unicodeNewHelper(BoxedClass* type, Box* string, Box* encoding_obj, Box** _args) {
+    Box* errors_obj = _args[0];
+
+    assert(type == unicode_cls);
+
+    char* encoding = NULL;
+    char* errors = NULL;
+    if (encoding_obj)
+        if (!PyArg_ParseSingle(encoding_obj, 1, "unicode", "s", &encoding))
+            throwCAPIException();
+    if (errors_obj)
+        if (!PyArg_ParseSingle(errors_obj, 1, "unicode", "s", &errors))
+            throwCAPIException();
+
+    Box* r = unicode_new_inner(string, encoding, errors);
+    if (!r)
+        throwCAPIException();
+    assert(r->cls == unicode_cls); // otherwise we'd need to call this object's init
+    return r;
+}
+
 static Box* typeCallInner(CallRewriteArgs* rewrite_args, ArgPassSpec argspec, Box* arg1, Box* arg2, Box* arg3,
                           Box** args, const std::vector<BoxedString*>* keyword_names) {
     int npassed_args = argspec.totalPassed();
+    int npositional = argspec.num_args;
 
+    // We need to know what the class is.  We could potentially call rearrangeArguments here
     assert(argspec.num_args >= 1);
     Box* _cls = arg1;
 
@@ -696,6 +724,42 @@ static Box* typeCallInner(CallRewriteArgs* rewrite_args, ArgPassSpec argspec, Bo
 
     BoxedClass* cls = static_cast<BoxedClass*>(_cls);
 
+    if (cls == unicode_cls && !argspec.has_kwargs && !argspec.has_starargs
+        && (argspec.num_args == 1 || (argspec.num_args == 2 && arg2->cls == str_cls))) {
+        // unicode() takes an "encoding" parameter which can cause the constructor to return unicode subclasses.
+
+        if (rewrite_args) {
+            rewrite_args->arg1->addGuard((intptr_t)cls);
+            if (argspec.num_args >= 2)
+                rewrite_args->arg2->addGuard((intptr_t)arg2->cls);
+        }
+
+        // Special-case unicode for now, maybe there's something about this that can eventually be generalized:
+        ParamReceiveSpec paramspec(4, 3, false, false);
+        bool rewrite_success = false;
+        Box* oarg1, *oarg2, *oarg3;
+        static ParamNames param_names({ "string", "encoding", "errors" }, "", "");
+        static Box* defaults[3] = { NULL, NULL, NULL };
+        Box* oargs[1];
+
+        rearrangeArguments(paramspec, &param_names, "unicode", defaults, rewrite_args, rewrite_success, argspec, arg1,
+                           arg2, arg3, args, keyword_names, oarg1, oarg2, oarg3, oargs);
+        assert(oarg1 == cls);
+
+        if (!rewrite_success)
+            rewrite_args = NULL;
+
+        if (rewrite_args) {
+            rewrite_args->out_rtn
+                = rewrite_args->rewriter->call(true, (void*)unicodeNewHelper, rewrite_args->arg1, rewrite_args->arg2,
+                                               rewrite_args->arg3, rewrite_args->args);
+            rewrite_args->out_success = true;
+        }
+
+        // TODO other encodings could return non-unicode?
+        return unicodeNewHelper(cls, oarg2, oarg3, oargs);
+    }
+
     if (cls->tp_new != object_cls->tp_new && cls->tp_new != slot_tp_new) {
         // Looks like we're calling an extension class and we're not going to be able to
         // separately rewrite the new + init calls.  But we can rewrite the fact that we
@@ -719,12 +783,16 @@ static Box* typeCallInner(CallRewriteArgs* rewrite_args, ArgPassSpec argspec, Bo
         return cpythonTypeCall(cls, oarg2, oarg3);
     }
 
+    if (argspec.has_starargs || argspec.has_kwargs)
+        rewrite_args = NULL;
+
     RewriterVar* r_ccls = NULL;
     RewriterVar* r_new = NULL;
     RewriterVar* r_init = NULL;
     Box* new_attr, *init_attr;
     if (rewrite_args) {
         assert(!argspec.has_starargs);
+        assert(!argspec.has_kwargs);
         assert(argspec.num_args > 0);
 
         r_ccls = rewrite_args->arg1;
@@ -782,20 +850,35 @@ static Box* typeCallInner(CallRewriteArgs* rewrite_args, ArgPassSpec argspec, Bo
 
     // typeCall is tricky to rewrite since it has complicated behavior: we are supposed to
     // call the __init__ method of the *result of the __new__ call*, not of the original
-    // class.  (And only if the result is an instance of the original class, but that's not
-    // even the tricky part here.)
+    // class.  (And only if the result is an instance of the original class (or a subclass),
+    // but that's not even the tricky part here.)
     //
     // By the time we know the type of the result of __new__(), it's too late to add traditional
     // guards.  So, instead of doing that, we're going to add a guard that makes sure that __new__
-    // has the property that __new__(kls) always returns an instance of kls.
-    //
-    // Whitelist a set of __new__ methods that we know work like this.  Most importantly: object.__new__.
+    // has the property that it will always return an instance where we know what __init__ has to be
+    // called on it.  There are a couple cases:
+    // - Some __new__ functions, such as object.__new__, always return an instance of the requested class.
+    //   We can whitelist these __new__ functions.
+    // - There are cls+arg pairs such that cls(arg) always returns an instance of cls.  For example,
+    //   str() of an int is always a str, but str of arbitrary types does not necessarily return a str
+    //   (could return a subtype of str)
+    // - There are cls+arg pairs where we know that we don't have to call an __init__, despite the return
+    //   value having variable type.  For instance, int(float) can return a long on overflow, but in either
+    //   case no __init__ should be called.
+    // - There's a final special case that type(obj) does not call __init__ even if type.__new__(type, obj)
+    //   happens to return a subclass of type.  This is a special case in cpython's code that we have as well.
     //
-    // Most builtin classes behave this way, but not all!
-    // Notably, "type" itself does not.  For instance, assuming M is a subclass of
-    // type, type.__new__(M, 1) will return the int class, which is not an instance of M.
 
-    // this is ok with not using StlCompatAllocator since we will manually register these objects with the GC
+    // For debugging, keep track of why we think we can rewrite this:
+    enum { NOT_ALLOWED, VERIFIED, NO_INIT, TYPE_NEW_SPECIAL_CASE, } why_rewrite_allowed = NOT_ALLOWED;
+
+    // These are __new__ functions that have the property that __new__(kls) always returns an instance of kls.
+    // These are ok to call regardless of what type was requested.
+    //
+    // TODO what if an extension type defines a tp_alloc that returns something that's not an instance of that
+    // type?  then object.__new__ would not be able to be here:
+    //
+    // this array is ok with not using StlCompatAllocator since we will manually register these objects with the GC
     static std::vector<Box*> allowable_news;
     if (allowable_news.empty()) {
         for (BoxedClass* allowed_cls : { object_cls, enumerate_cls, xrange_cls, tuple_cls, list_cls, dict_cls }) {
@@ -805,9 +888,6 @@ static Box* typeCallInner(CallRewriteArgs* rewrite_args, ArgPassSpec argspec, Bo
         }
     }
 
-    // For debugging, keep track of why we think we can rewrite this:
-    enum { NOT_ALLOWED, VERIFIED, NO_INIT, TYPE_NEW_SPECIAL_CASE, } why_rewrite_allowed = NOT_ALLOWED;
-
     if (rewrite_args) {
         for (auto b : allowable_news) {
             if (b == new_attr) {
@@ -816,13 +896,44 @@ static Box* typeCallInner(CallRewriteArgs* rewrite_args, ArgPassSpec argspec, Bo
             }
         }
 
-        if (cls == int_cls || cls == float_cls || cls == long_cls) {
-            if (npassed_args == 1) {
+        bool know_first_arg = !argspec.has_starargs && !argspec.has_kwargs && argspec.num_keywords == 0;
+
+        if (know_first_arg) {
+            if (argspec.num_args == 1
+                && (cls == int_cls || cls == float_cls || cls == long_cls || cls == str_cls || cls == unicode_cls))
                 why_rewrite_allowed = VERIFIED;
-            } else if (npassed_args == 2 && (arg2->cls == int_cls || arg2->cls == str_cls || arg2->cls == float_cls)) {
+
+            if (argspec.num_args == 2 && (cls == int_cls || cls == float_cls || cls == long_cls)
+                && (arg2->cls == int_cls || arg2->cls == str_cls || arg2->cls == float_cls
+                    || arg2->cls == unicode_cls)) {
                 why_rewrite_allowed = NO_INIT;
                 rewrite_args->arg2->addAttrGuard(offsetof(Box, cls), (intptr_t)arg2->cls);
             }
+
+            // str(obj) can return str-subtypes, but for builtin types it won't:
+            if (argspec.num_args == 2 && cls == str_cls && (arg2->cls == int_cls || arg2->cls == float_cls)) {
+                why_rewrite_allowed = VERIFIED;
+                rewrite_args->arg2->addAttrGuard(offsetof(Box, cls), (intptr_t)arg2->cls);
+            }
+
+            // int(str, base) can only return int/long
+            if (argspec.num_args == 3 && cls == int_cls) {
+                why_rewrite_allowed = NO_INIT;
+            }
+
+#if 0
+            if (why_rewrite_allowed == NOT_ALLOWED) {
+                std::string per_name_stat_name = "zzz_norewrite_" + std::string(cls->tp_name);
+                if (argspec.num_args == 1)
+                    per_name_stat_name += "_1arg";
+                else if (argspec.num_args == 2)
+                    per_name_stat_name += "_" + std::string(arg2->cls->tp_name);
+                else
+                    per_name_stat_name += "_narg";
+                uint64_t* counter = Stats::getStatCounter(per_name_stat_name);
+                Stats::log(counter);
+            }
+#endif
         }
 
         if (cls == type_cls && argspec == ArgPassSpec(2))

test/tests/encoding_test.py

+import codecs
+
+### Codec APIs
+
+class MyUnicode(unicode):
+    def __new__(*args):
+        print "MyUnicode.__new__", map(type, args)
+        return unicode.__new__(*args)
+
+    def __init__(*args):
+        print "MyUnicode.__init__", map(type, args)
+
+def encode(input, errors='strict'):
+    raise Exception()
+
+def decode(input, errors='strict'):
+    return (MyUnicode(u"."), 1)
+
+class IncrementalEncoder(codecs.IncrementalEncoder):
+    def encode(self, input, final=False):
+        return codecs.utf_8_encode(input, self.errors)[0]
+
+class IncrementalDecoder(codecs.BufferedIncrementalDecoder):
+    _buffer_decode = codecs.utf_8_decode
+
+class StreamWriter(codecs.StreamWriter):
+    encode = codecs.utf_8_encode
+
+class StreamReader(codecs.StreamReader):
+    decode = codecs.utf_8_decode
+
+codec = codecs.CodecInfo(
+    name='myunicode',
+    encode=encode,
+    decode=decode,
+    incrementalencoder=IncrementalEncoder,
+    incrementaldecoder=IncrementalDecoder,
+    streamreader=StreamReader,
+    streamwriter=StreamWriter,
+)
+
+def search(name):
+    if name == "myunicode":
+        return codec
+codecs.register(search)
+
+
+u = unicode("hello world", "myunicode", "strict")
+print type(u)

test/tests/intmethods.py

@@ -132,4 +132,9 @@ print
 
 # These return longs:
 print int("12938719238719827398172938712983791827938712987312")
+print int(u"12938719238719827398172938712983791827938712987312")
+print int("12938719238719827398172938712983791827938712987312", 16)
+print int(u"12938719238719827398172938712983791827938712987312", 16)
 print int(1e100)
+print int(*[1e100])
+print int(x=1e100)

test/tests/object_new_arguments.py

@@ -21,3 +21,8 @@ except TypeError as e:
 # are being passed, but really they are not.
 type.__call__(*[C2])
 type.__call__(C2, **{})
+
+try:
+    type.__call__(*[])
+except TypeError as e:
+    print "caught typeerror"

test/tests/str_subclassing.py

@@ -10,3 +10,13 @@ print
 
 print repr("hello" + MyStr("world"))
 print int(MyStr("2"))
+
+class MyStr(str):
+    def __init__(*args):
+        print "MyStr.__init__", map(type, args)
+
+class C(object):
+    def __str__(self):
+        return MyStr("hello world")
+
+print type(str(C()))

test/tests/unicode_test.py

@@ -155,3 +155,13 @@ print "".join([u"\xB2", u"\xB3"])
 
 import sys
 print type(sys.maxunicode)
+
+class MyUnicode(unicode):
+    def __init__(*args):
+        print "MyUnicode.__init__", map(type, args)
+
+class C(object):
+    def __unicode__(self):
+        return MyUnicode("hello world")
+
+print type(unicode(C()))