Make `import ctypes` work.

- Temporarily disable code for ctypes that requires class descriptors.
- Skip the attribute lookup for __new__ for the base type class and
  directly call a special function assigned to tp_new.
- Add a #define from CPython's ./configure
- Add test.

from_cpython/Include/pyconfig.h.in

@@ -20,6 +20,7 @@
 #define _GNU_SOURCE 1
 
 #define PY_LONG_LONG long long
+#define SIZEOF__BOOL 1
 #define SIZEOF_VOID_P 8
 #define SIZEOF_SIZE_T 8
 #define SIZEOF_INT 4

from_cpython/Modules/_ctypes/_ctypes.c

@@ -2044,13 +2044,21 @@ PyCSimpleType_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
             break;
         }
 
-        if (ml) {
+        // TODO: Pyston change:
+        // For now, don't run this code path because we don't support some of
+        // the descriptor CAPI functions.
+        // We do this to be able to run `import ctypes`, but this will need to be
+        // enabled again once we want CType to actually work.
+        // if (ml) {
+        if (false) {
 #if (PYTHON_API_VERSION >= 1012)
             PyObject *meth;
             int x;
+            /*
             meth = PyDescr_NewClassMethod(result, ml);
             if (!meth)
                 return NULL;
+            */
 #else
 #error
             PyObject *meth, *func;

src/runtime/objmodel.cpp

@@ -4846,34 +4846,7 @@ void assertValidSlotIdentifier(Box* s) {
     }
 }
 
-Box* typeNew(Box* _cls, Box* arg1, Box* arg2, Box** _args) {
-    STAT_TIMER(t0, "us_timer_typeNew", 10);
-
-    Box* arg3 = _args[0];
-
-    if (!isSubclass(_cls->cls, type_cls))
-        raiseExcHelper(TypeError, "type.__new__(X): X is not a type object (%s)", getTypeName(_cls));
-
-    BoxedClass* metatype = static_cast<BoxedClass*>(_cls);
-    if (!isSubclass(metatype, type_cls))
-        raiseExcHelper(TypeError, "type.__new__(%s): %s is not a subtype of type", getNameOfClass(metatype),
-                       getNameOfClass(metatype));
-
-    if (arg2 == NULL) {
-        assert(arg3 == NULL);
-        BoxedClass* rtn = arg1->cls;
-        return rtn;
-    }
-
-    RELEASE_ASSERT(PyDict_Check(arg3), "%s", getTypeName(arg3));
-    BoxedDict* attr_dict = static_cast<BoxedDict*>(arg3);
-
-    RELEASE_ASSERT(arg2->cls == tuple_cls, "");
-    BoxedTuple* bases = static_cast<BoxedTuple*>(arg2);
-
-    RELEASE_ASSERT(arg1->cls == str_cls, "");
-    BoxedString* name = static_cast<BoxedString*>(arg1);
-
+Box* _typeNew(BoxedClass* metatype, BoxedString* name, BoxedTuple* bases, BoxedDict* attr_dict) {
     if (bases->size() == 0) {
         bases = BoxedTuple::create({ object_cls });
     }
@@ -4903,7 +4876,8 @@ Box* typeNew(Box* _cls, Box* arg1, Box* arg2, Box** _args) {
         if (getattr(winner, new_box) != getattr(type_cls, new_box)) {
             CallattrFlags callattr_flags
                 = {.cls_only = false, .null_on_nonexistent = false, .argspec = ArgPassSpec(4) };
-            return callattr(winner, new_box, callattr_flags, winner, arg1, arg2, _args, NULL);
+            Box* args[1] = { (Box*)attr_dict };
+            return callattr(winner, new_box, callattr_flags, winner, name, bases, args, NULL);
         }
         metatype = winner;
     }
@@ -5137,6 +5111,100 @@ Box* typeNew(Box* _cls, Box* arg1, Box* arg2, Box** _args) {
     return made;
 }
 
+// Analogous to CPython's type_new.
+// This is assigned directly to type_cls's (PyType_Type's) tp_new slot and skips
+// doing an attribute lookup for __new__.
+//
+// We need this to support some edge cases. For example, in ctypes, we have a function:
+// PyCSimpleType_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
+// {
+//     ...
+//     result = (PyTypeObject *)PyType_Type.tp_new(type, args, kwds);
+//     ...
+// }
+//
+// Assigned to the tp_new of PyCSimpleType, a metaclass. By calling PyType_Type.tp_new,
+// we don't want to do an attribute lookup for __new__, because that would end up calling
+// the tp_new of the subclass PyCSimpleType (PyCSimpleType_new again) and end up in a loop.
+Box* type_new(BoxedClass* metatype, Box* args, Box* kwds) noexcept {
+    PyObject* name, *bases, *dict;
+    static const char* kwlist[] = { "name", "bases", "dict", 0 };
+
+    // Copied from CPython.
+    assert(args != NULL && PyTuple_Check(args));
+    assert(kwds == NULL || PyDict_Check(kwds));
+
+    /* Special case: type(x) should return x->ob_type */
+    {
+        const Py_ssize_t nargs = PyTuple_GET_SIZE(args);
+        const Py_ssize_t nkwds = kwds == NULL ? 0 : PyDict_Size(kwds);
+
+        if (PyType_CheckExact(metatype) && nargs == 1 && nkwds == 0) {
+            PyObject* x = PyTuple_GET_ITEM(args, 0);
+            Py_INCREF(Py_TYPE(x));
+            return (PyObject*)Py_TYPE(x);
+        }
+
+        /* SF bug 475327 -- if that didn't trigger, we need 3
+           arguments. but PyArg_ParseTupleAndKeywords below may give
+           a msg saying type() needs exactly 3. */
+        if (nargs + nkwds != 3) {
+            PyErr_SetString(PyExc_TypeError, "type() takes 1 or 3 arguments");
+            return NULL;
+        }
+    }
+
+    // Check arguments: (name, bases, dict)
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "SO!O!:type", const_cast<char**>(kwlist), &name, &PyTuple_Type, &bases,
+                                     &PyDict_Type, &dict))
+        return NULL;
+
+    try {
+        RELEASE_ASSERT(name->cls == str_cls, "");
+        RELEASE_ASSERT(bases->cls == tuple_cls, "");
+        RELEASE_ASSERT(dict->cls == dict_cls, "");
+
+        return _typeNew(metatype, static_cast<BoxedString*>(name), static_cast<BoxedTuple*>(bases),
+                        static_cast<BoxedDict*>(dict));
+    } catch (ExcInfo e) {
+        setCAPIException(e);
+        return NULL;
+    }
+}
+
+// This is the function we want uses of __new__ to call.
+Box* typeNewGeneric(Box* _cls, Box* arg1, Box* arg2, Box** _args) {
+    STAT_TIMER(t0, "us_timer_typeNew", 10);
+
+    Box* arg3 = _args[0];
+
+    if (!isSubclass(_cls->cls, type_cls))
+        raiseExcHelper(TypeError, "type.__new__(X): X is not a type object (%s)", getTypeName(_cls));
+
+    BoxedClass* metatype = static_cast<BoxedClass*>(_cls);
+    if (!isSubclass(metatype, type_cls))
+        raiseExcHelper(TypeError, "type.__new__(%s): %s is not a subtype of type", getNameOfClass(metatype),
+                       getNameOfClass(metatype));
+
+    if (arg2 == NULL) {
+        assert(arg3 == NULL);
+        BoxedClass* rtn = arg1->cls;
+
+        return rtn;
+    }
+
+    RELEASE_ASSERT(PyDict_Check(arg3), "%s", getTypeName(arg3));
+    BoxedDict* attr_dict = static_cast<BoxedDict*>(arg3);
+
+    RELEASE_ASSERT(arg2->cls == tuple_cls, "");
+    BoxedTuple* bases = static_cast<BoxedTuple*>(arg2);
+
+    RELEASE_ASSERT(arg1->cls == str_cls, "");
+    BoxedString* name = static_cast<BoxedString*>(arg1);
+
+    return _typeNew(metatype, name, bases, attr_dict);
+}
+
 extern "C" void delGlobal(Box* globals, BoxedString* name) {
     if (globals->cls == module_cls) {
         BoxedModule* m = static_cast<BoxedModule*>(globals);

src/runtime/objmodel.h

@@ -160,7 +160,8 @@ extern "C" void raiseNotIterableError(const char* typeName) __attribute__((__nor
 extern "C" void raiseIndexErrorStr(const char* typeName) __attribute__((__noreturn__));
 
 Box* typeCall(Box*, BoxedTuple*, BoxedDict*);
-Box* typeNew(Box* cls, Box* arg1, Box* arg2, Box** _args);
+Box* type_new(BoxedClass* metatype, Box* args, Box* kwds) noexcept;
+Box* typeNewGeneric(Box* cls, Box* arg1, Box* arg2, Box** _args);
 
 // These process a potential descriptor, differing in their behavior if the object was not a descriptor.
 // the OrNull variant returns NULL to signify it wasn't a descriptor, and the processDescriptor version

src/runtime/types.cpp

@@ -3425,8 +3425,8 @@ void setupRuntime() {
     type_cls->giveAttr("__bases__", new (pyston_getset_cls) BoxedGetsetDescriptor(typeBases, typeSetBases, NULL));
     type_cls->giveAttr("__call__", new BoxedFunction(typeCallObj));
 
-    type_cls->giveAttr("__new__",
-                       new BoxedFunction(boxRTFunction((void*)typeNew, UNKNOWN, 4, 2, false, false), { NULL, NULL }));
+    type_cls->giveAttr("__new__", new BoxedFunction(boxRTFunction((void*)typeNewGeneric, UNKNOWN, 4, 2, false, false),
+                                                    { NULL, NULL }));
     type_cls->giveAttr("__repr__", new BoxedFunction(boxRTFunction((void*)typeRepr, STR, 1)));
     type_cls->tp_hash = (hashfunc)_Py_HashPointer;
     type_cls->giveAttr("__module__", new (pyston_getset_cls) BoxedGetsetDescriptor(typeModule, typeSetModule, NULL));
@@ -3437,6 +3437,7 @@ void setupRuntime() {
     type_cls->tp_richcompare = type_richcompare;
     add_operators(type_cls);
     type_cls->freeze();
+    type_cls->tp_new = type_new;
     type_cls->tpp_call = &typeTppCall;
 
     none_cls->giveAttr("__repr__", new BoxedFunction(boxRTFunction((void*)noneRepr, STR, 1)));

test/tests/ctypes_import.py

+# We can only import ctypes for now - it would be nice to have tests later,
+# which will probably involve running the existing test suite for ctype rather
+# than add our own here (unless there's special Pyston-related edge cases).
+import ctypes