use cpythons 'import' implementation

this fixes some smaller differences we had and adds some previously unsupported stuff to the imp module
(we could not load C extensions and were too pedantic about some args before, ...)
In addition this gives us:
 - import lock
 - we load now a lot of the builtin modules lazely
 - support for printing out which files get loaded in verbose mode
 - we can now delete a builtin module from sys.modules and import it again (some cpython tests use this)

It also made a classobj.__delattr__ problem appear

from_cpython/CMakeLists.txt

@@ -91,6 +91,7 @@ file(GLOB_RECURSE STDOBJECT_SRCS Objects
     exceptions.c
     floatobject.c
     fileobject.c
+    import.c
     iterobject.c
     memoryobject.c
     stringobject.c

from_cpython/Include/import.h

@@ -11,7 +11,7 @@ extern "C" {
 PyAPI_FUNC(long) PyImport_GetMagicNumber(void) PYSTON_NOEXCEPT;
 PyAPI_FUNC(PyObject *) PyImport_ExecCodeModule(const char *name, PyObject *co) PYSTON_NOEXCEPT;
 PyAPI_FUNC(PyObject *) PyImport_ExecCodeModuleEx(
-	char *name, PyObject *co, char *pathname) PYSTON_NOEXCEPT;
+    const char *name, PyObject *co, char *pathname) PYSTON_NOEXCEPT;
 PyAPI_FUNC(PyObject *) PyImport_GetModuleDict(void) PYSTON_NOEXCEPT;
 PyAPI_FUNC(PyObject *) PyImport_AddModule(const char *name) PYSTON_NOEXCEPT;
 PyAPI_FUNC(PyObject *) PyImport_ImportModule(const char *name) PYSTON_NOEXCEPT;
@@ -26,7 +26,7 @@ PyAPI_FUNC(PyObject *) PyImport_GetImporter(PyObject *path) PYSTON_NOEXCEPT;
 PyAPI_FUNC(PyObject *) PyImport_Import(PyObject *name) PYSTON_NOEXCEPT;
 PyAPI_FUNC(PyObject *) PyImport_ReloadModule(PyObject *m) PYSTON_NOEXCEPT;
 PyAPI_FUNC(void) PyImport_Cleanup(void) PYSTON_NOEXCEPT;
-PyAPI_FUNC(int) PyImport_ImportFrozenModule(char *) PYSTON_NOEXCEPT;
+PyAPI_FUNC(int) PyImport_ImportFrozenModule(const char *) PYSTON_NOEXCEPT;
 
 #ifdef WITH_THREAD
 PyAPI_FUNC(void) _PyImport_AcquireLock(void) PYSTON_NOEXCEPT;
@@ -45,12 +45,11 @@ PyAPI_FUNC(PyObject *) _PyImport_FindExtension(char *, char *) PYSTON_NOEXCEPT;
 PyAPI_FUNC(PyObject *) _PyImport_FixupExtension(char *, char *) PYSTON_NOEXCEPT;
 
 struct _inittab {
-    char *name;
+    const char *name;
     void (*initfunc)(void);
 };
 
-// Pyston change: this is no longer a static object
-//PyAPI_DATA(PyTypeObject) PyNullImporter_Type;
+PyAPI_DATA(PyTypeObject) PyNullImporter_Type;
 PyAPI_DATA(struct _inittab *) PyImport_Inittab;
 
 PyAPI_FUNC(int) PyImport_AppendInittab(const char *name, void (*initfunc)(void)) PYSTON_NOEXCEPT;

from_cpython/Lib/test/test_atexit.py

-# expected: fail
 import sys
 import unittest
 import StringIO

src/capi/modsupport.cpp

@@ -433,7 +433,7 @@ extern "C" PyObject* Py_InitModule4(const char* name, PyMethodDef* methods, cons
 
 extern "C" PyObject* PyModule_GetDict(PyObject* _m) noexcept {
     BoxedModule* m = static_cast<BoxedModule*>(_m);
-    assert(m->cls == module_cls);
+    assert(PyModule_Check(m));
 
     return m->getAttrWrapper();
 }

src/jit.cpp

@@ -188,9 +188,10 @@ int handleArg(char code) {
         TRAP = true;
     else if (code == 'q')
         GLOBAL_VERBOSITY = 0;
-    else if (code == 'v')
+    else if (code == 'v') {
+        Py_VerboseFlag++;
         GLOBAL_VERBOSITY++;
-    else if (code == 'd')
+    } else if (code == 'd')
         SHOW_DISASM = true;
     else if (code == 'I')
         FORCE_INTERPRETER = true;
@@ -276,7 +277,7 @@ static int RunMainFromImporter(const char* filename) {
     PyObject* argv0 = NULL, * importer = NULL;
 
     if ((argv0 = PyString_FromString(filename)) && (importer = PyImport_GetImporter(argv0))
-        && (importer->cls != null_importer_cls)) {
+        && (importer->cls != &PyNullImporter_Type)) {
         /* argv0 is usable as an import source, so
                put it in sys.path[0] and import __main__ */
         PyObject* sys_path = NULL;
@@ -436,8 +437,9 @@ static int main(int argc, char** argv) {
 
         if (!Py_NoSiteFlag) {
             try {
-                std::string module_name = "site";
-                importModuleLevel(module_name, None, None, 0);
+                Box* module = PyImport_ImportModule("site");
+                if (!module)
+                    throwCAPIException();
             } catch (ExcInfo e) {
                 e.printExcAndTraceback();
                 return 1;

src/runtime/builtin_modules/builtins.cpp

@@ -478,8 +478,11 @@ Box* bltinImport(Box* name, Box* globals, Box* locals, Box** args) {
         raiseExcHelper(TypeError, "an integer is required");
     }
 
-    std::string _name = static_cast<BoxedString*>(name)->s();
-    return importModuleLevel(_name, globals, fromlist, ((BoxedInt*)level)->n);
+    Box* rtn
+        = PyImport_ImportModuleLevel(((BoxedString*)name)->c_str(), globals, NULL, fromlist, ((BoxedInt*)level)->n);
+    if (!rtn)
+        throwCAPIException();
+    return rtn;
 }
 
 Box* delattrFunc(Box* obj, Box* _str) {
@@ -1992,7 +1995,7 @@ void setupBuiltins() {
                                    ParamNames({ "name", "globals", "locals", "fromlist", "level" }, "", ""));
     builtins_module->giveAttr("__import__",
                               new BoxedBuiltinFunctionOrMethod(import_func, "__import__",
-                                                               { None, None, None, boxInt(-1) }, NULL, import_doc));
+                                                               { NULL, NULL, NULL, boxInt(-1) }, NULL, import_doc));
 
     enumerate_cls = BoxedClass::create(type_cls, object_cls, &BoxedEnumerate::gcHandler, 0, 0, sizeof(BoxedEnumerate),
                                        false, "enumerate");

src/runtime/builtin_modules/sys.cpp

@@ -681,10 +681,6 @@ void setupSys() {
                                                   FunctionMetadata::create((void*)sysGetRecursionLimit, UNKNOWN, 0),
                                                   "getrecursionlimit", getrecursionlimit_doc));
 
-    sys_module->giveAttr("meta_path", new BoxedList());
-    sys_module->giveAttr("path_hooks", new BoxedList());
-    sys_module->giveAttr("path_importer_cache", new BoxedDict());
-
     // As we don't support compile() etc yet force 'dont_write_bytecode' to true.
     sys_module->giveAttr("dont_write_bytecode", True);
 
@@ -742,15 +738,30 @@ void setupSys() {
 
 void setupSysEnd() {
     std::vector<Box*, StlCompatAllocator<Box*>> builtin_module_names;
-    for (const auto& p : *sys_modules_dict) {
-        builtin_module_names.push_back(p.first);
-    }
+    for (int i = 0; PyImport_Inittab[i].name != NULL; i++)
+        builtin_module_names.push_back(boxString(PyImport_Inittab[i].name));
 
     std::sort<decltype(builtin_module_names)::iterator, PyLt>(builtin_module_names.begin(), builtin_module_names.end(),
                                                               PyLt());
 
     sys_module->giveAttr("builtin_module_names",
                          BoxedTuple::create(builtin_module_names.size(), &builtin_module_names[0]));
+
+#ifndef NDEBUG
+    for (const auto& p : *sys_modules_dict) {
+        assert(PyString_Check(p.first));
+
+        bool found = false;
+        for (int i = 0; PyImport_Inittab[i].name != NULL; i++) {
+            if (((BoxedString*)p.first)->s() == PyImport_Inittab[i].name) {
+                found = true;
+            }
+        }
+        if (!found)
+            assert(0 && "found a module which is inside sys.modules but not listed inside PyImport_Inittab!");
+    }
+#endif
+
     sys_flags_cls->finishInitialization();
 
     /* version_info */

src/runtime/capi.cpp

@@ -1443,17 +1443,6 @@ extern "C" void makePendingCalls() {
         throwCAPIException();
 }
 
-extern "C" PyObject* _PyImport_FixupExtension(char* name, char* filename) noexcept {
-    // Don't have to do anything here, since we will error in _PyImport_FindExtension
-    // TODO is this ok?
-    return NULL;
-}
-
-extern "C" PyObject* _PyImport_FindExtension(char* name, char* filename) noexcept {
-    fatalOrError(PyExc_NotImplementedError, "unimplemented");
-    return nullptr;
-}
-
 static PyObject* listmethodchain(PyMethodChain* chain) noexcept {
     PyMethodChain* c;
     PyMethodDef* ml;

src/runtime/dict.cpp

@@ -176,8 +176,11 @@ extern "C" void PyDict_Clear(PyObject* op) noexcept {
 }
 
 extern "C" PyObject* PyDict_Copy(PyObject* o) noexcept {
-    RELEASE_ASSERT(PyDict_Check(o), "");
+    RELEASE_ASSERT(PyDict_Check(o) || o->cls == attrwrapper_cls, "");
     try {
+        if (o->cls == attrwrapper_cls)
+            return attrwrapperToDict(o);
+
         return dictCopy(static_cast<BoxedDict*>(o));
     } catch (ExcInfo e) {
         setCAPIException(e);
@@ -651,15 +654,23 @@ void dictMergeFromSeq2(BoxedDict* self, Box* other) {
 }
 
 extern "C" int PyDict_Merge(PyObject* a, PyObject* b, int override_) noexcept {
-    if (a == NULL || !PyDict_Check(a) || b == NULL) {
-        PyErr_BadInternalCall();
-        return -1;
-    }
+    try {
+        if (a == NULL || !PyDict_Check(a) || b == NULL) {
+            if (a && b && a->cls == attrwrapper_cls) {
+                RELEASE_ASSERT(PyDict_Check(b) && override_ == 1, "");
+                for (auto&& item : *(BoxedDict*)b) {
+                    setitem(a, item.first, item.second);
+                }
+                return 0;
+            }
 
-    if (override_ != 1)
-        Py_FatalError("unimplemented");
+            PyErr_BadInternalCall();
+            return -1;
+        }
+
+        if (override_ != 1)
+            Py_FatalError("unimplemented");
 
-    try {
         dictMerge(static_cast<BoxedDict*>(a), b);
         return 0;
     } catch (ExcInfo e) {

src/runtime/import.h

@@ -21,7 +21,6 @@ namespace pyston {
 
 extern "C" PyObject* PyImport_GetImporter(PyObject* path) noexcept;
 extern "C" Box* import(int level, Box* from_imports, llvm::StringRef module_name);
-extern Box* importModuleLevel(llvm::StringRef module_name, Box* globals, Box* from_imports, int level);
 BoxedModule* importCExtension(BoxedString* full_name, const std::string& last_name, const std::string& path);
 }
 

src/runtime/types.cpp

@@ -88,6 +88,7 @@ extern "C" void init_sqlite3();
 extern "C" void PyMarshal_Init();
 extern "C" void initstrop();
 extern "C" void init_ast();
+extern "C" void initimp();
 
 namespace pyston {
 
@@ -1566,8 +1567,8 @@ void BoxedClosure::gcHandler(GCVisitor* v, Box* b) {
 extern "C" {
 BoxedClass* object_cls, *type_cls, *none_cls, *bool_cls, *int_cls, *float_cls,
     * str_cls = NULL, *function_cls, *instancemethod_cls, *list_cls, *slice_cls, *module_cls, *dict_cls, *tuple_cls,
-      *member_descriptor_cls, *closure_cls, *generator_cls, *null_importer_cls, *complex_cls, *basestring_cls,
-      *property_cls, *staticmethod_cls, *classmethod_cls, *attrwrapper_cls, *pyston_getset_cls, *capi_getset_cls,
+      *member_descriptor_cls, *closure_cls, *generator_cls, *complex_cls, *basestring_cls, *property_cls,
+      *staticmethod_cls, *classmethod_cls, *attrwrapper_cls, *pyston_getset_cls, *capi_getset_cls,
       *builtin_function_or_method_cls, *attrwrapperiter_cls, *set_cls, *frozenset_cls;
 
 BoxedTuple* EmptyTuple;
@@ -3546,6 +3547,10 @@ static void setTypeGCNone(BoxedClass* cls) {
 }
 
 static void setupDefaultClassGCParticipation() {
+    // we need to make sure this types are initalized
+    init_sre();
+    inititertools();
+
     // some additional setup to ensure weakrefs participate in our GC
     setTypeGCProxy(&_PyWeakref_RefType);
     setTypeGCProxy(&_PyWeakref_ProxyType);
@@ -3632,6 +3637,60 @@ static int _check_and_flush(FILE* stream) {
     return fflush(stream) || prev_fail ? EOF : 0;
 }
 
+extern "C" {
+struct _inittab _PyImport_Inittab[] = { { "array", initarray },
+                                        { "_ast", init_ast },
+                                        { "binascii", initbinascii },
+                                        { "_codecs", init_codecs },
+                                        { "_collections", init_collections },
+                                        { "cStringIO", initcStringIO },
+                                        { "_csv", init_csv },
+                                        { "datetime", initdatetime },
+                                        { "errno", initerrno },
+                                        { "fcntl", initfcntl },
+                                        { "_functools", init_functools },
+                                        { "imp", initimp },
+                                        { "_io", init_io },
+                                        { "itertools", inititertools },
+                                        { "marshal", PyMarshal_Init },
+                                        { "math", initmath },
+                                        { "_md5", init_md5 },
+                                        { "operator", initoperator },
+                                        { "posix", initposix },
+                                        { "pwd", initpwd },
+                                        { "_random", init_random },
+                                        { "resource", initresource },
+                                        { "select", initselect },
+                                        { "_sha", init_sha },
+                                        { "_sha256", init_sha256 },
+                                        { "_sha512", init_sha512 },
+                                        { "signal", initsignal },
+                                        { "_socket", init_socket },
+                                        { "_sqlite3", init_sqlite3 },
+                                        { "_sre", init_sre },
+                                        { "_ssl", init_ssl },
+                                        { "strop", initstrop },
+                                        { "_struct", init_struct },
+                                        { "time", inittime },
+                                        { "unicodedata", initunicodedata },
+                                        { "_warnings", _PyWarnings_Init },
+                                        { "_weakref", init_weakref },
+                                        { "zipimport", initzipimport },
+                                        { "zlib", initzlib },
+
+                                        { "gc", setupGC },
+                                        { "__pyston__", setupPyston },
+                                        { "thread", setupThread },
+
+                                        { "__main__", NULL },
+                                        { "__builtin__", NULL },
+                                        { "sys", NULL },
+                                        { "exceptions", NULL },
+
+
+                                        { 0, 0 } };
+}
+
 bool TRACK_ALLOCATIONS = false;
 void setupRuntime() {
 
@@ -4115,10 +4174,8 @@ void setupRuntime() {
 
     setupBuiltins();
     _PyExc_Init();
-    setupThread();
-    setupGC();
-    setupImport();
-    setupPyston();
+    _PyImport_Init();
+    _PyImportHooks_Init();
 
     PyType_Ready(&PyByteArrayIter_Type);
     PyType_Ready(&PyCapsule_Type);
@@ -4130,44 +4187,6 @@ void setupRuntime() {
     PyType_Ready(&PyDictProxy_Type);
     PyType_Ready(&PyTraceBack_Type);
 
-
-    initerrno();
-    init_sha();
-    init_sha256();
-    init_sha512();
-    init_md5();
-    init_random();
-    init_sre();
-    initmath();
-    initoperator();
-    initbinascii();
-    initpwd();
-    initposix();
-    init_struct();
-    initdatetime();
-    init_functools();
-    init_collections();
-    inititertools();
-    initresource();
-    initsignal();
-    initselect();
-    initfcntl();
-    inittime();
-    initarray();
-    initzlib();
-    init_codecs();
-    init_socket();
-    initunicodedata();
-    initcStringIO();
-    init_io();
-    initzipimport();
-    init_csv();
-    init_ssl();
-    init_sqlite3();
-    PyMarshal_Init();
-    initstrop();
-    init_ast();
-
     setupDefaultClassGCParticipation();
 
     assert(object_cls->tp_setattro == PyObject_GenericSetAttr);

src/runtime/types.h

@@ -89,10 +89,10 @@ extern "C" BoxedString* EmptyString;
 extern "C" {
 extern BoxedClass* object_cls, *type_cls, *bool_cls, *int_cls, *long_cls, *float_cls, *str_cls, *function_cls,
     *none_cls, *instancemethod_cls, *list_cls, *slice_cls, *module_cls, *dict_cls, *tuple_cls, *enumerate_cls,
-    *xrange_cls, *member_descriptor_cls, *null_importer_cls, *method_cls, *closure_cls, *generator_cls, *complex_cls,
-    *basestring_cls, *property_cls, *staticmethod_cls, *classmethod_cls, *attrwrapper_cls, *pyston_getset_cls,
-    *capi_getset_cls, *builtin_function_or_method_cls, *set_cls, *frozenset_cls, *code_cls, *frame_cls, *capifunc_cls,
-    *wrapperdescr_cls, *wrapperobject_cls;
+    *xrange_cls, *member_descriptor_cls, *method_cls, *closure_cls, *generator_cls, *complex_cls, *basestring_cls,
+    *property_cls, *staticmethod_cls, *classmethod_cls, *attrwrapper_cls, *pyston_getset_cls, *capi_getset_cls,
+    *builtin_function_or_method_cls, *set_cls, *frozenset_cls, *code_cls, *frame_cls, *capifunc_cls, *wrapperdescr_cls,
+    *wrapperobject_cls;
 }
 #define unicode_cls (&PyUnicode_Type)
 #define memoryview_cls (&PyMemoryView_Type)

test/CPYTHON_TEST_NOTES.md

@@ -24,7 +24,6 @@ test_al                 No module named al
 test_applesingle        Not really a failure, but it tries to skip itself and we don't support that
 test_argparse           [unknown]
 test_ascii_formatd      segfault in ctypes (but only on CI)
-test_atexit             [unknown]
 test_audioop            [unknown]
 test_bigmem             [unknown]
 test_bisect             somehow sys.modules['_bisect'] is getting set to 0

test/tests/imp_test.py

@@ -13,7 +13,9 @@ for a in (1, "", "/proc", "nonexisting_dir"):
     except Exception as e:
         print e
 
+print imp.lock_held()
 imp.acquire_lock()
+print imp.lock_held()
 imp.release_lock()
 
 import os

test/tests/import_test.py

@@ -35,3 +35,9 @@ import _multiprocessing
 del _multiprocessing
 del sys.modules["_multiprocessing"]
 import _multiprocessing
+
+import time
+del time
+del sys.modules["time"]
+import time
+print time.sleep(0)