Skip to content

P
Pyston
Commit 2d0eaa3d authored by Marius Wachtler's avatar Marius Wachtler
Browse files
Options

use cpythons 'import' implementation 

this fixes some smaller differences we had and adds some previously unsupported stuff to the imp module
(we could not load C extensions and were too pedantic about some args before, ...)
In addition this gives us:
 - import lock
 - we load now a lot of the builtin modules lazely
 - support for printing out which files get loaded in verbose mode
 - we can now delete a builtin module from sys.modules and import it again (some cpython tests use this)

It also made a classobj.__delattr__ problem appear
parent c90c9b1f
Hide whitespace changes
Inline Side-by-side
Showing with 254 additions and 985 deletions
from_cpython/CMakeLists.txt
View file @ 2d0eaa3d
......@@ -91,6 +91,7 @@ file(GLOB_RECURSE STDOBJECT_SRCS Objects
exceptions.c
floatobject.c
fileobject.c
import.c
iterobject.c
memoryobject.c
stringobject.c
......
from_cpython/Include/import.h
View file @ 2d0eaa3d
......@@ -11,7 +11,7 @@ extern "C" {
PyAPI_FUNC(long) PyImport_GetMagicNumber(void) PYSTON_NOEXCEPT;
PyAPI_FUNC(PyObject *) PyImport_ExecCodeModule(const char *name, PyObject *co) PYSTON_NOEXCEPT;
PyAPI_FUNC(PyObject *) PyImport_ExecCodeModuleEx(
char *name, PyObject *co, char *pathname) PYSTON_NOEXCEPT;
const char *name, PyObject *co, char *pathname) PYSTON_NOEXCEPT;
PyAPI_FUNC(PyObject *) PyImport_GetModuleDict(void) PYSTON_NOEXCEPT;
PyAPI_FUNC(PyObject *) PyImport_AddModule(const char *name) PYSTON_NOEXCEPT;
PyAPI_FUNC(PyObject *) PyImport_ImportModule(const char *name) PYSTON_NOEXCEPT;
......@@ -26,7 +26,7 @@ PyAPI_FUNC(PyObject *) PyImport_GetImporter(PyObject *path) PYSTON_NOEXCEPT;
PyAPI_FUNC(PyObject *) PyImport_Import(PyObject *name) PYSTON_NOEXCEPT;
PyAPI_FUNC(PyObject *) PyImport_ReloadModule(PyObject *m) PYSTON_NOEXCEPT;
PyAPI_FUNC(void) PyImport_Cleanup(void) PYSTON_NOEXCEPT;
PyAPI_FUNC(int) PyImport_ImportFrozenModule(char *) PYSTON_NOEXCEPT;
PyAPI_FUNC(int) PyImport_ImportFrozenModule(const char *) PYSTON_NOEXCEPT;
#ifdef WITH_THREAD
PyAPI_FUNC(void) _PyImport_AcquireLock(void) PYSTON_NOEXCEPT;
......@@ -45,12 +45,11 @@ PyAPI_FUNC(PyObject *) _PyImport_FindExtension(char *, char *) PYSTON_NOEXCEPT;
PyAPI_FUNC(PyObject *) _PyImport_FixupExtension(char *, char *) PYSTON_NOEXCEPT;
struct _inittab {
char *name;
const char *name;
void (*initfunc)(void);
};
// Pyston change: this is no longer a static object
//PyAPI_DATA(PyTypeObject) PyNullImporter_Type;
PyAPI_DATA(PyTypeObject) PyNullImporter_Type;
PyAPI_DATA(struct _inittab *) PyImport_Inittab;
PyAPI_FUNC(int) PyImport_AppendInittab(const char *name, void (*initfunc)(void)) PYSTON_NOEXCEPT;
......
from_cpython/Lib/test/test_atexit.py
View file @ 2d0eaa3d
# expected: fail
import sys
import unittest
import StringIO
......
from_cpython/Python/import.c
View file @ 2d0eaa3d
// This file is originally from CPython 2.7, with modifications for Pyston
/* Module definition and import implementation */
......@@ -81,6 +82,19 @@ typedef unsigned short mode_t;
*/
#define MAGIC (62211 | ((long)'\r'<<16) | ((long)'\n'<<24))
// Pyston change: add this definitions
const struct filedescr _PyImport_DynLoadFiletab[] = {
{".pyston.so", "rb", C_EXTENSION},
{0, 0}
};
static struct _frozen _PyImport_FrozenModules[] = {
{0, 0, 0}
};
struct _frozen *PyImport_FrozenModules = _PyImport_FrozenModules;
// Pyston change: add this static variable instead of storing it inside the interpreter state
static PyObject *modules_reloading = NULL;
/* Magic word as global; note that _PyImport_Init() can change the
value of this global to accommodate for alterations of how the
compiler works which are enabled by command line switches. */
......@@ -106,10 +120,13 @@ static const struct filedescr _PyImport_StandardFiletab[] = {
#else
static const struct filedescr _PyImport_StandardFiletab[] = {
{".py", "U", PY_SOURCE},
// Pyston change: disable the other file extensions
/*
#ifdef MS_WINDOWS
{".pyw", "U", PY_SOURCE},
#endif
{".pyc", "rb", PY_COMPILED},
*/
{0, 0}
};
#endif
......@@ -175,6 +192,9 @@ _PyImport_Init(void)
_PyImport_Filetab = filetab;
// Pyston change:
PyGC_AddPotentialRoot(&_PyImport_Filetab, sizeof(void*));
if (Py_OptimizeFlag) {
/* Replace ".pyc" with ".pyo" in _PyImport_Filetab */
for (; filetab->suffix != NULL; filetab++) {
......@@ -382,13 +402,17 @@ imp_release_lock(PyObject *self, PyObject *noargs)
static void
imp_modules_reloading_clear(void)
{
PyInterpreterState *interp = PyThreadState_Get()->interp;
if (interp->modules_reloading != NULL)
PyDict_Clear(interp->modules_reloading);
// Pyston change: use static variable instead of storing it inside the interpreter state
// PyInterpreterState *interp = PyThreadState_Get()->interp;
// if (interp->modules_reloading != NULL)
// PyDict_Clear(interp->modules_reloading);
if (modules_reloading != NULL)
PyDict_Clear(modules_reloading);
}
/* Helper for sys */
// Pyston change: use diferent implementation from import.cpp
#if 0
PyObject *
PyImport_GetModuleDict(void)
{
......@@ -397,7 +421,7 @@ PyImport_GetModuleDict(void)
Py_FatalError("PyImport_GetModuleDict: no module dictionary!");
return interp->modules;
}
#endif
/* List of names to clear in sys */
static char* sys_deletes[] = {
......@@ -419,7 +443,8 @@ static char* sys_files[] = {
/* Un-initialize things, as good as we can */
// Pyston change: we don't support calling cleanup currently
#if 0
void
PyImport_Cleanup(void)
{
......@@ -559,7 +584,7 @@ PyImport_GetMagicNumber(void)
{
return pyc_magic;
}
#endif
/* Magic for extension modules (built-in as well as dynamically
loaded). To prevent initializing an extension module more than
......@@ -576,7 +601,7 @@ _PyImport_FixupExtension(char *name, char *filename)
{
PyObject *modules, *mod, *dict, *copy;
if (extensions == NULL) {
extensions = PyDict_New();
extensions = PyGC_AddRoot(PyDict_New());
if (extensions == NULL)
return NULL;
}
......@@ -669,11 +694,13 @@ remove_module(const char *name)
* example.
*/
PyObject *
PyImport_ExecCodeModule(char *name, PyObject *co)
PyImport_ExecCodeModule(const char *name, PyObject *co)
{
return PyImport_ExecCodeModuleEx(name, co, (char *)NULL);
}
// Pyston change: use our implementation from import.cpp and disable compiled module support
#if 0
PyObject *
PyImport_ExecCodeModuleEx(char *name, PyObject *co, char *pathname)
{
......@@ -727,7 +754,6 @@ PyImport_ExecCodeModuleEx(char *name, PyObject *co, char *pathname)
return NULL;
}
/* Given a pathname for a Python source file, fill a buffer with the
pathname for the corresponding compiled file. Return the pathname
for the compiled file, or NULL if there's no space in the buffer.
......@@ -862,6 +888,14 @@ parse_source_module(const char *pathname, FILE *fp)
PyArena_Free(arena);
return co;
}
#else
static PyObject *
load_compiled_module(char *name, char *cpathname, FILE *fp) {
// we don't support compiled aka bytecode files currently
assert(0);
abort();
}
#endif
/* Helper to open a bytecode file for writing in exclusive mode */
......@@ -902,7 +936,7 @@ open_exclusive(char *filename, mode_t mode)
modification of its source into the header.
Errors are ignored, if a write error occurs an attempt is made to
remove the file. */
#if 0
static void
write_compiled_module(PyCodeObject *co, char *cpathname, struct stat *srcstat, time_t mtime)
{
......@@ -991,6 +1025,7 @@ update_compiled_module(PyCodeObject *co, char *pathname)
Py_DECREF(newname);
return 1;
}
#endif
#ifdef MS_WINDOWS
......@@ -1028,6 +1063,8 @@ win32_mtime(FILE *fp, char *pathname)
object WITH INCREMENTED REFERENCE COUNT. If there's a matching
byte-compiled file, use that instead. */
// Pyston change: use our implementation from import.cpp
#if 0
static PyObject *
load_source_module(char *name, char *pathname, FILE *fp)
{
......@@ -1107,13 +1144,17 @@ error_exit:
PyMem_FREE(buf);
return NULL;
}
#else
PyObject *
load_source_module(char *name, char *pathname, FILE *fp);
#endif
/* Forward */
static PyObject *load_module(char *, FILE *, char *, int, PyObject *);
static struct filedescr *find_module(char *, char *, PyObject *,
char *, size_t, FILE **, PyObject **);
static struct _frozen *find_frozen(char *name);
static struct _frozen *find_frozen(const char *name);
/* Load a package and return its module object WITH INCREMENTED
REFERENCE COUNT */
......@@ -2011,7 +2052,7 @@ init_builtin(char *name)
/* Frozen modules */
static struct _frozen *
find_frozen(char *name)
find_frozen(const char *name)
{
struct _frozen *p;
......@@ -2054,7 +2095,7 @@ get_frozen_object(char *name)
This function is also used from frozenmain.c */
int
PyImport_ImportFrozenModule(char *name)
PyImport_ImportFrozenModule(const char *name)
{
struct _frozen *p = find_frozen(name);
PyObject *co;
......@@ -2187,7 +2228,7 @@ PyImport_ImportModuleNoBlock(const char *name)
static PyObject *get_parent(PyObject *globals, char *buf,
Py_ssize_t *p_buflen, int level);
static PyObject *load_next(PyObject *mod, PyObject *altmod,
char **p_name, char *buf, Py_ssize_t *p_buflen);
const char **p_name, char *buf, Py_ssize_t *p_buflen);
static int mark_miss(char *name);
static int ensure_fromlist(PyObject *mod, PyObject *fromlist,
char *buf, Py_ssize_t buflen, int recursive);
......@@ -2196,7 +2237,7 @@ static PyObject * import_submodule(PyObject *mod, char *name, char *fullname);
/* The Magnum Opus of dotted-name import :-) */
static PyObject *
import_module_level(char *name, PyObject *globals, PyObject *locals,
import_module_level(const char *name, PyObject *globals, PyObject *locals,
PyObject *fromlist, int level)
{
char *buf;
......@@ -2280,7 +2321,7 @@ error_exit:
}
PyObject *
PyImport_ImportModuleLevel(char *name, PyObject *globals, PyObject *locals,
PyImport_ImportModuleLevel(const char *name, PyObject *globals, PyObject *locals,
PyObject *fromlist, int level)
{
PyObject *result;
......@@ -2315,21 +2356,23 @@ get_parent(PyObject *globals, char *buf, Py_ssize_t *p_buflen, int level)
PyObject *pkgname, *modname, *modpath, *modules, *parent;
int orig_level = level;
if (globals == NULL || !PyDict_Check(globals) || !level)
// Pyston change: support attrwrapper as globals
// if (globals == NULL || !PyDict_Check(globals) || !level)
if (globals == NULL || (!PyDict_Check(globals) && globals->ob_type != &PyAttrWrapper_Type) || !level)
return Py_None;
if (namestr == NULL) {
namestr = PyString_InternFromString("__name__");
namestr = PyGC_AddRoot(PyString_InternFromString("__name__"));
if (namestr == NULL)
return NULL;
}
if (pathstr == NULL) {
pathstr = PyString_InternFromString("__path__");
pathstr = PyGC_AddRoot(PyString_InternFromString("__path__"));
if (pathstr == NULL)
return NULL;
}
if (pkgstr == NULL) {
pkgstr = PyString_InternFromString("__package__");
pkgstr = PyGC_AddRoot(PyString_InternFromString("__package__"));
if (pkgstr == NULL)
return NULL;
}
......@@ -2469,10 +2512,10 @@ get_parent(PyObject *globals, char *buf, Py_ssize_t *p_buflen, int level)
/* altmod is either None or same as mod */
static PyObject *
load_next(PyObject *mod, PyObject *altmod, char **p_name, char *buf,
load_next(PyObject *mod, PyObject *altmod, const char **p_name, char *buf,
Py_ssize_t *p_buflen)
{
char *name = *p_name;
const char *name = *p_name;
char *dot = strchr(name, '.');
size_t len;
char *p;
......@@ -2715,11 +2758,13 @@ import_submodule(PyObject *mod, char *subname, char *fullname)
/* Re-import a module of any kind and return its module object, WITH
INCREMENTED REFERENCE COUNT */
PyObject *
PyImport_ReloadModule(PyObject *m)
{
PyInterpreterState *interp = PyThreadState_Get()->interp;
PyObject *modules_reloading = interp->modules_reloading;
// Pyston change: use static variable instead of storing it inside the interpreter state
// PyInterpreterState *interp = PyThreadState_Get()->interp;
// PyObject *modules_reloading = interp->modules_reloading;*/
PyObject *modules = PyImport_GetModuleDict();
PyObject *path = NULL, *loader = NULL, *existing_m = NULL;
char *name, *subname;
......@@ -2728,6 +2773,10 @@ PyImport_ReloadModule(PyObject *m)
FILE *fp = NULL;
PyObject *newm;
// Pyston change: initalize static variable
if (!modules_reloading)
modules_reloading = PyGC_AddRoot(PyDict_New());
if (modules_reloading == NULL) {
Py_FatalError("PyImport_ReloadModule: "
"no modules_reloading dictionary!");
......@@ -2840,13 +2889,13 @@ PyImport_Import(PyObject *module_name)
/* Initialize constant string objects */
if (silly_list == NULL) {
import_str = PyString_InternFromString("__import__");
import_str = PyGC_AddRoot(PyString_InternFromString("__import__"));
if (import_str == NULL)
return NULL;
builtins_str = PyString_InternFromString("__builtins__");
builtins_str = PyGC_AddRoot(PyString_InternFromString("__builtins__"));
if (builtins_str == NULL)
return NULL;
silly_list = Py_BuildValue("[s]", "__doc__");
silly_list = PyGC_AddRoot(Py_BuildValue("[s]", "__doc__"));
if (silly_list == NULL)
return NULL;
}
......@@ -2871,7 +2920,9 @@ PyImport_Import(PyObject *module_name)
}
/* Get the __import__ function from the builtins */
if (PyDict_Check(builtins)) {
// Pyston change: add support for attrwrapper
// if (PyDict_Check(builtins)) {
if (PyDict_Check(builtins) || builtins->ob_type == &PyAttrWrapper_Type) {
import = PyObject_GetItem(builtins, import_str);
if (import == NULL)
PyErr_SetObject(PyExc_KeyError, import_str);
......@@ -2904,6 +2955,10 @@ imp_get_magic(PyObject *self, PyObject *noargs)
{
char buf[4];
// Pyston change: we have different pyc
assert(0);
abort();
buf[0] = (char) ((pyc_magic >> 0) & 0xff);
buf[1] = (char) ((pyc_magic >> 8) & 0xff);
buf[2] = (char) ((pyc_magic >> 16) & 0xff);
......@@ -3426,7 +3481,6 @@ initimp(void)
returned and the existing table is unchanged.
After a similar function by Just van Rossum. */
int
PyImport_ExtendInittab(struct _inittab *newtab)
{
......@@ -3442,6 +3496,10 @@ PyImport_ExtendInittab(struct _inittab *newtab)
for (i = 0; PyImport_Inittab[i].name != NULL; i++)
;
// Pyston change: Let the GC know about the variable
if (!our_copy)
PyGC_AddPotentialRoot(&our_copy, sizeof(void*));
/* Allocate new memory for the combined table */
p = our_copy;
PyMem_RESIZE(p, struct _inittab, i+n+1);
......
src/capi/modsupport.cpp
View file @ 2d0eaa3d
......@@ -433,7 +433,7 @@ extern "C" PyObject* Py_InitModule4(const char* name, PyMethodDef* methods, cons
extern "C" PyObject* PyModule_GetDict(PyObject* _m) noexcept {
BoxedModule* m = static_cast<BoxedModule*>(_m);
assert(m->cls == module_cls);
assert(PyModule_Check(m));
return m->getAttrWrapper();
}
......
src/jit.cpp
View file @ 2d0eaa3d
......@@ -188,9 +188,10 @@ int handleArg(char code) {
TRAP = true;
else if (code == 'q')
GLOBAL_VERBOSITY = 0;
else if (code == 'v')
else if (code == 'v') {
Py_VerboseFlag++;
GLOBAL_VERBOSITY++;
else if (code == 'd')
} else if (code == 'd')
SHOW_DISASM = true;
else if (code == 'I')
FORCE_INTERPRETER = true;
......@@ -276,7 +277,7 @@ static int RunMainFromImporter(const char* filename) {
PyObject* argv0 = NULL, * importer = NULL;
if ((argv0 = PyString_FromString(filename)) && (importer = PyImport_GetImporter(argv0))
&& (importer->cls != null_importer_cls)) {
&& (importer->cls != &PyNullImporter_Type)) {
/* argv0 is usable as an import source, so
put it in sys.path[0] and import __main__ */
PyObject* sys_path = NULL;
......@@ -436,8 +437,9 @@ static int main(int argc, char** argv) {
if (!Py_NoSiteFlag) {
try {
std::string module_name = "site";
importModuleLevel(module_name, None, None, 0);
Box* module = PyImport_ImportModule("site");
if (!module)
throwCAPIException();
} catch (ExcInfo e) {
e.printExcAndTraceback();
return 1;
......
src/runtime/builtin_modules/builtins.cpp
View file @ 2d0eaa3d
......@@ -478,8 +478,11 @@ Box* bltinImport(Box* name, Box* globals, Box* locals, Box** args) {
raiseExcHelper(TypeError, "an integer is required");
}
std::string _name = static_cast<BoxedString*>(name)->s();
return importModuleLevel(_name, globals, fromlist, ((BoxedInt*)level)->n);
Box* rtn
= PyImport_ImportModuleLevel(((BoxedString*)name)->c_str(), globals, NULL, fromlist, ((BoxedInt*)level)->n);
if (!rtn)
throwCAPIException();
return rtn;
}
Box* delattrFunc(Box* obj, Box* _str) {
......@@ -1992,7 +1995,7 @@ void setupBuiltins() {
ParamNames({ "name", "globals", "locals", "fromlist", "level" }, "", ""));
builtins_module->giveAttr("__import__",
new BoxedBuiltinFunctionOrMethod(import_func, "__import__",
{ None, None, None, boxInt(-1) }, NULL, import_doc));
{ NULL, NULL, NULL, boxInt(-1) }, NULL, import_doc));
enumerate_cls = BoxedClass::create(type_cls, object_cls, &BoxedEnumerate::gcHandler, 0, 0, sizeof(BoxedEnumerate),
false, "enumerate");
......
src/runtime/builtin_modules/sys.cpp
View file @ 2d0eaa3d
......@@ -681,10 +681,6 @@ void setupSys() {
FunctionMetadata::create((void*)sysGetRecursionLimit, UNKNOWN, 0),
"getrecursionlimit", getrecursionlimit_doc));
sys_module->giveAttr("meta_path", new BoxedList());
sys_module->giveAttr("path_hooks", new BoxedList());
sys_module->giveAttr("path_importer_cache", new BoxedDict());
// As we don't support compile() etc yet force 'dont_write_bytecode' to true.
sys_module->giveAttr("dont_write_bytecode", True);
......@@ -742,15 +738,30 @@ void setupSys() {
void setupSysEnd() {
std::vector<Box*, StlCompatAllocator<Box*>> builtin_module_names;
for (const auto& p : *sys_modules_dict) {
builtin_module_names.push_back(p.first);
}
for (int i = 0; PyImport_Inittab[i].name != NULL; i++)
builtin_module_names.push_back(boxString(PyImport_Inittab[i].name));
std::sort<decltype(builtin_module_names)::iterator, PyLt>(builtin_module_names.begin(), builtin_module_names.end(),
PyLt());
sys_module->giveAttr("builtin_module_names",
BoxedTuple::create(builtin_module_names.size(), &builtin_module_names[0]));
#ifndef NDEBUG
for (const auto& p : *sys_modules_dict) {
assert(PyString_Check(p.first));
bool found = false;
for (int i = 0; PyImport_Inittab[i].name != NULL; i++) {
if (((BoxedString*)p.first)->s() == PyImport_Inittab[i].name) {
found = true;
}
}
if (!found)
assert(0 && "found a module which is inside sys.modules but not listed inside PyImport_Inittab!");
}
#endif
sys_flags_cls->finishInitialization();
/* version_info */
......
src/runtime/capi.cpp
View file @ 2d0eaa3d
......@@ -1443,17 +1443,6 @@ extern "C" void makePendingCalls() {
throwCAPIException();
}
extern "C" PyObject* _PyImport_FixupExtension(char* name, char* filename) noexcept {
// Don't have to do anything here, since we will error in _PyImport_FindExtension
// TODO is this ok?
return NULL;
}
extern "C" PyObject* _PyImport_FindExtension(char* name, char* filename) noexcept {
fatalOrError(PyExc_NotImplementedError, "unimplemented");
return nullptr;
}
static PyObject* listmethodchain(PyMethodChain* chain) noexcept {
PyMethodChain* c;
PyMethodDef* ml;
......
src/runtime/dict.cpp
View file @ 2d0eaa3d
......@@ -176,8 +176,11 @@ extern "C" void PyDict_Clear(PyObject* op) noexcept {
}
extern "C" PyObject* PyDict_Copy(PyObject* o) noexcept {
RELEASE_ASSERT(PyDict_Check(o), "");
RELEASE_ASSERT(PyDict_Check(o) || o->cls == attrwrapper_cls, "");
try {
if (o->cls == attrwrapper_cls)
return attrwrapperToDict(o);
return dictCopy(static_cast<BoxedDict*>(o));
} catch (ExcInfo e) {
setCAPIException(e);
......@@ -651,15 +654,23 @@ void dictMergeFromSeq2(BoxedDict* self, Box* other) {
}
extern "C" int PyDict_Merge(PyObject* a, PyObject* b, int override_) noexcept {
if (a == NULL || !PyDict_Check(a) || b == NULL) {
PyErr_BadInternalCall();
return -1;
}
try {
if (a == NULL || !PyDict_Check(a) || b == NULL) {
if (a && b && a->cls == attrwrapper_cls) {
RELEASE_ASSERT(PyDict_Check(b) && override_ == 1, "");
for (auto&& item : *(BoxedDict*)b) {
setitem(a, item.first, item.second);
}
return 0;
}
if (override_ != 1)
Py_FatalError("unimplemented");
PyErr_BadInternalCall();
return -1;
}
if (override_ != 1)
Py_FatalError("unimplemented");
try {
dictMerge(static_cast<BoxedDict*>(a), b);
return 0;
} catch (ExcInfo e) {
......
src/runtime/import.cpp
View file @ 2d0eaa3d
......@@ -32,645 +32,62 @@
namespace pyston {
static const std::string all_str("__all__");
static const std::string name_str("__name__");
static const std::string package_str("__package__");
static void removeModule(BoxedString* name) {
BoxedDict* d = getSysModulesDict();
d->d.erase(name);
}
Box* createAndRunModule(BoxedString* name, const std::string& fn) {
BoxedModule* module = createModule(name, fn.c_str());
AST_Module* ast = caching_parse_file(fn.c_str(), /* future_flags = */ 0);
assert(ast);
try {
compileAndRunModule(ast, module);
} catch (ExcInfo e) {
removeModule(name);
throw e;
}
Box* r = getSysModulesDict()->getOrNull(name);
if (!r)
raiseExcHelper(ImportError, "Loaded module %.200s not found in sys.modules", name->c_str());
return r;
}
static Box* createAndRunModule(BoxedString* name, const std::string& fn, const std::string& module_path) {
BoxedModule* module = createModule(name, fn.c_str());
Box* b_path = boxString(module_path);
BoxedList* path_list = new BoxedList();
listAppendInternal(path_list, b_path);
static BoxedString* path_str = internStringImmortal("__path__");
module->setattr(path_str, path_list, NULL);
AST_Module* ast = caching_parse_file(fn.c_str(), /* future_flags = */ 0);
assert(ast);
try {
compileAndRunModule(ast, module);
} catch (ExcInfo e) {
removeModule(name);
throw e;
}
Box* r = getSysModulesDict()->getOrNull(name);
if (!r)
raiseExcHelper(ImportError, "Loaded module %.200s not found in sys.modules", name->c_str());
return r;
}
#if LLVMREV < 210072
#define LLVM_SYS_FS_EXISTS_CODE_OKAY(code) ((code) == 0)
#else
#define LLVM_SYS_FS_EXISTS_CODE_OKAY(code) (!(code))
#endif
static bool pathExists(const std::string& path) {
#if LLVMREV < 217625
bool exists;
llvm_error_code code = llvm::sys::fs::exists(path, exists);
assert(LLVM_SYS_FS_EXISTS_CODE_OKAY(code));
#else
bool exists = llvm::sys::fs::exists(path);
#endif
return exists;
}
/* Return an importer object for a sys.path/pkg.__path__ item 'p',
possibly by fetching it from the path_importer_cache dict. If it
wasn't yet cached, traverse path_hooks until a hook is found
that can handle the path item. Return None if no hook could;
this tells our caller it should fall back to the builtin
import mechanism. Cache the result in path_importer_cache.
Returns a borrowed reference. */
extern "C" PyObject* get_path_importer(PyObject* path_importer_cache, PyObject* path_hooks, PyObject* p) noexcept {
PyObject* importer;
Py_ssize_t j, nhooks;
/* These conditions are the caller's responsibility: */
assert(PyList_Check(path_hooks));
assert(PyDict_Check(path_importer_cache));
nhooks = PyList_Size(path_hooks);
if (nhooks < 0)
return NULL; /* Shouldn't happen */
importer = PyDict_GetItem(path_importer_cache, p);
if (importer != NULL)
return importer;
/* set path_importer_cache[p] to None to avoid recursion */
if (PyDict_SetItem(path_importer_cache, p, Py_None) != 0)
return NULL;
for (j = 0; j < nhooks; j++) {
PyObject* hook = PyList_GetItem(path_hooks, j);
if (hook == NULL)
return NULL;
importer = PyObject_CallFunctionObjArgs(hook, p, NULL);
if (importer != NULL)
break;
if (!PyErr_ExceptionMatches(PyExc_ImportError)) {
return NULL;
}
PyErr_Clear();
}
if (importer == NULL) {
importer = PyObject_CallFunctionObjArgs(null_importer_cls, p, NULL);
if (importer == NULL) {
if (PyErr_ExceptionMatches(PyExc_ImportError)) {
PyErr_Clear();
return Py_None;
}
}
}
if (importer != NULL) {
int err = PyDict_SetItem(path_importer_cache, p, importer);
Py_DECREF(importer);
if (err != 0)
return NULL;
}
return importer;
}
struct SearchResult {
// Each of these fields are only valid/used for certain filetypes:
std::string path;
Box* loader;
enum filetype {
SEARCH_ERROR,
PY_SOURCE,
PY_COMPILED,
C_EXTENSION,
PY_RESOURCE, /* Mac only */
PKG_DIRECTORY,
C_BUILTIN,
PY_FROZEN,
PY_CODERESOURCE, /* Mac only */
IMP_HOOK
} type;
SearchResult(const std::string& path, filetype type) : path(path), type(type) {}
SearchResult(std::string&& path, filetype type) : path(std::move(path)), type(type) {}
SearchResult(Box* loader) : loader(loader), type(IMP_HOOK) {}
};
SearchResult findModule(const std::string& name, BoxedString* full_name, BoxedList* path_list, bool call_import_hooks) {
static BoxedString* findmodule_str = internStringImmortal("find_module");
if (call_import_hooks) {
static BoxedString* meta_path_str = internStringImmortal("meta_path");
BoxedList* meta_path = static_cast<BoxedList*>(sys_module->getattr(meta_path_str));
if (!meta_path || meta_path->cls != list_cls)
raiseExcHelper(RuntimeError, "sys.meta_path must be a list of import hooks");
for (int i = 0; i < meta_path->size; i++) {
Box* finder = meta_path->elts->elts[i];
auto path_pass = path_list ? path_list : None;
CallattrFlags callattr_flags{.cls_only = false, .null_on_nonexistent = false, .argspec = ArgPassSpec(2) };
Box* loader = callattr(finder, findmodule_str, callattr_flags, full_name, path_pass, NULL, NULL, NULL);
if (loader != None)
return SearchResult(loader);
}
}
if (!path_list)
path_list = getSysPath();
if (path_list == NULL || path_list->cls != list_cls) {
raiseExcHelper(RuntimeError, "sys.path must be a list of directory names");
}
static BoxedString* path_hooks_str = internStringImmortal("path_hooks");
BoxedList* path_hooks = static_cast<BoxedList*>(sys_module->getattr(path_hooks_str));
if (!path_hooks || path_hooks->cls != list_cls)
raiseExcHelper(RuntimeError, "sys.path_hooks must be a list of import hooks");
static BoxedString* path_importer_cache_str = internStringImmortal("path_importer_cache");
BoxedDict* path_importer_cache = static_cast<BoxedDict*>(sys_module->getattr(path_importer_cache_str));
if (!path_importer_cache || path_importer_cache->cls != dict_cls)
raiseExcHelper(RuntimeError, "sys.path_importer_cache must be a dict");
llvm::SmallString<128> joined_path;
for (int i = 0; i < path_list->size; i++) {
Box* _p = path_list->elts->elts[i];
if (isSubclass(_p->cls, unicode_cls)) {
_p = PyUnicode_AsEncodedString(_p, Py_FileSystemDefaultEncoding, NULL);
if (_p == NULL) {
continue;
}
}
if (!isSubclass(_p->cls, str_cls)) {
continue;
}
BoxedString* p = static_cast<BoxedString*>(_p);
joined_path.clear();
llvm::sys::path::append(joined_path, p->s(), name);
std::string dn(joined_path.str());
llvm::sys::path::append(joined_path, "__init__.py");
std::string fn(joined_path.str());
if (call_import_hooks) {
PyObject* importer = get_path_importer(path_importer_cache, path_hooks, _p);
if (importer == NULL)
throwCAPIException();
if (importer != None) {
CallattrFlags callattr_flags{.cls_only = false,
.null_on_nonexistent = false,
.argspec = ArgPassSpec(1) };
Box* loader = callattr(importer, findmodule_str, callattr_flags, full_name, NULL, NULL, NULL, NULL);
if (loader != None)
return SearchResult(loader);
}
}
if (pathExists(fn))
return SearchResult(std::move(dn), SearchResult::PKG_DIRECTORY);
joined_path.clear();
llvm::sys::path::append(joined_path, std::string(p->s()), name + ".py");
fn = joined_path.str();
if (pathExists(fn))
return SearchResult(std::move(fn), SearchResult::PY_SOURCE);
joined_path.clear();
llvm::sys::path::append(joined_path, p->s(), name + ".pyston.so");
fn = joined_path.str();
if (pathExists(fn))
return SearchResult(std::move(fn), SearchResult::C_EXTENSION);
}
return SearchResult("", SearchResult::SEARCH_ERROR);
}
PyObject* PyImport_GetImporter(PyObject* path) noexcept {
PyObject* importer = NULL, * path_importer_cache = NULL, * path_hooks = NULL;
if ((path_importer_cache = PySys_GetObject("path_importer_cache"))) {
if ((path_hooks = PySys_GetObject("path_hooks"))) {
importer = get_path_importer(path_importer_cache, path_hooks, path);
}
}
Py_XINCREF(importer); /* get_path_importer returns a borrowed reference */
return importer;
}
/* Return the package that an import is being performed in. If globals comes
from the module foo.bar.bat (not itself a package), this returns the
sys.modules entry for foo.bar. If globals is from a package's __init__.py,
the package's entry in sys.modules is returned, as a borrowed reference.
The *name* of the returned package is returned in buf.
If globals doesn't come from a package or a module in a package, or a
corresponding entry is not found in sys.modules, Py_None is returned.
*/
static Box* getParent(Box* globals, int level, std::string& buf) {
int orig_level = level;
if (globals == NULL || globals == None || level == 0)
return None;
static BoxedString* package_str = internStringImmortal("__package__");
BoxedString* pkgname = static_cast<BoxedString*>(getFromGlobals(globals, package_str));
if (pkgname != NULL && pkgname != None) {
/* __package__ is set, so use it */
if (pkgname->cls != str_cls) {
raiseExcHelper(ValueError, "__package__ set to non-string");
}
size_t len = pkgname->size();
if (len == 0) {
if (level > 0) {
raiseExcHelper(ValueError, "Attempted relative import in non-package");
}
return None;
}
if (len > PATH_MAX) {
raiseExcHelper(ValueError, "Package name too long");
}
buf += pkgname->s();
} else {
static BoxedString* name_str = internStringImmortal("__name__");
/* __package__ not set, so figure it out and set it */
BoxedString* modname = static_cast<BoxedString*>(getFromGlobals(globals, name_str));
if (modname == NULL || modname->cls != str_cls)
return None;
static BoxedString* path_str = internStringImmortal("__path__");
Box* modpath = getFromGlobals(globals, path_str);
if (modpath != NULL) {
/* __path__ is set, so modname is already the package name */
if (modname->size() > PATH_MAX) {
raiseExcHelper(ValueError, "Module name too long");
}
buf += modname->s();
setGlobal(globals, package_str, modname);
} else {
/* Normal module, so work out the package name if any */
size_t lastdot = modname->s().rfind('.');
if (lastdot == std::string::npos && level > 0) {
raiseExcHelper(ValueError, "Attempted relative import in non-package");
}
if (lastdot == std::string::npos) {
setGlobal(globals, package_str, None);
return None;
}
if (lastdot >= PATH_MAX) {
raiseExcHelper(ValueError, "Module name too long");
}
buf = std::string(modname->s(), 0, lastdot);
setGlobal(globals, package_str, boxString(buf));
}
}
size_t dot = buf.size() - 1;
while (--level > 0) {
dot = buf.rfind('.', dot);
if (dot == std::string::npos) {
raiseExcHelper(ValueError, "Attempted relative import beyond toplevel package");
}
dot--;
}
buf = std::string(buf, 0, dot + 1);
BoxedDict* sys_modules = getSysModulesDict();
Box* boxed_name = boxString(buf);
Box* parent = sys_modules->d.find(boxed_name) != sys_modules->d.end() ? sys_modules->d[boxed_name] : NULL;
if (parent == NULL) {
if (orig_level < 1) {
printf("Warning: Parent module '%.200s' not found "
"while handling absolute import\n",
buf.c_str());
} else {
raiseExcHelper(SystemError, "Parent module '%.200s' not loaded, "
"cannot perform relative import",
buf.c_str());
}
}
return parent;
/* We expect, but can't guarantee, if parent != None, that:
- parent.__name__ == buf
- parent.__dict__ is globals
If this is violated... Who cares? */
}
static Box* importSub(const std::string& name, BoxedString* full_name, Box* parent_module) {
BoxedDict* sys_modules = getSysModulesDict();
if (sys_modules->d.find(full_name) != sys_modules->d.end()) {
return sys_modules->d[full_name];
}
BoxedList* path_list;
if (parent_module == NULL || parent_module == None) {
path_list = NULL;
} else {
static BoxedString* path_str = internStringImmortal("__path__");
path_list = static_cast<BoxedList*>(getattrInternal<ExceptionStyle::CXX>(parent_module, path_str));
if (path_list == NULL || path_list->cls != list_cls) {
return None;
}
}
SearchResult sr = findModule(name, full_name, path_list, true /* call_import_hooks */);
if (sr.type != SearchResult::SEARCH_ERROR) {
Box* module;
try {
if (sr.type == SearchResult::PY_SOURCE)
module = createAndRunModule(full_name, sr.path);
else if (sr.type == SearchResult::PKG_DIRECTORY)
module = createAndRunModule(full_name, sr.path + "/__init__.py", sr.path);
else if (sr.type == SearchResult::C_EXTENSION)
module = importCExtension(full_name, name, sr.path);
else if (sr.type == SearchResult::IMP_HOOK) {
static BoxedString* loadmodule_str = internStringImmortal("load_module");
CallattrFlags callattr_flags{.cls_only = false,
.null_on_nonexistent = false,
.argspec = ArgPassSpec(1) };
module = callattr(sr.loader, loadmodule_str, callattr_flags, full_name, NULL, NULL, NULL, NULL);
} else
RELEASE_ASSERT(0, "%d", sr.type);
} catch (ExcInfo e) {
removeModule(full_name);
throw e;
}
if (parent_module && parent_module != None)
parent_module->setattr(internStringMortal(name), module, NULL);
return module;
}
return None;
}
static void markMiss(std::string& name) {
BoxedDict* modules = getSysModulesDict();
Box* b_name = boxString(name);
modules->d[b_name] = None;
}
/* altmod is either None or same as mod */
static bool loadNext(Box* mod, Box* altmod, std::string& name, std::string& buf, Box** rtn) {
size_t dot = name.find('.');
size_t len;
Box* result;
bool call_again = true;
if (name.size() == 0) {
/* completely empty module name should only happen in
'from . import' (or '__import__("")')*/
*rtn = mod;
return false;
}
std::string local_name(name);
if (dot == std::string::npos) {
len = name.size();
call_again = false;
} else {
name = name.substr(dot + 1);
len = dot;
}
if (len == 0) {
raiseExcHelper(ValueError, "Empty module name");
}
if (buf.size() != 0)
buf += ".";
if (buf.size() >= PATH_MAX) {
raiseExcHelper(ValueError, "Module name too long");
}
std::string subname(local_name.substr(0, len));
buf += subname;
result = importSub(subname, boxString(buf), mod);
if (result == None && altmod != mod) {
/* Here, altmod must be None and mod must not be None */
result = importSub(subname, boxString(subname), altmod);
if (result != NULL && result != None) {
markMiss(buf);
buf = subname;
}
}
if (result == NULL)
throwCAPIException();
if (result == None)
raiseExcHelper(ImportError, "No module named %.200s", local_name.c_str());
*rtn = result;
return call_again;
}
static void ensureFromlist(Box* module, Box* fromlist, std::string& buf, bool recursive);
Box* importModuleLevel(llvm::StringRef name, Box* globals, Box* from_imports, int level) {
bool return_first = from_imports == None;
static StatCounter slowpath_import("slowpath_import");
slowpath_import.log();
std::string buf;
Box* parent = getParent(globals, level, buf);
if (!parent)
return NULL;
// make a copy of the string we pass in, since loadNext while
// modify the string as we load the modules along the path.
std::string _name = name;
Box* head;
bool again = loadNext(parent, level < 0 ? None : parent, _name, buf, &head);
if (head == NULL)
return NULL;
Box* tail = head;
while (again) {
Box* next;
again = loadNext(tail, tail, _name, buf, &next);
if (next == NULL) {
return NULL;
}
tail = next;
}
if (tail == None) {
/* If tail is Py_None, both get_parent and load_next found
an empty module name: someone called __import__("") or
doctored faulty bytecode */
raiseExcHelper(ValueError, "Empty module name");
}
Box* module = return_first ? head : tail;
if (from_imports != None) {
ensureFromlist(module, from_imports, buf, false);
}
return module;
}
extern "C" void _PyImport_AcquireLock() noexcept {
// TODO: currently no import lock!
}
extern "C" int _PyImport_ReleaseLock() noexcept {
// TODO: currently no import lock!
return 1;
}
extern "C" void _PyImport_ReInitLock() noexcept {
// TODO: currently no import lock!
}
extern "C" PyObject* PyImport_ImportModuleNoBlock(const char* name) noexcept {
return PyImport_ImportModule(name);
}
// This function has the same behaviour as __import__()
extern "C" PyObject* PyImport_ImportModuleLevel(const char* name, PyObject* globals, PyObject* locals,
PyObject* fromlist, int level) noexcept {
extern "C" PyObject* PyImport_GetModuleDict(void) noexcept {
try {
std::string module_name = name;
return importModuleLevel(module_name, globals, fromlist ? fromlist : None, level);
return getSysModulesDict();
} catch (ExcInfo e) {
setCAPIException(e);
return NULL;
}
}
static void ensureFromlist(Box* module, Box* fromlist, std::string& buf, bool recursive) {
static BoxedString* path_str = internStringImmortal("__path__");
Box* pathlist = NULL;
extern "C" PyObject* _PyImport_LoadDynamicModule(char* name, char* pathname, FILE* fp) noexcept {
BoxedString* name_boxed = boxString(name);
try {
pathlist = getattrInternal<ExceptionStyle::CXX>(module, path_str);
} catch (ExcInfo e) {
if (!e.matches(AttributeError))
throw e;
}
if (pathlist == NULL) {
// If it's not a package, then there's no sub-importing to do
return;
}
for (Box* _s : fromlist->pyElements()) {
RELEASE_ASSERT(PyString_Check(_s), "");
BoxedString* s = static_cast<BoxedString*>(_s);
internStringMortalInplace(s);
if (s->s()[0] == '*') {
// If __all__ contains a '*', just skip it:
if (recursive)
continue;
static BoxedString* all_str = internStringImmortal("__all__");
Box* all = getattrInternal<ExceptionStyle::CXX>(module, all_str);
if (all) {
ensureFromlist(module, all, buf, true);
}
continue;
const char* lastdot = strrchr(name, '.');
const char* shortname;
if (lastdot == NULL) {
shortname = name;
} else {
shortname = lastdot + 1;
}
Box* attr = getattrInternal<ExceptionStyle::CXX>(module, s);
if (attr != NULL)
continue;
// Just want to import it and add it to the modules list for now:
importSub(s->s(), boxStringTwine(llvm::Twine(buf) + "." + s->s()), module);
}
}
extern "C" PyObject* PyImport_Import(PyObject* module_name) noexcept {
RELEASE_ASSERT(module_name, "");
RELEASE_ASSERT(module_name->cls == str_cls, "");
try {
// TODO: check if this has the same behaviour as the cpython implementation
BoxedList* silly_list = new BoxedList();
listAppendInternal(silly_list, boxString("__doc__"));
return import(0, silly_list, ((BoxedString*)module_name)->s());
return importCExtension(name_boxed, shortname, pathname);
} catch (ExcInfo e) {
removeModule(name_boxed);
setCAPIException(e);
return NULL;
}
}
extern "C" PyObject* PyImport_ImportModule(const char* name) noexcept {
return PyImport_Import(boxString(name));
}
extern "C" PyObject* PyImport_GetModuleDict(void) noexcept {
extern "C" PyObject* load_source_module(char* name, char* pathname, FILE* fp) noexcept {
BoxedString* name_boxed = boxString(name);
try {
return getSysModulesDict();
} catch (ExcInfo e) {
setCAPIException(e);
return NULL;
}
}
/* Get the module object corresponding to a module name.
First check the modules dictionary if there's one there,
if not, create a new one and insert it in the modules dictionary.
Because the former action is most common, THIS DOES NOT RETURN A
'NEW' REFERENCE! */
extern "C" PyObject* PyImport_AddModule(const char* name) noexcept {
try {
PyObject* modules = getSysModulesDict();
BoxedString* s = boxString(name);
PyObject* m = PyDict_GetItem(modules, s);
if (m != NULL && m->cls == module_cls)
return m;
return createModule(s);
BoxedModule* module = createModule(name_boxed, pathname);
AST_Module* ast = caching_parse_file(pathname, /* future_flags = */ 0);
assert(ast);
compileAndRunModule(ast, module);
Box* r = getSysModulesDict()->getOrNull(name_boxed);
if (!r) {
PyErr_Format(ImportError, "Loaded module %.200s not found in sys.modules", name);
return NULL;
}
if (Py_VerboseFlag)
PySys_WriteStderr("import %s # from %s\n", name, pathname);
return r;
} catch (ExcInfo e) {
removeModule(name_boxed);
setCAPIException(e);
return NULL;
}
}
extern "C" PyObject* PyImport_ExecCodeModuleEx(char* name, PyObject* co, char* pathname) noexcept {
extern "C" PyObject* PyImport_ExecCodeModuleEx(const char* name, PyObject* co, char* pathname) noexcept {
BoxedString* s = boxString(name);
try {
RELEASE_ASSERT(co->cls == str_cls, "");
......@@ -692,131 +109,11 @@ extern "C" PyObject* PyImport_ExecCodeModuleEx(char* name, PyObject* co, char* p
}
}
static int isdir(const char* path) {
struct stat statbuf;
return stat(path, &statbuf) == 0 && S_ISDIR(statbuf.st_mode);
}
Box* nullImporterInit(Box* self, Box* _path) {
RELEASE_ASSERT(self->cls == null_importer_cls, "");
if (!isSubclass(_path->cls, str_cls))
raiseExcHelper(TypeError, "must be string, not %s", getTypeName(_path));
BoxedString* path = (BoxedString*)_path;
if (path->s().empty())
raiseExcHelper(ImportError, "empty pathname");
if (isdir(path->data()))
raiseExcHelper(ImportError, "existing directory");
return None;
}
Box* nullImporterFindModule(Box* self, Box* fullname, Box* path) {
return None;
}
extern "C" Box* import(int level, Box* from_imports, llvm::StringRef module_name) {
return importModuleLevel(module_name, getGlobals(), from_imports, level);
}
Box* impFindModule(Box* _name, BoxedList* path) {
_name = coerceUnicodeToStr<CXX>(_name);
RELEASE_ASSERT(_name->cls == str_cls, "");
BoxedString* name = static_cast<BoxedString*>(_name);
BoxedList* path_list = path && path != None ? path : getSysPath();
SearchResult sr = findModule(name->s(), name, path_list, false /* call_import_hooks */);
if (sr.type == SearchResult::SEARCH_ERROR)
raiseExcHelper(ImportError, "%s", name->data());
if (sr.type == SearchResult::PY_SOURCE) {
Box* path = boxString(sr.path);
Box* mode = boxString("r");
Box* f = runtimeCall(&PyFile_Type, ArgPassSpec(2), path, mode, NULL, NULL, NULL);
return BoxedTuple::create({ f, path, BoxedTuple::create({ boxString(".py"), mode, boxInt(sr.type) }) });
}
if (sr.type == SearchResult::PKG_DIRECTORY) {
Box* path = boxString(sr.path);
Box* mode = EmptyString;
return BoxedTuple::create({ None, path, BoxedTuple::create({ mode, mode, boxInt(sr.type) }) });
}
if (sr.type == SearchResult::C_EXTENSION) {
Box* path = boxString(sr.path);
Box* mode = boxString("rb");
Box* f = runtimeCall(&PyFile_Type, ArgPassSpec(2), path, mode, NULL, NULL, NULL);
return BoxedTuple::create({ f, path, BoxedTuple::create({ boxString(".so"), mode, boxInt(sr.type) }) });
}
RELEASE_ASSERT(0, "unknown type: %d", sr.type);
}
Box* impLoadModule(Box* _name, Box* _file, Box* _pathname, Box** args) {
Box* _description = args[0];
RELEASE_ASSERT(_name->cls == str_cls, "");
RELEASE_ASSERT(_pathname->cls == str_cls, "");
RELEASE_ASSERT(_description->cls == tuple_cls, "");
BoxedString* name = (BoxedString*)_name;
BoxedString* pathname = (BoxedString*)_pathname;
BoxedTuple* description = (BoxedTuple*)_description;
RELEASE_ASSERT(description->size() == 3, "");
BoxedString* suffix = (BoxedString*)description->elts[0];
BoxedString* mode = (BoxedString*)description->elts[1];
BoxedInt* type = (BoxedInt*)description->elts[2];
RELEASE_ASSERT(mode->cls == str_cls, "");
RELEASE_ASSERT(type->cls == int_cls, "");
RELEASE_ASSERT(pathname->cls == str_cls, "");
RELEASE_ASSERT(pathname->size(), "");
if (type->n == SearchResult::PKG_DIRECTORY) {
RELEASE_ASSERT(suffix->cls == str_cls, "");
RELEASE_ASSERT(suffix->s().empty(), "");
RELEASE_ASSERT(mode->s().empty(), "");
RELEASE_ASSERT(_file == None, "");
return createAndRunModule(name, (llvm::Twine(pathname->s()) + "/__init__.py").str(), pathname->s());
} else if (type->n == SearchResult::PY_SOURCE) {
RELEASE_ASSERT(_file->cls == &PyFile_Type, "");
return createAndRunModule(name, pathname->s());
}
Py_FatalError("unimplemented");
}
Box* impLoadSource(Box* _name, Box* _pathname, Box* _file) {
RELEASE_ASSERT(!_file, "'file' argument not support yet");
RELEASE_ASSERT(_name->cls == str_cls, "");
RELEASE_ASSERT(_pathname->cls == str_cls, "");
return createAndRunModule(static_cast<BoxedString*>(_name), static_cast<BoxedString*>(_pathname)->s());
}
Box* impLoadDynamic(Box* _name, Box* _pathname, Box* _file) {
RELEASE_ASSERT(_name->cls == str_cls, "");
RELEASE_ASSERT(_pathname->cls == str_cls, "");
RELEASE_ASSERT(_file == None, "");
BoxedString* name = (BoxedString*)_name;
BoxedString* pathname = (BoxedString*)_pathname;
const char* lastdot = strrchr(name->s().data(), '.');
const char* shortname;
if (lastdot == NULL) {
shortname = name->s().data();
} else {
shortname = lastdot + 1;
}
return importCExtension(name, shortname, pathname->s());
Box* rtn = PyImport_ImportModuleLevel(module_name.str().c_str(), getGlobalsDict(), NULL, from_imports, level);
if (!rtn)
throwCAPIException();
return rtn;
}
// Parses the memory map and registers all memory regions with "rwxp" permission and which contain the requested file
......@@ -922,136 +219,10 @@ BoxedModule* importCExtension(BoxedString* full_name, const std::string& last_na
BoxedModule* m = static_cast<BoxedModule*>(_m);
static BoxedString* file_str = internStringImmortal("__file__");
m->setattr(file_str, boxString(path), NULL);
return m;
}
Box* impGetSuffixes() {
BoxedList* list = new BoxedList;
// For now only add *.py
listAppendInternal(list, BoxedTuple::create({ boxString(".py"), boxString("U"), boxInt(SearchResult::PY_SOURCE) }));
return list;
}
if (Py_VerboseFlag)
PySys_WriteStderr("import %s # dynamically loaded from %s\n", full_name->c_str(), path.c_str());
Box* impAcquireLock() {
_PyImport_AcquireLock();
checkAndThrowCAPIException();
return None;
}
Box* impReleaseLock() {
_PyImport_ReleaseLock();
checkAndThrowCAPIException();
return None;
}
Box* impNewModule(Box* _name) {
if (!PyString_Check(_name))
raiseExcHelper(TypeError, "must be string, not %s", getTypeName(_name));
BoxedModule* module = new BoxedModule();
moduleInit(module, _name);
return module;
}
Box* impIsBuiltin(Box* _name) {
if (!PyString_Check(_name))
raiseExcHelper(TypeError, "must be string, not %s", getTypeName(_name));
static BoxedString* builtin_module_names_str = internStringImmortal("builtin_module_names");
BoxedTuple* builtin_modules = (BoxedTuple*)sys_module->getattr(builtin_module_names_str);
RELEASE_ASSERT(PyTuple_Check(builtin_modules), "");
for (Box* m : builtin_modules->pyElements()) {
if (compare(m, _name, AST_TYPE::Eq) == True)
return boxInt(-1); // CPython returns 1 for modules which can get reinitialized.
}
return boxInt(0);
}
Box* impIsFrozen(Box* name) {
if (!PyString_Check(name))
raiseExcHelper(TypeError, "must be string, not %s", getTypeName(name));
return False;
}
PyDoc_STRVAR(find_module_doc, "find_module(name, [path]) -> (file, filename, (suffix, mode, type))\n\
Search for a module. If path is omitted or None, search for a\n\
built-in, frozen or special module and continue search in sys.path.\n\
The module name cannot contain '.'; to search for a submodule of a\n\
package, pass the submodule name and the package's __path__.");
PyDoc_STRVAR(load_module_doc, "load_module(name, file, filename, (suffix, mode, type)) -> module\n\
Load a module, given information returned by find_module().\n\
The module name must include the full package name, if any.");
PyDoc_STRVAR(get_suffixes_doc, "get_suffixes() -> [(suffix, mode, type), ...]\n\
Return a list of (suffix, mode, type) tuples describing the files\n\
that find_module() looks for.");
PyDoc_STRVAR(acquire_lock_doc, "acquire_lock() -> None\n\
Acquires the interpreter's import lock for the current thread.\n\
This lock should be used by import hooks to ensure thread-safety\n\
when importing modules.\n\
On platforms without threads, this function does nothing.");
PyDoc_STRVAR(release_lock_doc, "release_lock() -> None\n\
Release the interpreter's import lock.\n\
On platforms without threads, this function does nothing.");
void setupImport() {
BoxedModule* imp_module
= createModule(boxString("imp"), NULL, "'This module provides the components needed to build your own\n"
"__import__ function. Undocumented functions are obsolete.'");
imp_module->giveAttr("PY_SOURCE", boxInt(SearchResult::PY_SOURCE));
imp_module->giveAttr("PY_COMPILED", boxInt(SearchResult::PY_COMPILED));
imp_module->giveAttr("C_EXTENSION", boxInt(SearchResult::C_EXTENSION));
imp_module->giveAttr("PKG_DIRECTORY", boxInt(SearchResult::PKG_DIRECTORY));
imp_module->giveAttr("C_BUILTIN", boxInt(SearchResult::C_BUILTIN));
imp_module->giveAttr("PY_FROZEN", boxInt(SearchResult::PY_FROZEN));
null_importer_cls = BoxedClass::create(type_cls, object_cls, NULL, 0, 0, sizeof(Box), false, "NullImporter");
null_importer_cls->giveAttr(
"__init__",
new BoxedFunction(FunctionMetadata::create((void*)nullImporterInit, NONE, 2, false, false), { None }));
null_importer_cls->giveAttr(
"find_module",
new BoxedBuiltinFunctionOrMethod(FunctionMetadata::create((void*)nullImporterFindModule, NONE, 2, false, false),
"find_module", { None }, NULL, find_module_doc));
null_importer_cls->freeze();
imp_module->giveAttr("NullImporter", null_importer_cls);
FunctionMetadata* find_module_func = FunctionMetadata::create((void*)impFindModule, UNKNOWN, 2, false, false,
ParamNames({ "name", "path" }, "", ""));
imp_module->giveAttr("find_module", new BoxedBuiltinFunctionOrMethod(find_module_func, "find_module", { None },
NULL, find_module_doc));
FunctionMetadata* load_module_func = FunctionMetadata::create(
(void*)impLoadModule, UNKNOWN, 4, ParamNames({ "name", "file", "pathname", "description" }, "", ""));
imp_module->giveAttr("load_module",
new BoxedBuiltinFunctionOrMethod(load_module_func, "load_module", load_module_doc));
imp_module->giveAttr("load_source", new BoxedBuiltinFunctionOrMethod(
FunctionMetadata::create((void*)impLoadSource, UNKNOWN, 3, false, false),
"load_source", { NULL }));
FunctionMetadata* load_dynamic_func = FunctionMetadata::create((void*)impLoadDynamic, UNKNOWN, 3, false, false,
ParamNames({ "name", "pathname", "file" }, "", ""));
imp_module->giveAttr("load_dynamic", new BoxedBuiltinFunctionOrMethod(load_dynamic_func, "load_dynamic", { None }));
imp_module->giveAttr("get_suffixes",
new BoxedBuiltinFunctionOrMethod(FunctionMetadata::create((void*)impGetSuffixes, UNKNOWN, 0),
"get_suffixes", get_suffixes_doc));
imp_module->giveAttr("acquire_lock",
new BoxedBuiltinFunctionOrMethod(FunctionMetadata::create((void*)impAcquireLock, NONE, 0),
"acquire_lock", acquire_lock_doc));
imp_module->giveAttr("release_lock",
new BoxedBuiltinFunctionOrMethod(FunctionMetadata::create((void*)impReleaseLock, NONE, 0),
"release_lock", release_lock_doc));
imp_module->giveAttr("new_module", new BoxedBuiltinFunctionOrMethod(
FunctionMetadata::create((void*)impNewModule, MODULE, 1), "new_module"));
imp_module->giveAttr("is_builtin", new BoxedBuiltinFunctionOrMethod(
FunctionMetadata::create((void*)impIsBuiltin, BOXED_INT, 1), "is_builtin"));
imp_module->giveAttr("is_frozen", new BoxedBuiltinFunctionOrMethod(
FunctionMetadata::create((void*)impIsFrozen, BOXED_BOOL, 1), "is_frozen"));
return m;
}
}
src/runtime/import.h
View file @ 2d0eaa3d
......@@ -21,7 +21,6 @@ namespace pyston {
extern "C" PyObject* PyImport_GetImporter(PyObject* path) noexcept;
extern "C" Box* import(int level, Box* from_imports, llvm::StringRef module_name);
extern Box* importModuleLevel(llvm::StringRef module_name, Box* globals, Box* from_imports, int level);
BoxedModule* importCExtension(BoxedString* full_name, const std::string& last_name, const std::string& path);
}
......
src/runtime/types.cpp
View file @ 2d0eaa3d
......@@ -88,6 +88,7 @@ extern "C" void init_sqlite3();
extern "C" void PyMarshal_Init();
extern "C" void initstrop();
extern "C" void init_ast();
extern "C" void initimp();
namespace pyston {
......@@ -1566,8 +1567,8 @@ void BoxedClosure::gcHandler(GCVisitor* v, Box* b) {
extern "C" {
BoxedClass* object_cls, *type_cls, *none_cls, *bool_cls, *int_cls, *float_cls,
* str_cls = NULL, *function_cls, *instancemethod_cls, *list_cls, *slice_cls, *module_cls, *dict_cls, *tuple_cls,
*member_descriptor_cls, *closure_cls, *generator_cls, *null_importer_cls, *complex_cls, *basestring_cls,
*property_cls, *staticmethod_cls, *classmethod_cls, *attrwrapper_cls, *pyston_getset_cls, *capi_getset_cls,
*member_descriptor_cls, *closure_cls, *generator_cls, *complex_cls, *basestring_cls, *property_cls,
*staticmethod_cls, *classmethod_cls, *attrwrapper_cls, *pyston_getset_cls, *capi_getset_cls,
*builtin_function_or_method_cls, *attrwrapperiter_cls, *set_cls, *frozenset_cls;
BoxedTuple* EmptyTuple;
......@@ -3546,6 +3547,10 @@ static void setTypeGCNone(BoxedClass* cls) {
}
static void setupDefaultClassGCParticipation() {
// we need to make sure this types are initalized
init_sre();
inititertools();
// some additional setup to ensure weakrefs participate in our GC
setTypeGCProxy(&_PyWeakref_RefType);
setTypeGCProxy(&_PyWeakref_ProxyType);
......@@ -3632,6 +3637,60 @@ static int _check_and_flush(FILE* stream) {
return fflush(stream) || prev_fail ? EOF : 0;
}
extern "C" {
struct _inittab _PyImport_Inittab[] = { { "array", initarray },
{ "_ast", init_ast },
{ "binascii", initbinascii },
{ "_codecs", init_codecs },
{ "_collections", init_collections },
{ "cStringIO", initcStringIO },
{ "_csv", init_csv },
{ "datetime", initdatetime },
{ "errno", initerrno },
{ "fcntl", initfcntl },
{ "_functools", init_functools },
{ "imp", initimp },
{ "_io", init_io },
{ "itertools", inititertools },
{ "marshal", PyMarshal_Init },
{ "math", initmath },
{ "_md5", init_md5 },
{ "operator", initoperator },
{ "posix", initposix },
{ "pwd", initpwd },
{ "_random", init_random },
{ "resource", initresource },
{ "select", initselect },
{ "_sha", init_sha },
{ "_sha256", init_sha256 },
{ "_sha512", init_sha512 },
{ "signal", initsignal },
{ "_socket", init_socket },
{ "_sqlite3", init_sqlite3 },
{ "_sre", init_sre },
{ "_ssl", init_ssl },
{ "strop", initstrop },
{ "_struct", init_struct },
{ "time", inittime },
{ "unicodedata", initunicodedata },
{ "_warnings", _PyWarnings_Init },
{ "_weakref", init_weakref },
{ "zipimport", initzipimport },
{ "zlib", initzlib },
{ "gc", setupGC },
{ "__pyston__", setupPyston },
{ "thread", setupThread },
{ "__main__", NULL },
{ "__builtin__", NULL },
{ "sys", NULL },
{ "exceptions", NULL },
{ 0, 0 } };
}
bool TRACK_ALLOCATIONS = false;
void setupRuntime() {
......@@ -4115,10 +4174,8 @@ void setupRuntime() {
setupBuiltins();
_PyExc_Init();
setupThread();
setupGC();
setupImport();
setupPyston();
_PyImport_Init();
_PyImportHooks_Init();
PyType_Ready(&PyByteArrayIter_Type);
PyType_Ready(&PyCapsule_Type);
......@@ -4130,44 +4187,6 @@ void setupRuntime() {
PyType_Ready(&PyDictProxy_Type);
PyType_Ready(&PyTraceBack_Type);
initerrno();
init_sha();
init_sha256();
init_sha512();
init_md5();
init_random();
init_sre();
initmath();
initoperator();
initbinascii();
initpwd();
initposix();
init_struct();
initdatetime();
init_functools();
init_collections();
inititertools();
initresource();
initsignal();
initselect();
initfcntl();
inittime();
initarray();
initzlib();
init_codecs();
init_socket();
initunicodedata();
initcStringIO();
init_io();
initzipimport();
init_csv();
init_ssl();
init_sqlite3();
PyMarshal_Init();
initstrop();
init_ast();
setupDefaultClassGCParticipation();
assert(object_cls->tp_setattro == PyObject_GenericSetAttr);
......
src/runtime/types.h
View file @ 2d0eaa3d
......@@ -89,10 +89,10 @@ extern "C" BoxedString* EmptyString;
extern "C" {
extern BoxedClass* object_cls, *type_cls, *bool_cls, *int_cls, *long_cls, *float_cls, *str_cls, *function_cls,
*none_cls, *instancemethod_cls, *list_cls, *slice_cls, *module_cls, *dict_cls, *tuple_cls, *enumerate_cls,
*xrange_cls, *member_descriptor_cls, *null_importer_cls, *method_cls, *closure_cls, *generator_cls, *complex_cls,
*basestring_cls, *property_cls, *staticmethod_cls, *classmethod_cls, *attrwrapper_cls, *pyston_getset_cls,
*capi_getset_cls, *builtin_function_or_method_cls, *set_cls, *frozenset_cls, *code_cls, *frame_cls, *capifunc_cls,
*wrapperdescr_cls, *wrapperobject_cls;
*xrange_cls, *member_descriptor_cls, *method_cls, *closure_cls, *generator_cls, *complex_cls, *basestring_cls,
*property_cls, *staticmethod_cls, *classmethod_cls, *attrwrapper_cls, *pyston_getset_cls, *capi_getset_cls,
*builtin_function_or_method_cls, *set_cls, *frozenset_cls, *code_cls, *frame_cls, *capifunc_cls, *wrapperdescr_cls,
*wrapperobject_cls;
}
#define unicode_cls (&PyUnicode_Type)
#define memoryview_cls (&PyMemoryView_Type)
......
test/CPYTHON_TEST_NOTES.md
View file @ 2d0eaa3d
......@@ -24,7 +24,6 @@ test_al No module named al
test_applesingle Not really a failure, but it tries to skip itself and we don't support that
test_argparse [unknown]
test_ascii_formatd segfault in ctypes (but only on CI)
test_atexit [unknown]
test_audioop [unknown]
test_bigmem [unknown]
test_bisect somehow sys.modules['_bisect'] is getting set to 0
......
test/tests/imp_test.py
View file @ 2d0eaa3d
......@@ -13,7 +13,9 @@ for a in (1, "", "/proc", "nonexisting_dir"):
except Exception as e:
print e
print imp.lock_held()
imp.acquire_lock()
print imp.lock_held()
imp.release_lock()
import os
......
test/tests/import_test.py
View file @ 2d0eaa3d
......@@ -35,3 +35,9 @@ import _multiprocessing
del _multiprocessing
del sys.modules["_multiprocessing"]
import _multiprocessing
import time
del time
del sys.modules["time"]
import time
print time.sleep(0)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7