Merge pull request #877 from kmod/execfile

3-arg execfile

src/analysis/scoping_analysis.cpp

@@ -171,8 +171,6 @@ private:
     bool globals_from_module;
 
 public:
-    EvalExprScopeInfo(bool globals_from_module) : globals_from_module(globals_from_module) {}
-
     EvalExprScopeInfo(AST* node, bool globals_from_module) : globals_from_module(globals_from_module) {
         // Find all the global statements in the node's scope (not delving into FuncitonDefs
         // or ClassDefs) and put the names in `forced_globals`.
@@ -959,21 +957,24 @@ ScopingAnalysis::ScopingAnalysis(AST* ast, bool globals_from_module)
     : parent_module(NULL), globals_from_module(globals_from_module) {
     switch (ast->type) {
         case AST_TYPE::Module:
-            assert(globals_from_module);
-            scopes[ast] = new ModuleScopeInfo();
             interned_strings = static_cast<AST_Module*>(ast)->interned_strings.get();
-            parent_module = static_cast<AST_Module*>(ast);
             break;
         case AST_TYPE::Expression:
-            scopes[ast] = new EvalExprScopeInfo(globals_from_module);
             interned_strings = static_cast<AST_Expression*>(ast)->interned_strings.get();
             break;
         case AST_TYPE::Suite:
-            scopes[ast] = new EvalExprScopeInfo(ast, globals_from_module);
             interned_strings = static_cast<AST_Suite*>(ast)->interned_strings.get();
             break;
         default:
             RELEASE_ASSERT(0, "%d", ast->type);
     }
+
+    if (globals_from_module) {
+        assert(ast->type == AST_TYPE::Module);
+        scopes[ast] = new ModuleScopeInfo();
+        parent_module = static_cast<AST_Module*>(ast);
+    } else {
+        scopes[ast] = new EvalExprScopeInfo(ast, globals_from_module);
+    }
 }
 }

src/codegen/ast_interpreter.cpp

@@ -1011,6 +1011,9 @@ Value ASTInterpreter::createFunction(AST* node, AST_arguments* args, const std::
     u.d.s = defaults.size() - 1;
 
     bool takes_closure;
+    if (!LAZY_SCOPING_ANALYSIS)
+        source_info->scoping->getScopeInfoForNode(node);
+
     // Optimization: when compiling a module, it's nice to not have to run analyses into the
     // entire module's source code.
     // If we call getScopeInfoForNode, that will trigger an analysis of that function tree,

src/codegen/irgen/hooks.cpp

@@ -370,10 +370,7 @@ static CLFunction* compileForEvalOrExec(AST* source, std::vector<AST_stmt*> body
     return cl_f;
 }
 
-// TODO: CPython parses execs as Modules, not as Suites.  This is probably not too hard to change,
-// but is non-trivial since we will later decide some things (ex in scoping_analysis) based off
-// the type of the root ast node.
-static AST_Suite* parseExec(llvm::StringRef source, bool interactive = false) {
+static AST_Module* parseExec(llvm::StringRef source, bool interactive = false) {
     // TODO error message if parse fails or if it isn't an expr
     // TODO should have a cleaner interface that can parse the Expression directly
     // TODO this memory leaks
@@ -393,13 +390,11 @@ static AST_Suite* parseExec(llvm::StringRef source, bool interactive = false) {
         }
     }
 
-    AST_Suite* parsedSuite = new AST_Suite(std::move(parsedModule->interned_strings));
-    parsedSuite->body = std::move(parsedModule->body);
-    return parsedSuite;
+    return parsedModule;
 }
 
-static CLFunction* compileExec(AST_Suite* parsedSuite, BoxedString* fn, PyCompilerFlags* flags) {
-    return compileForEvalOrExec(parsedSuite, parsedSuite->body, fn, flags);
+static CLFunction* compileExec(AST_Module* parsedModule, BoxedString* fn, PyCompilerFlags* flags) {
+    return compileForEvalOrExec(parsedModule, parsedModule->body, fn, flags);
 }
 
 static AST_Expression* parseEval(llvm::StringRef source) {
@@ -520,9 +515,9 @@ Box* compile(Box* source, Box* fn, Box* type, Box** _args) {
     CLFunction* cl;
     if (type_str->s() == "exec" || type_str->s() == "single") {
         // TODO: CPython parses execs as Modules
-        if (parsed->type != AST_TYPE::Suite)
-            raiseExcHelper(TypeError, "expected Suite node, got %s", boxAst(parsed)->cls->tp_name);
-        cl = compileExec(static_cast<AST_Suite*>(parsed), filename_str, &pcf);
+        if (parsed->type != AST_TYPE::Module)
+            raiseExcHelper(TypeError, "expected Module node, got %s", boxAst(parsed)->cls->tp_name);
+        cl = compileExec(static_cast<AST_Module*>(parsed), filename_str, &pcf);
     } else if (type_str->s() == "eval") {
         if (parsed->type != AST_TYPE::Expression)
             raiseExcHelper(TypeError, "expected Expression node, got %s", boxAst(parsed)->cls->tp_name);
@@ -534,7 +529,7 @@ Box* compile(Box* source, Box* fn, Box* type, Box** _args) {
     return (Box*)cl->getCode();
 }
 
-static Box* evalMain(Box* boxedCode, Box* globals, Box* locals, PyCompilerFlags* flags) {
+static void pickGlobalsAndLocals(Box*& globals, Box*& locals) {
     if (globals == None)
         globals = NULL;
 
@@ -561,6 +556,19 @@ static Box* evalMain(Box* boxedCode, Box* globals, Box* locals, PyCompilerFlags*
 
     assert(globals && (globals->cls == module_cls || globals->cls == dict_cls));
 
+    if (globals) {
+        // From CPython (they set it to be f->f_builtins):
+        Box* globals_dict = globals;
+        if (globals->cls == module_cls)
+            globals_dict = globals->getAttrWrapper();
+        if (PyDict_GetItemString(globals_dict, "__builtins__") == NULL)
+            PyDict_SetItemString(globals_dict, "__builtins__", builtins_module);
+    }
+}
+
+static Box* evalMain(Box* boxedCode, Box* globals, Box* locals, PyCompilerFlags* flags) {
+    pickGlobalsAndLocals(globals, locals);
+
     if (boxedCode->cls == unicode_cls) {
         boxedCode = PyUnicode_AsUTF8String(boxedCode);
         if (!boxedCode)
@@ -593,6 +601,48 @@ Box* eval(Box* boxedCode, Box* globals, Box* locals) {
     return evalMain(boxedCode, globals, locals, &pcf);
 }
 
+Box* execfile(Box* _fn, Box* globals, Box* locals) {
+    if (!PyString_Check(_fn)) {
+        raiseExcHelper(TypeError, "must be string, not %s", getTypeName(_fn));
+    }
+
+    BoxedString* fn = static_cast<BoxedString*>(_fn);
+
+    pickGlobalsAndLocals(globals, locals);
+
+#if LLVMREV < 217625
+    bool exists;
+    llvm_error_code code = llvm::sys::fs::exists(fn->s, exists);
+
+#if LLVMREV < 210072
+    ASSERT(code == 0, "%s: %s", code.message().c_str(), fn->s.c_str());
+#else
+    assert(!code);
+#endif
+
+#else
+    bool exists = llvm::sys::fs::exists(std::string(fn->s()));
+#endif
+
+    if (!exists)
+        raiseExcHelper(IOError, "No such file or directory: '%s'", fn->s().data());
+
+    AST_Module* parsed = caching_parse_file(fn->s().data());
+    assert(parsed);
+
+    CLFunction* caller_cl = getTopPythonFunction();
+    assert(caller_cl != NULL);
+    assert(caller_cl->source != NULL);
+    FutureFlags caller_future_flags = caller_cl->source->future_flags;
+    PyCompilerFlags pcf;
+    pcf.cf_flags = caller_future_flags;
+
+    CLFunction* cl = compileForEvalOrExec(parsed, parsed->body, fn, &pcf);
+    assert(cl);
+
+    return evalOrExec(cl, globals, locals);
+}
+
 Box* execMain(Box* boxedCode, Box* globals, Box* locals, PyCompilerFlags* flags) {
     if (PyTuple_Check(boxedCode)) {
         RELEASE_ASSERT(!globals, "");
@@ -606,40 +656,7 @@ Box* execMain(Box* boxedCode, Box* globals, Box* locals, PyCompilerFlags* flags)
             locals = t->elts[2];
     }
 
-    if (globals == None)
-        globals = NULL;
-
-    if (locals == None)
-        locals = NULL;
-
-    if (locals == NULL) {
-        locals = globals;
-    }
-
-    if (locals == NULL) {
-        locals = fastLocalsToBoxedLocals();
-    }
-
-    if (globals == NULL)
-        globals = getGlobals();
-
-    BoxedModule* module = getCurrentModule();
-    if (globals && globals->cls == attrwrapper_cls && unwrapAttrWrapper(globals) == module)
-        globals = module;
-
-    if (globals->cls == attrwrapper_cls)
-        globals = unwrapAttrWrapper(globals);
-
-    assert(globals && (globals->cls == module_cls || globals->cls == dict_cls));
-
-    if (globals) {
-        // From CPython (they set it to be f->f_builtins):
-        Box* globals_dict = globals;
-        if (globals->cls == module_cls)
-            globals_dict = globals->getAttrWrapper();
-        if (PyDict_GetItemString(globals_dict, "__builtins__") == NULL)
-            PyDict_SetItemString(globals_dict, "__builtins__", builtins_module);
-    }
+    pickGlobalsAndLocals(globals, locals);
 
     if (boxedCode->cls == unicode_cls) {
         boxedCode = PyUnicode_AsUTF8String(boxedCode);
@@ -650,7 +667,7 @@ Box* execMain(Box* boxedCode, Box* globals, Box* locals, PyCompilerFlags* flags)
 
     CLFunction* cl;
     if (boxedCode->cls == str_cls) {
-        AST_Suite* parsed = parseExec(static_cast<BoxedString*>(boxedCode)->s());
+        auto parsed = parseExec(static_cast<BoxedString*>(boxedCode)->s());
         static BoxedString* string_string = internStringImmortal("<string>");
         cl = compileExec(parsed, string_string, flags);
     } else if (boxedCode->cls == code_cls) {

src/codegen/irgen/hooks.h

@@ -40,6 +40,7 @@ void compileAndRunModule(AST_Module* m, BoxedModule* bm);
 CompiledFunction* cfForMachineFunctionName(const std::string&);
 
 extern "C" Box* exec(Box* boxedCode, Box* globals, Box* locals, FutureFlags caller_future_flags);
+extern "C" Box* execfile(Box* fn, Box* globals, Box* locals);
 extern "C" Box* eval(Box* boxedCode, Box* globals, Box* locals);
 extern "C" Box* compile(Box* source, Box* filename, Box* mode, Box** _args /* flags, dont_inherit */);
 }

src/core/ast.h

@@ -1204,7 +1204,7 @@ public:
     virtual bool visit_excepthandler(AST_ExceptHandler* node) { return false; }
     virtual bool visit_exec(AST_Exec* node) { return false; }
     virtual bool visit_expr(AST_Expr* node) { return false; }
-    virtual bool visit_expr(AST_Expression* node) { return false; }
+    virtual bool visit_expression(AST_Expression* node) { return false; }
     virtual bool visit_suite(AST_Suite* node) { return false; }
     virtual bool visit_extslice(AST_ExtSlice* node) { return false; }
     virtual bool visit_for(AST_For* node) { return false; }

src/core/options.cpp

@@ -82,6 +82,7 @@ bool ENABLE_PYSTON_PASSES = 1 && _GLOBAL_ENABLE;
 bool ENABLE_TYPE_FEEDBACK = 1 && _GLOBAL_ENABLE;
 bool ENABLE_RUNTIME_ICS = 1 && _GLOBAL_ENABLE;
 bool ENABLE_JIT_OBJECT_CACHE = 1 && _GLOBAL_ENABLE;
+bool LAZY_SCOPING_ANALYSIS = 1;
 
 bool ENABLE_FRAME_INTROSPECTION = 1;
 bool BOOLS_AS_I64 = ENABLE_FRAME_INTROSPECTION;

src/core/options.h

@@ -44,7 +44,8 @@ extern bool SHOW_DISASM, FORCE_INTERPRETER, FORCE_OPTIMIZE, PROFILE, DUMPJIT, TR
 extern bool ENABLE_ICS, ENABLE_ICGENERICS, ENABLE_ICGETITEMS, ENABLE_ICSETITEMS, ENABLE_ICDELITEMS, ENABLE_ICBINEXPS,
     ENABLE_ICNONZEROS, ENABLE_ICCALLSITES, ENABLE_ICSETATTRS, ENABLE_ICGETATTRS, ENALBE_ICDELATTRS, ENABLE_ICGETGLOBALS,
     ENABLE_SPECULATION, ENABLE_OSR, ENABLE_LLVMOPTS, ENABLE_INLINING, ENABLE_REOPT, ENABLE_PYSTON_PASSES,
-    ENABLE_TYPE_FEEDBACK, ENABLE_FRAME_INTROSPECTION, ENABLE_RUNTIME_ICS, ENABLE_JIT_OBJECT_CACHE;
+    ENABLE_TYPE_FEEDBACK, ENABLE_FRAME_INTROSPECTION, ENABLE_RUNTIME_ICS, ENABLE_JIT_OBJECT_CACHE,
+    LAZY_SCOPING_ANALYSIS;
 
 // Due to a temporary LLVM limitation, represent bools as i64's instead of i1's.
 extern bool BOOLS_AS_I64;

src/runtime/builtin_modules/builtins.cpp

@@ -1249,40 +1249,6 @@ Box* powFunc(Box* x, Box* y, Box* z) {
     return rtn;
 }
 
-Box* execfile(Box* _fn) {
-    // The "globals" and "locals" arguments aren't implemented for now
-    if (!PyString_Check(_fn)) {
-        raiseExcHelper(TypeError, "must be string, not %s", getTypeName(_fn));
-    }
-
-    BoxedString* fn = static_cast<BoxedString*>(_fn);
-
-#if LLVMREV < 217625
-    bool exists;
-    llvm_error_code code = llvm::sys::fs::exists(fn->s, exists);
-
-#if LLVMREV < 210072
-    ASSERT(code == 0, "%s: %s", code.message().c_str(), fn->s.c_str());
-#else
-    assert(!code);
-#endif
-
-#else
-    bool exists = llvm::sys::fs::exists(std::string(fn->s()));
-#endif
-
-    if (!exists)
-        raiseExcHelper(IOError, "No such file or directory: '%s'", fn->data());
-
-    // Run directly inside the current module:
-    AST_Module* ast = caching_parse_file(fn->data());
-
-    ASSERT(getTopPythonFunction()->source->scoping->areGlobalsFromModule(), "need to pass custom globals in");
-    compileAndRunModule(ast, getCurrentModule());
-
-    return None;
-}
-
 Box* print(BoxedTuple* args, BoxedDict* kwargs) {
     assert(args->cls == tuple_cls);
     assert(!kwargs || kwargs->cls == dict_cls);
@@ -1718,8 +1684,10 @@ void setupBuiltins() {
                                             boxRTFunction((void*)coerceFunc, UNKNOWN, 2, 0, false, false), "coerce"));
     builtins_module->giveAttr("divmod",
                               new BoxedBuiltinFunctionOrMethod(boxRTFunction((void*)divmod, UNKNOWN, 2), "divmod"));
-    builtins_module->giveAttr("execfile",
-                              new BoxedBuiltinFunctionOrMethod(boxRTFunction((void*)execfile, UNKNOWN, 1), "execfile"));
+
+    builtins_module->giveAttr(
+        "execfile", new BoxedBuiltinFunctionOrMethod(boxRTFunction((void*)execfile, UNKNOWN, 3, 2, false, false),
+                                                     "execfile", { NULL, NULL }));
 
     CLFunction* compile_func = createRTFunction(
         5, 2, false, false, ParamNames({ "source", "filename", "mode", "flags", "dont_inherit" }, "", ""));

src/runtime/builtin_modules/pyston.cpp

@@ -45,6 +45,7 @@ static Box* setOption(Box* option, Box* value) {
     else CHECK(SPECULATION_THRESHOLD);
     else CHECK(ENABLE_ICS);
     else CHECK(ENABLE_ICGETATTRS);
+    else CHECK(LAZY_SCOPING_ANALYSIS);
     else raiseExcHelper(ValueError, "unknown option name '%s", option_string->data());
 
     return None;

test/tests/del_closure_var_syntax_error.py

+try:
+    import __pyston__
+    __pyston__.setOption("LAZY_SCOPING_ANALYSIS", 0)
+except ImportError:
+    pass
+
 cases = [
 
 """

test/tests/execfile_test.py

-# expected: fail
-# - we throw some very weird error here
-
 try:
     execfile("doesnt_exist.py")
 except IOError, e:
-    print e
+    print type(e)
 
 import os
 fn = os.path.join(os.path.dirname(__file__), 'execfile_target.py')
@@ -14,3 +11,9 @@ execfile(fn)
 
 print test_name
 print type(execfile_target)
+
+g = {}
+l = {}
+execfile(fn, g, l)
+print sorted(g.keys())
+print sorted(l.keys())

test/tests/global_and_local.py

 # I would have expected this to be valid, but cPython and pypy err out saying "name 'x' is local and global"
 
+try:
+    import __pyston__
+    __pyston__.setOption("LAZY_SCOPING_ANALYSIS", 0)
+except ImportError:
+    pass
+
 try:
     exec """
 x = 1
@@ -40,7 +46,7 @@ except SyntaxError as e:
 
 try:
     exec """
-def f(*kwargs):
+def f(**kwargs):
     global kwargs
 
 print "calling"

test/tests/name_forcing_syntax_error.py

@@ -4,6 +4,12 @@
 
 # The logic beyond this error message is oddly complicated.
 
+try:
+    import __pyston__
+    __pyston__.setOption("LAZY_SCOPING_ANALYSIS", 0)
+except ImportError:
+    pass
+
 cases = [
 
 # protip: delete this first """ to get your editor to syntax-highlight the code