Parse exec's more like normal code

src/analysis/scoping_analysis.cpp

@@ -171,8 +171,6 @@ private:
     bool globals_from_module;
 
 public:
-    EvalExprScopeInfo(bool globals_from_module) : globals_from_module(globals_from_module) {}
-
     EvalExprScopeInfo(AST* node, bool globals_from_module) : globals_from_module(globals_from_module) {
         // Find all the global statements in the node's scope (not delving into FuncitonDefs
         // or ClassDefs) and put the names in `forced_globals`.
@@ -959,21 +957,24 @@ ScopingAnalysis::ScopingAnalysis(AST* ast, bool globals_from_module)
     : parent_module(NULL), globals_from_module(globals_from_module) {
     switch (ast->type) {
         case AST_TYPE::Module:
-            assert(globals_from_module);
-            scopes[ast] = new ModuleScopeInfo();
             interned_strings = static_cast<AST_Module*>(ast)->interned_strings.get();
-            parent_module = static_cast<AST_Module*>(ast);
             break;
         case AST_TYPE::Expression:
-            scopes[ast] = new EvalExprScopeInfo(globals_from_module);
             interned_strings = static_cast<AST_Expression*>(ast)->interned_strings.get();
             break;
         case AST_TYPE::Suite:
-            scopes[ast] = new EvalExprScopeInfo(ast, globals_from_module);
             interned_strings = static_cast<AST_Suite*>(ast)->interned_strings.get();
             break;
         default:
             RELEASE_ASSERT(0, "%d", ast->type);
     }
+
+    if (globals_from_module) {
+        assert(ast->type == AST_TYPE::Module);
+        scopes[ast] = new ModuleScopeInfo();
+        parent_module = static_cast<AST_Module*>(ast);
+    } else {
+        scopes[ast] = new EvalExprScopeInfo(ast, globals_from_module);
+    }
 }
 }

src/codegen/ast_interpreter.cpp

@@ -1011,6 +1011,9 @@ Value ASTInterpreter::createFunction(AST* node, AST_arguments* args, const std::
     u.d.s = defaults.size() - 1;
 
     bool takes_closure;
+    if (!LAZY_SCOPING_ANALYSIS)
+        source_info->scoping->getScopeInfoForNode(node);
+
     // Optimization: when compiling a module, it's nice to not have to run analyses into the
     // entire module's source code.
     // If we call getScopeInfoForNode, that will trigger an analysis of that function tree,

src/codegen/irgen/hooks.cpp

@@ -370,10 +370,7 @@ static CLFunction* compileForEvalOrExec(AST* source, std::vector<AST_stmt*> body
     return cl_f;
 }
 
-// TODO: CPython parses execs as Modules, not as Suites.  This is probably not too hard to change,
-// but is non-trivial since we will later decide some things (ex in scoping_analysis) based off
-// the type of the root ast node.
-static AST_Suite* parseExec(llvm::StringRef source, bool interactive = false) {
+static AST_Module* parseExec(llvm::StringRef source, bool interactive = false) {
     // TODO error message if parse fails or if it isn't an expr
     // TODO should have a cleaner interface that can parse the Expression directly
     // TODO this memory leaks
@@ -393,13 +390,11 @@ static AST_Suite* parseExec(llvm::StringRef source, bool interactive = false) {
         }
     }
 
-    AST_Suite* parsedSuite = new AST_Suite(std::move(parsedModule->interned_strings));
-    parsedSuite->body = std::move(parsedModule->body);
-    return parsedSuite;
+    return parsedModule;
 }
 
-static CLFunction* compileExec(AST_Suite* parsedSuite, BoxedString* fn, PyCompilerFlags* flags) {
-    return compileForEvalOrExec(parsedSuite, parsedSuite->body, fn, flags);
+static CLFunction* compileExec(AST_Module* parsedModule, BoxedString* fn, PyCompilerFlags* flags) {
+    return compileForEvalOrExec(parsedModule, parsedModule->body, fn, flags);
 }
 
 static AST_Expression* parseEval(llvm::StringRef source) {
@@ -520,9 +515,9 @@ Box* compile(Box* source, Box* fn, Box* type, Box** _args) {
     CLFunction* cl;
     if (type_str->s() == "exec" || type_str->s() == "single") {
         // TODO: CPython parses execs as Modules
-        if (parsed->type != AST_TYPE::Suite)
-            raiseExcHelper(TypeError, "expected Suite node, got %s", boxAst(parsed)->cls->tp_name);
-        cl = compileExec(static_cast<AST_Suite*>(parsed), filename_str, &pcf);
+        if (parsed->type != AST_TYPE::Module)
+            raiseExcHelper(TypeError, "expected Module node, got %s", boxAst(parsed)->cls->tp_name);
+        cl = compileExec(static_cast<AST_Module*>(parsed), filename_str, &pcf);
     } else if (type_str->s() == "eval") {
         if (parsed->type != AST_TYPE::Expression)
             raiseExcHelper(TypeError, "expected Expression node, got %s", boxAst(parsed)->cls->tp_name);
@@ -534,7 +529,7 @@ Box* compile(Box* source, Box* fn, Box* type, Box** _args) {
     return (Box*)cl->getCode();
 }
 
-static Box* evalMain(Box* boxedCode, Box* globals, Box* locals, PyCompilerFlags* flags) {
+static void pickGlobalsAndLocals(Box*& globals, Box*& locals) {
     if (globals == None)
         globals = NULL;
 
@@ -561,6 +556,19 @@ static Box* evalMain(Box* boxedCode, Box* globals, Box* locals, PyCompilerFlags*
 
     assert(globals && (globals->cls == module_cls || globals->cls == dict_cls));
 
+    if (globals) {
+        // From CPython (they set it to be f->f_builtins):
+        Box* globals_dict = globals;
+        if (globals->cls == module_cls)
+            globals_dict = globals->getAttrWrapper();
+        if (PyDict_GetItemString(globals_dict, "__builtins__") == NULL)
+            PyDict_SetItemString(globals_dict, "__builtins__", builtins_module);
+    }
+}
+
+static Box* evalMain(Box* boxedCode, Box* globals, Box* locals, PyCompilerFlags* flags) {
+    pickGlobalsAndLocals(globals, locals);
+
     if (boxedCode->cls == unicode_cls) {
         boxedCode = PyUnicode_AsUTF8String(boxedCode);
         if (!boxedCode)
@@ -606,40 +614,7 @@ Box* execMain(Box* boxedCode, Box* globals, Box* locals, PyCompilerFlags* flags)
             locals = t->elts[2];
     }
 
-    if (globals == None)
-        globals = NULL;
-
-    if (locals == None)
-        locals = NULL;
-
-    if (locals == NULL) {
-        locals = globals;
-    }
-
-    if (locals == NULL) {
-        locals = fastLocalsToBoxedLocals();
-    }
-
-    if (globals == NULL)
-        globals = getGlobals();
-
-    BoxedModule* module = getCurrentModule();
-    if (globals && globals->cls == attrwrapper_cls && unwrapAttrWrapper(globals) == module)
-        globals = module;
-
-    if (globals->cls == attrwrapper_cls)
-        globals = unwrapAttrWrapper(globals);
-
-    assert(globals && (globals->cls == module_cls || globals->cls == dict_cls));
-
-    if (globals) {
-        // From CPython (they set it to be f->f_builtins):
-        Box* globals_dict = globals;
-        if (globals->cls == module_cls)
-            globals_dict = globals->getAttrWrapper();
-        if (PyDict_GetItemString(globals_dict, "__builtins__") == NULL)
-            PyDict_SetItemString(globals_dict, "__builtins__", builtins_module);
-    }
+    pickGlobalsAndLocals(globals, locals);
 
     if (boxedCode->cls == unicode_cls) {
         boxedCode = PyUnicode_AsUTF8String(boxedCode);
@@ -650,7 +625,7 @@ Box* execMain(Box* boxedCode, Box* globals, Box* locals, PyCompilerFlags* flags)
 
     CLFunction* cl;
     if (boxedCode->cls == str_cls) {
-        AST_Suite* parsed = parseExec(static_cast<BoxedString*>(boxedCode)->s());
+        auto parsed = parseExec(static_cast<BoxedString*>(boxedCode)->s());
         static BoxedString* string_string = internStringImmortal("<string>");
         cl = compileExec(parsed, string_string, flags);
     } else if (boxedCode->cls == code_cls) {

src/core/ast.h

@@ -1204,7 +1204,7 @@ public:
     virtual bool visit_excepthandler(AST_ExceptHandler* node) { return false; }
     virtual bool visit_exec(AST_Exec* node) { return false; }
     virtual bool visit_expr(AST_Expr* node) { return false; }
-    virtual bool visit_expr(AST_Expression* node) { return false; }
+    virtual bool visit_expression(AST_Expression* node) { return false; }
     virtual bool visit_suite(AST_Suite* node) { return false; }
     virtual bool visit_extslice(AST_ExtSlice* node) { return false; }
     virtual bool visit_for(AST_For* node) { return false; }

src/core/options.cpp

@@ -82,6 +82,7 @@ bool ENABLE_PYSTON_PASSES = 1 && _GLOBAL_ENABLE;
 bool ENABLE_TYPE_FEEDBACK = 1 && _GLOBAL_ENABLE;
 bool ENABLE_RUNTIME_ICS = 1 && _GLOBAL_ENABLE;
 bool ENABLE_JIT_OBJECT_CACHE = 1 && _GLOBAL_ENABLE;
+bool LAZY_SCOPING_ANALYSIS = 1;
 
 bool ENABLE_FRAME_INTROSPECTION = 1;
 bool BOOLS_AS_I64 = ENABLE_FRAME_INTROSPECTION;

src/core/options.h

@@ -44,7 +44,8 @@ extern bool SHOW_DISASM, FORCE_INTERPRETER, FORCE_OPTIMIZE, PROFILE, DUMPJIT, TR
 extern bool ENABLE_ICS, ENABLE_ICGENERICS, ENABLE_ICGETITEMS, ENABLE_ICSETITEMS, ENABLE_ICDELITEMS, ENABLE_ICBINEXPS,
     ENABLE_ICNONZEROS, ENABLE_ICCALLSITES, ENABLE_ICSETATTRS, ENABLE_ICGETATTRS, ENALBE_ICDELATTRS, ENABLE_ICGETGLOBALS,
     ENABLE_SPECULATION, ENABLE_OSR, ENABLE_LLVMOPTS, ENABLE_INLINING, ENABLE_REOPT, ENABLE_PYSTON_PASSES,
-    ENABLE_TYPE_FEEDBACK, ENABLE_FRAME_INTROSPECTION, ENABLE_RUNTIME_ICS, ENABLE_JIT_OBJECT_CACHE;
+    ENABLE_TYPE_FEEDBACK, ENABLE_FRAME_INTROSPECTION, ENABLE_RUNTIME_ICS, ENABLE_JIT_OBJECT_CACHE,
+    LAZY_SCOPING_ANALYSIS;
 
 // Due to a temporary LLVM limitation, represent bools as i64's instead of i1's.
 extern bool BOOLS_AS_I64;

src/runtime/builtin_modules/pyston.cpp

@@ -45,6 +45,7 @@ static Box* setOption(Box* option, Box* value) {
     else CHECK(SPECULATION_THRESHOLD);
     else CHECK(ENABLE_ICS);
     else CHECK(ENABLE_ICGETATTRS);
+    else CHECK(LAZY_SCOPING_ANALYSIS);
     else raiseExcHelper(ValueError, "unknown option name '%s", option_string->data());
 
     return None;

test/tests/del_closure_var_syntax_error.py

+try:
+    import __pyston__
+    __pyston__.setOption("LAZY_SCOPING_ANALYSIS", 0)
+except ImportError:
+    pass
+
 cases = [
 
 """

test/tests/global_and_local.py

 # I would have expected this to be valid, but cPython and pypy err out saying "name 'x' is local and global"
 
+try:
+    import __pyston__
+    __pyston__.setOption("LAZY_SCOPING_ANALYSIS", 0)
+except ImportError:
+    pass
+
 try:
     exec """
 x = 1
@@ -40,7 +46,7 @@ except SyntaxError as e:
 
 try:
     exec """
-def f(*kwargs):
+def f(**kwargs):
     global kwargs
 
 print "calling"

test/tests/name_forcing_syntax_error.py

@@ -4,6 +4,12 @@
 
 # The logic beyond this error message is oddly complicated.
 
+try:
+    import __pyston__
+    __pyston__.setOption("LAZY_SCOPING_ANALYSIS", 0)
+except ImportError:
+    pass
+
 cases = [
 
 # protip: delete this first """ to get your editor to syntax-highlight the code