Merge pull request #1351 from kmod/scoping_branch

Revert "Merge pull request #1349 from kmod/scoping"

src/analysis/function_analysis.cpp

@@ -214,8 +214,10 @@ class DefinednessBBAnalyzer : public BBAnalyzer<DefinednessAnalysis::DefinitionL
 private:
     typedef DefinednessAnalysis::DefinitionLevel DefinitionLevel;
 
+    ScopeInfo* scope_info;
+
 public:
-    DefinednessBBAnalyzer() {}
+    DefinednessBBAnalyzer(ScopeInfo* scope_info) : scope_info(scope_info) {}
 
     virtual DefinitionLevel merge(DefinitionLevel from, DefinitionLevel into) const {
         assert(from != DefinitionLevel::Unknown);
@@ -372,7 +374,8 @@ void DefinednessBBAnalyzer::processBB(Map& starting, CFGBlock* block) const {
     }
 }
 
-void DefinednessAnalysis::run(VRegMap<DefinednessAnalysis::DefinitionLevel> initial_map, CFGBlock* initial_block) {
+void DefinednessAnalysis::run(VRegMap<DefinednessAnalysis::DefinitionLevel> initial_map, CFGBlock* initial_block,
+                              ScopeInfo* scope_info) {
     Timer _t("DefinednessAnalysis()", 10);
 
     // Don't run this twice:
@@ -383,8 +386,8 @@ void DefinednessAnalysis::run(VRegMap<DefinednessAnalysis::DefinitionLevel> init
     assert(initial_map.numVregs() == nvregs);
 
     auto&& vreg_info = cfg->getVRegInfo();
-    computeFixedPoint(std::move(initial_map), initial_block, DefinednessBBAnalyzer(), false, defined_at_beginning,
-                      defined_at_end);
+    computeFixedPoint(std::move(initial_map), initial_block, DefinednessBBAnalyzer(scope_info), false,
+                      defined_at_beginning, defined_at_end);
 
     for (const auto& p : defined_at_end) {
         assert(p.second.numVregs() == nvregs);
@@ -395,6 +398,14 @@ void DefinednessAnalysis::run(VRegMap<DefinednessAnalysis::DefinitionLevel> init
         // required.resize(nvregs, /* value= */ false);
 
         for (int vreg = 0; vreg < nvregs; vreg++) {
+#ifndef NDEBUG
+            if (vreg_info.vregHasName(vreg)) {
+                ScopeInfo::VarScopeType vst = scope_info->getScopeTypeOfName(vreg_info.getName(vreg));
+                ASSERT(vst != ScopeInfo::VarScopeType::GLOBAL && vst != ScopeInfo::VarScopeType::NAME, "%s",
+                       vreg_info.getName(vreg).c_str());
+            }
+#endif
+
             auto status = p.second[vreg];
             // assert(p.second.count(name));
             // auto status = p.second.find(name)->second;
@@ -420,7 +431,7 @@ const VRegSet& DefinednessAnalysis::getDefinedVregsAtEnd(CFGBlock* block) {
 }
 
 PhiAnalysis::PhiAnalysis(VRegMap<DefinednessAnalysis::DefinitionLevel> initial_map, CFGBlock* initial_block,
-                         bool initials_need_phis, LivenessAnalysis* liveness)
+                         bool initials_need_phis, LivenessAnalysis* liveness, ScopeInfo* scope_info)
     : definedness(), empty_set(initial_map.numVregs()), liveness(liveness) {
     auto cfg = initial_block->cfg;
     auto&& vreg_info = cfg->getVRegInfo();
@@ -432,7 +443,7 @@ PhiAnalysis::PhiAnalysis(VRegMap<DefinednessAnalysis::DefinitionLevel> initial_m
     int num_vregs = initial_map.numVregs();
     assert(num_vregs == vreg_info.getTotalNumOfVRegs());
 
-    definedness.run(std::move(initial_map), initial_block);
+    definedness.run(std::move(initial_map), initial_block, scope_info);
 
     Timer _t("PhiAnalysis()", 10);
 
@@ -523,7 +534,8 @@ std::unique_ptr<LivenessAnalysis> computeLivenessInfo(CFG* cfg) {
     return std::unique_ptr<LivenessAnalysis>(new LivenessAnalysis(cfg));
 }
 
-std::unique_ptr<PhiAnalysis> computeRequiredPhis(const ParamNames& args, CFG* cfg, LivenessAnalysis* liveness) {
+std::unique_ptr<PhiAnalysis> computeRequiredPhis(const ParamNames& args, CFG* cfg, LivenessAnalysis* liveness,
+                                                 ScopeInfo* scope_info) {
     static StatCounter counter("num_phi_analysis");
     counter.log();
 
@@ -550,11 +562,11 @@ std::unique_ptr<PhiAnalysis> computeRequiredPhis(const ParamNames& args, CFG* cf
     assert(initial_map.numVregs() == vreg_info.getTotalNumOfVRegs());
 
     return std::unique_ptr<PhiAnalysis>(
-        new PhiAnalysis(std::move(initial_map), cfg->getStartingBlock(), false, liveness));
+        new PhiAnalysis(std::move(initial_map), cfg->getStartingBlock(), false, liveness, scope_info));
 }
 
-std::unique_ptr<PhiAnalysis> computeRequiredPhis(const OSREntryDescriptor* entry_descriptor,
-                                                 LivenessAnalysis* liveness) {
+std::unique_ptr<PhiAnalysis> computeRequiredPhis(const OSREntryDescriptor* entry_descriptor, LivenessAnalysis* liveness,
+                                                 ScopeInfo* scope_info) {
     static StatCounter counter("num_phi_analysis");
     counter.log();
 
@@ -576,6 +588,6 @@ std::unique_ptr<PhiAnalysis> computeRequiredPhis(const OSREntryDescriptor* entry
     }
 
     return std::unique_ptr<PhiAnalysis>(
-        new PhiAnalysis(std::move(initial_map), entry_descriptor->backedge->target, true, liveness));
+        new PhiAnalysis(std::move(initial_map), entry_descriptor->backedge->target, true, liveness, scope_info));
 }
 }

src/analysis/function_analysis.h

@@ -31,6 +31,7 @@ class AST_Jump;
 class AST_Name;
 class CFG;
 class CFGBlock;
+class ScopeInfo;
 class LivenessBBVisitor;
 
 class LivenessAnalysis {
@@ -71,7 +72,7 @@ private:
 public:
     DefinednessAnalysis() {}
 
-    void run(VRegMap<DefinitionLevel> initial_map, CFGBlock* initial_block);
+    void run(VRegMap<DefinitionLevel> initial_map, CFGBlock* initial_block, ScopeInfo* scope_info);
 
     DefinitionLevel isDefinedAtEnd(int vreg, CFGBlock* block);
     const VRegSet& getDefinedVregsAtEnd(CFGBlock* block);
@@ -93,7 +94,7 @@ public:
     // Initials_need_phis specifies that initial_map should count as an additional entry point
     // that may require phis.
     PhiAnalysis(VRegMap<DefinednessAnalysis::DefinitionLevel> initial_map, CFGBlock* initial_block,
-                bool initials_need_phis, LivenessAnalysis* liveness);
+                bool initials_need_phis, LivenessAnalysis* liveness, ScopeInfo* scope_info);
 
     bool isRequired(int vreg, CFGBlock* block);
     bool isRequiredAfter(int vreg, CFGBlock* block);
@@ -106,8 +107,8 @@ public:
 };
 
 std::unique_ptr<LivenessAnalysis> computeLivenessInfo(CFG*);
-std::unique_ptr<PhiAnalysis> computeRequiredPhis(const ParamNames&, CFG*, LivenessAnalysis*);
-std::unique_ptr<PhiAnalysis> computeRequiredPhis(const OSREntryDescriptor*, LivenessAnalysis*);
+std::unique_ptr<PhiAnalysis> computeRequiredPhis(const ParamNames&, CFG*, LivenessAnalysis*, ScopeInfo* scope_info);
+std::unique_ptr<PhiAnalysis> computeRequiredPhis(const OSREntryDescriptor*, LivenessAnalysis*, ScopeInfo* scope_info);
 }
 
 #endif

src/analysis/scoping_analysis.cpp

@@ -24,28 +24,6 @@
 
 namespace pyston {
 
-ScopingResults::ScopingResults(ScopeInfo* scope_info, bool globals_from_module)
-    : are_locals_from_module(scope_info->areLocalsFromModule()),
-      are_globals_from_module(globals_from_module),
-      creates_closure(scope_info->createsClosure()),
-      takes_closure(scope_info->takesClosure()),
-      passes_through_closure(scope_info->passesThroughClosure()),
-      uses_name_lookup(scope_info->usesNameLookup()),
-      closure_size(creates_closure ? scope_info->getClosureSize() : 0) {
-    deref_info = scope_info->getAllDerefVarsAndInfo();
-}
-
-DerefInfo ScopingResults::getDerefInfo(AST_Name* node) const {
-    assert(node->lookup_type == ScopeInfo::VarScopeType::DEREF);
-    assert(node->deref_info.offset != INT_MAX);
-    return node->deref_info;
-}
-size_t ScopingResults::getClosureOffset(AST_Name* node) const {
-    assert(node->lookup_type == ScopeInfo::VarScopeType::CLOSURE);
-    assert(node->closure_offset != -1);
-    return node->closure_offset;
-}
-
 class YieldVisitor : public NoopASTVisitor {
 public:
     AST* starting_node;
@@ -925,13 +903,13 @@ void ScopingAnalysis::processNameUsages(ScopingAnalysis::NameUsageMap* usages) {
         ScopeNameUsage* usage = sorted_usages[i];
         AST* node = usage->node;
 
-        ScopeInfo* parent_info
-            = this->scopes[(usage->parent == NULL) ? this->parent_module : usage->parent->node].get();
+        ScopeInfo* parent_info = this->scopes[(usage->parent == NULL) ? this->parent_module : usage->parent->node];
 
         switch (node->type) {
             case AST_TYPE::ClassDef: {
-                this->scopes[node] = llvm::make_unique<ScopeInfoBase>(parent_info, usage, usage->node,
-                                                                      true /* usesNameLookup */, globals_from_module);
+                ScopeInfoBase* scopeInfo = new ScopeInfoBase(parent_info, usage, usage->node, true /* usesNameLookup */,
+                                                             globals_from_module);
+                this->scopes[node] = scopeInfo;
                 break;
             }
             case AST_TYPE::FunctionDef:
@@ -939,9 +917,10 @@ void ScopingAnalysis::processNameUsages(ScopingAnalysis::NameUsageMap* usages) {
             case AST_TYPE::GeneratorExp:
             case AST_TYPE::DictComp:
             case AST_TYPE::SetComp: {
-                this->scopes[node] = llvm::make_unique<ScopeInfoBase>(
-                    parent_info, usage, usage->node, usage->hasNameForcingSyntax() /* usesNameLookup */,
-                    globals_from_module);
+                ScopeInfoBase* scopeInfo
+                    = new ScopeInfoBase(parent_info, usage, usage->node,
+                                        usage->hasNameForcingSyntax() /* usesNameLookup */, globals_from_module);
+                this->scopes[node] = scopeInfo;
                 break;
             }
             default:
@@ -955,22 +934,45 @@ InternedStringPool& ScopingAnalysis::getInternedStrings() {
     return *interned_strings;
 }
 
-void ScopingAnalysis::analyzeSubtree(AST* node) {
+ScopeInfo* ScopingAnalysis::analyzeSubtree(AST* node) {
     NameUsageMap usages;
     usages[node] = new ScopeNameUsage(node, NULL, this);
     NameCollectorVisitor::collect(node, &usages, this);
 
     processNameUsages(&usages);
+
+    ScopeInfo* rtn = scopes[node];
+    assert(rtn);
+    return rtn;
+}
+
+void ScopingAnalysis::registerScopeReplacement(AST* original_node, AST* new_node) {
+    assert(scope_replacements.count(original_node) == 0);
+    assert(scope_replacements.count(new_node) == 0);
+    assert(scopes.count(new_node) == 0);
+
+#ifndef NDEBUG
+    // NULL this out just to make sure it doesn't get accessed:
+    scopes[new_node] = NULL;
+#endif
+
+    scope_replacements[new_node] = original_node;
 }
 
 ScopeInfo* ScopingAnalysis::getScopeInfoForNode(AST* node) {
     assert(node);
 
-    if (!scopes.count(node))
-        analyzeSubtree(node);
+    auto it = scope_replacements.find(node);
+    if (it != scope_replacements.end())
+        node = it->second;
+
+    auto rtn = scopes.find(node);
+    if (rtn != scopes.end()) {
+        assert(rtn->second);
+        return rtn->second;
+    }
 
-    assert(scopes.count(node));
-    return scopes[node].get();
+    return analyzeSubtree(node);
 }
 
 ScopingAnalysis::ScopingAnalysis(AST* ast, bool globals_from_module)
@@ -991,10 +993,10 @@ ScopingAnalysis::ScopingAnalysis(AST* ast, bool globals_from_module)
 
     if (globals_from_module) {
         assert(ast->type == AST_TYPE::Module);
-        scopes[ast] = llvm::make_unique<ModuleScopeInfo>();
+        scopes[ast] = new ModuleScopeInfo();
         parent_module = static_cast<AST_Module*>(ast);
     } else {
-        scopes[ast] = llvm::make_unique<EvalExprScopeInfo>(ast, globals_from_module);
+        scopes[ast] = new EvalExprScopeInfo(ast, globals_from_module);
     }
 }
 }

src/analysis/scoping_analysis.h

@@ -19,7 +19,6 @@
 
 #include "core/common.h"
 #include "core/stringpool.h"
-#include "core/types.h"
 
 namespace pyston {
 
@@ -28,6 +27,16 @@ class AST_Module;
 class AST_Expression;
 class AST_Suite;
 
+// Each closure has an array (fixed-size for that particular scope) of variables
+// and a parent pointer to a parent closure. To look up a variable from the passed-in
+// closure (i.e., DEREF), you just need to know (i) how many parents up to go and
+// (ii) what offset into the array to find the variable. This struct stores that
+// information. You can query the ScopeInfo with a name to get this info.
+struct DerefInfo {
+    size_t num_parents_from_passed_closure;
+    size_t offset;
+};
+
 class ScopeInfo {
 public:
     ScopeInfo() {}
@@ -142,16 +151,27 @@ public:
     typedef llvm::DenseMap<AST*, ScopeNameUsage*> NameUsageMap;
 
 private:
-    llvm::DenseMap<AST*, std::unique_ptr<ScopeInfo>> scopes;
+    llvm::DenseMap<AST*, ScopeInfo*> scopes;
     AST_Module* parent_module;
     InternedStringPool* interned_strings;
 
-    void analyzeSubtree(AST* node);
+    llvm::DenseMap<AST*, AST*> scope_replacements;
+
+    ScopeInfo* analyzeSubtree(AST* node);
     void processNameUsages(NameUsageMap* usages);
 
     bool globals_from_module;
 
 public:
+    // The scope-analysis is done before any CFG-ization is done,
+    // but many of the queries will be done post-CFG-ization.
+    // The CFG process can replace scope AST nodes with others (ex:
+    // generator expressions with generator functions), so we need to
+    // have a way of mapping the original analysis with the new queries.
+    // This is a hook for the CFG process to register when it has replaced
+    // a scope-node with a different node.
+    void registerScopeReplacement(AST* original_node, AST* new_node);
+
     ScopingAnalysis(AST* ast, bool globals_from_module);
     ScopeInfo* getScopeInfoForNode(AST* node);
 

src/analysis/type_analysis.cpp

@@ -93,14 +93,17 @@ private:
     ExprTypeMap& expr_types;
     TypeSpeculations& type_speculations;
     TypeAnalysis::SpeculationLevel speculation;
+    ScopeInfo* scope_info;
 
     BasicBlockTypePropagator(CFGBlock* block, TypeMap& initial, ExprTypeMap& expr_types,
-                             TypeSpeculations& type_speculations, TypeAnalysis::SpeculationLevel speculation)
+                             TypeSpeculations& type_speculations, TypeAnalysis::SpeculationLevel speculation,
+                             ScopeInfo* scope_info)
         : block(block),
           sym_table(initial),
           expr_types(expr_types),
           type_speculations(type_speculations),
-          speculation(speculation) {}
+          speculation(speculation),
+          scope_info(scope_info) {}
 
     void run() {
         for (int i = 0; i < block->body.size(); i++) {
@@ -426,7 +429,8 @@ private:
     }
 
     void* visit_name(AST_Name* node) override {
-        assert(node->lookup_type != ScopeInfo::VarScopeType::UNKNOWN);
+        if (node->lookup_type == ScopeInfo::VarScopeType::UNKNOWN)
+            node->lookup_type = scope_info->getScopeTypeOfName(node->id);
         auto name_scope = node->lookup_type;
 
         if (name_scope == ScopeInfo::VarScopeType::GLOBAL) {
@@ -667,9 +671,10 @@ private:
 
 public:
     static TypeMap propagate(CFGBlock* block, const TypeMap& starting, ExprTypeMap& expr_types,
-                             TypeSpeculations& type_speculations, TypeAnalysis::SpeculationLevel speculation) {
+                             TypeSpeculations& type_speculations, TypeAnalysis::SpeculationLevel speculation,
+                             ScopeInfo* scope_info) {
         TypeMap ending = starting;
-        BasicBlockTypePropagator(block, ending, expr_types, type_speculations, speculation).run();
+        BasicBlockTypePropagator(block, ending, expr_types, type_speculations, speculation, scope_info).run();
         return ending;
     }
 };
@@ -748,8 +753,8 @@ public:
         return changed;
     }
 
-    static PropagatingTypeAnalysis* doAnalysis(SpeculationLevel speculation, TypeMap&& initial_types,
-                                               CFGBlock* initial_block) {
+    static PropagatingTypeAnalysis* doAnalysis(SpeculationLevel speculation, ScopeInfo* scope_info,
+                                               TypeMap&& initial_types, CFGBlock* initial_block) {
         Timer _t("PropagatingTypeAnalysis::doAnalysis()");
 
         CFG* cfg = initial_block->cfg;
@@ -790,7 +795,7 @@ public:
             }
 
             TypeMap ending = BasicBlockTypePropagator::propagate(block, starting_types.find(block)->second, expr_types,
-                                                                 type_speculations, speculation);
+                                                                 type_speculations, speculation, scope_info);
 
             if (VERBOSITY("types") >= 3) {
                 printf("before (after):\n");
@@ -846,7 +851,7 @@ public:
 
 // public entry point:
 TypeAnalysis* doTypeAnalysis(CFG* cfg, const ParamNames& arg_names, const std::vector<ConcreteCompilerType*>& arg_types,
-                             EffortLevel effort, TypeAnalysis::SpeculationLevel speculation) {
+                             EffortLevel effort, TypeAnalysis::SpeculationLevel speculation, ScopeInfo* scope_info) {
     // if (effort == EffortLevel::INTERPRETED) {
     // return new NullTypeAnalysis();
     //}
@@ -865,11 +870,12 @@ TypeAnalysis* doTypeAnalysis(CFG* cfg, const ParamNames& arg_names, const std::v
 
     assert(i == arg_types.size());
 
-    return PropagatingTypeAnalysis::doAnalysis(speculation, std::move(initial_types), cfg->getStartingBlock());
+    return PropagatingTypeAnalysis::doAnalysis(speculation, scope_info, std::move(initial_types),
+                                               cfg->getStartingBlock());
 }
 
 TypeAnalysis* doTypeAnalysis(const OSREntryDescriptor* entry_descriptor, EffortLevel effort,
-                             TypeAnalysis::SpeculationLevel speculation) {
+                             TypeAnalysis::SpeculationLevel speculation, ScopeInfo* scope_info) {
     auto cfg = entry_descriptor->md->source->cfg;
     auto&& vreg_info = cfg->getVRegInfo();
     TypeMap initial_types(vreg_info.getTotalNumOfVRegs());
@@ -878,7 +884,7 @@ TypeAnalysis* doTypeAnalysis(const OSREntryDescriptor* entry_descriptor, EffortL
         initial_types[p.first] = p.second;
     }
 
-    return PropagatingTypeAnalysis::doAnalysis(speculation, std::move(initial_types),
+    return PropagatingTypeAnalysis::doAnalysis(speculation, scope_info, std::move(initial_types),
                                                entry_descriptor->backedge->target);
 }
 }

src/analysis/type_analysis.h

@@ -23,6 +23,7 @@
 
 namespace pyston {
 
+class ScopeInfo;
 class CFGBlock;
 class BoxedClass;
 class AST_expr;
@@ -46,9 +47,9 @@ public:
 
 TypeAnalysis* doTypeAnalysis(CFG* cfg, const ParamNames& param_names,
                              const std::vector<ConcreteCompilerType*>& arg_types, EffortLevel effort,
-                             TypeAnalysis::SpeculationLevel speculation);
+                             TypeAnalysis::SpeculationLevel speculation, ScopeInfo* scope_info);
 TypeAnalysis* doTypeAnalysis(const OSREntryDescriptor* entry_descriptor, EffortLevel effort,
-                             TypeAnalysis::SpeculationLevel speculation);
+                             TypeAnalysis::SpeculationLevel speculation, ScopeInfo* scope_info);
 }
 
 #endif

src/codegen/ast_interpreter.h

@@ -47,11 +47,11 @@ struct ASTInterpreterJitInterface {
     static int getGlobalsOffset();
 
     static void delNameHelper(void* _interpreter, InternedString name);
-    static Box* derefHelper(void* interp, AST_Name* node);
+    static Box* derefHelper(void* interp, InternedString s);
     static Box* landingpadHelper(void* interp);
     static void pendingCallsCheckHelper();
     static void setExcInfoHelper(void* interp, STOLEN(Box*) type, STOLEN(Box*) value, STOLEN(Box*) traceback);
-    static void setLocalClosureHelper(void* interp, AST_Name* name, Box* v);
+    static void setLocalClosureHelper(void* interp, long vreg, InternedString id, Box* v);
     static void uncacheExcInfoHelper(void* interp);
     static void raise0Helper(void* interp) __attribute__((noreturn));
     static Box* yieldHelper(void* interp, STOLEN(Box*) value);

src/codegen/baseline_jit.cpp

@@ -372,8 +372,14 @@ RewriterVar* JitFragmentWriter::emitCreateTuple(const llvm::ArrayRef<RewriterVar
     return r;
 }
 
-RewriterVar* JitFragmentWriter::emitDeref(AST_Name* name) {
-    return call(false, (void*)ASTInterpreterJitInterface::derefHelper, getInterp(), imm(name))->setType(RefType::OWNED);
+RewriterVar* JitFragmentWriter::emitDeref(InternedString s) {
+    return call(false, (void*)ASTInterpreterJitInterface::derefHelper, getInterp(),
+#ifndef NDEBUG
+                imm(asUInt(s).first), imm(asUInt(s).second))
+#else
+                imm(asUInt(s)))
+#endif
+        ->setType(RefType::OWNED);
 }
 
 RewriterVar* JitFragmentWriter::emitExceptionMatches(RewriterVar* v, RewriterVar* cls) {
@@ -385,22 +391,20 @@ RewriterVar* JitFragmentWriter::emitGetAttr(RewriterVar* obj, BoxedString* s, AS
         .first->setType(RefType::OWNED);
 }
 
-RewriterVar* JitFragmentWriter::emitGetBlockLocal(AST_Name* name) {
-    auto s = name->id;
-    auto vreg = name->vreg;
+RewriterVar* JitFragmentWriter::emitGetBlockLocal(InternedString s, int vreg) {
     auto it = local_syms.find(s);
     if (it == local_syms.end()) {
-        auto r = emitGetLocal(name);
+        auto r = emitGetLocal(s, vreg);
         assert(r->reftype == RefType::OWNED);
-        emitSetBlockLocal(name, r);
+        emitSetBlockLocal(s, vreg, r);
         return r;
     }
     return it->second;
 }
 
-void JitFragmentWriter::emitKillTemporary(AST_Name* name) {
-    if (!local_syms.count(name->id))
-        emitSetLocal(name, false, imm(nullptr));
+void JitFragmentWriter::emitKillTemporary(InternedString s, int vreg) {
+    if (!local_syms.count(s))
+        emitSetLocal(s, vreg, false, imm(nullptr));
 }
 
 RewriterVar* JitFragmentWriter::emitGetBoxedLocal(BoxedString* s) {
@@ -432,11 +436,9 @@ RewriterVar* JitFragmentWriter::emitGetItem(AST_expr* node, RewriterVar* value,
         .first->setType(RefType::OWNED);
 }
 
-RewriterVar* JitFragmentWriter::emitGetLocal(AST_Name* name) {
+RewriterVar* JitFragmentWriter::emitGetLocal(InternedString s, int vreg) {
     if (LOG_BJIT_ASSEMBLY)
         comment("BJIT: emitGetLocal start");
-    auto vreg = name->vreg;
-    auto s = name->id;
     assert(vreg >= 0);
     // TODO Can we use BORROWED here? Not sure if there are cases when we can't rely on borrowing the ref
     // from the vregs array.  Safer like this.
@@ -684,16 +686,14 @@ void JitFragmentWriter::emitSetAttr(AST_expr* node, RewriterVar* obj, BoxedStrin
     attr->refConsumed(rtn.second);
 }
 
-void JitFragmentWriter::emitSetBlockLocal(AST_Name* name, STOLEN(RewriterVar*) v) {
+void JitFragmentWriter::emitSetBlockLocal(InternedString s, int vreg, STOLEN(RewriterVar*) v) {
     if (LOG_BJIT_ASSEMBLY)
         comment("BJIT: emitSetBlockLocal() start");
-    auto vreg = name->vreg;
-    auto s = name->id;
-    RewriterVar* prev = local_syms[name->id];
+    RewriterVar* prev = local_syms[s];
     // if we never set this sym before in this BB and the symbol gets accessed in several blocks clear it because it
     // could have been set in a previous block.
     if (!prev && !block->cfg->getVRegInfo().isBlockLocalVReg(vreg))
-        emitSetLocal(name, false, imm(nullptr)); // clear out the vreg
+        emitSetLocal(s, vreg, false, imm(nullptr)); // clear out the vreg
     local_syms[s] = v;
     if (LOG_BJIT_ASSEMBLY)
         comment("BJIT: emitSetBlockLocal() end");
@@ -728,13 +728,18 @@ void JitFragmentWriter::emitSetItemName(BoxedString* s, RewriterVar* v) {
     emitSetItem(emitGetBoxedLocals(), imm(s), v);
 }
 
-void JitFragmentWriter::emitSetLocal(AST_Name* name, bool set_closure, STOLEN(RewriterVar*) v) {
+void JitFragmentWriter::emitSetLocal(InternedString s, int vreg, bool set_closure, STOLEN(RewriterVar*) v) {
     if (LOG_BJIT_ASSEMBLY)
         comment("BJIT: emitSetLocal() start");
-    auto vreg = name->vreg;
     assert(vreg >= 0);
     if (set_closure) {
-        call(false, (void*)ASTInterpreterJitInterface::setLocalClosureHelper, getInterp(), imm(name), v);
+        call(false, (void*)ASTInterpreterJitInterface::setLocalClosureHelper, getInterp(), imm(vreg),
+#ifndef NDEBUG
+             imm(asUInt(s).first), imm(asUInt(s).second),
+#else
+             imm(asUInt(s)),
+#endif
+             v);
         v->refConsumed();
     } else {
         // TODO With definedness analysis, we could know whether we needed to emit an decref/xdecref/neither.

src/codegen/baseline_jit.h

@@ -278,17 +278,17 @@ public:
     RewriterVar* emitCreateSet(const llvm::ArrayRef<RewriterVar*> values);
     RewriterVar* emitCreateSlice(RewriterVar* start, RewriterVar* stop, RewriterVar* step);
     RewriterVar* emitCreateTuple(const llvm::ArrayRef<RewriterVar*> values);
-    RewriterVar* emitDeref(AST_Name* name);
+    RewriterVar* emitDeref(InternedString s);
     RewriterVar* emitExceptionMatches(RewriterVar* v, RewriterVar* cls);
     RewriterVar* emitGetAttr(RewriterVar* obj, BoxedString* s, AST_expr* node);
-    RewriterVar* emitGetBlockLocal(AST_Name* name);
-    void emitKillTemporary(AST_Name* name);
+    RewriterVar* emitGetBlockLocal(InternedString s, int vreg);
+    void emitKillTemporary(InternedString s, int vreg);
     RewriterVar* emitGetBoxedLocal(BoxedString* s);
     RewriterVar* emitGetBoxedLocals();
     RewriterVar* emitGetClsAttr(RewriterVar* obj, BoxedString* s);
     RewriterVar* emitGetGlobal(BoxedString* s);
     RewriterVar* emitGetItem(AST_expr* node, RewriterVar* value, RewriterVar* slice);
-    RewriterVar* emitGetLocal(AST_Name* name);
+    RewriterVar* emitGetLocal(InternedString s, int vreg);
     RewriterVar* emitGetPystonIter(RewriterVar* v);
     RewriterVar* emitHasnext(RewriterVar* v);
     RewriterVar* emitImportFrom(RewriterVar* module, BoxedString* name);
@@ -318,13 +318,13 @@ public:
     void emitRaise3(RewriterVar* arg0, RewriterVar* arg1, RewriterVar* arg2);
     void emitReturn(RewriterVar* v);
     void emitSetAttr(AST_expr* node, RewriterVar* obj, BoxedString* s, STOLEN(RewriterVar*) attr);
-    void emitSetBlockLocal(AST_Name* name, STOLEN(RewriterVar*) v);
+    void emitSetBlockLocal(InternedString s, int vreg, STOLEN(RewriterVar*) v);
     void emitSetCurrentInst(AST_stmt* node);
     void emitSetExcInfo(RewriterVar* type, RewriterVar* value, RewriterVar* traceback);
     void emitSetGlobal(BoxedString* s, STOLEN(RewriterVar*) v, bool are_globals_from_module);
     void emitSetItemName(BoxedString* s, RewriterVar* v);
     void emitSetItem(RewriterVar* target, RewriterVar* slice, RewriterVar* value);
-    void emitSetLocal(AST_Name* name, bool set_closure, STOLEN(RewriterVar*) v);
+    void emitSetLocal(InternedString s, int vreg, bool set_closure, STOLEN(RewriterVar*) v);
     // emitSideExit steals a full ref from v, not just a vref
     void emitSideExit(STOLEN(RewriterVar*) v, Box* cmp_value, CFGBlock* next_block);
     void emitUncacheExcInfo();

src/codegen/codegen.cpp

@@ -38,10 +38,10 @@
 namespace pyston {
 
 FunctionMetadata::FunctionMetadata(int num_args, bool takes_varargs, bool takes_kwargs,
-                                   std::unique_ptr<SourceInfo> source, ParamNames param_names)
+                                   std::unique_ptr<SourceInfo> source)
     : code_obj(NULL),
       source(std::move(source)),
-      param_names(std::move(param_names)),
+      param_names(this->source->ast, this->source->getInternedStrings()),
       takes_varargs(takes_varargs),
       takes_kwargs(takes_kwargs),
       num_args(num_args),
@@ -91,8 +91,8 @@ void FunctionMetadata::addVersion(CompiledFunction* compiled) {
     }
 }
 
-SourceInfo::SourceInfo(BoxedModule* m, ScopingResults scoping, FutureFlags future_flags, AST* ast, BoxedString* fn)
-    : parent_module(m), scoping(std::move(scoping)), ast(ast), cfg(NULL), future_flags(future_flags) {
+SourceInfo::SourceInfo(BoxedModule* m, ScopingAnalysis* scoping, FutureFlags future_flags, AST* ast, BoxedString* fn)
+    : parent_module(m), scoping(scoping), scope_info(NULL), ast(ast), cfg(NULL), future_flags(future_flags) {
     assert(fn);
 
     // TODO: this is a very bad way of handling this:

src/codegen/irgen.cpp

@@ -408,7 +408,7 @@ static void emitBBs(IRGenState* irstate, TypeAnalysis* types, const OSREntryDesc
         irstate->getRefcounts()->setType(osr_created_closure, RefType::BORROWED);
         if (source->is_generator)
             irstate->setPassedGenerator(osr_generator);
-        if (source->scoping.createsClosure())
+        if (source->getScopeInfo()->createsClosure())
             irstate->setCreatedClosure(osr_created_closure);
 
         int arg_num = -1;
@@ -730,16 +730,13 @@ static void emitBBs(IRGenState* irstate, TypeAnalysis* types, const OSREntryDesc
                         auto asgn = ast_cast<AST_Assign>(stmt);
                         assert(asgn->targets.size() == 1);
                         if (asgn->targets[0]->type == AST_TYPE::Name) {
-                            auto asname = ast_cast<AST_Name>(asgn->targets[0]);
-                            assert(asname->lookup_type != ScopeInfo::VarScopeType::UNKNOWN);
-
-                            InternedString name = asname->id;
+                            InternedString name = ast_cast<AST_Name>(asgn->targets[0])->id;
                             int vreg = ast_cast<AST_Name>(asgn->targets[0])->vreg;
                             assert(name.c_str()[0] == '#'); // it must be a temporary
                             // You might think I need to check whether `name' is being assigned globally or locally,
                             // since a global assign doesn't affect the symbol table. However, the CFG pass only
                             // generates invoke-assigns to temporary variables. Just to be sure, we assert:
-                            assert(asname->lookup_type != ScopeInfo::VarScopeType::GLOBAL);
+                            assert(source->getScopeInfo()->getScopeTypeOfName(name) != ScopeInfo::VarScopeType::GLOBAL);
 
                             // TODO: inefficient
                             sym_table = new SymbolTable(*sym_table);
@@ -1066,13 +1063,13 @@ std::pair<CompiledFunction*, llvm::Function*> doCompile(FunctionMetadata* md, So
         int nargs = param_names->totalParameters();
         ASSERT(nargs == spec->arg_types.size(), "%d %ld", nargs, spec->arg_types.size());
 
-        if (source->scoping.takesClosure())
+        if (source->getScopeInfo()->takesClosure())
             llvm_arg_types.push_back(g.llvm_closure_type_ptr);
 
         if (source->is_generator)
             llvm_arg_types.push_back(g.llvm_generator_type_ptr);
 
-        if (!source->scoping.areGlobalsFromModule())
+        if (!source->scoping->areGlobalsFromModule())
             llvm_arg_types.push_back(g.llvm_value_type_ptr);
 
         for (int i = 0; i < nargs; i++) {
@@ -1112,9 +1109,10 @@ std::pair<CompiledFunction*, llvm::Function*> doCompile(FunctionMetadata* md, So
         speculation_level = TypeAnalysis::SOME;
     TypeAnalysis* types;
     if (entry_descriptor)
-        types = doTypeAnalysis(entry_descriptor, effort, speculation_level);
+        types = doTypeAnalysis(entry_descriptor, effort, speculation_level, source->getScopeInfo());
     else
-        types = doTypeAnalysis(source->cfg, *param_names, spec->arg_types, effort, speculation_level);
+        types = doTypeAnalysis(source->cfg, *param_names, spec->arg_types, effort, speculation_level,
+                               source->getScopeInfo());
 
 
     _t2.split();
@@ -1133,9 +1131,9 @@ std::pair<CompiledFunction*, llvm::Function*> doCompile(FunctionMetadata* md, So
     std::unique_ptr<PhiAnalysis> phis;
 
     if (entry_descriptor)
-        phis = computeRequiredPhis(entry_descriptor, liveness);
+        phis = computeRequiredPhis(entry_descriptor, liveness, source->getScopeInfo());
     else
-        phis = computeRequiredPhis(*param_names, source->cfg, liveness);
+        phis = computeRequiredPhis(*param_names, source->cfg, liveness, source->getScopeInfo());
 
     RefcountTracker refcounter;
 

src/codegen/irgen/hooks.cpp

@@ -54,6 +54,78 @@
 
 namespace pyston {
 
+// TODO terrible place for these!
+ParamNames::ParamNames(AST* ast, InternedStringPool& pool)
+    : all_args_contains_names(1), takes_param_names(1), has_vararg_name(0), has_kwarg_name(0) {
+    if (ast->type == AST_TYPE::Module || ast->type == AST_TYPE::ClassDef || ast->type == AST_TYPE::Expression
+        || ast->type == AST_TYPE::Suite) {
+    } else if (ast->type == AST_TYPE::FunctionDef || ast->type == AST_TYPE::Lambda) {
+        AST_arguments* arguments = ast->type == AST_TYPE::FunctionDef ? ast_cast<AST_FunctionDef>(ast)->args
+                                                                      : ast_cast<AST_Lambda>(ast)->args;
+        for (int i = 0; i < arguments->args.size(); i++) {
+            AST_expr* arg = arguments->args[i];
+            if (arg->type == AST_TYPE::Name) {
+                AST_Name* name = ast_cast<AST_Name>(arg);
+                all_args.emplace_back(name);
+            } else {
+                InternedString dot_arg_name = pool.get("." + std::to_string(i));
+                all_args.emplace_back(new AST_Name(dot_arg_name, AST_TYPE::Param, arg->lineno, arg->col_offset));
+            }
+        }
+
+        auto vararg_name = arguments->vararg;
+        if (vararg_name) {
+            has_vararg_name = 1;
+            all_args.emplace_back(vararg_name);
+        }
+
+        auto kwarg_name = arguments->kwarg;
+        if (kwarg_name) {
+            has_kwarg_name = 1;
+            all_args.emplace_back(kwarg_name);
+        }
+    } else {
+        RELEASE_ASSERT(0, "%d", ast->type);
+    }
+}
+
+ParamNames::ParamNames(const std::vector<const char*>& args, const char* vararg, const char* kwarg)
+    : all_args_contains_names(0),
+      takes_param_names(1),
+      has_vararg_name(vararg && *vararg),
+      has_kwarg_name(kwarg && *kwarg) {
+    all_args.reserve(args.size() + has_vararg_name + has_kwarg_name);
+    for (auto&& arg : args) {
+        all_args.emplace_back(arg);
+    }
+    if (has_vararg_name)
+        all_args.emplace_back(vararg);
+    if (has_kwarg_name)
+        all_args.emplace_back(kwarg);
+}
+
+std::vector<const char*> ParamNames::allArgsAsStr() const {
+    std::vector<const char*> ret;
+    ret.reserve(all_args.size());
+    if (all_args_contains_names) {
+        for (auto&& arg : all_args) {
+            ret.push_back(arg.name->id.c_str());
+        }
+    } else {
+        for (auto&& arg : all_args) {
+            ret.push_back(arg.str);
+        }
+    }
+    return ret;
+}
+
+InternedString SourceInfo::mangleName(InternedString id) {
+    assert(ast);
+    if (ast->type == AST_TYPE::Module)
+        return id;
+    return getScopeInfo()->mangleName(id);
+}
+
 llvm::ArrayRef<AST_stmt*> SourceInfo::getBody() const {
     switch (ast->type) {
         case AST_TYPE::ClassDef:
@@ -69,6 +141,10 @@ llvm::ArrayRef<AST_stmt*> SourceInfo::getBody() const {
     };
 }
 
+InternedStringPool& SourceInfo::getInternedStrings() {
+    return scoping->getInternedStrings();
+}
+
 BORROWED(BoxedString*) SourceInfo::getFn() {
     assert(fn->ob_refcnt >= 1);
     return fn;
@@ -106,6 +182,12 @@ Box* SourceInfo::getDocString() {
     return incref(Py_None);
 }
 
+ScopeInfo* SourceInfo::getScopeInfo() {
+    if (!scope_info)
+        scope_info = scoping->getScopeInfoForNode(ast);
+    return scope_info;
+}
+
 LivenessAnalysis* SourceInfo::getLiveness() {
     if (!liveness_info)
         liveness_info = computeLivenessInfo(cfg);
@@ -228,7 +310,10 @@ CompiledFunction* compileFunction(FunctionMetadata* f, FunctionSpecialization* s
         printf("%s", ss.str().c_str());
     }
 
-    assert(source->cfg);
+    // Do the analysis now if we had deferred it earlier:
+    if (source->cfg == NULL) {
+        source->cfg = computeCFG(source, f->param_names);
+    }
 
 
     CompiledFunction* cf = NULL;
@@ -277,32 +362,42 @@ CompiledFunction* compileFunction(FunctionMetadata* f, FunctionSpecialization* s
 }
 
 void compileAndRunModule(AST_Module* m, BoxedModule* bm) {
+    FunctionMetadata* md;
+
     Timer _t("for compileModule()");
 
     const char* fn = PyModule_GetFilename(bm);
     RELEASE_ASSERT(fn, "");
 
     FutureFlags future_flags = getFutureFlags(m->body, fn);
-    computeAllCFGs(m, /* globals_from_module */ true, future_flags, autoDecref(boxString(fn)), bm);
+    ScopingAnalysis* scoping = new ScopingAnalysis(m, true);
 
-    FunctionMetadata* md = metadataForAST(m);
-    assert(md);
+    auto fn_str = boxString(fn);
+    AUTO_DECREF(fn_str);
+    std::unique_ptr<SourceInfo> si(new SourceInfo(bm, scoping, future_flags, m, fn_str));
 
     static BoxedString* doc_str = getStaticString("__doc__");
-    bm->setattr(doc_str, autoDecref(md->source->getDocString()), NULL);
+    bm->setattr(doc_str, autoDecref(si->getDocString()), NULL);
 
     static BoxedString* builtins_str = getStaticString("__builtins__");
     if (!bm->hasattr(builtins_str))
         bm->setattr(builtins_str, PyModule_GetDict(builtins_module), NULL);
 
+    md = new FunctionMetadata(0, false, false, std::move(si));
+
     UNAVOIDABLE_STAT_TIMER(t0, "us_timer_interpreted_module_toplevel");
     Box* r = astInterpretFunction(md, NULL, NULL, NULL, NULL, NULL, NULL, NULL);
     assert(r == Py_None);
     Py_DECREF(r);
+
+    // XXX for bjit testing
+    // r = astInterpretFunction(md, NULL, NULL, NULL, NULL, NULL, NULL, NULL);
+    // assert(r == None);
+    // Py_DECREF(r);
 }
 
 Box* evalOrExec(FunctionMetadata* md, Box* globals, Box* boxedLocals) {
-    RELEASE_ASSERT(!md->source->scoping.areGlobalsFromModule(), "");
+    RELEASE_ASSERT(!md->source->scoping->areGlobalsFromModule(), "");
 
     assert(globals && (globals->cls == module_cls || globals->cls == dict_cls));
 
@@ -321,6 +416,8 @@ static FunctionMetadata* compileForEvalOrExec(AST* source, llvm::ArrayRef<AST_st
                                               PyCompilerFlags* flags) {
     Timer _t("for evalOrExec()");
 
+    ScopingAnalysis* scoping = new ScopingAnalysis(source, false);
+
     // `my_future_flags` are the future flags enabled in the exec's code.
     // `caller_future_flags` are the future flags of the source that the exec statement is in.
     // We need to enable features that are enabled in either.
@@ -332,8 +429,10 @@ static FunctionMetadata* compileForEvalOrExec(AST* source, llvm::ArrayRef<AST_st
         flags->cf_flags = future_flags;
     }
 
-    computeAllCFGs(source, /* globals_from_module */ false, future_flags, fn, getCurrentModule());
-    return metadataForAST(source);
+    std::unique_ptr<SourceInfo> si(new SourceInfo(getCurrentModule(), scoping, future_flags, source, fn));
+
+    FunctionMetadata* md = new FunctionMetadata(0, false, false, std::move(si));
+    return md;
 }
 
 static FunctionMetadata* compileExec(AST_Module* parsedModule, BoxedString* fn, PyCompilerFlags* flags) {

src/codegen/irgen/irgenerator.h

@@ -118,7 +118,8 @@ public:
     LivenessAnalysis* getLiveness() { return source_info->getLiveness(); }
     PhiAnalysis* getPhis() { return phis.get(); }
 
-    const ScopingResults& getScopeInfo();
+    ScopeInfo* getScopeInfo();
+    ScopeInfo* getScopeInfoForNode(AST* node);
 
     llvm::MDNode* getFuncDbgInfo() { return func_dbg_info; }
 
@@ -204,6 +205,7 @@ IREmitter* createIREmitter(IRGenState* irstate, llvm::BasicBlock*& curblock, IRG
 IRGenerator* createIRGenerator(IRGenState* irstate, std::unordered_map<CFGBlock*, llvm::BasicBlock*>& entry_blocks,
                                CFGBlock* myblock, TypeAnalysis* types);
 
+FunctionMetadata* wrapFunction(AST* node, AST_arguments* args, SourceInfo* source);
 std::vector<BoxedString*>* getKeywordNameStorage(AST_Call* node);
 }
 

src/codegen/patchpoints.cpp

@@ -98,9 +98,9 @@ void PatchpointInfo::parseLocationMap(StackMap::Record* r, LocationMap* map) {
     auto&& source = parentFunction()->md->source;
     if (source->is_generator)
         map->generator.locations.push_back(parse_type(GENERATOR));
-    if (source->scoping.takesClosure())
+    if (source->getScopeInfo()->takesClosure())
         map->passed_closure.locations.push_back(parse_type(CLOSURE));
-    if (source->scoping.createsClosure())
+    if (source->getScopeInfo()->createsClosure())
         map->created_closure.locations.push_back(parse_type(CLOSURE));
 
     for (FrameVarInfo& frame_var : frame_info_desc.vars) {

src/codegen/unwinding.cpp

@@ -985,12 +985,12 @@ BORROWED(Box*) FrameInfo::updateBoxedLocals() {
 
     FrameInfo* frame_info = this;
     FunctionMetadata* md = frame_info->md;
-    const ScopingResults& scope_info = md->source->scoping;
+    ScopeInfo* scope_info = md->source->getScopeInfo();
 
-    if (scope_info.areLocalsFromModule()) {
+    if (scope_info->areLocalsFromModule()) {
         // TODO we should cache this in frame_info->locals or something so that locals()
         // (and globals() too) will always return the same dict
-        RELEASE_ASSERT(md->source->scoping.areGlobalsFromModule(), "");
+        RELEASE_ASSERT(md->source->scoping->areGlobalsFromModule(), "");
         return md->source->parent_module->getAttrWrapper();
     }
 
@@ -1000,7 +1000,7 @@ BORROWED(Box*) FrameInfo::updateBoxedLocals() {
     // Add the locals from the closure
     // TODO in a ClassDef scope, we aren't supposed to add these
     size_t depth = 0;
-    for (auto& p : scope_info.getAllDerefVarsAndInfo()) {
+    for (auto& p : scope_info->getAllDerefVarsAndInfo()) {
         InternedString name = p.first;
         DerefInfo derefInfo = p.second;
         while (depth < derefInfo.num_parents_from_passed_closure) {

src/core/ast.cpp

@@ -1639,24 +1639,8 @@ bool PrintVisitor::visit_suite(AST_Suite* node) {
 
 bool PrintVisitor::visit_name(AST_Name* node) {
     stream << node->id.s();
-#if 0
-    if (node->lookup_type == ScopeInfo::VarScopeType::UNKNOWN)
-        stream << "<U>";
-    else if (node->lookup_type == ScopeInfo::VarScopeType::FAST)
-        stream << "<F>";
-    else if (node->lookup_type == ScopeInfo::VarScopeType::DEREF)
-        stream << "<D>";
-    else if (node->lookup_type == ScopeInfo::VarScopeType::CLOSURE)
-        stream << "<C>";
-    else if (node->lookup_type == ScopeInfo::VarScopeType::GLOBAL)
-        stream << "<G>";
-    else
-        stream << "<?>";
-#endif
-
-#if 0
-    if (node->is_kill) stream << "<k>";
-#endif
+    // Uncomment this line to see which names are kills:
+    // if (node->is_kill) stream << "<k>";
     return false;
 }
 

src/core/ast.h

@@ -400,8 +400,6 @@ public:
     std::vector<AST_stmt*> body;
     InternedString name;
 
-    FunctionMetadata* md;
-
     AST_ClassDef() : AST_stmt(AST_TYPE::ClassDef) {}
 
     static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::ClassDef;
@@ -511,8 +509,6 @@ public:
     // this should be an expr but we convert it into a AST_Return(AST_expr) to make the code simpler
     AST_stmt* body;
 
-    FunctionMetadata* md;
-
     virtual void accept(ASTVisitor* v);
 
     AST_Expression(std::unique_ptr<InternedStringPool> interned_strings)
@@ -553,8 +549,6 @@ public:
     InternedString name; // if the name is not set this is a lambda
     AST_arguments* args;
 
-    FunctionMetadata* md;
-
     virtual void accept(ASTVisitor* v);
     virtual void accept_stmt(StmtVisitor* v);
 
@@ -710,8 +704,6 @@ public:
     // no lineno, col_offset attributes
     std::vector<AST_stmt*> body;
 
-    FunctionMetadata* md;
-
     virtual void accept(ASTVisitor* v);
 
     AST_Module(std::unique_ptr<InternedStringPool> interned_strings)
@@ -744,16 +736,11 @@ public:
     // different bytecodes.
     ScopeInfo::VarScopeType lookup_type;
 
-    // These are only valid for lookup_type == FAST or CLOSURE
     // The interpreter and baseline JIT store variables with FAST and CLOSURE scopes in an array (vregs) this specifies
     // the zero based index of this variable inside the vregs array. If uninitialized it's value is -1.
     int vreg;
-    bool is_kill = false;
 
-    // Only valid for lookup_type == DEREF:
-    DerefInfo deref_info = DerefInfo({ INT_MAX, INT_MAX });
-    // Only valid for lookup_type == CLOSURE:
-    int closure_offset = -1;
+    bool is_kill = false;
 
     virtual void accept(ASTVisitor* v);
     virtual void* accept_expr(ExprVisitor* v);
@@ -1134,7 +1121,7 @@ public:
 };
 
 template <typename T> T* ast_cast(AST* node) {
-    ASSERT(!node || node->type == T::TYPE, "%d", node ? node->type : 0);
+    assert(!node || node->type == T::TYPE);
     return static_cast<T*>(node);
 }
 

src/core/cfg.h

@@ -169,7 +169,7 @@ public:
     int getNumOfCrossBlockVRegs() const { return num_vregs_cross_block; }
 
     bool hasVRegsAssigned() const { return num_vregs != -1; }
-    void assignVRegs(CFG* cfg, const ParamNames& param_names);
+    void assignVRegs(CFG* cfg, const ParamNames& param_names, ScopeInfo* scope_info);
 };
 
 // Control Flow Graph
@@ -330,10 +330,8 @@ public:
     iterator end() const { return iterator(*this, this->v.size()); }
 };
 
-FunctionMetadata*& metadataForAST(AST* ast);
-InternedStringPool& stringpoolForAST(AST* ast);
-
-void computeAllCFGs(AST* ast, bool globals_from_module, FutureFlags future_flags, BoxedString* fn, BoxedModule* bm);
+class SourceInfo;
+CFG* computeCFG(SourceInfo* source, const ParamNames& param_names);
 void printCFG(CFG* cfg);
 }
 

src/core/options.cpp

@@ -80,6 +80,7 @@ bool ENABLE_PYSTON_PASSES = 0 && _GLOBAL_ENABLE;
 bool ENABLE_TYPE_FEEDBACK = 1 && _GLOBAL_ENABLE;
 bool ENABLE_RUNTIME_ICS = 1 && _GLOBAL_ENABLE;
 bool ENABLE_JIT_OBJECT_CACHE = 1 && _GLOBAL_ENABLE;
+bool LAZY_SCOPING_ANALYSIS = 1;
 
 bool ENABLE_FRAME_INTROSPECTION = 1;
 

src/core/options.h

@@ -44,7 +44,8 @@ extern bool LOG_IC_ASSEMBLY, LOG_BJIT_ASSEMBLY;
 extern bool ENABLE_ICS, ENABLE_ICGENERICS, ENABLE_ICGETITEMS, ENABLE_ICSETITEMS, ENABLE_ICDELITEMS, ENABLE_ICBINEXPS,
     ENABLE_ICNONZEROS, ENABLE_ICCALLSITES, ENABLE_ICSETATTRS, ENABLE_ICGETATTRS, ENALBE_ICDELATTRS, ENABLE_ICGETGLOBALS,
     ENABLE_SPECULATION, ENABLE_OSR, ENABLE_LLVMOPTS, ENABLE_INLINING, ENABLE_REOPT, ENABLE_PYSTON_PASSES,
-    ENABLE_TYPE_FEEDBACK, ENABLE_FRAME_INTROSPECTION, ENABLE_RUNTIME_ICS, ENABLE_JIT_OBJECT_CACHE;
+    ENABLE_TYPE_FEEDBACK, ENABLE_FRAME_INTROSPECTION, ENABLE_RUNTIME_ICS, ENABLE_JIT_OBJECT_CACHE,
+    LAZY_SCOPING_ANALYSIS;
 
 // Due to a temporary LLVM limitation, represent bools as i64's instead of i1's.
 #define BOOLS_AS_I64 1

src/core/types.h

@@ -161,6 +161,7 @@ class AST_stmt;
 
 class PhiAnalysis;
 class LivenessAnalysis;
+class ScopingAnalysis;
 
 class FunctionMetadata;
 class OSREntryDescriptor;
@@ -429,60 +430,10 @@ public:
 typedef int FutureFlags;
 
 class BoxedModule;
+class ScopeInfo;
 class InternedStringPool;
 class LivenessAnalysis;
 
-// Each closure has an array (fixed-size for that particular scope) of variables
-// and a parent pointer to a parent closure. To look up a variable from the passed-in
-// closure (i.e., DEREF), you just need to know (i) how many parents up to go and
-// (ii) what offset into the array to find the variable. This struct stores that
-// information. You can query the ScopeInfo with a name to get this info.
-struct DerefInfo {
-    size_t num_parents_from_passed_closure;
-    size_t offset;
-};
-
-class ScopeInfo;
-// The results of our scoping analysis.
-// A ScopeInfo is a component of the analysis itself and contains a lot of other
-// metadata that is necessary during the analysis, after which we can throw it
-// away and only keep a ScopingResults object.
-struct ScopingResults {
-private:
-    bool are_locals_from_module : 1;
-    bool are_globals_from_module : 1;
-    bool creates_closure : 1;
-    bool takes_closure : 1;
-    bool passes_through_closure : 1;
-    bool uses_name_lookup : 1;
-
-    int closure_size;
-    std::vector<std::pair<InternedString, DerefInfo>> deref_info;
-
-public:
-    ScopingResults(ScopingResults&&) = default;
-    // Delete these just to make sure we avoid extra copies
-    ScopingResults(const ScopingResults&) = delete;
-    void operator=(const ScopingResults&) = delete;
-
-    bool areLocalsFromModule() const { return are_locals_from_module; }
-    bool areGlobalsFromModule() const { return are_globals_from_module; }
-    bool createsClosure() const { return creates_closure; }
-    bool takesClosure() const { return takes_closure; }
-    bool passesThroughClosure() const { return passes_through_closure; }
-    bool usesNameLookup() const { return uses_name_lookup; }
-
-    int getClosureSize() const {
-        assert(createsClosure());
-        return closure_size;
-    }
-    const std::vector<std::pair<InternedString, DerefInfo>>& getAllDerefVarsAndInfo() const { return deref_info; }
-    DerefInfo getDerefInfo(AST_Name*) const;
-    size_t getClosureOffset(AST_Name*) const;
-
-    ScopingResults(ScopeInfo* scope_info, bool globals_from_module);
-};
-
 // Data about a single textual function definition.
 class SourceInfo {
 private:
@@ -491,23 +442,29 @@ private:
 
 public:
     BoxedModule* parent_module;
-    ScopingResults scoping;
+    ScopingAnalysis* scoping;
+    ScopeInfo* scope_info;
     AST* ast;
     CFG* cfg;
     FutureFlags future_flags;
     bool is_generator;
 
+    InternedStringPool& getInternedStrings();
+
+    ScopeInfo* getScopeInfo();
     LivenessAnalysis* getLiveness();
 
     // does not throw CXX or CAPI exceptions:
     BORROWED(BoxedString*) getName() noexcept;
     BORROWED(BoxedString*) getFn();
 
+    InternedString mangleName(InternedString id);
+
     llvm::ArrayRef<AST_stmt*> getBody() const;
 
     Box* getDocString();
 
-    SourceInfo(BoxedModule* m, ScopingResults scoping, FutureFlags future_flags, AST* ast, BoxedString* fn);
+    SourceInfo(BoxedModule* m, ScopingAnalysis* scoping, FutureFlags future_flags, AST* ast, BoxedString* fn);
     ~SourceInfo();
 };
 
@@ -562,8 +519,7 @@ public:
                                         Box**, const std::vector<BoxedString*>*> InternalCallable;
     InternalCallable internal_callable;
 
-    FunctionMetadata(int num_args, bool takes_varargs, bool takes_kwargs, std::unique_ptr<SourceInfo> source,
-                     ParamNames param_names);
+    FunctionMetadata(int num_args, bool takes_varargs, bool takes_kwargs, std::unique_ptr<SourceInfo> source);
     FunctionMetadata(int num_args, bool takes_varargs, bool takes_kwargs,
                      const ParamNames& param_names = ParamNames::empty());
     ~FunctionMetadata();

src/runtime/builtin_modules/pyston.cpp

@@ -45,6 +45,7 @@ static Box* setOption(Box* option, Box* value) {
     else CHECK(SPECULATION_THRESHOLD);
     else CHECK(ENABLE_ICS);
     else CHECK(ENABLE_ICGETATTRS);
+    else CHECK(LAZY_SCOPING_ANALYSIS);
     else raiseExcHelper(ValueError, "unknown option name '%s", option_string->data());
 
     Py_RETURN_NONE;

src/runtime/code.cpp

@@ -197,7 +197,7 @@ extern "C" BORROWED(PyObject*) PyCode_GetName(PyCodeObject* op) noexcept {
 
 extern "C" int PyCode_HasFreeVars(PyCodeObject* _code) noexcept {
     BoxedCode* code = (BoxedCode*)_code;
-    return code->f->source->scoping.takesClosure() ? 1 : 0;
+    return code->f->source->getScopeInfo()->takesClosure() ? 1 : 0;
 }
 
 void setupCode() {

src/runtime/objmodel.cpp

@@ -4771,7 +4771,7 @@ static Box* callChosenCF(CompiledFunction* chosen_cf, BoxedClosure* closure, Box
         }
     }
 
-    assert((globals == NULL) == (!chosen_cf->md->source || chosen_cf->md->source->scoping.areGlobalsFromModule()));
+    assert((globals == NULL) == (!chosen_cf->md->source || chosen_cf->md->source->scoping->areGlobalsFromModule()));
 
     Box* maybe_args[3];
     int nmaybe_args = 0;

src/runtime/types.cpp

@@ -325,7 +325,7 @@ extern "C" BoxedFunctionBase::BoxedFunctionBase(FunctionMetadata* md, llvm::Arra
       modname(NULL),
       name(NULL),
       doc(NULL) {
-    assert((!globals) == (!md->source || md->source->scoping.areGlobalsFromModule()));
+    assert((!globals) == (!md->source || md->source->scoping->areGlobalsFromModule()));
 
     if (globals)
         ASSERT(globals->cls == dict_cls || globals->cls == module_cls, "%s", globals->cls->tp_name);
@@ -352,7 +352,7 @@ extern "C" BoxedFunctionBase::BoxedFunctionBase(FunctionMetadata* md, llvm::Arra
 
         Box* globals_for_name = globals;
         if (!globals_for_name) {
-            assert(md->source->scoping.areGlobalsFromModule());
+            assert(md->source->scoping->areGlobalsFromModule());
             globals_for_name = md->source->parent_module;
         }
 
@@ -1647,7 +1647,7 @@ static Box* function_new(BoxedClass* cls, Box* code, Box* globals, Box** _args)
 
     FunctionMetadata* md = static_cast<BoxedCode*>(code)->f;
     RELEASE_ASSERT(md->source, "");
-    if (md->source->scoping.areGlobalsFromModule()) {
+    if (md->source->scoping->areGlobalsFromModule()) {
         RELEASE_ASSERT(unwrapAttrWrapper(globals) == md->source->parent_module, "");
         globals = NULL;
     } else {
@@ -1729,13 +1729,13 @@ static Box* function_globals(Box* self, void*) noexcept {
     assert(self->cls == function_cls);
     BoxedFunction* func = static_cast<BoxedFunction*>(self);
     if (func->globals) {
-        assert(!func->md->source || !func->md->source->scoping.areGlobalsFromModule());
+        assert(!func->md->source || !func->md->source->scoping->areGlobalsFromModule());
         if (func->globals->cls == module_cls)
             return incref(func->globals->getAttrWrapper());
         return incref(func->globals);
     }
     assert(func->md->source);
-    assert(func->md->source->scoping.areGlobalsFromModule());
+    assert(func->md->source->scoping->areGlobalsFromModule());
 
     static BoxedString* dict_str = getStaticString("__dict__");
     return getattrInternal<CAPI>(func->md->source->parent_module, dict_str);

test/tests/del_closure_var_syntax_error.py

+try:
+    import __pyston__
+    __pyston__.setOption("LAZY_SCOPING_ANALYSIS", 0)
+except ImportError:
+    pass
+
 cases = [
 
 """

test/tests/global_and_local.py

 # I would have expected this to be valid, but cPython and pypy err out saying "name 'x' is local and global"
+
+try:
+    import __pyston__
+    __pyston__.setOption("LAZY_SCOPING_ANALYSIS", 0)
+except ImportError:
+    pass
+
 try:
     exec """
 x = 1

test/tests/name_forcing_syntax_error.py

@@ -4,6 +4,12 @@
 
 # The logic beyond this error message is oddly complicated.
 
+try:
+    import __pyston__
+    __pyston__.setOption("LAZY_SCOPING_ANALYSIS", 0)
+except ImportError:
+    pass
+
 cases = [
 
 # protip: delete this first """ to get your editor to syntax-highlight the code

test/tests/name_scopes.py

@@ -14,20 +14,3 @@ def f2():
 
     f3(*[1, 2], **dict(a=1, b=2))
 f2()
-
-__module__ = 1
-__name__ = 2
-__doc__ = 3
-print __module__ # prints "1"
-print __name__ # prints "2"
-print __doc__ # prints "3"
-class C(object):
-    "hello world"
-
-    # C's implicit setting of "__module__" will end up propagating to the global scope:
-    global __module__
-    global __name__
-    global __doc__
-print __module__ # prints "2"
-print __name__ # prints "2"
-print __doc__ # prints "hello world"

test/unittests/analysis.cpp

@@ -31,25 +31,21 @@ TEST_F(AnalysisTest, augassign) {
     AST_Module* module = caching_parse_file(fn.c_str(), 0);
     assert(module);
 
-    FutureFlags future_flags = getFutureFlags(module->body, fn.c_str());
-
-    auto scoping = std::make_shared<ScopingAnalysis>(module, true);
-    computeAllCFGs(module, true, future_flags, boxString(fn), NULL);
+    ScopingAnalysis *scoping = new ScopingAnalysis(module, true);
 
     assert(module->body[0]->type == AST_TYPE::FunctionDef);
     AST_FunctionDef* func = static_cast<AST_FunctionDef*>(module->body[0]);
 
     ScopeInfo* scope_info = scoping->getScopeInfoForNode(func);
-
-    ASSERT_NE(scope_info->getScopeTypeOfName(module->interned_strings->get("a")), ScopeInfo::VarScopeType::GLOBAL);
+    ASSERT_FALSE(scope_info->getScopeTypeOfName(module->interned_strings->get("a")) == ScopeInfo::VarScopeType::GLOBAL);
     ASSERT_FALSE(scope_info->getScopeTypeOfName(module->interned_strings->get("b")) == ScopeInfo::VarScopeType::GLOBAL);
 
-    ParamNames param_names(func, *module->interned_strings.get());
+    FutureFlags future_flags = getFutureFlags(module->body, fn.c_str());
 
-    // Hack to get at the cfg:
-    auto node = module->md->source->cfg->blocks[0]->body[0];
-    CFG* cfg = ast_cast<AST_MakeFunction>(ast_cast<AST_Assign>(node)->value)->function_def->md->source->cfg;
+    SourceInfo* si = new SourceInfo(createModule(boxString("augassign"), fn.c_str()), scoping, future_flags, func, boxString(fn));
 
+    ParamNames param_names(si->ast, si->getInternedStrings());
+    CFG* cfg = computeCFG(si, param_names);
     std::unique_ptr<LivenessAnalysis> liveness = computeLivenessInfo(cfg);
     auto&& vregs = cfg->getVRegInfo();
 
@@ -61,7 +57,7 @@ TEST_F(AnalysisTest, augassign) {
             ASSERT_TRUE(liveness->isLiveAtEnd(vregs.getVReg(module->interned_strings->get("a")), block));
     }
 
-    std::unique_ptr<PhiAnalysis> phis = computeRequiredPhis(ParamNames(func, *module->interned_strings.get()), cfg, liveness.get());
+    std::unique_ptr<PhiAnalysis> phis = computeRequiredPhis(ParamNames(func, si->getInternedStrings()), cfg, liveness.get(), scope_info);
 }
 
 void doOsrTest(bool is_osr, bool i_maybe_undefined) {
@@ -69,22 +65,20 @@ void doOsrTest(bool is_osr, bool i_maybe_undefined) {
     AST_Module* module = caching_parse_file(fn.c_str(), 0);
     assert(module);
 
-    ParamNames param_names(module, *module->interned_strings.get());
+    ScopingAnalysis *scoping = new ScopingAnalysis(module, true);
 
     assert(module->body[0]->type == AST_TYPE::FunctionDef);
     AST_FunctionDef* func = static_cast<AST_FunctionDef*>(module->body[0]);
 
-    auto scoping = std::make_shared<ScopingAnalysis>(module, true);
-    ScopeInfo* scope_info = scoping->getScopeInfoForNode(func);
-
     FutureFlags future_flags = getFutureFlags(module->body, fn.c_str());
 
-    computeAllCFGs(module, true, future_flags, boxString(fn), NULL);
+    ScopeInfo* scope_info = scoping->getScopeInfoForNode(func);
+    std::unique_ptr<SourceInfo> si(new SourceInfo(createModule(boxString("osr" + std::to_string((is_osr << 1) + i_maybe_undefined)),
+                    fn.c_str()), scoping, future_flags, func, boxString(fn)));
+    FunctionMetadata* clfunc = new FunctionMetadata(0, false, false, std::move(si));
 
-    // Hack to get at the cfg:
-    auto node = module->md->source->cfg->blocks[0]->body[0];
-    auto md = ast_cast<AST_MakeFunction>(ast_cast<AST_Assign>(node)->value)->function_def->md;
-    CFG* cfg = md->source->cfg;
+    CFG* cfg = computeCFG(clfunc->source.get(), clfunc->param_names);
+    clfunc->source->cfg = cfg;
     std::unique_ptr<LivenessAnalysis> liveness = computeLivenessInfo(cfg);
 
     // cfg->print();
@@ -107,16 +101,16 @@ void doOsrTest(bool is_osr, bool i_maybe_undefined) {
 
     if (is_osr) {
         int vreg = vregs.getVReg(i_str);
-        OSREntryDescriptor* entry_descriptor = OSREntryDescriptor::create(md, backedge, CXX);
+        OSREntryDescriptor* entry_descriptor = OSREntryDescriptor::create(clfunc, backedge, CXX);
         // need to set it to non-null
         ConcreteCompilerType* fake_type = (ConcreteCompilerType*)1;
         entry_descriptor->args[vreg] = fake_type;
         if (i_maybe_undefined)
             entry_descriptor->potentially_undefined.set(vreg);
         entry_descriptor->args[vregs.getVReg(iter_str)] = fake_type;
-        phis = computeRequiredPhis(entry_descriptor, liveness.get());
+        phis = computeRequiredPhis(entry_descriptor, liveness.get(), scope_info);
     } else {
-        phis = computeRequiredPhis(ParamNames(func, *module->interned_strings), cfg, liveness.get());
+        phis = computeRequiredPhis(ParamNames(func, clfunc->source->getInternedStrings()), cfg, liveness.get(), scope_info);
     }
 
     // First, verify that we require phi nodes for the block we enter into.