Intern most codegen strings

Most importantly, intern all the strings we put into the AST* nodes.
(the AST_Module* owns them)

This should save us some memory, but it also improves performance pretty
substantially since now we can do string comparisons very cheaply.  Performance
of the interpreter tier is up by something like 30%, and JIT-compilation times
are down as well (though not by as much as I was hoping).

The overall effect on perf is more muted since we tier out of the interpreter
pretty quickly; to see more benefit, we'll have to retune the OSR/reopt thresholds.

For better or worse (mostly better IMO), the interned-ness is encoded in the type
system, and things will not automatically convert between an InternedString and
a std::string.  It means that this diff is quite large, but it also makes it a lot
more clear where we are making our string copies or have other room for optimization.

src/analysis/fpc.h

@@ -25,7 +25,7 @@ namespace pyston {
 
 template <typename T> class BBAnalyzer {
 public:
-    typedef std::unordered_map<std::string, T> Map;
+    typedef std::unordered_map<InternedString, T> Map;
     typedef std::unordered_map<CFGBlock*, Map> AllMap;
 
     virtual ~BBAnalyzer() {}

src/analysis/function_analysis.cpp

@@ -51,27 +51,23 @@ private:
     };
 
     std::unordered_set<AST_Name*> kills;
-    std::unordered_map<int, AST_Name*> last_uses;
+    std::unordered_map<InternedString, AST_Name*> last_uses;
 
-    std::unordered_map<int, Status> statuses;
+    std::unordered_map<InternedString, Status> statuses;
     LivenessAnalysis* analysis;
 
-    void _doLoad(const std::string& name, AST_Name* node) {
-        int id = analysis->getStringIndex(name);
-
-        Status& status = statuses[id];
+    void _doLoad(InternedString name, AST_Name* node) {
+        Status& status = statuses[name];
         status.addUsage(Status::USED);
 
-        last_uses[id] = node;
+        last_uses[name] = node;
     }
 
-    void _doStore(const std::string& name) {
-        int id = analysis->getStringIndex(name);
-
-        Status& status = statuses[id];
+    void _doStore(InternedString name) {
+        Status& status = statuses[name];
         status.addUsage(Status::DEFINED);
 
-        auto it = last_uses.find(id);
+        auto it = last_uses.find(name);
         if (it != last_uses.end()) {
             kills.insert(it->second);
             last_uses.erase(it);
@@ -81,9 +77,9 @@ private:
 public:
     LivenessBBVisitor(LivenessAnalysis* analysis) : analysis(analysis) {}
 
-    bool firstIsUse(int idx) { return statuses[idx].first == Status::USED; }
+    bool firstIsUse(InternedString name) { return statuses[name].first == Status::USED; }
 
-    bool firstIsDef(int idx) { return statuses[idx].first == Status::DEFINED; }
+    bool firstIsDef(InternedString name) { return statuses[name].first == Status::DEFINED; }
 
     bool isKilledAt(AST_Name* node, bool is_live_at_end) {
         if (kills.count(node))
@@ -93,7 +89,7 @@ public:
         // even though we weren't able to determine that in a single
         // pass
         if (!is_live_at_end) {
-            auto it = last_uses.find(analysis->getStringIndex(node->id));
+            auto it = last_uses.find(node->id);
             if (it != last_uses.end() && node == it->second)
                 return true;
         }
@@ -141,23 +137,15 @@ public:
         return true;
     }
     bool visit_alias(AST_alias* node) {
-        const std::string* name = &node->name;
-        if (node->asname.size())
-            name = &node->asname;
+        InternedString name = node->name;
+        if (node->asname.str().size())
+            name = node->asname;
 
-        _doStore(*name);
+        _doStore(name);
         return true;
     }
 };
 
-int LivenessAnalysis::getStringIndex(const std::string& s) {
-    int& r = string_index_map[s];
-    if (r == 0) {
-        r = string_index_map.size(); // includes the '0' entry we just put in there
-    }
-    return r;
-}
-
 LivenessAnalysis::LivenessAnalysis(CFG* cfg) : cfg(cfg) {
     Timer _t("LivenessAnalysis()", 10);
 
@@ -174,31 +162,30 @@ LivenessAnalysis::LivenessAnalysis(CFG* cfg) : cfg(cfg) {
 }
 
 bool LivenessAnalysis::isKill(AST_Name* node, CFGBlock* parent_block) {
-    if (node->id[0] != '#')
+    if (node->id.str()[0] != '#')
         return false;
 
     return liveness_cache[parent_block]->isKilledAt(node, isLiveAtEnd(node->id, parent_block));
 }
 
-bool LivenessAnalysis::isLiveAtEnd(const std::string& name, CFGBlock* block) {
+bool LivenessAnalysis::isLiveAtEnd(InternedString name, CFGBlock* block) {
     Timer _t("LivenessAnalysis()", 10);
 
-    if (name[0] != '#')
+    if (name.str()[0] != '#')
         return true;
 
     if (block->successors.size() == 0)
         return false;
 
-    int idx = getStringIndex(name);
-    if (!result_cache.count(idx)) {
-        std::unordered_map<CFGBlock*, bool>& map = result_cache[idx];
+    if (!result_cache.count(name)) {
+        std::unordered_map<CFGBlock*, bool>& map = result_cache[name];
 
         // Approach:
         // - Find all uses (blocks where the status is USED)
         // - Trace backwards, marking all blocks as live-at-end
         // - If we hit a block that is DEFINED, stop
         for (CFGBlock* b : cfg->blocks) {
-            if (!liveness_cache[b]->firstIsUse(idx))
+            if (!liveness_cache[b]->firstIsUse(name))
                 continue;
 
             std::deque<CFGBlock*> q;
@@ -214,7 +201,7 @@ bool LivenessAnalysis::isLiveAtEnd(const std::string& name, CFGBlock* block) {
                     continue;
 
                 map[thisblock] = true;
-                if (!liveness_cache[thisblock]->firstIsDef(idx)) {
+                if (!liveness_cache[thisblock]->firstIsDef(name)) {
                     for (CFGBlock* pred : thisblock->predecessors) {
                         q.push_back(pred);
                     }
@@ -227,7 +214,7 @@ bool LivenessAnalysis::isLiveAtEnd(const std::string& name, CFGBlock* block) {
     static StatCounter us_liveness("us_compiling_analysis_liveness");
     us_liveness.log(_t.end());
 
-    return result_cache[idx][block];
+    return result_cache[name][block];
 }
 
 class DefinednessBBAnalyzer : public BBAnalyzer<DefinednessAnalysis::DefinitionLevel> {
@@ -259,7 +246,7 @@ private:
     typedef DefinednessBBAnalyzer::Map Map;
     Map& state;
 
-    void _doSet(const std::string& s) { state[s] = DefinednessAnalysis::Defined; }
+    void _doSet(InternedString s) { state[s] = DefinednessAnalysis::Defined; }
 
     void _doSet(AST* t) {
         switch (t->type) {
@@ -309,11 +296,11 @@ public:
     }
 
     virtual bool visit_alias(AST_alias* node) {
-        const std::string* name = &node->name;
-        if (node->asname.size())
-            name = &node->asname;
+        InternedString name = node->name;
+        if (node->asname.str().size())
+            name = node->asname;
 
-        _doSet(*name);
+        _doSet(name);
         return true;
     }
     virtual bool visit_import(AST_Import* node) { return false; }
@@ -327,9 +314,9 @@ public:
     }
 
     virtual bool visit_arguments(AST_arguments* node) {
-        if (node->kwarg.size())
+        if (node->kwarg.str().size())
             _doSet(node->kwarg);
-        if (node->vararg.size())
+        if (node->vararg.str().size())
             _doSet(node->vararg);
         for (int i = 0; i < node->args.size(); i++) {
             _doSet(node->args[i]);
@@ -346,10 +333,10 @@ void DefinednessBBAnalyzer::processBB(Map& starting, CFGBlock* block) const {
     if (block == cfg->getStartingBlock() && arg_names.args) {
         for (auto e : (*arg_names.args))
             visitor._doSet(e);
-        if (arg_names.vararg->size())
-            visitor._doSet(*arg_names.vararg);
-        if (arg_names.kwarg->size())
-            visitor._doSet(*arg_names.kwarg);
+        if (arg_names.vararg.str().size())
+            visitor._doSet(arg_names.vararg);
+        if (arg_names.kwarg.str().size())
+            visitor._doSet(arg_names.kwarg);
     }
 
     for (int i = 0; i < block->body.size(); i++) {
@@ -386,8 +373,8 @@ DefinednessAnalysis::DefinednessAnalysis(const SourceInfo::ArgNames& arg_names,
     us_definedness.log(_t.end());
 }
 
-DefinednessAnalysis::DefinitionLevel DefinednessAnalysis::isDefinedAtEnd(const std::string& name, CFGBlock* block) {
-    std::unordered_map<std::string, DefinitionLevel>& map = results[block];
+DefinednessAnalysis::DefinitionLevel DefinednessAnalysis::isDefinedAtEnd(InternedString name, CFGBlock* block) {
+    auto& map = results[block];
     if (map.count(name) == 0)
         return Undefined;
     return map[name];
@@ -436,13 +423,13 @@ const PhiAnalysis::RequiredSet& PhiAnalysis::getAllRequiredFor(CFGBlock* block)
     return required_phis[block];
 }
 
-bool PhiAnalysis::isRequired(const std::string& name, CFGBlock* block) {
-    assert(!startswith(name, "!"));
+bool PhiAnalysis::isRequired(InternedString name, CFGBlock* block) {
+    assert(!startswith(name.str(), "!"));
     return required_phis[block].count(name) != 0;
 }
 
-bool PhiAnalysis::isRequiredAfter(const std::string& name, CFGBlock* block) {
-    assert(!startswith(name, "!"));
+bool PhiAnalysis::isRequiredAfter(InternedString name, CFGBlock* block) {
+    assert(!startswith(name.str(), "!"));
     // If there are multiple successors, then none of them are allowed
     // to require any phi nodes
     if (block->successors.size() != 1)
@@ -452,8 +439,8 @@ bool PhiAnalysis::isRequiredAfter(const std::string& name, CFGBlock* block) {
     return isRequired(name, block->successors[0]);
 }
 
-bool PhiAnalysis::isPotentiallyUndefinedAfter(const std::string& name, CFGBlock* block) {
-    assert(!startswith(name, "!"));
+bool PhiAnalysis::isPotentiallyUndefinedAfter(InternedString name, CFGBlock* block) {
+    assert(!startswith(name.str(), "!"));
 
     if (block->successors.size() != 1)
         return false;
@@ -461,8 +448,8 @@ bool PhiAnalysis::isPotentiallyUndefinedAfter(const std::string& name, CFGBlock*
     return isPotentiallyUndefinedAt(name, block->successors[0]);
 }
 
-bool PhiAnalysis::isPotentiallyUndefinedAt(const std::string& name, CFGBlock* block) {
-    assert(!startswith(name, "!"));
+bool PhiAnalysis::isPotentiallyUndefinedAt(InternedString name, CFGBlock* block) {
+    assert(!startswith(name.str(), "!"));
 
     for (CFGBlock* pred : block->predecessors) {
         DefinednessAnalysis::DefinitionLevel dlevel = definedness.isDefinedAtEnd(name, pred);

src/analysis/function_analysis.h

@@ -19,6 +19,7 @@
 #include <unordered_map>
 #include <unordered_set>
 
+#include "core/stringpool.h"
 #include "core/types.h"
 
 namespace pyston {
@@ -39,17 +40,7 @@ private:
     typedef std::unordered_map<CFGBlock*, std::unique_ptr<LivenessBBVisitor>> LivenessCacheMap;
     LivenessCacheMap liveness_cache;
 
-    std::unordered_map<int, std::unordered_map<CFGBlock*, bool>> result_cache;
-
-    // Map strings to unique indices.  For a given CFG, the set of strings should be fairly small
-    // (a constant fraction max of the CFG itself), so just store all of them.  The theory is that
-    // for any particular name, we will do many lookups on it in different hash tables, and by
-    // converting to a string only once, the extra hashtable lookup will be profitable since it
-    // can make all the rest faster (int hashes vs string hashes).
-    //
-    // Haven't validated this, though.
-    std::unordered_map<std::string, int> string_index_map;
-    int getStringIndex(const std::string& s);
+    std::unordered_map<InternedString, std::unordered_map<CFGBlock*, bool>> result_cache;
 
 public:
     LivenessAnalysis(CFG* cfg);
@@ -57,7 +48,7 @@ public:
     // we don't keep track of node->parent_block relationships, so you have to pass both:
     bool isKill(AST_Name* node, CFGBlock* parent_block);
 
-    bool isLiveAtEnd(const std::string& name, CFGBlock* block);
+    bool isLiveAtEnd(InternedString name, CFGBlock* block);
 };
 
 class DefinednessAnalysis {
@@ -67,22 +58,22 @@ public:
         PotentiallyDefined,
         Defined,
     };
-    typedef std::unordered_set<std::string> RequiredSet;
+    typedef std::unordered_set<InternedString> RequiredSet;
 
 private:
-    std::unordered_map<CFGBlock*, std::unordered_map<std::string, DefinitionLevel>> results;
+    std::unordered_map<CFGBlock*, std::unordered_map<InternedString, DefinitionLevel>> results;
     std::unordered_map<CFGBlock*, const RequiredSet> defined_at_end;
     ScopeInfo* scope_info;
 
 public:
     DefinednessAnalysis(const SourceInfo::ArgNames& args, CFG* cfg, ScopeInfo* scope_info);
 
-    DefinitionLevel isDefinedAtEnd(const std::string& name, CFGBlock* block);
+    DefinitionLevel isDefinedAtEnd(InternedString name, CFGBlock* block);
     const RequiredSet& getDefinedNamesAtEnd(CFGBlock* block);
 };
 class PhiAnalysis {
 public:
-    typedef std::unordered_set<std::string> RequiredSet;
+    typedef std::unordered_set<InternedString> RequiredSet;
 
     DefinednessAnalysis definedness;
 
@@ -93,12 +84,12 @@ private:
 public:
     PhiAnalysis(const SourceInfo::ArgNames&, CFG* cfg, LivenessAnalysis* liveness, ScopeInfo* scope_info);
 
-    bool isRequired(const std::string& name, CFGBlock* block);
-    bool isRequiredAfter(const std::string& name, CFGBlock* block);
+    bool isRequired(InternedString name, CFGBlock* block);
+    bool isRequiredAfter(InternedString name, CFGBlock* block);
     const RequiredSet& getAllRequiredAfter(CFGBlock* block);
     const RequiredSet& getAllRequiredFor(CFGBlock* block);
-    bool isPotentiallyUndefinedAfter(const std::string& name, CFGBlock* block);
-    bool isPotentiallyUndefinedAt(const std::string& name, CFGBlock* block);
+    bool isPotentiallyUndefinedAfter(InternedString name, CFGBlock* block);
+    bool isPotentiallyUndefinedAt(InternedString name, CFGBlock* block);
 };
 
 LivenessAnalysis* computeLivenessInfo(CFG*);

src/analysis/scoping_analysis.h

@@ -16,6 +16,7 @@
 #define PYSTON_ANALYSIS_SCOPINGANALYSIS_H
 
 #include "core/common.h"
+#include "core/stringpool.h"
 
 namespace pyston {
 
@@ -32,16 +33,11 @@ public:
     virtual bool takesClosure() = 0;
     virtual bool passesThroughClosure() = 0;
 
-    virtual bool refersToGlobal(const std::string& name) = 0;
-    virtual bool refersToClosure(const std::string& name) = 0;
-    virtual bool saveInClosure(const std::string& name) = 0;
+    virtual bool refersToGlobal(InternedString name) = 0;
+    virtual bool refersToClosure(InternedString name) = 0;
+    virtual bool saveInClosure(InternedString name) = 0;
 
-    // Get the names set within a classdef that should be forwarded on to
-    // the metaclass constructor.
-    // An error to call this on a non-classdef node.
-    virtual const std::unordered_set<std::string>& getClassDefLocalNames() = 0;
-
-    virtual std::string mangleName(const std::string& id) = 0;
+    virtual InternedString mangleName(InternedString id) = 0;
 };
 
 class ScopingAnalysis {
@@ -52,6 +48,7 @@ public:
 private:
     std::unordered_map<AST*, ScopeInfo*> scopes;
     AST_Module* parent_module;
+    InternedStringPool& interned_strings;
 
     std::unordered_map<AST*, AST*> scope_replacements;
 
@@ -70,6 +67,8 @@ public:
 
     ScopingAnalysis(AST_Module* m);
     ScopeInfo* getScopeInfoForNode(AST* node);
+
+    InternedStringPool& getInternedStrings();
 };
 
 ScopingAnalysis* runScopingAnalysis(AST_Module* m);

src/analysis/type_analysis.cpp

@@ -36,17 +36,17 @@ namespace pyston {
 
 class NullTypeAnalysis : public TypeAnalysis {
 public:
-    ConcreteCompilerType* getTypeAtBlockStart(const std::string& name, CFGBlock* block) override;
-    ConcreteCompilerType* getTypeAtBlockEnd(const std::string& name, CFGBlock* block) override;
+    ConcreteCompilerType* getTypeAtBlockStart(InternedString name, CFGBlock* block) override;
+    ConcreteCompilerType* getTypeAtBlockEnd(InternedString name, CFGBlock* block) override;
 
     BoxedClass* speculatedExprClass(AST_expr*) override { return NULL; }
 };
 
-ConcreteCompilerType* NullTypeAnalysis::getTypeAtBlockStart(const std::string& name, CFGBlock* block) {
+ConcreteCompilerType* NullTypeAnalysis::getTypeAtBlockStart(InternedString name, CFGBlock* block) {
     return UNKNOWN;
 }
 
-ConcreteCompilerType* NullTypeAnalysis::getTypeAtBlockEnd(const std::string& name, CFGBlock* block) {
+ConcreteCompilerType* NullTypeAnalysis::getTypeAtBlockEnd(InternedString name, CFGBlock* block) {
     assert(block->successors.size() > 0);
     return getTypeAtBlockStart(name, block->successors[0]);
 }
@@ -68,7 +68,7 @@ static BoxedClass* simpleCallSpeculation(AST_Call* node, CompilerType* rtn_type,
         return NULL;
     }
 
-    if (node->func->type == AST_TYPE::Name && ast_cast<AST_Name>(node->func)->id == "xrange")
+    if (node->func->type == AST_TYPE::Name && ast_cast<AST_Name>(node->func)->id.str() == "xrange")
         return xrange_cls;
 
     // if (node->func->type == AST_TYPE::Attribute && ast_cast<AST_Attribute>(node->func)->attr == "dot")
@@ -77,7 +77,7 @@ static BoxedClass* simpleCallSpeculation(AST_Call* node, CompilerType* rtn_type,
     return NULL;
 }
 
-typedef std::unordered_map<std::string, CompilerType*> TypeMap;
+typedef std::unordered_map<InternedString, CompilerType*> TypeMap;
 typedef std::unordered_map<CFGBlock*, TypeMap> AllTypeMap;
 typedef std::unordered_map<AST_expr*, CompilerType*> ExprTypeMap;
 typedef std::unordered_map<AST_expr*, BoxedClass*> TypeSpeculations;
@@ -140,7 +140,7 @@ private:
         return rtn;
     }
 
-    void _doSet(std::string target, CompilerType* t) {
+    void _doSet(InternedString target, CompilerType* t) {
         if (t)
             sym_table[target] = t;
     }
@@ -170,7 +170,7 @@ private:
 
     void* visit_attribute(AST_Attribute* node) override {
         CompilerType* t = getType(node->value);
-        CompilerType* rtn = t->getattrType(&node->attr, false);
+        CompilerType* rtn = t->getattrType(&node->attr.str(), false);
 
         // if (speculation != TypeAnalysis::NONE && (node->attr == "x" || node->attr == "y" || node->attr == "z")) {
         // rtn = processSpeculation(float_cls, node, rtn);
@@ -192,7 +192,7 @@ private:
 
     void* visit_clsattribute(AST_ClsAttribute* node) override {
         CompilerType* t = getType(node->value);
-        CompilerType* rtn = t->getattrType(&node->attr, true);
+        CompilerType* rtn = t->getattrType(&node->attr.str(), true);
         if (VERBOSITY() >= 2 && rtn == UNDEF) {
             printf("Think %s.%s is undefined, at %d:%d\n", t->debugName().c_str(), node->attr.c_str(), node->lineno,
                    node->col_offset);
@@ -288,9 +288,9 @@ private:
             arg_types.push_back(getType(node->args[i]));
         }
 
-        std::vector<std::pair<const std::string&, CompilerType*>> kw_types;
+        std::vector<std::pair<InternedString, CompilerType*>> kw_types;
         for (AST_keyword* kw : node->keywords) {
-            kw_types.push_back(std::make_pair<const std::string&, CompilerType*>(kw->arg, getType(kw->value)));
+            kw_types.push_back(std::make_pair(kw->arg, getType(kw->value)));
         }
 
         CompilerType* starargs = node->starargs ? getType(node->starargs) : NULL;
@@ -393,7 +393,7 @@ private:
 
     void* visit_name(AST_Name* node) override {
         if (scope_info->refersToGlobal(node->id)) {
-            if (node->id == "xrange") {
+            if (node->id.str() == "xrange") {
                 // printf("TODO guard here and return the classobj\n");
                 // return typeOfClassobj(xrange_cls);
             }
@@ -543,11 +543,11 @@ private:
 
     // not part of the visitor api:
     void _visit_alias(AST_alias* node) {
-        const std::string* name = &node->name;
-        if (node->asname.size())
-            name = &node->asname;
+        InternedString name = node->name;
+        if (node->asname.str().size())
+            name = node->asname;
 
-        _doSet(*name, UNKNOWN);
+        _doSet(name, UNKNOWN);
     }
 
     void visit_import(AST_Import* node) override {
@@ -616,11 +616,11 @@ private:
           speculation(speculation) {}
 
 public:
-    ConcreteCompilerType* getTypeAtBlockEnd(const std::string& name, CFGBlock* block) override {
+    ConcreteCompilerType* getTypeAtBlockEnd(InternedString name, CFGBlock* block) override {
         assert(block->successors.size() > 0);
         return getTypeAtBlockStart(name, block->successors[0]);
     }
-    ConcreteCompilerType* getTypeAtBlockStart(const std::string& name, CFGBlock* block) override {
+    ConcreteCompilerType* getTypeAtBlockStart(InternedString name, CFGBlock* block) override {
         CompilerType* base = starting_types[block][name];
         ASSERT(base != NULL, "%s %d", name.c_str(), block->idx);
 
@@ -692,13 +692,13 @@ public:
                 initial_types[arg_name->id] = unboxedType(arg_types[i]);
             }
 
-            if (arg_names.vararg->size()) {
-                initial_types[*arg_names.vararg] = unboxedType(arg_types[i]);
+            if (arg_names.vararg.str().size()) {
+                initial_types[arg_names.vararg] = unboxedType(arg_types[i]);
                 i++;
             }
 
-            if (arg_names.kwarg->size()) {
-                initial_types[*arg_names.kwarg] = unboxedType(arg_types[i]);
+            if (arg_names.kwarg.str().size()) {
+                initial_types[arg_names.kwarg] = unboxedType(arg_types[i]);
                 i++;
             }
 

src/analysis/type_analysis.h

@@ -18,6 +18,7 @@
 #include <unordered_map>
 #include <vector>
 
+#include "core/stringpool.h"
 #include "core/types.h"
 
 namespace pyston {
@@ -36,12 +37,11 @@ public:
 
     virtual ~TypeAnalysis() {}
 
-    virtual ConcreteCompilerType* getTypeAtBlockStart(const std::string& name, CFGBlock* block) = 0;
-    virtual ConcreteCompilerType* getTypeAtBlockEnd(const std::string& name, CFGBlock* block) = 0;
+    virtual ConcreteCompilerType* getTypeAtBlockStart(InternedString name, CFGBlock* block) = 0;
+    virtual ConcreteCompilerType* getTypeAtBlockEnd(InternedString name, CFGBlock* block) = 0;
     virtual BoxedClass* speculatedExprClass(AST_expr*) = 0;
 };
 
-// TypeAnalysis* analyze(CFG *cfg, std::unordered_map<std::string, ConcreteCompilerType*> arg_types);
 TypeAnalysis* doTypeAnalysis(CFG* cfg, const SourceInfo::ArgNames& arg_names,
                              const std::vector<ConcreteCompilerType*>& arg_types, EffortLevel::EffortLevel effort,
                              TypeAnalysis::SpeculationLevel speculation, ScopeInfo* scope_info);

src/codegen/codegen.cpp

@@ -34,7 +34,8 @@ namespace pyston {
 DS_DEFINE_RWLOCK(codegen_rwlock);
 
 SourceInfo::SourceInfo(BoxedModule* m, ScopingAnalysis* scoping, AST* ast, const std::vector<AST_stmt*>& body)
-    : parent_module(m), scoping(scoping), ast(ast), cfg(NULL), liveness(NULL), phis(NULL), arg_names(ast), body(body) {
+    : parent_module(m), scoping(scoping), ast(ast), cfg(NULL), liveness(NULL), phis(NULL), arg_names(ast, scoping),
+      body(body) {
     switch (ast->type) {
         case AST_TYPE::ClassDef:
         case AST_TYPE::Lambda:

src/codegen/irgen.cpp

@@ -272,14 +272,14 @@ computeBlockTraversalOrder(const BlockSet& full_blocks, const BlockSet& partial_
     return rtn;
 }
 
-static ConcreteCompilerType* getTypeAtBlockStart(TypeAnalysis* types, const std::string& name, CFGBlock* block) {
-    if (isIsDefinedName(name))
+static ConcreteCompilerType* getTypeAtBlockStart(TypeAnalysis* types, InternedString name, CFGBlock* block) {
+    if (isIsDefinedName(name.str()))
         return BOOL;
-    else if (name == PASSED_GENERATOR_NAME)
+    else if (name.str() == PASSED_GENERATOR_NAME)
         return GENERATOR;
-    else if (name == PASSED_CLOSURE_NAME)
+    else if (name.str() == PASSED_CLOSURE_NAME)
         return CLOSURE;
-    else if (name == CREATED_CLOSURE_NAME)
+    else if (name.str() == CREATED_CLOSURE_NAME)
         return CLOSURE;
     else
         return types->getTypeAtBlockStart(name, block);
@@ -450,9 +450,10 @@ static void emitBBs(IRGenState* irstate, const char* bb_type, GuardList& out_gua
                 }
                 ASSERT(speculated_class, "%s", phi_type->debugName().c_str());
 
-                assert(p.first[0] != '!');
+                assert(p.first.str()[0] != '!');
 
-                std::string is_defined_name = getIsDefinedName(p.first);
+                // TODO cache this
+                InternedString is_defined_name = getIsDefinedName(p.first, source->getInternedStrings());
                 llvm::Value* prev_guard_val = NULL;
 
                 ConcreteCompilerVariable* is_defined_var = NULL;
@@ -507,7 +508,7 @@ static void emitBBs(IRGenState* irstate, const char* bb_type, GuardList& out_gua
             }
 
             if (VERBOSITY("irgen"))
-                v->setName("prev_" + p.first);
+                v->setName("prev_" + p.first.str());
 
             (*osr_syms)[p.first] = new ConcreteCompilerVariable(phi_type, v, true);
         }
@@ -543,7 +544,7 @@ static void emitBBs(IRGenState* irstate, const char* bb_type, GuardList& out_gua
 
     std::unordered_map<CFGBlock*, SymbolTable*> ending_symbol_tables;
     std::unordered_map<CFGBlock*, ConcreteSymbolTable*> phi_ending_symbol_tables;
-    typedef std::unordered_map<std::string, std::pair<ConcreteCompilerType*, llvm::PHINode*>> PHITable;
+    typedef std::unordered_map<InternedString, std::pair<ConcreteCompilerType*, llvm::PHINode*>> PHITable;
     std::unordered_map<CFGBlock*, PHITable*> created_phis;
 
     CFGBlock* initial_block = NULL;
@@ -680,7 +681,7 @@ static void emitBBs(IRGenState* irstate, const char* bb_type, GuardList& out_gua
                 // analyzed_type->debugName().c_str());
 
                 llvm::PHINode* phi = emitter->getBuilder()->CreatePHI(analyzed_type->llvmType(),
-                                                                      block->predecessors.size() + 1, p.first);
+                                                                      block->predecessors.size() + 1, p.first.str());
                 ConcreteCompilerVariable* var = new ConcreteCompilerVariable(analyzed_type, phi, true);
                 generator->giveLocalSymbol(p.first, var);
                 (*phis)[p.first] = std::make_pair(analyzed_type, phi);
@@ -696,27 +697,28 @@ static void emitBBs(IRGenState* irstate, const char* bb_type, GuardList& out_gua
             }
 
 
-            std::unordered_set<std::string> names;
+            std::unordered_set<InternedString> names;
             for (const auto& s : source->phis->getAllRequiredFor(block)) {
                 names.insert(s);
                 if (source->phis->isPotentiallyUndefinedAfter(s, block->predecessors[0])) {
-                    names.insert(getIsDefinedName(s));
+                    names.insert(getIsDefinedName(s, source->getInternedStrings()));
                 }
             }
 
             if (source->getScopeInfo()->createsClosure())
-                names.insert(CREATED_CLOSURE_NAME);
+                names.insert(source->getInternedStrings().get(CREATED_CLOSURE_NAME));
 
             if (source->getScopeInfo()->takesClosure())
-                names.insert(PASSED_CLOSURE_NAME);
+                names.insert(source->getInternedStrings().get(PASSED_CLOSURE_NAME));
 
             if (source->is_generator)
-                names.insert(PASSED_GENERATOR_NAME);
+                names.insert(source->getInternedStrings().get(PASSED_GENERATOR_NAME));
 
             for (const auto& s : names) {
                 // printf("adding guessed phi for %s\n", s.c_str());
                 ConcreteCompilerType* type = getTypeAtBlockStart(types, s, block);
-                llvm::PHINode* phi = emitter->getBuilder()->CreatePHI(type->llvmType(), block->predecessors.size(), s);
+                llvm::PHINode* phi
+                    = emitter->getBuilder()->CreatePHI(type->llvmType(), block->predecessors.size(), s.str());
                 ConcreteCompilerVariable* var = new ConcreteCompilerVariable(type, phi, true);
                 generator->giveLocalSymbol(s, var);
 
@@ -776,7 +778,7 @@ static void emitBBs(IRGenState* irstate, const char* bb_type, GuardList& out_gua
                 for (ConcreteSymbolTable::iterator it = pred_st->begin(); it != pred_st->end(); it++) {
                     // printf("adding phi for %s\n", it->first.c_str());
                     llvm::PHINode* phi = emitter->getBuilder()->CreatePHI(it->second->getType()->llvmType(),
-                                                                          block->predecessors.size(), it->first);
+                                                                          block->predecessors.size(), it->first.str());
                     // emitter->getBuilder()->CreateCall(g.funcs.dump, phi);
                     ConcreteCompilerVariable* var = new ConcreteCompilerVariable(it->second->getType(), phi, true);
                     generator->giveLocalSymbol(it->first, var);
@@ -882,7 +884,7 @@ static void emitBBs(IRGenState* irstate, const char* bb_type, GuardList& out_gua
                 llvm_phi->addIncoming(v->getValue(), osr_unbox_block_end);
             }
 
-            std::string is_defined_name = getIsDefinedName(it->first);
+            InternedString is_defined_name = getIsDefinedName(it->first, source->getInternedStrings());
 
             for (int i = 0; i < block_guards.size(); i++) {
                 GuardList::BlockEntryGuard* guard = block_guards[i];

src/codegen/irgen.h

@@ -91,7 +91,7 @@ extern const std::string CREATED_CLOSURE_NAME;
 extern const std::string PASSED_CLOSURE_NAME;
 extern const std::string PASSED_GENERATOR_NAME;
 
-std::string getIsDefinedName(const std::string& name);
+InternedString getIsDefinedName(InternedString name, InternedStringPool& interned_strings);
 bool isIsDefinedName(const std::string& name);
 
 CompiledFunction* doCompile(SourceInfo* source, const OSREntryDescriptor* entry_descriptor,

src/codegen/irgen/future.cpp

@@ -69,7 +69,7 @@ void raiseFutureImportErrorNotBeginning(const char* file, AST* node) {
 class BadFutureImportVisitor : public NoopASTVisitor {
 public:
     virtual bool visit_importfrom(AST_ImportFrom* node) {
-        if (node->module == "__future__") {
+        if (node->module.str() == "__future__") {
             raiseFutureImportErrorNotBeginning(file, node);
         }
         return true;
@@ -103,14 +103,14 @@ FutureFlags getFutureFlags(AST_Module* m, const char* file) {
     for (int i = 0; i < m->body.size(); i++) {
         AST_stmt* stmt = m->body[i];
 
-        if (stmt->type == AST_TYPE::ImportFrom && static_cast<AST_ImportFrom*>(stmt)->module == "__future__") {
+        if (stmt->type == AST_TYPE::ImportFrom && static_cast<AST_ImportFrom*>(stmt)->module.str() == "__future__") {
             if (future_import_allowed) {
                 // We have a `from __future__` import statement, and we are
                 // still at the top of the file, so just set the appropriate
                 // future flag for each imported option.
 
                 for (AST_alias* alias : static_cast<AST_ImportFrom*>(stmt)->names) {
-                    const std::string& option_name = alias->name;
+                    const std::string& option_name = alias->name.str();
                     auto iter = future_options.find(option_name);
                     if (iter == future_options.end()) {
                         // If it's not one of the available options, throw an error.

src/codegen/irgen/hooks.cpp

@@ -42,39 +42,44 @@
 namespace pyston {
 
 // TODO terrible place for these!
-SourceInfo::ArgNames::ArgNames(AST* ast) {
+SourceInfo::ArgNames::ArgNames(AST* ast, ScopingAnalysis* scoping) {
     if (ast->type == AST_TYPE::Module || ast->type == AST_TYPE::ClassDef) {
         args = NULL;
-        kwarg = vararg = NULL;
+        kwarg = vararg = scoping->getInternedStrings().get("");
     } else if (ast->type == AST_TYPE::FunctionDef) {
         AST_FunctionDef* f = ast_cast<AST_FunctionDef>(ast);
         args = &f->args->args;
-        vararg = &f->args->vararg;
-        kwarg = &f->args->kwarg;
+        vararg = f->args->vararg;
+        kwarg = f->args->kwarg;
     } else if (ast->type == AST_TYPE::Lambda) {
         AST_Lambda* l = ast_cast<AST_Lambda>(ast);
         args = &l->args->args;
-        vararg = &l->args->vararg;
-        kwarg = &l->args->kwarg;
+        vararg = l->args->vararg;
+        kwarg = l->args->kwarg;
     } else {
         RELEASE_ASSERT(0, "%d", ast->type);
     }
 }
 
-std::string SourceInfo::mangleName(const std::string& id) {
+InternedString SourceInfo::mangleName(InternedString id) {
+    // TODO should cache the results of this
     assert(ast);
     if (ast->type == AST_TYPE::Module)
         return id;
     return getScopeInfo()->mangleName(id);
 }
 
+InternedStringPool& SourceInfo::getInternedStrings() {
+    return scoping->getInternedStrings();
+}
+
 const std::string SourceInfo::getName() {
     assert(ast);
     switch (ast->type) {
         case AST_TYPE::ClassDef:
-            return ast_cast<AST_ClassDef>(ast)->name;
+            return ast_cast<AST_ClassDef>(ast)->name.str();
         case AST_TYPE::FunctionDef:
-            return ast_cast<AST_FunctionDef>(ast)->name;
+            return ast_cast<AST_FunctionDef>(ast)->name.str();
         case AST_TYPE::Lambda:
             return "<lambda>";
         case AST_TYPE::Module:

src/codegen/irgen/irgenerator.h

@@ -20,6 +20,7 @@
 #include "llvm/ADT/iterator_range.h"
 #include "llvm/IR/Instructions.h"
 
+#include "core/stringpool.h"
 #include "core/types.h"
 
 namespace llvm {
@@ -38,9 +39,9 @@ struct PatchpointInfo;
 class ScopeInfo;
 class TypeAnalysis;
 
-typedef std::unordered_map<std::string, CompilerVariable*> SymbolTable;
-typedef std::map<std::string, CompilerVariable*> SortedSymbolTable;
-typedef std::unordered_map<std::string, ConcreteCompilerVariable*> ConcreteSymbolTable;
+typedef std::unordered_map<InternedString, CompilerVariable*> SymbolTable;
+typedef std::map<InternedString, CompilerVariable*> SortedSymbolTable;
+typedef std::unordered_map<InternedString, ConcreteCompilerVariable*> ConcreteSymbolTable;
 
 extern const std::string CREATED_CLOSURE_NAME;
 extern const std::string PASSED_CLOSURE_NAME;
@@ -197,7 +198,7 @@ public:
     virtual void doFunctionEntry(const SourceInfo::ArgNames& arg_names,
                                  const std::vector<ConcreteCompilerType*>& arg_types) = 0;
 
-    virtual void giveLocalSymbol(const std::string& name, CompilerVariable* var) = 0;
+    virtual void giveLocalSymbol(InternedString name, CompilerVariable* var) = 0;
     virtual void copySymbolsFrom(SymbolTable* st) = 0;
     virtual void run(const CFGBlock* block) = 0;
     virtual EndingState getEndingSymbolTable() = 0;

src/codegen/osrentry.h

@@ -17,6 +17,8 @@
 
 #include <vector>
 
+#include "core/stringpool.h"
+
 namespace llvm {
 class Function;
 }
@@ -33,7 +35,7 @@ private:
 public:
     CompiledFunction* const cf;
     AST_Jump* const backedge;
-    typedef std::map<std::string, ConcreteCompilerType*> ArgMap;
+    typedef std::map<InternedString, ConcreteCompilerType*> ArgMap;
     ArgMap args;
 
     static OSREntryDescriptor* create(CompiledFunction* from_cf, AST_Jump* backedge) {

src/codegen/parser.cpp

@@ -43,6 +43,8 @@ private:
     int start, end;
     FILE* fp;
 
+    InternedStringPool* intern_pool;
+
     void ensure(int num) {
         if (end - start < num) {
             fill();
@@ -59,7 +61,7 @@ public:
             printf("filled, now at %d-%d\n", start, end);
     }
 
-    BufferedReader(FILE* fp) : start(0), end(0), fp(fp) {}
+    BufferedReader(FILE* fp) : start(0), end(0), fp(fp), intern_pool(NULL) {}
 
     int bytesBuffered() { return (end - start); }
 
@@ -81,8 +83,18 @@ public:
         raw = readULL();
         return d;
     }
+
+    std::unique_ptr<InternedStringPool> createInternedPool();
+    InternedString readAndInternString();
+    void readAndInternStringVector(std::vector<InternedString>& v);
 };
 
+std::unique_ptr<InternedStringPool> BufferedReader::createInternedPool() {
+    assert(this->intern_pool == NULL);
+    this->intern_pool = new InternedStringPool();
+    return std::unique_ptr<InternedStringPool>(this->intern_pool);
+}
+
 AST* readASTMisc(BufferedReader* reader);
 AST_expr* readASTExpr(BufferedReader* reader);
 AST_stmt* readASTStmt(BufferedReader* reader);
@@ -96,6 +108,20 @@ static std::string readString(BufferedReader* reader) {
     return std::string(chars.begin(), chars.end());
 }
 
+InternedString BufferedReader::readAndInternString() {
+    std::string str = readString(this);
+    return intern_pool->get(std::move(str));
+}
+
+void BufferedReader::readAndInternStringVector(std::vector<InternedString>& v) {
+    int num_elts = readShort();
+    if (VERBOSITY("parsing") >= 2)
+        printf("%d elts to read\n", num_elts);
+    for (int i = 0; i < num_elts; i++) {
+        v.push_back(readAndInternString());
+    }
+}
+
 static void readStringVector(std::vector<std::string>& vec, BufferedReader* reader) {
     int num_elts = reader->readShort();
     if (VERBOSITY("parsing") >= 2)
@@ -142,8 +168,8 @@ static int readColOffset(BufferedReader* reader) {
 }
 
 AST_alias* read_alias(BufferedReader* reader) {
-    std::string asname = readString(reader);
-    std::string name = readString(reader);
+    InternedString asname = reader->readAndInternString();
+    InternedString name = reader->readAndInternString();
 
     AST_alias* rtn = new AST_alias(name, asname);
     rtn->col_offset = -1;
@@ -155,14 +181,15 @@ AST_alias* read_alias(BufferedReader* reader) {
 AST_arguments* read_arguments(BufferedReader* reader) {
     if (VERBOSITY("parsing") >= 2)
         printf("reading arguments\n");
+
     AST_arguments* rtn = new AST_arguments();
 
     readExprVector(rtn->args, reader);
     rtn->col_offset = -1;
     readExprVector(rtn->defaults, reader);
-    rtn->kwarg = readString(reader);
+    rtn->kwarg = reader->readAndInternString();
     rtn->lineno = -1;
-    rtn->vararg = readString(reader);
+    rtn->vararg = reader->readAndInternString();
     return rtn;
 }
 
@@ -200,7 +227,7 @@ AST_AugAssign* read_augassign(BufferedReader* reader) {
 AST_Attribute* read_attribute(BufferedReader* reader) {
     AST_Attribute* rtn = new AST_Attribute();
 
-    rtn->attr = readString(reader);
+    rtn->attr = reader->readAndInternString();
     rtn->col_offset = readColOffset(reader);
     rtn->ctx_type = (AST_TYPE::AST_TYPE)reader->readByte();
     rtn->lineno = reader->readULL();
@@ -293,7 +320,7 @@ AST_ClassDef* read_classdef(BufferedReader* reader) {
     rtn->col_offset = readColOffset(reader);
     readExprVector(rtn->decorator_list, reader);
     rtn->lineno = reader->readULL();
-    rtn->name = readString(reader);
+    rtn->name = reader->readAndInternString();
 
     return rtn;
 }
@@ -395,7 +422,7 @@ AST_FunctionDef* read_functiondef(BufferedReader* reader) {
     rtn->col_offset = readColOffset(reader);
     readExprVector(rtn->decorator_list, reader);
     rtn->lineno = reader->readULL();
-    rtn->name = readString(reader);
+    rtn->name = reader->readAndInternString();
     return rtn;
 }
 
@@ -414,7 +441,7 @@ AST_Global* read_global(BufferedReader* reader) {
 
     rtn->col_offset = readColOffset(reader);
     rtn->lineno = reader->readULL();
-    readStringVector(rtn->names, reader);
+    reader->readAndInternStringVector(rtn->names);
     return rtn;
 }
 
@@ -455,7 +482,7 @@ AST_ImportFrom* read_importfrom(BufferedReader* reader) {
     rtn->col_offset = readColOffset(reader);
     rtn->level = reader->readULL();
     rtn->lineno = reader->readULL();
-    rtn->module = readString(reader);
+    rtn->module = reader->readAndInternString();
     readMiscVector(rtn->names, reader);
     return rtn;
 }
@@ -473,7 +500,7 @@ AST_Index* read_index(BufferedReader* reader) {
 AST_keyword* read_keyword(BufferedReader* reader) {
     AST_keyword* rtn = new AST_keyword();
 
-    rtn->arg = readString(reader);
+    rtn->arg = reader->readAndInternString();
     rtn->col_offset = -1;
     rtn->lineno = -1;
     rtn->value = readASTExpr(reader);
@@ -513,7 +540,8 @@ AST_ListComp* read_listcomp(BufferedReader* reader) {
 AST_Module* read_module(BufferedReader* reader) {
     if (VERBOSITY("parsing") >= 2)
         printf("reading module\n");
-    AST_Module* rtn = new AST_Module();
+
+    AST_Module* rtn = new AST_Module(reader->createInternedPool());
 
     readStmtVector(rtn->body, reader);
     rtn->col_offset = -1;
@@ -524,7 +552,7 @@ AST_Module* read_module(BufferedReader* reader) {
 AST_Name* read_name(BufferedReader* reader) {
     auto col_offset = readColOffset(reader);
     auto ctx_type = (AST_TYPE::AST_TYPE)reader->readByte();
-    auto id = readString(reader);
+    auto id = reader->readAndInternString();
     auto lineno = reader->readULL();
 
     return new AST_Name(std::move(id), ctx_type, lineno, col_offset);

src/core/ast.cpp

@@ -967,7 +967,7 @@ void PrintVisitor::printIndent() {
 
 bool PrintVisitor::visit_alias(AST_alias* node) {
     printf("%s", node->name.c_str());
-    if (node->asname.size())
+    if (node->asname.str().size())
         printf(" as %s", node->asname.c_str());
     return true;
 }

src/core/ast.h

@@ -24,6 +24,7 @@
 #include "llvm/ADT/StringRef.h"
 
 #include "core/common.h"
+#include "core/stringpool.h"
 
 namespace pyston {
 
@@ -181,11 +182,11 @@ public:
 
 class AST_alias : public AST {
 public:
-    std::string name, asname;
+    InternedString name, asname;
 
     virtual void accept(ASTVisitor* v);
 
-    AST_alias(const std::string& name, const std::string& asname) : AST(AST_TYPE::alias), name(name), asname(asname) {}
+    AST_alias(InternedString name, InternedString asname) : AST(AST_TYPE::alias), name(name), asname(asname) {}
 
     static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::alias;
 };
@@ -197,7 +198,7 @@ public:
 
     // These are represented as strings, not names; not sure why.
     // If they don't exist, the string is empty.
-    std::string kwarg, vararg;
+    InternedString kwarg, vararg;
 
     virtual void accept(ASTVisitor* v);
 
@@ -262,14 +263,14 @@ class AST_Attribute : public AST_expr {
 public:
     AST_expr* value;
     AST_TYPE::AST_TYPE ctx_type;
-    std::string attr;
+    InternedString attr;
 
     virtual void accept(ASTVisitor* v);
     virtual void* accept_expr(ExprVisitor* v);
 
     AST_Attribute() : AST_expr(AST_TYPE::Attribute) {}
 
-    AST_Attribute(AST_expr* value, AST_TYPE::AST_TYPE ctx_type, const std::string& attr)
+    AST_Attribute(AST_expr* value, AST_TYPE::AST_TYPE ctx_type, InternedString attr)
         : AST_expr(AST_TYPE::Attribute), value(value), ctx_type(ctx_type), attr(attr) {}
 
     static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::Attribute;
@@ -359,7 +360,7 @@ public:
 
     std::vector<AST_expr*> bases, decorator_list;
     std::vector<AST_stmt*> body;
-    std::string name;
+    InternedString name;
 
     AST_ClassDef() : AST_stmt(AST_TYPE::ClassDef) {}
 
@@ -490,7 +491,7 @@ class AST_FunctionDef : public AST_stmt {
 public:
     std::vector<AST_stmt*> body;
     std::vector<AST_expr*> decorator_list;
-    std::string name;
+    InternedString name;
     AST_arguments* args;
 
     virtual void accept(ASTVisitor* v);
@@ -516,7 +517,7 @@ public:
 
 class AST_Global : public AST_stmt {
 public:
-    std::vector<std::string> names;
+    std::vector<InternedString> names;
 
     virtual void accept(ASTVisitor* v);
     virtual void accept_stmt(StmtVisitor* v);
@@ -565,7 +566,7 @@ public:
 
 class AST_ImportFrom : public AST_stmt {
 public:
-    std::string module;
+    InternedString module;
     std::vector<AST_alias*> names;
     int level;
 
@@ -593,7 +594,7 @@ class AST_keyword : public AST {
 public:
     // no lineno, col_offset attributes
     AST_expr* value;
-    std::string arg;
+    InternedString arg;
 
     virtual void accept(ASTVisitor* v);
 
@@ -643,12 +644,15 @@ public:
 
 class AST_Module : public AST {
 public:
+    std::unique_ptr<InternedStringPool> interned_strings;
+
     // no lineno, col_offset attributes
     std::vector<AST_stmt*> body;
 
     virtual void accept(ASTVisitor* v);
 
-    AST_Module() : AST(AST_TYPE::Module) {}
+    AST_Module(std::unique_ptr<InternedStringPool> interned_strings)
+        : AST(AST_TYPE::Module), interned_strings(std::move(interned_strings)) {}
 
     static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::Module;
 };
@@ -656,12 +660,12 @@ public:
 class AST_Name : public AST_expr {
 public:
     AST_TYPE::AST_TYPE ctx_type;
-    std::string id;
+    InternedString id;
 
     virtual void accept(ASTVisitor* v);
     virtual void* accept_expr(ExprVisitor* v);
 
-    AST_Name(const std::string& id, AST_TYPE::AST_TYPE ctx_type, int lineno, int col_offset = 0)
+    AST_Name(InternedString id, AST_TYPE::AST_TYPE ctx_type, int lineno, int col_offset = 0)
         : AST_expr(AST_TYPE::Name, lineno, col_offset), ctx_type(ctx_type), id(id) {}
 
     static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::Name;
@@ -939,7 +943,7 @@ public:
 class AST_ClsAttribute : public AST_expr {
 public:
     AST_expr* value;
-    std::string attr;
+    InternedString attr;
 
     virtual void accept(ASTVisitor* v);
     virtual void* accept_expr(ExprVisitor* v);

src/core/cfg.h

@@ -31,6 +31,7 @@
 
 #include "core/ast.h"
 #include "core/common.h"
+#include "core/stringpool.h"
 
 namespace pyston {
 

src/core/stringpool.h

+// Copyright (c) 2014-2015 Dropbox, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef PYSTON_CORE_STRINGPOOL_H
+#define PYSTON_CORE_STRINGPOOL_H
+
+#include <algorithm>
+#include <cstdio>
+#include <sys/time.h>
+
+#include "llvm/ADT/Hashing.h"
+#include "llvm/ADT/StringRef.h"
+
+#include "core/common.h"
+
+namespace std {
+template <> struct hash<llvm::StringRef> {
+    size_t operator()(const llvm::StringRef s) const { return llvm::hash_value(s); }
+};
+}
+
+namespace pyston {
+
+class InternedStringPool;
+class InternedString {
+private:
+    const std::string* _str;
+
+#ifndef NDEBUG
+    // Only for testing purposes:
+    InternedStringPool* pool;
+    InternedString(const std::string* str, InternedStringPool* pool) : _str(str), pool(pool) {}
+#else
+    InternedString(const std::string* str) : _str(str) {}
+#endif
+
+public:
+#ifndef NDEBUG
+    InternedString() : _str(NULL), pool(NULL) {}
+#else
+    InternedString() : _str(NULL) {}
+#endif
+
+    // operator const std::string&() { return *_str; }
+    const std::string& str() const {
+        assert(this->_str);
+        return *_str;
+    }
+
+    const char* c_str() const {
+        assert(this->_str);
+        return _str->c_str();
+    }
+
+    bool operator==(InternedString rhs) const {
+        assert(this->_str);
+        assert(this->pool == rhs.pool);
+        return this->_str == rhs._str;
+    }
+
+    bool operator<(InternedString rhs) const {
+        assert(this->_str);
+        assert(this->pool == rhs.pool);
+        return this->_str < rhs._str;
+    }
+
+    friend class InternedStringPool;
+    friend struct std::hash<InternedString>;
+    friend struct std::less<InternedString>;
+};
+
+class InternedStringPool {
+private:
+    // We probably don't need to pull in llvm::StringRef as the key, but it's better than std::string
+    // which I assume forces extra allocations.
+    // (We could define a custom string-pointer container but is it worth it?)
+    std::unordered_map<llvm::StringRef, std::string*> interned;
+
+public:
+    ~InternedStringPool() {
+        for (auto& p : interned) {
+            delete p.second;
+        }
+    }
+
+    template <class T> InternedString get(T&& arg) {
+        auto it = interned.find(llvm::StringRef(arg));
+
+        std::string* s;
+        if (it != interned.end()) {
+            s = it->second;
+        } else {
+            s = new std::string(std::forward<T>(arg));
+            interned.insert(it, std::make_pair(llvm::StringRef(*s), s));
+        }
+
+#ifndef NDEBUG
+        return InternedString(s, this);
+#else
+        return InternedString(s);
+#endif
+    }
+};
+
+} // namespace pyston
+
+namespace std {
+template <> struct hash<pyston::InternedString> {
+    size_t operator()(const pyston::InternedString s) const { return reinterpret_cast<intptr_t>(s._str) >> 3; }
+};
+template <> struct less<pyston::InternedString> {
+    bool operator()(const pyston::InternedString lhs, const pyston::InternedString rhs) const {
+        assert(lhs.pool && lhs.pool == rhs.pool);
+        // TODO: we should be able to do this comparison on the pointer value, not on the string value,
+        // but there are apparently parts of the code that rely on string sorting being actually alphabetical.
+        // We could create a faster "consistent ordering but not alphabetical" comparator if it makes a difference.
+        return *lhs._str < *rhs._str;
+    }
+};
+}
+
+#endif

src/core/types.h

@@ -28,6 +28,7 @@
 
 #include "core/common.h"
 #include "core/stats.h"
+#include "core/stringpool.h"
 
 namespace llvm {
 class Function;
@@ -218,6 +219,7 @@ public:
 
 class BoxedModule;
 class ScopeInfo;
+class InternedStringPool;
 class SourceInfo {
 public:
     BoxedModule* parent_module;
@@ -228,18 +230,20 @@ public:
     PhiAnalysis* phis;
     bool is_generator;
 
+    InternedStringPool& getInternedStrings();
+
     ScopeInfo* getScopeInfo();
 
     struct ArgNames {
         const std::vector<AST_expr*>* args;
-        const std::string* vararg, *kwarg;
+        InternedString vararg, kwarg;
 
-        explicit ArgNames(AST* ast);
+        explicit ArgNames(AST* ast, ScopingAnalysis* scoping);
 
         int totalParameters() const {
             if (!args)
                 return 0;
-            return args->size() + (vararg->size() == 0 ? 0 : 1) + (kwarg->size() == 0 ? 0 : 1);
+            return args->size() + (vararg.str().size() == 0 ? 0 : 1) + (kwarg.str().size() == 0 ? 0 : 1);
         }
     };
 
@@ -249,7 +253,7 @@ public:
     const std::vector<AST_stmt*> body;
 
     const std::string getName();
-    std::string mangleName(const std::string& id);
+    InternedString mangleName(InternedString id);
 
     SourceInfo(BoxedModule* m, ScopingAnalysis* scoping, AST* ast, const std::vector<AST_stmt*>& body);
 };

src/core/util.h

@@ -20,6 +20,7 @@
 #include <sys/time.h>
 
 #include "core/common.h"
+#include "core/stringpool.h"
 
 namespace pyston {
 
@@ -52,7 +53,7 @@ bool endswith(const std::string& s, const std::string& pattern);
 void removeDirectoryIfExists(const std::string& path);
 
 template <class T1, class T2> void compareKeyset(T1* lhs, T2* rhs) {
-    std::vector<std::string> lv, rv;
+    std::vector<InternedString> lv, rv;
     for (typename T1::iterator it = lhs->begin(); it != lhs->end(); it++) {
         lv.push_back(it->first);
     }
@@ -63,8 +64,8 @@ template <class T1, class T2> void compareKeyset(T1* lhs, T2* rhs) {
     std::sort(lv.begin(), lv.end());
     std::sort(rv.begin(), rv.end());
 
-    std::vector<std::string> lextra(lv.size());
-    std::vector<std::string>::iterator diffend
+    std::vector<InternedString> lextra(lv.size());
+    std::vector<InternedString>::iterator diffend
         = std::set_difference(lv.begin(), lv.end(), rv.begin(), rv.end(), lextra.begin());
     lextra.resize(diffend - lextra.begin());
 
@@ -77,7 +78,7 @@ template <class T1, class T2> void compareKeyset(T1* lhs, T2* rhs) {
         good = false;
     }
 
-    std::vector<std::string> rextra(rv.size());
+    std::vector<InternedString> rextra(rv.size());
     diffend = std::set_difference(rv.begin(), rv.end(), lv.begin(), lv.end(), rextra.begin());
     rextra.resize(diffend - rextra.begin());
 

src/jit.cpp

@@ -202,7 +202,7 @@ int main(int argc, char** argv) {
                 if (m->body.size() > 0 && m->body[0]->type == AST_TYPE::Expr) {
                     AST_Expr* e = ast_cast<AST_Expr>(m->body[0]);
                     AST_Call* c = new AST_Call();
-                    AST_Name* r = new AST_Name("repr", AST_TYPE::Load, 0);
+                    AST_Name* r = new AST_Name(m->interned_strings->get("repr"), AST_TYPE::Load, 0);
                     c->func = r;
                     c->starargs = NULL;
                     c->kwargs = NULL;

src/runtime/objmodel.h

@@ -72,8 +72,8 @@ extern "C" BoxedInt* len(Box* obj);
 extern "C" i64 unboxedLen(Box* obj);
 extern "C" Box* binop(Box* lhs, Box* rhs, int op_type);
 extern "C" Box* augbinop(Box* lhs, Box* rhs, int op_type);
-extern "C" Box* getGlobal(BoxedModule* m, std::string* name);
-extern "C" void delGlobal(BoxedModule* m, std::string* name);
+extern "C" Box* getGlobal(BoxedModule* m, const std::string* name);
+extern "C" void delGlobal(BoxedModule* m, const std::string* name);
 extern "C" Box* getitem(Box* value, Box* slice);
 extern "C" void setitem(Box* target, Box* slice, Box* value);
 extern "C" void delitem(Box* target, Box* slice);

src/runtime/types.cpp

@@ -508,7 +508,7 @@ BoxedClass* object_cls, *type_cls, *none_cls, *bool_cls, *int_cls, *float_cls, *
 BoxedTuple* EmptyTuple;
 }
 
-extern "C" Box* createUserClass(std::string* name, Box* _bases, Box* _attr_dict) {
+extern "C" Box* createUserClass(const std::string* name, Box* _bases, Box* _attr_dict) {
     ASSERT(_attr_dict->cls == dict_cls, "%s", getTypeName(_attr_dict)->c_str());
     BoxedDict* attr_dict = static_cast<BoxedDict*>(_attr_dict);
 

src/runtime/types.h

@@ -118,7 +118,7 @@ extern "C" void listAppendArrayInternal(Box* self, Box** v, int nelts);
 extern "C" Box* boxCLFunction(CLFunction* f, BoxedClosure* closure, bool isGenerator,
                               std::initializer_list<Box*> defaults);
 extern "C" CLFunction* unboxCLFunction(Box* b);
-extern "C" Box* createUserClass(std::string* name, Box* base, Box* attr_dict);
+extern "C" Box* createUserClass(const std::string* name, Box* base, Box* attr_dict);
 extern "C" double unboxFloat(Box* b);
 extern "C" Box* createDict();
 extern "C" Box* createList();

test/unittests/analysis.cpp

@@ -32,8 +32,8 @@ TEST_F(AnalysisTest, augassign) {
     AST_FunctionDef* func = static_cast<AST_FunctionDef*>(module->body[0]);
 
     ScopeInfo* scope_info = scoping->getScopeInfoForNode(func);
-    ASSERT_FALSE(scope_info->refersToGlobal("a"));
-    ASSERT_FALSE(scope_info->refersToGlobal("b"));
+    ASSERT_FALSE(scope_info->refersToGlobal(module->interned_strings->get("a")));
+    ASSERT_FALSE(scope_info->refersToGlobal(module->interned_strings->get("b")));
 
     SourceInfo* si = new SourceInfo(createModule("__main__", fn), scoping, func, func->body);
 
@@ -45,9 +45,9 @@ TEST_F(AnalysisTest, augassign) {
     for (CFGBlock* block : cfg->blocks) {
         //printf("%d\n", block->idx);
         if (block->body.back()->type != AST_TYPE::Return)
-            ASSERT_TRUE(liveness->isLiveAtEnd("a", block));
+            ASSERT_TRUE(liveness->isLiveAtEnd(module->interned_strings->get("a"), block));
     }
 
-    PhiAnalysis* phis = computeRequiredPhis(SourceInfo::ArgNames(func), cfg, liveness, scope_info);
+    PhiAnalysis* phis = computeRequiredPhis(SourceInfo::ArgNames(func, scoping), cfg, liveness, scope_info);
 }