Fairly large refactor: do much more during the CFG lowering.

In particular, lower all control-flow-involving expressions (such as boolops,
list comprehensions, if expressions) into actual control flow.  This requires
unfolding the AST into a flat structure, since the control-flow-expression can
be nested inside other expressions, including other control-flow-expressions.

The IRGenerator was doing this before, but was leading to duplication because
it's very similar to what the CFG generator already has to do.

Doing this earlier in the pipeline means that more analysis passes can work on
the lowered expressions, rather than having to be taught how they work.  In
particular, the name analysis does not need to have to special case the fact
that list comprehensions may set names -- but aren't guaranteed to if they don't
get executed or the iterator is empty.  I tried implementing list comprehensions
without doing this, and a bunch of the work was in reimplementing the logic that
the analyzers already implement.

As a side benefit, deopts become much easier since the AST is already unfolded;
this change gets rid of the messy fake-variable-setting way that the IRGenerator
was unfolding the AST to get this same effect.

src/Makefile

@@ -133,6 +133,9 @@ COMMON_LDFLAGS := -B../tools/build_system -L/usr/local/lib -lpthread -ldl -lcurs
 # Make sure that we put all symbols in the dynamic symbol table so that MCJIT can load them;
 # TODO should probably do the linking before MCJIT
 COMMON_LDFLAGS += -Wl,-E
+# For now, link in libstdc++ statically, since the system libstdc++ is likely to be too old:
+COMMON_LDFLAGS += -static-libstdc++
+
 LDFLAGS := $(LLVM_LDFLAGS) $(COMMON_LDFLAGS)
 LDFLAGS_DEBUG := $(LLVM_DEBUG_LDFLAGS) $(COMMON_LDFLAGS)
 LDFLAGS_PROFILE = $(LLVM_PROFILE_LDFLAGS) -pg $(COMMON_LDFLAGS)
@@ -238,6 +241,10 @@ run_gcunittests: unittests/gc
 run_unittests:: run_gcunittests
 dbg_unittests:: dbg_gcunittests
 
+define checksha
+	test "$$($1 | sha1sum)" = "$2  -"
+endef
+
 .PHONY: test test_debug test_prof test_release
 test_debug: pyston_dbg ext
 	python ../tools/tester.py -j$(TEST_THREADS) -k $(TESTS_DIR) $(ARGS)
@@ -251,7 +258,7 @@ test_prof: pyston_prof ext
 	python ../tools/tester.py -P -j$(TEST_THREADS) -k $(TESTS_DIR) $(ARGS)
 	python ../tools/tester.py -P -j$(TEST_THREADS) -a -n -k $(TESTS_DIR) $(ARGS)
 	python ../tools/tester.py -P -j$(TEST_THREADS) -a -O -k $(TESTS_DIR) $(ARGS)
-test: ext
+test: ext pyston_dbg
 	$(MAKE) pyston_dbg
 	# $(MAKE) run_unittests
 	$(MAKE) test_debug
@@ -259,13 +266,12 @@ test: ext
 	@# jit_prof forces the use of GCC as the compiler, which can expose other errors, so just build it and see what happens:
 	# $(MAKE) test_prof
 	$(MAKE) pyston_prof
-	./pyston_prof -q $(TESTS_DIR)/raytrace_small.py | sha1sum
+	$(call checksha,./pyston_prof -cq  $(TESTS_DIR)/raytrace_small.py,0544f4621dd45fe94205219488a2576b84dc044d)
+	$(call checksha,./pyston_prof -cqn $(TESTS_DIR)/raytrace_small.py,0544f4621dd45fe94205219488a2576b84dc044d)
+	$(call checksha,./pyston_prof -cqO $(TESTS_DIR)/raytrace_small.py,0544f4621dd45fe94205219488a2576b84dc044d)
 
 	echo "All tests passed"
 
-define checksha
-	test "$$($1 | sha1sum)" = "$2  -"
-endef
 check:
 	$(MAKE) pyston_dbg
 	$(call checksha,./pyston_dbg -cq  $(TESTS_DIR)/raytrace_small.py,0544f4621dd45fe94205219488a2576b84dc044d)

src/codegen/compvars.cpp

@@ -310,6 +310,7 @@ static ConcreteCompilerVariable* _call(IREmitter &emitter, llvm::Value* func, vo
     std::vector<BoxedClass*> guaranteed_classes;
     std::vector<ConcreteCompilerVariable*> converted_args;
     for (int i = 0; i < args.size(); i++) {
+        assert(args[i]);
         converted_args.push_back(args[i]->makeConverted(emitter, args[i]->getBoxType()));
         guaranteed_classes.push_back(converted_args.back()->guaranteedClass());
     }
@@ -545,6 +546,14 @@ class AbstractFunctionType : public CompilerType {
         }
 };
 
+CompilerType* makeFuncType(ConcreteCompilerType* rtn_type, const std::vector<ConcreteCompilerType*> &arg_types) {
+    std::vector<AbstractFunctionType::Sig*> sigs;
+    AbstractFunctionType::Sig *sig = new AbstractFunctionType::Sig();
+    sig->rtn_type = rtn_type;
+    sig->arg_types = arg_types;
+    return AbstractFunctionType::get(sigs);
+}
+
 class IntType : public ConcreteCompilerType {
     public:
         IntType() {}

src/codegen/compvars.h

@@ -314,6 +314,7 @@ CompilerVariable* makeTuple(const std::vector<CompilerVariable*> &elts);
 ConcreteCompilerType* typeFromClass(BoxedClass*);
 CompilerType* typeOfClassobj(BoxedClass*);
 CompilerType* makeTupleType(const std::vector<CompilerType*> &elt_types);
+CompilerType* makeFuncType(ConcreteCompilerType* rtn_type, const std::vector<ConcreteCompilerType*> &arg_types);
 
 } // namespace pyston
 

src/codegen/opt/inliner.cpp

@@ -59,11 +59,6 @@ class MyMaterializer : public llvm::ValueMaterializer {
                 //llvm::errs() << " is gv\n";
                 assert(gv->getLinkage() != llvm::GlobalVariable::PrivateLinkage);
                 r = new_module->getOrInsertGlobal(gv->getName(), gv->getType()->getElementType());
-            } else if (llvm::LoadInst *load = llvm::dyn_cast<llvm::LoadInst>(v)) {
-                //llvm::errs() << " is LoadInst\n";
-                //load->getPointerOperand()->dump();
-                llvm::GlobalVariable *gv = llvm::dyn_cast<llvm::GlobalVariable>(load->getPointerOperand());
-                r = v;
             } else if (llvm::GlobalAlias *alias = llvm::dyn_cast<llvm::GlobalAlias>(v)) {
                 llvm::Value* addressee = llvm::cast<llvm::Constant>(materializeValueFor(alias->getAliasedGlobal()));
                 assert(addressee);

src/codegen/parse_ast.py

@@ -118,7 +118,7 @@ def convert(n, f):
         else:
             raise Exception(type(n.n))
 
-    # print n, sorted(n.__dict__.items())
+    # print >>sys.stderr, n, sorted(n.__dict__.items())
     for k, v in sorted(n.__dict__.items()):
         if k.startswith('_'):
             continue

src/codegen/parser.cpp

@@ -380,6 +380,17 @@ AST_If* read_if(BufferedReader *reader) {
     return rtn;
 }
 
+AST_IfExp* read_ifexp(BufferedReader *reader) {
+    AST_IfExp *rtn = new AST_IfExp();
+
+    rtn->body = readASTExpr(reader);
+    rtn->col_offset = readColOffset(reader);
+    rtn->lineno = reader->readULL();
+    rtn->orelse = readASTExpr(reader);
+    rtn->test = readASTExpr(reader);
+    return rtn;
+}
+
 AST_Import* read_import(BufferedReader *reader) {
     AST_Import *rtn = new AST_Import();
 
@@ -595,6 +606,8 @@ AST_expr* readASTExpr(BufferedReader *reader) {
             return read_compare(reader);
         case AST_TYPE::Dict:
             return read_dict(reader);
+        case AST_TYPE::IfExp:
+            return read_ifexp(reader);
         case AST_TYPE::Index:
             return read_index(reader);
         case AST_TYPE::List:

src/core/ast.cpp

@@ -383,6 +383,19 @@ void AST_If::accept_stmt(StmtVisitor *v) {
     v->visit_if(this);
 }
 
+void AST_IfExp::accept(ASTVisitor *v) {
+    bool skip = v->visit_ifexp(this);
+    if (skip) return;
+
+    this->test->accept(v);
+    this->body->accept(v);
+    this->orelse->accept(v);
+}
+
+void* AST_IfExp::accept_expr(ExprVisitor *v) {
+    return v->visit_ifexp(this);
+}
+
 void AST_Import::accept(ASTVisitor *v) {
     bool skip = v->visit_import(this);
     if (skip) return;
@@ -595,6 +608,7 @@ void* AST_ClsAttribute::accept_expr(ExprVisitor *v) {
 
 
 
+
 void print_ast(AST* ast) {
     PrintVisitor v;
     ast->accept(&v);
@@ -881,6 +895,15 @@ bool PrintVisitor::visit_if(AST_If *node) {
     return true;
 }
 
+bool PrintVisitor::visit_ifexp(AST_IfExp *node) {
+    node->body->accept(this);
+    printf(" if ");
+    node->test->accept(this);
+    printf(" else ");
+    node->orelse->accept(this);
+    return true;
+}
+
 bool PrintVisitor::visit_import(AST_Import *node) {
     printf("import ");
     for (int i = 0; i < node->names.size(); i++) {

src/core/ast.h

@@ -350,6 +350,17 @@ class AST_If : public AST_stmt {
         AST_If() : AST_stmt(AST_TYPE::If) {}
 };
 
+class AST_IfExp : public AST_expr {
+    public:
+        const static AST_TYPE::AST_TYPE TYPE = AST_TYPE::IfExp;
+        AST_expr *body, *test, *orelse;
+
+        virtual void accept(ASTVisitor *v);
+        virtual void* accept_expr(ExprVisitor *v);
+
+        AST_IfExp() : AST_expr(AST_TYPE::IfExp) {}
+};
+
 class AST_Import : public AST_stmt {
     public:
         std::vector<AST_alias*> names;
@@ -580,7 +591,6 @@ class AST_ClsAttribute : public AST_expr {
         AST_ClsAttribute() : AST_expr(AST_TYPE::ClsAttribute) {}
 };
 
-
 class ASTVisitor {
     protected:
     public:
@@ -605,6 +615,7 @@ class ASTVisitor {
         virtual bool visit_functiondef(AST_FunctionDef *node) { assert(0); abort(); }
         virtual bool visit_global(AST_Global *node) { assert(0); abort(); }
         virtual bool visit_if(AST_If *node) { assert(0); abort(); }
+        virtual bool visit_ifexp(AST_IfExp *node) { assert(0); abort(); }
         virtual bool visit_import(AST_Import *node) { assert(0); abort(); }
         virtual bool visit_index(AST_Index *node) { assert(0); abort(); }
         virtual bool visit_keyword(AST_keyword *node) { assert(0); abort(); }
@@ -651,6 +662,7 @@ class NoopASTVisitor : public ASTVisitor {
         virtual bool visit_functiondef(AST_FunctionDef *node) { return false; }
         virtual bool visit_global(AST_Global *node) { return false; }
         virtual bool visit_if(AST_If *node) { return false; }
+        virtual bool visit_ifexp(AST_IfExp *node) { return false; }
         virtual bool visit_import(AST_Import *node) { return false; }
         virtual bool visit_index(AST_Index *node) { return false; }
         virtual bool visit_keyword(AST_keyword *node) { return false; }
@@ -685,6 +697,7 @@ class ExprVisitor {
         virtual void* visit_clsattribute(AST_ClsAttribute *node) { assert(0); abort(); }
         virtual void* visit_compare(AST_Compare *node) { assert(0); abort(); }
         virtual void* visit_dict(AST_Dict *node) { assert(0); abort(); }
+        virtual void* visit_ifexp(AST_IfExp *node) { assert(0); abort(); }
         virtual void* visit_index(AST_Index *node) { assert(0); abort(); }
         virtual void* visit_list(AST_List *node) { assert(0); abort(); }
         virtual void* visit_name(AST_Name *node) { assert(0); abort(); }
@@ -750,6 +763,7 @@ class PrintVisitor : public ASTVisitor {
         virtual bool visit_functiondef(AST_FunctionDef *node);
         virtual bool visit_global(AST_Global *node);
         virtual bool visit_if(AST_If *node);
+        virtual bool visit_ifexp(AST_IfExp *node);
         virtual bool visit_import(AST_Import *node);
         virtual bool visit_index(AST_Index *node);
         virtual bool visit_keyword(AST_keyword *node);

test/tests/list.py

@@ -26,3 +26,5 @@ for i in xrange(-5, 4):
     l3 = range(5)
     l3[:i] = [7, 8]
     print l3
+
+print [1, 2, 3, 4, 5]

test/tests/t2.py

-def f(c):
-    c.x = None
-    c.x = 1
-    # return None
-
-class C(object):
-    pass
-print f(C())
+t = (1, "h")
+print t, str(t), repr(t)
+if 1:
+    t = (3,)
+print t
+
+def f():
+    t = (1, 3)
+    print t
+f()
+
+print ()
+print (1,)
+print (1, 2)
+print (1, 2, 3)
+
+t = 1, 3
+print t
+
+print (2,) < (2,)
+print (2,) < (2, 3)
+print (3,) < (2, 3)
+
+print

tools/tester.py

@@ -275,10 +275,6 @@ if __name__ == "__main__":
         nostat(["nondirectly_callable_ics"]) # WIP
         skip(["finalization_cycles", "resurrection", "nonself_resurrection"]) # WIP
 
-        if '-O' in EXTRA_JIT_ARGS:
-            # Have guard issues:
-            skip(["dict", "tuple_iteration"])
-
     if datetime.datetime.now() < datetime.datetime(2014,4,28):
         skip(["class_noctor", "non_function_ctors"]) # object.__new__
         skip(["setattr_patching_under"]) # requires __del__