Found some bugs in the name analyzer around listcomps; got the unittests back...

Found some bugs in the name analyzer around listcomps; got the unittests back up and running so I can write some more

docs/INSTALLING.md

@@ -110,7 +110,7 @@ cd ~/pyston_deps
 wget https://googletest.googlecode.com/files/gtest-1.7.0.zip
 unzip gtest-1.7.0.zip
 cd gtest-1.7.0
-./configure
+./configure CXXFLAGS=-fno-omit-frame-pointer
 make -j4
 ```
 

src/Makefile

@@ -228,7 +228,11 @@ OPT_OBJS := $(STDLIB_RELEASE_OBJS) $(SRCS:.cpp=.release.o)
 
 OPTIONAL_SRCS := codegen/profiling/oprofile.cpp codegen/profiling/pprof.cpp
 TOOL_SRCS := $(wildcard $(TOOLS_DIR)/*.cpp)
-NONSTDLIB_SRCS := $(MAIN_SRCS) $(OPTIONAL_SRCS) $(TOOL_SRCS)
+
+UNITTEST_DIR := ../test/unittests
+UNITTEST_SRCS := $(wildcard $(UNITTEST_DIR)/*.cpp)
+
+NONSTDLIB_SRCS := $(MAIN_SRCS) $(OPTIONAL_SRCS) $(TOOL_SRCS) $(UNITTEST_SRCS)
 
 .DEFAULT_GOAL := pyston_dbg
 # _ :
@@ -245,18 +249,15 @@ tags: $(SRCS) $(OPTIONAL_SRCS) $(ALL_HEADERS)
 	$(ECHO) Computing tags...
 	$(VERB) ctags $^
 
-UNITTEST_SRCS := $(wildcard unittests/*.cpp)
-GC_OBJS := $(patsubst %.cpp,%.o,$(wildcard gc/*.cpp))
-$(UNITTEST_SRCS:.cpp=.o): %.o: %.cpp $(BUILD_SYSTEM_DEPS)
-	$(ECHO) Compiling $@
-	$(VERB) $(CXX) $(CXXFLAGS) -I$(GTEST_DIR)/include -MMD -MP -MF $(patsubst %.o,%.d,$@) $< -c -o $@
-unittests/gc: $(GTEST_DIR)/src/gtest-all.o $(UNITTEST_SRCS:.cpp=.o) $(BUILD_SYSTEM_DEPS) $(GC_OBJS)
+$(UNITTEST_SRCS:.cpp=.o): CXXFLAGS := $(CXXFLAGS) -I$(GTEST_DIR)/include
+NON_ENTRY_OBJS := $(filter-out jit.o,$(OBJS))
+$(UNITTEST_DIR)/gc: $(GTEST_DIR)/src/gtest-all.o $(NON_ENTRY_OBJS) $(BUILD_SYSTEM_DEPS) $(UNITTEST_DIR)/gc.o
 	$(ECHO) Linking $@
-	$(VERB) $(CXX) $(GTEST_DIR)/src/gtest-all.o $(GTEST_DIR)/src/gtest_main.o $(UNITTEST_SRCS:.cpp=.o) $(GC_OBJS) $(LDFLAGS) -o $@
-dbg_gcunittests: unittests/gc
-	zsh -c 'ulimit -v $(MAX_MEM_KB); ulimit -d $(MAX_MEM_KB); time gdb $(GDB_CMDS) --args ./unittests/gc --gtest_break_on_failure $(ARGS)'
-run_gcunittests: unittests/gc
-	zsh -c 'ulimit -v $(MAX_MEM_KB); ulimit -d $(MAX_MEM_KB); time ./unittests/gc $(ARGS)'
+	$(VERB) $(CXX) $(NON_ENTRY_OBJS) $(GTEST_DIR)/src/gtest-all.o $(GTEST_DIR)/src/gtest_main.o $(UNITTEST_DIR)/gc.o $(LDFLAGS) -o $@
+dbg_gcunittests: $(UNITTEST_DIR)/gc
+	zsh -c 'ulimit -v $(MAX_MEM_KB); ulimit -d $(MAX_MEM_KB); time gdb $(GDB_CMDS) --args ./$(UNITTEST_DIR)/gc --gtest_break_on_failure $(ARGS)'
+run_gcunittests: $(UNITTEST_DIR)/gc
+	zsh -c 'ulimit -v $(MAX_MEM_KB); ulimit -d $(MAX_MEM_KB); time ./$(UNITTEST_DIR)/gc $(ARGS)'
 run_unittests:: run_gcunittests
 dbg_unittests:: dbg_gcunittests
 
@@ -289,6 +290,8 @@ check test: ext pyston_dbg
 	$(call checksha,./pyston_prof -cqn $(TESTS_DIR)/raytrace_small.py,0544f4621dd45fe94205219488a2576b84dc044d)
 	$(call checksha,./pyston_prof -cqO $(TESTS_DIR)/raytrace_small.py,0544f4621dd45fe94205219488a2576b84dc044d)
 
+	$(MAKE) run_unittests
+
 	echo "All tests passed"
 
 quick_check:
@@ -597,7 +600,7 @@ pyston_profile: $(PROFILE_OBJS) $(LLVM_PROFILE_DEPS)
 	$(VERB) $(CXX) $(PROFILE_OBJS) $(LDFLAGS_PROFILE) -o $@
 
 
--include $(wildcard *.d) $(wildcard */*.d) $(wildcard */*/*.d)
+-include $(wildcard *.d) $(wildcard */*.d) $(wildcard */*/*.d) $(wildcard $(UNITTEST_DIR)/*.d)
 
 .PHONY: clean
 clean:

src/analysis/function_analysis.cpp

@@ -72,6 +72,10 @@ class LivenessBBVisitor : public NoopASTVisitor {
             }
             return true;
         }
+
+        bool visit_augassign(AST_AugAssign* node) {
+            assert(0 && "need to set it as a load");
+        }
 };
 
 bool LivenessAnalysis::isLiveAtEnd(const std::string &name, CFGBlock *block) {
@@ -99,9 +103,14 @@ bool LivenessAnalysis::isLiveAtEnd(const std::string &name, CFGBlock *block) {
         for (int i = 0; i < thisblock->body.size(); i++) {
             thisblock->body[i]->accept(&visitor);
         }
-        if (visitor.loads().count(name))
+
+        if (visitor.loads().count(name)) {
+            assert(!visitor.stores().count(name));
             return true;
+        }
+
         if (!visitor.stores().count(name)) {
+            assert(!visitor.loads().count(name));
             for (int i = 0; i < thisblock->successors.size(); i++) {
                 q.push_back(thisblock->successors[i]);
             }

src/runtime/types.h

@@ -197,7 +197,7 @@ class StlCompatAllocator {
 
         pointer allocate(size_t n) {
             size_t to_allocate = n * sizeof(value_type);
-            assert(to_allocate < (1<<16));
+            //assert(to_allocate < (1<<16));
 
             ConservativeWrapper* rtn = new (to_allocate) ConservativeWrapper(to_allocate);
             return (pointer)&rtn->data[0];

test/tests/augassign_analysis.py

+# Augassigns are somewhat odd because they encode both a use and a def of a variable.
+# So for the sake of determining if a scope sets a variable, they count as a def, but
+# for the sake of whether or not a variable is live, they count as a use.
+
+
+def f0(a, b):
+    a += b
+    return a
+print f0(1, 2)
+
+def f1(a, b):
+    a += b
+    if 1:
+        pass
+    return a
+print f1(1, 2)
+
+def f2(a, b):
+    if 1:
+        pass
+    a += b
+    return a
+print f2(1, 2)
+
+def f3(a, b):
+    if 1:
+        pass
+    a += b
+    if 1:
+        pass
+    return a
+print f3(1, 2)
+
+def f4():
+    lists = [[], [], []]
+
+    for i in xrange(3):
+        [l for l in lists[:i+1]][-1] += [i]
+
+    print lists
+    print l
+f4()

test/tests/osr_into_failed_speculation.py

-# expected: fail
-
 # Try to trick the JIT into OSR'ing into an optimized version with a speculation
 # that has already failed.
 # In the f() function, y will have type int, but when we OSR from the while loop,

test/unittests/.gitignore

+gc
+analysis

test/unittests/analysis.cpp

+#include <memory>
+#include <vector>
+#include <unordered_set>
+
+#include "gtest/gtest.h"
+
+#include "gc/gc_alloc.h"
+
+using namespace pyston;
+using namespace pyston::gc;
+
+TEST(func_analysis, augassign) {
+}
+

test/unittests/gc.cpp

@@ -4,16 +4,19 @@
 
 #include "gtest/gtest.h"
 
+#include "core/types.h"
 #include "gc/gc_alloc.h"
+#include "runtime/types.h"
 
 using namespace pyston;
 using namespace pyston::gc;
 
-void testAlloc(int B) {
-    struct S {
-        int data[0];
-    };
+struct S {
+    GCObjectHeader header;
+    int data[0];
+};
 
+void testAlloc(int B) {
     std::unique_ptr<int> masks(new int[B/4]);
     masks.get()[0] = 0;
     for (int j = 1; j < B/4; j++) {
@@ -21,17 +24,20 @@ void testAlloc(int B) {
     }
 
     for (int l = 0; l < 10; l++) {
-        std::vector<S*> allocd;
-        std::unordered_set<S*> seen;
+        std::vector<S*, StlCompatAllocator<S*>> allocd;
+        std::unordered_set<S*, std::hash<S*>, std::equal_to<S*>, StlCompatAllocator<S*>> seen;
 
-        const int N = l * 1000;
+        int N = l * 1000;
+        if (B > 1024)
+            N /= 10;
         for (int i = 0; i < N; i++) {
             S* t = static_cast<S*>(gc_alloc(B));
+            t->header.kind_id = untracked_kind.kind_id;
 
             ASSERT_TRUE(t != NULL);
             ASSERT_EQ(0, seen.count(t));
 
-            for (int j = 0; j < B/4; j++) {
+            for (int j = 0; j < (B - sizeof(S))/4; j++) {
                 t->data[j] = i ^ masks.get()[j];
             }
 
@@ -40,7 +46,7 @@ void testAlloc(int B) {
         }
 
         for (int i = 0; i < N; i++) {
-            for (int j = 0; j < B/4; j++) {
+            for (int j = 0; j < (B - sizeof(S))/4; j++) {
                 ASSERT_EQ(i ^ masks.get()[j], allocd[i]->data[j]);
             }
             gc_free(allocd[i]);
@@ -48,52 +54,55 @@ void testAlloc(int B) {
     }
 }
 
-TEST(gc, alloc16) { testAlloc(16); }
-TEST(gc, alloc24) { testAlloc(24); }
-TEST(gc, alloc32) { testAlloc(32); }
-TEST(gc, alloc48) { testAlloc(48); }
-TEST(gc, alloc64) { testAlloc(64); }
-TEST(gc, alloc128) { testAlloc(128); }
-TEST(gc, alloc258) { testAlloc(258); }
-TEST(gc, alloc3584) { testAlloc(3584); }
+TEST(alloc, alloc16) { testAlloc(16); }
+TEST(alloc, alloc24) { testAlloc(24); }
+TEST(alloc, alloc32) { testAlloc(32); }
+TEST(alloc, alloc48) { testAlloc(48); }
+TEST(alloc, alloc64) { testAlloc(64); }
+TEST(alloc, alloc128) { testAlloc(128); }
+TEST(alloc, alloc258) { testAlloc(258); }
+TEST(alloc, alloc3584) { testAlloc(3584); }
 
-TEST(gc, largeallocs) {
+TEST(alloc, largeallocs) {
     int s1 = 1 << 20;
-    char* d1 = (char*)gc_alloc(s1);
-    memset(d1, 1, s1);
+    S* d1 = (S*)gc_alloc(s1);
+    d1->header.kind_id = untracked_kind.kind_id;
+    memset(d1->data, 1, s1 - sizeof(S));
 
     int s2 = 2 << 20;
-    char* d2 = (char*)gc_alloc(s2);
-    memset(d2, 2, s2);
+    S* d2 = (S*)gc_alloc(s2);
+    d2->header.kind_id = untracked_kind.kind_id;
+    memset(d2->data, 2, s2 - sizeof(S));
 
     int s3 = 4 << 20;
-    char* d3 = (char*)gc_alloc(s3);
-    memset(d3, 3, s3);
+    S* d3 = (S*)gc_alloc(s3);
+    d3->header.kind_id = untracked_kind.kind_id;
+    memset(d3->data, 3, s3 - sizeof(S));
 
-    for (int i = 0; i < s1; i++) {
-        ASSERT_EQ(1, d1[i]);
+    for (int i = sizeof(S); i < s1; i++) {
+        ASSERT_EQ(1, *(i + (char*)d1));
     }
 
-    for (int i = 0; i < s2; i++) {
-        ASSERT_EQ(2, d2[i]);
+    for (int i = sizeof(S); i < s2; i++) {
+        ASSERT_EQ(2, *(i + (char*)d2));
     }
 
-    for (int i = 0; i < s3; i++) {
-        ASSERT_EQ(3, d3[i]);
+    for (int i = sizeof(S); i < s3; i++) {
+        ASSERT_EQ(3, *(i + (char*)d3));
     }
 }
 
-TEST(gc, freeing) {
+TEST(alloc, freeing) {
     // Not sure this is enough to crash if it doesn't get freed:
-    for (int i = 0; i < 1000000; i++) {
+    for (int i = 0; i < 100000; i++) {
         void* a = gc_alloc(1024);
         gc_free(a);
     }
 }
 
-TEST(gc, freeingLarge) {
-    for (int i = 0; i < 100000; i++) {
-        void* a = gc_alloc(1<<24);
+TEST(alloc, freeingLarge) {
+    for (int i = 0; i < 200; i++) {
+        void* a = gc_alloc(1<<26);
         gc_free(a);
     }
 }