Some work on the Grand CAPI Unification

ie trying to start running real C modules.  The goal has always been to
run them without modification and without a proxy layer, and this is the first step.

The first issue is conversion between PyObject and our internal representation,
"Box" (needs to be renamed).  Once they have the same layout, some hacks were
needed to make them seem like the same thing, even though we can attach C++
methods on the Box side, but use PyObject in C code.

The next major issue was PyTypeObject, since the structure is directly exposed to
extensions.  I just added the similar fields to BoxedClass, so they can be
used interchangeably; the "real" ones are the BoxedClass ones, but we'll migrate
them incrementally.

There's also the issue that PyTypeObject's are typically created statically,
so I added registerStaticRootMemory to the GC interface.

Also add in a smattering of other API functions, and the _sha module works
(who cares if it's normally disabled).

The C API is quite constraining about choices of data structure implementation; it's
in direct conflict with the std::string API, for example.  For now, close our eyes
and let the C API modify the internal bytes of std::string.

src/Makefile

@@ -267,12 +267,15 @@ STDLIB_SRCS := $(wildcard runtime/inline/*.cpp)
 SRCS := $(MAIN_SRCS) $(STDLIB_SRCS)
 STDLIB_OBJS := stdlib.bc.o stdlib.stripped.bc.o
 STDLIB_RELEASE_OBJS := stdlib.release.bc.o
-STDMODULES_SRCS := ../lib_python/2.7_Modules/errnomodule.c
+
+STDMODULES_SRCS := errnomodule.c shamodule.c
+STDMODULES_SRCS := $(addprefix ../lib_python/2.7_Modules/,$(STDMODULES_SRCS))
+
 # The stdlib objects have slightly longer dependency chains,
 # so put them first in the list:
 OBJS := $(STDLIB_OBJS) $(SRCS:.cpp=.o) $(STDMODULES_SRCS:.c=.o)
-PROFILE_OBJS := $(STDLIB_RELEASE_OBJS) $(MAIN_SRCS:.cpp=.prof.o) $(STDLIB_SRCS:.cpp=.release.o) $(STDMODULES_SRCS:.c=.o)
-OPT_OBJS := $(STDLIB_RELEASE_OBJS) $(SRCS:.cpp=.release.o) $(STDMODULES_SRCS:.c=.o)
+PROFILE_OBJS := $(STDLIB_RELEASE_OBJS) $(MAIN_SRCS:.cpp=.prof.o) $(STDLIB_SRCS:.cpp=.release.o) $(STDMODULES_SRCS:.c=.release.o)
+OPT_OBJS := $(STDLIB_RELEASE_OBJS) $(SRCS:.cpp=.release.o) $(STDMODULES_SRCS:.c=.release.o)
 
 OPTIONAL_SRCS := codegen/profiling/oprofile.cpp codegen/profiling/pprof.cpp
 TOOL_SRCS := $(wildcard $(TOOLS_DIR)/*.cpp)
@@ -879,5 +882,9 @@ ext: ../test/test_extension/test.so
 	$(CLANG_EXE) -O2 -fPIC -Wimplicit -I../include -c $< -o $@ -g
 
 ../lib_python/2.7_Modules/%.o: ../lib_python/2.7_Modules/%.c $(BUILD_SYSTEM_DEPS)
+	$(ECHO) Compiling extension file $@
+	$(VERB) $(CC) $(EXT_CFLAGS) -c $< -o $@ -g -MMD -MP -MF $<.d -O0
+
+../lib_python/2.7_Modules/%.release.o: ../lib_python/2.7_Modules/%.c $(BUILD_SYSTEM_DEPS)
 	$(ECHO) Compiling extension file $@
 	$(VERB) $(CC) $(EXT_CFLAGS) -c $< -o $@ -g -MMD -MP -MF $<.d

src/asm_writing/assembler.cpp

@@ -558,12 +558,15 @@ void Assembler::cmp(Indirect mem, Register reg) {
     emitRex(rex);
     emitByte(0x3B);
 
-    assert(-0x80 <= mem.offset && mem.offset < 0x80);
     if (mem.offset == 0) {
         emitModRM(0b00, reg_idx, mem_idx);
-    } else {
+    } else if (-0x80 <= mem.offset && mem.offset < 0x80) {
         emitModRM(0b01, reg_idx, mem_idx);
         emitByte(mem.offset);
+    } else {
+        assert((-1L << 31) <= mem.offset && mem.offset < (1L << 31) - 1);
+        emitModRM(0b10, reg_idx, mem_idx);
+        emitInt(mem.offset, 4);
     }
 }
 

src/codegen/opt/const_classes.cpp

@@ -124,9 +124,9 @@ private:
                     errs() << "attrs_offset; replacing with " << cls->attrs_offset << "\n";
                     replaceUsesWithConstant(gep_load, cls->attrs_offset);
                     changed = true;
-                } else if (offset == offsetof(BoxedClass, instance_size)) {
-                    errs() << "instance_size; replacing with " << cls->instance_size << "\n";
-                    replaceUsesWithConstant(gep_load, cls->instance_size);
+                } else if (offset == offsetof(BoxedClass, tp_basicsize)) {
+                    errs() << "tp_basicsize; replacing with " << cls->tp_basicsize << "\n";
+                    replaceUsesWithConstant(gep_load, cls->tp_basicsize);
                     changed = true;
                 }
             }

src/codegen/patchpoints.cpp

@@ -120,7 +120,7 @@ PatchpointSetupInfo* createGenericPatchpoint(CompiledFunction* parent_cf, TypeRe
 }
 
 PatchpointSetupInfo* createGetattrPatchpoint(CompiledFunction* parent_cf, TypeRecorder* type_recorder) {
-    return PatchpointSetupInfo::initialize(true, 1, 144, parent_cf, Getattr, type_recorder);
+    return PatchpointSetupInfo::initialize(true, 1, 196, parent_cf, Getattr, type_recorder);
 }
 
 PatchpointSetupInfo* createGetitemPatchpoint(CompiledFunction* parent_cf, TypeRecorder* type_recorder) {

src/core/types.h

@@ -19,7 +19,10 @@
 // over having them spread randomly in different files, this should probably be split again
 // but in a way that makes more sense.
 
-#include <llvm/ADT/iterator_range.h>
+#include <stddef.h>
+
+#include "llvm/ADT/iterator_range.h"
+#include "Python.h"
 
 #include "core/common.h"
 #include "core/stats.h"
@@ -379,6 +382,9 @@ extern "C" const std::string* getTypeName(Box* o);
 
 class BoxedClass : public Box {
 public:
+    Py_ssize_t ob_size; // CPython API compatibility
+    PyTypeObject_BODY;
+
     HCAttrs attrs;
 
     // If the user sets __getattribute__ or __getattr__, we will have to invalidate
@@ -389,7 +395,7 @@ public:
 
     // Only a single base supported for now.
     // Is NULL iff this is object_cls
-    BoxedClass* const base;
+    BoxedClass* base;
 
     typedef void (*gcvisit_func)(GCVisitor*, Box*);
     gcvisit_func gc_visit;
@@ -397,8 +403,6 @@ public:
     // Offset of the HCAttrs object or 0 if there are no hcattrs.
     // Analogous to tp_dictoffset
     const int attrs_offset;
-    // Analogous to tp_basicsize
-    const int instance_size;
 
     bool instancesHaveAttrs() { return attrs_offset != 0; }
 
@@ -422,6 +426,17 @@ public:
     }
 };
 
+static_assert(sizeof(pyston::Box) == sizeof(struct _object), "");
+static_assert(offsetof(pyston::Box, cls) == offsetof(struct _object, ob_type), "");
+
+static_assert(offsetof(pyston::BoxedClass, cls) == offsetof(struct _typeobject, ob_type), "");
+static_assert(offsetof(pyston::BoxedClass, tp_name) == offsetof(struct _typeobject, tp_name), "");
+static_assert(offsetof(pyston::BoxedClass, attrs) == offsetof(struct _typeobject, _hcls), "");
+static_assert(offsetof(pyston::BoxedClass, dependent_icgetattrs) == offsetof(struct _typeobject, _dep_getattrs), "");
+static_assert(offsetof(pyston::BoxedClass, base) == offsetof(struct _typeobject, _base), "");
+static_assert(offsetof(pyston::BoxedClass, gc_visit) == offsetof(struct _typeobject, _gcvisit_func), "");
+static_assert(sizeof(pyston::BoxedClass) == sizeof(struct _typeobject), "");
+
 // TODO these shouldn't be here
 void setupRuntime();
 void teardownRuntime();

src/gc/collector.cpp

@@ -42,6 +42,17 @@ void registerStaticRootObj(void* obj) {
     roots.push(obj);
 }
 
+std::vector<std::pair<void*, void*> > static_root_memory;
+void registerStaticRootMemory(void* start, void* end) {
+    assert(start < end);
+
+    // While these aren't necessary to work correctly, they are the anticipated use case:
+    assert(global_heap.getAllocationFromInteriorPointer(start) == NULL);
+    assert(global_heap.getAllocationFromInteriorPointer(end) == NULL);
+
+    static_root_memory.push_back(std::make_pair(start, end));
+}
+
 static std::unordered_set<StaticRootHandle*>* getRootHandles() {
     static std::unordered_set<StaticRootHandle*> root_handles;
     return &root_handles;
@@ -102,6 +113,10 @@ static void markPhase() {
 
     TraceStackGCVisitor visitor(&stack);
 
+    for (const auto& p : static_root_memory) {
+        visitor.visitPotentialRange((void**)p.first, (void**)p.second);
+    }
+
     for (auto h : *getRootHandles()) {
         visitor.visitPotential(h->value);
     }

src/gc/collector.h

@@ -60,11 +60,13 @@ public:
     void visitPotentialRange(void* const* start, void* const* end) override;
 };
 
-// Call it a "root obj" because this function takes the pointer to the object, not a pointer
-// to a storage location where we might store different objects.
-// ie this only works for constant roots, and not out-of-gc-knowledge storage locations
-// (that should be registerStaticRootPtr)
+// Mark this gc-allocated object as being a root, even if there are no visible references to it.
+// (Note: this marks the gc allocation itself, not the pointer that points to one.  For that, use
+// a StaticRootHandle or registerStaticRootMemory)
 void registerStaticRootObj(void* root_obj);
+// Register a non-gc region of memory (such as statically-allocated memory) as a source of potential
+// GC roots.
+void registerStaticRootMemory(void* start, void* end);
 void runCollection();
 
 // If you want to have a static root "location" where multiple values could be stored, use this:

src/gc/heap.cpp

@@ -390,6 +390,16 @@ GCAllocation* Heap::getAllocationFromInteriorPointer(void* ptr) {
     return reinterpret_cast<GCAllocation*>(&b->atoms[atom_idx]);
 }
 
+static void _doFree(GCAllocation* al) {
+    if (VERBOSITY() >= 2)
+        printf("Freeing %p\n", al->user_data);
+
+    if (al->kind_id == GCKind::PYTHON) {
+        Box* b = (Box*)al->user_data;
+        ASSERT(b->cls->tp_dealloc == NULL, "%s", getTypeName(b)->c_str());
+    }
+}
+
 static Block** freeChain(Block** head) {
     while (Block* b = *head) {
         int num_objects = b->numObjects();
@@ -411,8 +421,7 @@ static Block** freeChain(Block** head) {
             if (isMarked(al)) {
                 clearMark(al);
             } else {
-                if (VERBOSITY() >= 2)
-                    printf("Freeing %p\n", al->user_data);
+                _doFree(al);
 
                 // assert(p != (void*)0x127000d960); // the main module
                 b->isfree[bitmap_idx] |= mask;
@@ -471,8 +480,7 @@ void Heap::freeUnmarked() {
         if (isMarked(al)) {
             clearMark(al);
         } else {
-            if (VERBOSITY() >= 2)
-                printf("Freeing %p\n", al->user_data);
+            _doFree(al);
 
             *cur->prev = cur->next;
             if (cur->next)

src/runtime/builtin_modules/builtins.cpp

@@ -410,7 +410,7 @@ public:
 };
 
 Box* exceptionNew2(BoxedClass* cls, Box* message) {
-    assert(cls->instance_size == sizeof(BoxedException));
+    assert(cls->tp_basicsize == sizeof(BoxedException));
     Box* r = new BoxedException(cls);
     r->giveAttr("message", message);
     return r;

src/runtime/inline/boxing.cpp

@@ -32,6 +32,10 @@ extern "C" BoxedString* boxStrConstant(const char* chars) {
     return new BoxedString(chars);
 }
 
+extern "C" BoxedString* boxStrConstantSize(const char* chars, size_t n) {
+    return new BoxedString(chars, n);
+}
+
 extern "C" Box* boxStringPtr(const std::string* s) {
     return new BoxedString(*s);
 }
@@ -39,6 +43,9 @@ extern "C" Box* boxStringPtr(const std::string* s) {
 Box* boxString(const std::string& s) {
     return new BoxedString(s);
 }
+Box* boxString(std::string&& s) {
+    return new BoxedString(std::move(s));
+}
 
 extern "C" double unboxFloat(Box* b) {
     ASSERT(b->cls == float_cls, "%s", getTypeName(b)->c_str());

src/runtime/int.cpp

@@ -31,6 +31,11 @@
 
 namespace pyston {
 
+extern "C" long PyInt_AsLong(PyObject* obj) {
+    assert(obj->cls == int_cls);
+    return static_cast<BoxedInt*>(obj)->n;
+}
+
 BoxedInt* interned_ints[NUM_INTERNED_INTS];
 
 // If we don't have fast overflow-checking builtins, provide some slow variants:
@@ -588,8 +593,8 @@ extern "C" Box* intNew(Box* _cls, Box* val) {
         raiseExcHelper(TypeError, "int.__new__(%s): %s is not a subtype of int", getNameOfClass(cls)->c_str(),
                        getNameOfClass(cls)->c_str());
 
-    assert(cls->instance_size >= sizeof(BoxedInt));
-    void* mem = gc_alloc(cls->instance_size, gc::GCKind::PYTHON);
+    assert(cls->tp_basicsize >= sizeof(BoxedInt));
+    void* mem = gc_alloc(cls->tp_basicsize, gc::GCKind::PYTHON);
     BoxedInt* rtn = ::new (mem) BoxedInt(cls, 0);
     initUserAttrs(rtn, cls);
 

src/runtime/long.cpp

@@ -55,8 +55,8 @@ extern "C" Box* longNew(Box* _cls, Box* val) {
         raiseExcHelper(TypeError, "long.__new__(%s): %s is not a subtype of long", getNameOfClass(cls)->c_str(),
                        getNameOfClass(cls)->c_str());
 
-    assert(cls->instance_size >= sizeof(BoxedInt));
-    void* mem = gc_alloc(cls->instance_size, gc::GCKind::PYTHON);
+    assert(cls->tp_basicsize >= sizeof(BoxedInt));
+    void* mem = gc_alloc(cls->tp_basicsize, gc::GCKind::PYTHON);
     BoxedLong* rtn = ::new (mem) BoxedLong(cls);
     initUserAttrs(rtn, cls);
 

src/runtime/objmodel.cpp

@@ -361,8 +361,9 @@ extern "C" void checkUnpackingLength(i64 expected, i64 given) {
 
 BoxedClass::BoxedClass(BoxedClass* base, gcvisit_func gc_visit, int attrs_offset, int instance_size,
                        bool is_user_defined)
-    : Box(type_cls), base(base), gc_visit(gc_visit), attrs_offset(attrs_offset), instance_size(instance_size),
-      is_constant(false), is_user_defined(is_user_defined) {
+    : Box(type_cls), tp_basicsize(instance_size), tp_dealloc(NULL), base(base), gc_visit(gc_visit),
+      attrs_offset(attrs_offset), is_constant(false), is_user_defined(is_user_defined) {
+    assert(tp_dealloc == NULL);
 
     if (gc_visit == NULL) {
         assert(base);
@@ -378,15 +379,15 @@ BoxedClass::BoxedClass(BoxedClass* base, gcvisit_func gc_visit, int attrs_offset
         assert(object_cls);
         if (base->attrs_offset)
             RELEASE_ASSERT(attrs_offset == base->attrs_offset, "");
-        assert(instance_size >= base->instance_size);
+        assert(tp_basicsize >= base->tp_basicsize);
     }
 
     if (base && cls && str_cls)
         giveAttr("__base__", base);
 
-    assert(instance_size % sizeof(void*) == 0); // Not critical I suppose, but probably signals a bug
+    assert(tp_basicsize % sizeof(void*) == 0); // Not critical I suppose, but probably signals a bug
     if (attrs_offset) {
-        assert(instance_size >= attrs_offset + sizeof(HCAttrs));
+        assert(tp_basicsize >= attrs_offset + sizeof(HCAttrs));
         assert(attrs_offset % sizeof(void*) == 0); // Not critical I suppose, but probably signals a bug
     }
 
@@ -651,7 +652,7 @@ void Box::setattr(const std::string& attr, Box* val, SetattrRewriteArgs* rewrite
 }
 
 static Box* _handleClsAttr(Box* obj, Box* attr) {
-    if (attr->cls == function_cls) {
+    if (attr->cls == function_cls || attr->cls == method_cls) {
         Box* rtn = boxInstanceMethod(obj, attr);
         return rtn;
     }
@@ -1568,6 +1569,11 @@ extern "C" Box* callattrInternal(Box* obj, const std::string* attr, LookupScope
             r_clsattr.ensureDoneUsing();
         }
 
+        auto old_clsattr = clsattr;
+        clsattr = _handleClsAttr(obj, clsattr);
+        if (clsattr != old_clsattr)
+            rewrite_args = NULL;
+
         if (rewrite_args) {
             CallRewriteArgs srewrite_args(rewrite_args->rewriter, std::move(r_clsattr), rewrite_args->destination,
                                           rewrite_args->more_guards_after);

src/runtime/types.cpp

@@ -32,6 +32,7 @@
 #include "runtime/set.h"
 
 extern "C" void initerrno();
+extern "C" void init_sha();
 
 namespace pyston {
 
@@ -298,10 +299,10 @@ extern "C" Box* createUserClass(std::string* name, Box* _base, Box* _attr_dict)
     BoxedClass* made;
 
     if (base->instancesHaveAttrs()) {
-        made = new BoxedClass(base, NULL, base->attrs_offset, base->instance_size, true);
+        made = new BoxedClass(base, NULL, base->attrs_offset, base->tp_basicsize, true);
     } else {
-        assert(base->instance_size % sizeof(void*) == 0);
-        made = new BoxedClass(base, NULL, base->instance_size, base->instance_size + sizeof(HCAttrs), true);
+        assert(base->tp_basicsize % sizeof(void*) == 0);
+        made = new BoxedClass(base, NULL, base->tp_basicsize, base->tp_basicsize + sizeof(HCAttrs), true);
     }
 
     for (const auto& p : attr_dict->d) {
@@ -473,8 +474,8 @@ Box* objectNew(BoxedClass* cls, BoxedTuple* args) {
             raiseExcHelper(TypeError, "object.__new__() takes no parameters");
     }
 
-    assert(cls->instance_size >= sizeof(Box));
-    void* mem = gc::gc_alloc(cls->instance_size, gc::GCKind::PYTHON);
+    assert(cls->tp_basicsize >= sizeof(Box));
+    void* mem = gc::gc_alloc(cls->tp_basicsize, gc::GCKind::PYTHON);
 
     Box* rtn = ::new (mem) Box(cls);
     initUserAttrs(rtn, cls);
@@ -628,6 +629,7 @@ void setupRuntime() {
     setupCAPI();
 
     initerrno();
+    init_sha();
 
     setupSysEnd();
 

src/runtime/types.h

@@ -72,7 +72,7 @@ BoxedList* getSysPath();
 extern "C" {
 extern BoxedClass* object_cls, *type_cls, *bool_cls, *int_cls, *float_cls, *str_cls, *function_cls, *none_cls,
     *instancemethod_cls, *list_cls, *slice_cls, *module_cls, *dict_cls, *tuple_cls, *file_cls, *xrange_cls, *member_cls,
-    *closure_cls, *generator_cls;
+    *method_cls, *closure_cls, *generator_cls;
 }
 extern "C" { extern Box* None, *NotImplemented, *True, *False; }
 extern "C" {
@@ -88,7 +88,9 @@ extern "C" Box* boxFloat(double d);
 extern "C" Box* boxInstanceMethod(Box* obj, Box* func);
 extern "C" Box* boxStringPtr(const std::string* s);
 Box* boxString(const std::string& s);
+Box* boxString(std::string&& s);
 extern "C" BoxedString* boxStrConstant(const char* chars);
+extern "C" BoxedString* boxStrConstantSize(const char* chars, size_t n);
 extern "C" void listAppendInternal(Box* self, Box* v);
 extern "C" void listAppendArrayInternal(Box* self, Box** v, int nelts);
 extern "C" Box* boxCLFunction(CLFunction* f, BoxedClosure* closure, bool isGenerator,
@@ -199,6 +201,7 @@ public:
     // const std::basic_string<char, std::char_traits<char>, StlCompatAllocator<char> > s;
     const std::string s;
 
+    BoxedString(const char* s, size_t n) __attribute__((visibility("default"))) : Box(str_cls), s(s, n) {}
     BoxedString(const std::string&& s) __attribute__((visibility("default"))) : Box(str_cls), s(std::move(s)) {}
     BoxedString(const std::string& s) __attribute__((visibility("default"))) : Box(str_cls), s(s) {}
 };

test/tests/shatest.py

+# expected: fail
+# - warnings about PyString_AsString(), since that is allowed to be modified
+
+try:
+    import _sha as sha
+except ImportError:
+    import sha
+
+s = sha.new()
+print s.hexdigest()
+s.update("aoeu")
+print s.hexdigest()