Some work on the Grand CAPI Unification

ie trying to start running real C modules.  The goal has always been to
run them without modification and without a proxy layer, and this is the first step.

The first issue is conversion between PyObject and our internal representation,
"Box" (needs to be renamed).  Once they have the same layout, some hacks were
needed to make them seem like the same thing, even though we can attach C++
methods on the Box side, but use PyObject in C code.

The next major issue was PyTypeObject, since the structure is directly exposed to
extensions.  I just added the similar fields to BoxedClass, so they can be
used interchangeably; the "real" ones are the BoxedClass ones, but we'll migrate
them incrementally.

There's also the issue that PyTypeObject's are typically created statically,
so I added registerStaticRootMemory to the GC interface.

Also add in a smattering of other API functions, and the _sha module works
(who cares if it's normally disabled).

The C API is quite constraining about choices of data structure implementation; it's
in direct conflict with the std::string API, for example.  For now, close our eyes
and let the C API modify the internal bytes of std::string.

src/Makefile

@@ -267,12 +267,15 @@ STDLIB_SRCS := $(wildcard runtime/inline/*.cpp)
 SRCS := $(MAIN_SRCS) $(STDLIB_SRCS)
 STDLIB_OBJS := stdlib.bc.o stdlib.stripped.bc.o
 STDLIB_RELEASE_OBJS := stdlib.release.bc.o
-STDMODULES_SRCS := ../lib_python/2.7_Modules/errnomodule.c
+
+STDMODULES_SRCS := errnomodule.c shamodule.c
+STDMODULES_SRCS := $(addprefix ../lib_python/2.7_Modules/,$(STDMODULES_SRCS))
+
 # The stdlib objects have slightly longer dependency chains,
 # so put them first in the list:
 OBJS := $(STDLIB_OBJS) $(SRCS:.cpp=.o) $(STDMODULES_SRCS:.c=.o)
-PROFILE_OBJS := $(STDLIB_RELEASE_OBJS) $(MAIN_SRCS:.cpp=.prof.o) $(STDLIB_SRCS:.cpp=.release.o) $(STDMODULES_SRCS:.c=.o)
-OPT_OBJS := $(STDLIB_RELEASE_OBJS) $(SRCS:.cpp=.release.o) $(STDMODULES_SRCS:.c=.o)
+PROFILE_OBJS := $(STDLIB_RELEASE_OBJS) $(MAIN_SRCS:.cpp=.prof.o) $(STDLIB_SRCS:.cpp=.release.o) $(STDMODULES_SRCS:.c=.release.o)
+OPT_OBJS := $(STDLIB_RELEASE_OBJS) $(SRCS:.cpp=.release.o) $(STDMODULES_SRCS:.c=.release.o)
 
 OPTIONAL_SRCS := codegen/profiling/oprofile.cpp codegen/profiling/pprof.cpp
 TOOL_SRCS := $(wildcard $(TOOLS_DIR)/*.cpp)
@@ -879,5 +882,9 @@ ext: ../test/test_extension/test.so
 	$(CLANG_EXE) -O2 -fPIC -Wimplicit -I../include -c $< -o $@ -g
 
 ../lib_python/2.7_Modules/%.o: ../lib_python/2.7_Modules/%.c $(BUILD_SYSTEM_DEPS)
+	$(ECHO) Compiling extension file $@
+	$(VERB) $(CC) $(EXT_CFLAGS) -c $< -o $@ -g -MMD -MP -MF $<.d -O0
+
+../lib_python/2.7_Modules/%.release.o: ../lib_python/2.7_Modules/%.c $(BUILD_SYSTEM_DEPS)
 	$(ECHO) Compiling extension file $@
 	$(VERB) $(CC) $(EXT_CFLAGS) -c $< -o $@ -g -MMD -MP -MF $<.d

src/asm_writing/assembler.cpp

@@ -558,12 +558,15 @@ void Assembler::cmp(Indirect mem, Register reg) {
     emitRex(rex);
     emitByte(0x3B);
 
-    assert(-0x80 <= mem.offset && mem.offset < 0x80);
     if (mem.offset == 0) {
         emitModRM(0b00, reg_idx, mem_idx);
-    } else {
+    } else if (-0x80 <= mem.offset && mem.offset < 0x80) {
         emitModRM(0b01, reg_idx, mem_idx);
         emitByte(mem.offset);
+    } else {
+        assert((-1L << 31) <= mem.offset && mem.offset < (1L << 31) - 1);
+        emitModRM(0b10, reg_idx, mem_idx);
+        emitInt(mem.offset, 4);
     }
 }
 

src/codegen/opt/const_classes.cpp

@@ -124,9 +124,9 @@ private:
                     errs() << "attrs_offset; replacing with " << cls->attrs_offset << "\n";
                     replaceUsesWithConstant(gep_load, cls->attrs_offset);
                     changed = true;
-                } else if (offset == offsetof(BoxedClass, instance_size)) {
-                    errs() << "instance_size; replacing with " << cls->instance_size << "\n";
-                    replaceUsesWithConstant(gep_load, cls->instance_size);
+                } else if (offset == offsetof(BoxedClass, tp_basicsize)) {
+                    errs() << "tp_basicsize; replacing with " << cls->tp_basicsize << "\n";
+                    replaceUsesWithConstant(gep_load, cls->tp_basicsize);
                     changed = true;
                 }
             }

src/codegen/patchpoints.cpp

@@ -120,7 +120,7 @@ PatchpointSetupInfo* createGenericPatchpoint(CompiledFunction* parent_cf, TypeRe
 }
 
 PatchpointSetupInfo* createGetattrPatchpoint(CompiledFunction* parent_cf, TypeRecorder* type_recorder) {
-    return PatchpointSetupInfo::initialize(true, 1, 144, parent_cf, Getattr, type_recorder);
+    return PatchpointSetupInfo::initialize(true, 1, 196, parent_cf, Getattr, type_recorder);
 }
 
 PatchpointSetupInfo* createGetitemPatchpoint(CompiledFunction* parent_cf, TypeRecorder* type_recorder) {

src/core/types.h

@@ -19,7 +19,10 @@
 // over having them spread randomly in different files, this should probably be split again
 // but in a way that makes more sense.
 
-#include <llvm/ADT/iterator_range.h>
+#include <stddef.h>
+
+#include "llvm/ADT/iterator_range.h"
+#include "Python.h"
 
 #include "core/common.h"
 #include "core/stats.h"
@@ -379,6 +382,9 @@ extern "C" const std::string* getTypeName(Box* o);
 
 class BoxedClass : public Box {
 public:
+    Py_ssize_t ob_size; // CPython API compatibility
+    PyTypeObject_BODY;
+
     HCAttrs attrs;
 
     // If the user sets __getattribute__ or __getattr__, we will have to invalidate
@@ -389,7 +395,7 @@ public:
 
     // Only a single base supported for now.
     // Is NULL iff this is object_cls
-    BoxedClass* const base;
+    BoxedClass* base;
 
     typedef void (*gcvisit_func)(GCVisitor*, Box*);
     gcvisit_func gc_visit;
@@ -397,8 +403,6 @@ public:
     // Offset of the HCAttrs object or 0 if there are no hcattrs.
     // Analogous to tp_dictoffset
     const int attrs_offset;
-    // Analogous to tp_basicsize
-    const int instance_size;
 
     bool instancesHaveAttrs() { return attrs_offset != 0; }
 
@@ -422,6 +426,17 @@ public:
     }
 };
 
+static_assert(sizeof(pyston::Box) == sizeof(struct _object), "");
+static_assert(offsetof(pyston::Box, cls) == offsetof(struct _object, ob_type), "");
+
+static_assert(offsetof(pyston::BoxedClass, cls) == offsetof(struct _typeobject, ob_type), "");
+static_assert(offsetof(pyston::BoxedClass, tp_name) == offsetof(struct _typeobject, tp_name), "");
+static_assert(offsetof(pyston::BoxedClass, attrs) == offsetof(struct _typeobject, _hcls), "");
+static_assert(offsetof(pyston::BoxedClass, dependent_icgetattrs) == offsetof(struct _typeobject, _dep_getattrs), "");
+static_assert(offsetof(pyston::BoxedClass, base) == offsetof(struct _typeobject, _base), "");
+static_assert(offsetof(pyston::BoxedClass, gc_visit) == offsetof(struct _typeobject, _gcvisit_func), "");
+static_assert(sizeof(pyston::BoxedClass) == sizeof(struct _typeobject), "");
+
 // TODO these shouldn't be here
 void setupRuntime();
 void teardownRuntime();

src/gc/collector.cpp

@@ -42,6 +42,17 @@ void registerStaticRootObj(void* obj) {
     roots.push(obj);
 }
 
+std::vector<std::pair<void*, void*> > static_root_memory;
+void registerStaticRootMemory(void* start, void* end) {
+    assert(start < end);
+
+    // While these aren't necessary to work correctly, they are the anticipated use case:
+    assert(global_heap.getAllocationFromInteriorPointer(start) == NULL);
+    assert(global_heap.getAllocationFromInteriorPointer(end) == NULL);
+
+    static_root_memory.push_back(std::make_pair(start, end));
+}
+
 static std::unordered_set<StaticRootHandle*>* getRootHandles() {
     static std::unordered_set<StaticRootHandle*> root_handles;
     return &root_handles;
@@ -102,6 +113,10 @@ static void markPhase() {
 
     TraceStackGCVisitor visitor(&stack);
 
+    for (const auto& p : static_root_memory) {
+        visitor.visitPotentialRange((void**)p.first, (void**)p.second);
+    }
+
     for (auto h : *getRootHandles()) {
         visitor.visitPotential(h->value);
     }

src/gc/collector.h

@@ -60,11 +60,13 @@ public:
     void visitPotentialRange(void* const* start, void* const* end) override;
 };
 
-// Call it a "root obj" because this function takes the pointer to the object, not a pointer
-// to a storage location where we might store different objects.
-// ie this only works for constant roots, and not out-of-gc-knowledge storage locations
-// (that should be registerStaticRootPtr)
+// Mark this gc-allocated object as being a root, even if there are no visible references to it.
+// (Note: this marks the gc allocation itself, not the pointer that points to one.  For that, use
+// a StaticRootHandle or registerStaticRootMemory)
 void registerStaticRootObj(void* root_obj);
+// Register a non-gc region of memory (such as statically-allocated memory) as a source of potential
+// GC roots.
+void registerStaticRootMemory(void* start, void* end);
 void runCollection();
 
 // If you want to have a static root "location" where multiple values could be stored, use this:

src/gc/heap.cpp

@@ -390,6 +390,16 @@ GCAllocation* Heap::getAllocationFromInteriorPointer(void* ptr) {
     return reinterpret_cast<GCAllocation*>(&b->atoms[atom_idx]);
 }
 
+static void _doFree(GCAllocation* al) {
+    if (VERBOSITY() >= 2)
+        printf("Freeing %p\n", al->user_data);
+
+    if (al->kind_id == GCKind::PYTHON) {
+        Box* b = (Box*)al->user_data;
+        ASSERT(b->cls->tp_dealloc == NULL, "%s", getTypeName(b)->c_str());
+    }
+}
+
 static Block** freeChain(Block** head) {
     while (Block* b = *head) {
         int num_objects = b->numObjects();
@@ -411,8 +421,7 @@ static Block** freeChain(Block** head) {
             if (isMarked(al)) {
                 clearMark(al);
             } else {
-                if (VERBOSITY() >= 2)
-                    printf("Freeing %p\n", al->user_data);
+                _doFree(al);
 
                 // assert(p != (void*)0x127000d960); // the main module
                 b->isfree[bitmap_idx] |= mask;
@@ -471,8 +480,7 @@ void Heap::freeUnmarked() {
         if (isMarked(al)) {
             clearMark(al);
         } else {
-            if (VERBOSITY() >= 2)
-                printf("Freeing %p\n", al->user_data);
+            _doFree(al);
 
             *cur->prev = cur->next;
             if (cur->next)

src/runtime/builtin_modules/builtins.cpp

@@ -410,7 +410,7 @@ public:
 };
 
 Box* exceptionNew2(BoxedClass* cls, Box* message) {
-    assert(cls->instance_size == sizeof(BoxedException));
+    assert(cls->tp_basicsize == sizeof(BoxedException));
     Box* r = new BoxedException(cls);
     r->giveAttr("message", message);
     return r;

src/runtime/capi.cpp

@@ -56,6 +56,39 @@ public:
     }
 };
 
+BoxedClass* method_cls;
+class BoxedMethodDescriptor : public Box {
+public:
+    PyMethodDef* method;
+
+    BoxedMethodDescriptor(PyMethodDef* method) : Box(method_cls), method(method) {}
+
+    static Box* __call__(BoxedMethodDescriptor* self, Box* obj, BoxedTuple* varargs, Box** _args) {
+        BoxedDict* kwargs = static_cast<BoxedDict*>(_args[0]);
+
+        assert(self->cls == method_cls);
+        assert(varargs->cls == tuple_cls);
+        assert(kwargs->cls == dict_cls);
+
+        threading::GLPromoteRegion _gil_lock;
+
+        int ml_flags = self->method->ml_flags;
+        Box* rtn;
+        if (ml_flags == METH_NOARGS) {
+            assert(varargs->elts.size() == 0);
+            assert(kwargs->d.size() == 0);
+            rtn = (Box*)self->method->ml_meth(obj, NULL);
+        } else if (ml_flags == METH_VARARGS) {
+            assert(kwargs->d.size() == 0);
+            rtn = (Box*)self->method->ml_meth(obj, varargs);
+        } else {
+            RELEASE_ASSERT(0, "0x%x", ml_flags);
+        }
+        assert(rtn);
+        return rtn;
+    }
+};
+
 extern "C" PyObject* PyModule_GetDict(PyObject* _m) {
     BoxedModule* m = static_cast<BoxedModule*>(_m);
     assert(m->cls == module_cls);
@@ -63,6 +96,14 @@ extern "C" PyObject* PyModule_GetDict(PyObject* _m) {
     return new AttrWrapper(m);
 }
 
+extern "C" int PyModule_AddIntConstant(PyObject* _m, const char* name, long value) {
+    BoxedModule* m = static_cast<BoxedModule*>(_m);
+    assert(m->cls == module_cls);
+
+    m->setattr(name, boxInt(value), NULL);
+    return 0;
+}
+
 extern "C" PyObject* PyDict_New() {
     return new BoxedDict();
 }
@@ -71,6 +112,28 @@ extern "C" PyObject* PyString_FromString(const char* s) {
     return boxStrConstant(s);
 }
 
+extern "C" PyObject* PyString_FromStringAndSize(const char* s, ssize_t n) {
+    if (s == NULL)
+        return boxString(std::string(n, '\x00'));
+    return boxStrConstantSize(s, n);
+}
+
+extern "C" char* PyString_AsString(PyObject* o) {
+    assert(o->cls == str_cls);
+
+    // TODO this is very brittle, since
+    // - you are very much not supposed to change the data, and
+    // - the pointer doesn't have great longevity guarantees
+    // To satisfy this API we might have to change the string representation?
+    printf("Warning: PyString_AsString() currently has risky behavior\n");
+    return const_cast<char*>(static_cast<BoxedString*>(o)->s.data());
+}
+
+extern "C" Py_ssize_t PyString_Size(PyObject* s) {
+    RELEASE_ASSERT(s->cls == str_cls, "");
+    return static_cast<BoxedString*>(s)->s.size();
+}
+
 extern "C" PyObject* PyInt_FromLong(long n) {
     return boxInt(n);
 }
@@ -81,7 +144,14 @@ extern "C" int PyDict_SetItem(PyObject* mp, PyObject* _key, PyObject* _item) {
     Box* item = static_cast<Box*>(_item);
 
     static std::string setitem_str("__setitem__");
-    Box* r = callattrInternal(b, &setitem_str, CLASS_ONLY, NULL, ArgPassSpec(2), key, item, NULL, NULL, NULL);
+    Box* r;
+    try {
+        // TODO should demote GIL?
+        r = callattrInternal(b, &setitem_str, CLASS_ONLY, NULL, ArgPassSpec(2), key, item, NULL, NULL, NULL);
+    } catch (Box* b) {
+        fprintf(stderr, "Error: uncaught error would be propagated to C code!\n");
+        abort();
+    }
 
     RELEASE_ASSERT(r, "");
     return 0;
@@ -95,26 +165,36 @@ extern "C" int PyDict_SetItemString(PyObject* mp, const char* key, PyObject* ite
 BoxedClass* capifunc_cls;
 class BoxedCApiFunction : public Box {
 private:
+    int ml_flags;
     Box* passthrough;
     const char* name;
     PyCFunction func;
 
 public:
-    BoxedCApiFunction(Box* passthrough, const char* name, PyCFunction func)
-        : Box(capifunc_cls), passthrough(passthrough), name(name), func(func) {}
+    BoxedCApiFunction(int ml_flags, Box* passthrough, const char* name, PyCFunction func)
+        : Box(capifunc_cls), ml_flags(ml_flags), passthrough(passthrough), name(name), func(func) {}
 
     static BoxedString* __repr__(BoxedCApiFunction* self) {
         assert(self->cls == capifunc_cls);
         return boxStrConstant(self->name);
     }
 
-    static Box* __call__(BoxedCApiFunction* self, BoxedTuple* varargs) {
+    static Box* __call__(BoxedCApiFunction* self, BoxedTuple* varargs, BoxedDict* kwargs) {
         assert(self->cls == capifunc_cls);
         assert(varargs->cls == tuple_cls);
+        assert(kwargs->cls == dict_cls);
 
         threading::GLPromoteRegion _gil_lock;
 
-        Box* rtn = (Box*)self->func(self->passthrough, varargs);
+        Box* rtn;
+        if (self->ml_flags == METH_VARARGS) {
+            assert(kwargs->d.size() == 0);
+            rtn = (Box*)self->func(self->passthrough, varargs);
+        } else if (self->ml_flags == (METH_VARARGS | METH_KEYWORDS)) {
+            rtn = (Box*)((PyCFunctionWithKeywords)self->func)(self->passthrough, varargs, kwargs);
+        } else {
+            RELEASE_ASSERT(0, "0x%x", self->ml_flags);
+        }
         assert(rtn);
         return rtn;
     }
@@ -131,9 +211,10 @@ extern "C" PyObject* Py_InitModule4(const char* name, PyMethodDef* methods, cons
     while (methods->ml_name) {
         if (VERBOSITY())
             printf("Loading method %s\n", methods->ml_name);
-        assert(methods->ml_flags == METH_VARARGS);
 
-        module->giveAttr(methods->ml_name, new BoxedCApiFunction(passthrough, methods->ml_name, methods->ml_meth));
+        assert((methods->ml_flags & (~(METH_VARARGS | METH_KEYWORDS))) == 0);
+        module->giveAttr(methods->ml_name,
+                         new BoxedCApiFunction(methods->ml_flags, passthrough, methods->ml_name, methods->ml_meth));
 
         methods++;
     }
@@ -145,29 +226,238 @@ extern "C" PyObject* Py_InitModule4(const char* name, PyMethodDef* methods, cons
     return module;
 }
 
+extern "C" void conservativeGCHandler(GCVisitor* v, Box* b) {
+    v->visitPotentialRange((void* const*)b, (void* const*)((char*)b + b->cls->tp_basicsize));
+}
+
+extern "C" int PyType_Ready(PyTypeObject* cls) {
+    gc::registerStaticRootMemory(cls, cls + 1);
+
+    // unhandled fields:
+    RELEASE_ASSERT(cls->tp_print == NULL, "");
+    RELEASE_ASSERT(cls->tp_getattr == NULL, "");
+    RELEASE_ASSERT(cls->tp_setattr == NULL, "");
+    RELEASE_ASSERT(cls->tp_compare == NULL, "");
+    RELEASE_ASSERT(cls->tp_repr == NULL, "");
+    RELEASE_ASSERT(cls->tp_as_number == NULL, "");
+    RELEASE_ASSERT(cls->tp_as_sequence == NULL, "");
+    RELEASE_ASSERT(cls->tp_as_mapping == NULL, "");
+    RELEASE_ASSERT(cls->tp_hash == NULL, "");
+    RELEASE_ASSERT(cls->tp_call == NULL, "");
+    RELEASE_ASSERT(cls->tp_str == NULL, "");
+    RELEASE_ASSERT(cls->tp_getattro == NULL, "");
+    RELEASE_ASSERT(cls->tp_setattro == NULL, "");
+    RELEASE_ASSERT(cls->tp_as_buffer == NULL, "");
+    RELEASE_ASSERT(cls->tp_flags == Py_TPFLAGS_DEFAULT, "");
+    RELEASE_ASSERT(cls->tp_traverse == NULL, "");
+    RELEASE_ASSERT(cls->tp_clear == NULL, "");
+    RELEASE_ASSERT(cls->tp_richcompare == NULL, "");
+    RELEASE_ASSERT(cls->tp_weaklistoffset == 0, "");
+    RELEASE_ASSERT(cls->tp_iter == NULL, "");
+    RELEASE_ASSERT(cls->tp_iternext == NULL, "");
+    RELEASE_ASSERT(cls->tp_members == NULL, "");
+    RELEASE_ASSERT(cls->tp_base == NULL, "");
+    RELEASE_ASSERT(cls->tp_dict == NULL, "");
+    RELEASE_ASSERT(cls->tp_descr_get == NULL, "");
+    RELEASE_ASSERT(cls->tp_descr_set == NULL, "");
+    RELEASE_ASSERT(cls->tp_init == NULL, "");
+    RELEASE_ASSERT(cls->tp_alloc == NULL, "");
+    RELEASE_ASSERT(cls->tp_new == NULL, "");
+    RELEASE_ASSERT(cls->tp_free == NULL, "");
+    RELEASE_ASSERT(cls->tp_is_gc == NULL, "");
+    RELEASE_ASSERT(cls->tp_base == NULL, "");
+    RELEASE_ASSERT(cls->tp_mro == NULL, "");
+    RELEASE_ASSERT(cls->tp_cache == NULL, "");
+    RELEASE_ASSERT(cls->tp_subclasses == NULL, "");
+    RELEASE_ASSERT(cls->tp_weaklist == NULL, "");
+    RELEASE_ASSERT(cls->tp_del == NULL, "");
+    RELEASE_ASSERT(cls->tp_version_tag == 0, "");
+
+#define INITIALIZE(a) new (&(a)) decltype(a)
+    INITIALIZE(cls->attrs);
+    INITIALIZE(cls->dependent_icgetattrs);
+#undef INITIALIZE
+
+    assert(cls->tp_name);
+    cls->giveAttr("__name__", boxStrConstant(cls->tp_name));
+    // tp_name
+    // tp_basicsize, tp_itemsize
+    // tp_doc
+
+    if (cls->tp_methods) {
+        PyMethodDef* method = cls->tp_methods;
+        while (method->ml_name) {
+            auto desc = new BoxedMethodDescriptor(method);
+            cls->giveAttr(method->ml_name, desc);
+            method++;
+        }
+    }
+
+    if (cls->tp_getset) {
+        if (VERBOSITY())
+            printf("warning: ignoring tp_getset for now\n");
+    }
+
+    cls->base = object_cls;
+
+    cls->gc_visit = &conservativeGCHandler;
+
+    // TODO not sure how we can handle extension types that manually
+    // specify a dict...
+    RELEASE_ASSERT(cls->tp_dictoffset == 0, "");
+    // this should get automatically initialized to 0 on this path:
+    assert(cls->attrs_offset == 0);
+
+    return 0;
+}
+
 extern "C" PyObject* Py_BuildValue(const char* arg0, ...) {
     assert(*arg0 == '\0');
     return None;
 }
 
-extern "C" int PyArg_ParseTuple(PyObject* tuple, const char* fmt, ...) {
-    if (strcmp("", fmt) == 0)
-        return 1;
+// copied from CPython's getargs.c:
+extern "C" int PyBuffer_FillInfo(Py_buffer* view, PyObject* obj, void* buf, Py_ssize_t len, int readonly, int flags) {
+    if (view == NULL)
+        return 0;
+    if (((flags & PyBUF_WRITABLE) == PyBUF_WRITABLE) && (readonly == 1)) {
+        // Don't support PyErr_SetString yet:
+        assert(0);
+        // PyErr_SetString(PyExc_BufferError, "Object is not writable.");
+        // return -1;
+    }
+
+    view->obj = obj;
+    if (obj)
+        Py_INCREF(obj);
+    view->buf = buf;
+    view->len = len;
+    view->readonly = readonly;
+    view->itemsize = 1;
+    view->format = NULL;
+    if ((flags & PyBUF_FORMAT) == PyBUF_FORMAT)
+        view->format = "B";
+    view->ndim = 1;
+    view->shape = NULL;
+    if ((flags & PyBUF_ND) == PyBUF_ND)
+        view->shape = &(view->len);
+    view->strides = NULL;
+    if ((flags & PyBUF_STRIDES) == PyBUF_STRIDES)
+        view->strides = &(view->itemsize);
+    view->suboffsets = NULL;
+    view->internal = NULL;
+    return 0;
+}
+
+extern "C" void PyBuffer_Release(Py_buffer* view) {
+    if (!view->buf) {
+        assert(!view->obj);
+        return;
+    }
+
+    PyObject* obj = view->obj;
+    assert(obj);
+    assert(obj->cls == str_cls);
+    if (obj && Py_TYPE(obj)->tp_as_buffer && Py_TYPE(obj)->tp_as_buffer->bf_releasebuffer)
+        Py_TYPE(obj)->tp_as_buffer->bf_releasebuffer(obj, view);
+    Py_XDECREF(obj);
+    view->obj = NULL;
+}
+
+int vPyArg_ParseTuple(PyObject* _tuple, const char* fmt, va_list ap) {
+    RELEASE_ASSERT(_tuple->cls == tuple_cls, "");
+    BoxedTuple* tuple = static_cast<BoxedTuple*>(_tuple);
 
-    assert(strcmp("O", fmt) == 0);
+    bool now_optional = false;
+    int arg_idx = 0;
 
-    BoxedTuple* varargs = (BoxedTuple*)tuple;
-    assert(varargs->elts.size() == 1);
+    int tuple_size = tuple->elts.size();
+
+    while (char c = *fmt) {
+        fmt++;
+
+        if (c == ':') {
+            break;
+        } else if (c == '|') {
+            now_optional = true;
+            continue;
+        } else {
+            if (arg_idx >= tuple_size) {
+                RELEASE_ASSERT(now_optional, "");
+                break;
+            }
+
+            PyObject* arg = tuple->elts[arg_idx];
+
+            switch (c) {
+                case 's': {
+                    if (*fmt == '*') {
+                        Py_buffer* p = (Py_buffer*)va_arg(ap, Py_buffer*);
+
+                        RELEASE_ASSERT(arg->cls == str_cls, "");
+                        PyBuffer_FillInfo(p, arg, PyString_AS_STRING(arg), PyString_GET_SIZE(arg), 1, 0);
+                        fmt++;
+                    } else if (*fmt == ':') {
+                        break;
+                    } else {
+                        RELEASE_ASSERT(0, "");
+                    }
+                    break;
+                }
+                case 'O': {
+                    PyObject** p = (PyObject**)va_arg(ap, PyObject**);
+                    *p = arg;
+                    break;
+                }
+                default:
+                    RELEASE_ASSERT(0, "Unhandled format character: '%c'", c);
+            }
+        }
+    }
+    return 1;
+}
 
+extern "C" int PyArg_ParseTuple(PyObject* _tuple, const char* fmt, ...) {
     va_list ap;
     va_start(ap, fmt);
 
-    Box** arg0 = va_arg(ap, Box**);
-    *arg0 = varargs->elts[0];
+    int r = vPyArg_ParseTuple(_tuple, fmt, ap);
 
     va_end(ap);
 
-    return 1;
+    return r;
+}
+
+extern "C" int PyArg_ParseTupleAndKeywords(PyObject* args, PyObject* kwargs, const char* format, char** kwlist, ...) {
+    assert(kwargs->cls == dict_cls);
+    RELEASE_ASSERT(static_cast<BoxedDict*>(kwargs)->d.size() == 0, "");
+
+    va_list ap;
+    va_start(ap, kwlist);
+
+    int r = vPyArg_ParseTuple(args, format, ap);
+
+    va_end(ap);
+
+    return r;
+}
+
+extern "C" PyObject* _PyObject_New(PyTypeObject* cls) {
+    assert(cls->tp_itemsize == 0);
+    auto rtn = (PyObject*)gc_alloc(cls->tp_basicsize, gc::GCKind::PYTHON);
+    rtn->cls = cls;
+    return rtn;
+}
+
+extern "C" void PyObject_Free(void* p) {
+    gc::gc_free(p);
+    ASSERT(0, "I think this is good enough but I'm not sure; should test");
+}
+
+extern "C" PyObject* PyErr_Occurred() {
+    printf("need to hook exception handling -- make sure errors dont propagate into C code, and error codes get "
+           "checked coming out\n");
+    return NULL;
 }
 
 BoxedModule* importTestExtension() {
@@ -211,7 +501,7 @@ void setupCAPI() {
     capifunc_cls->giveAttr("__str__", capifunc_cls->getattr("__repr__"));
 
     capifunc_cls->giveAttr(
-        "__call__", new BoxedFunction(boxRTFunction((void*)BoxedCApiFunction::__call__, UNKNOWN, 1, 0, true, false)));
+        "__call__", new BoxedFunction(boxRTFunction((void*)BoxedCApiFunction::__call__, UNKNOWN, 1, 0, true, true)));
 
     capifunc_cls->freeze();
 
@@ -219,6 +509,12 @@ void setupCAPI() {
     attrwrapper_cls->giveAttr("__name__", boxStrConstant("attrwrapper"));
     attrwrapper_cls->giveAttr("__setitem__", new BoxedFunction(boxRTFunction((void*)AttrWrapper::setitem, UNKNOWN, 3)));
     attrwrapper_cls->freeze();
+
+    method_cls = new BoxedClass(object_cls, NULL, 0, sizeof(BoxedMethodDescriptor), false);
+    method_cls->giveAttr("__name__", boxStrConstant("method"));
+    method_cls->giveAttr("__call__", new BoxedFunction(boxRTFunction((void*)BoxedMethodDescriptor::__call__, UNKNOWN, 2,
+                                                                     0, true, true)));
+    method_cls->freeze();
 }
 
 void teardownCAPI() {

src/runtime/inline/boxing.cpp

@@ -32,6 +32,10 @@ extern "C" BoxedString* boxStrConstant(const char* chars) {
     return new BoxedString(chars);
 }
 
+extern "C" BoxedString* boxStrConstantSize(const char* chars, size_t n) {
+    return new BoxedString(chars, n);
+}
+
 extern "C" Box* boxStringPtr(const std::string* s) {
     return new BoxedString(*s);
 }
@@ -39,6 +43,9 @@ extern "C" Box* boxStringPtr(const std::string* s) {
 Box* boxString(const std::string& s) {
     return new BoxedString(s);
 }
+Box* boxString(std::string&& s) {
+    return new BoxedString(std::move(s));
+}
 
 extern "C" double unboxFloat(Box* b) {
     ASSERT(b->cls == float_cls, "%s", getTypeName(b)->c_str());

src/runtime/int.cpp

@@ -31,6 +31,11 @@
 
 namespace pyston {
 
+extern "C" long PyInt_AsLong(PyObject* obj) {
+    assert(obj->cls == int_cls);
+    return static_cast<BoxedInt*>(obj)->n;
+}
+
 BoxedInt* interned_ints[NUM_INTERNED_INTS];
 
 // If we don't have fast overflow-checking builtins, provide some slow variants:
@@ -588,8 +593,8 @@ extern "C" Box* intNew(Box* _cls, Box* val) {
         raiseExcHelper(TypeError, "int.__new__(%s): %s is not a subtype of int", getNameOfClass(cls)->c_str(),
                        getNameOfClass(cls)->c_str());
 
-    assert(cls->instance_size >= sizeof(BoxedInt));
-    void* mem = gc_alloc(cls->instance_size, gc::GCKind::PYTHON);
+    assert(cls->tp_basicsize >= sizeof(BoxedInt));
+    void* mem = gc_alloc(cls->tp_basicsize, gc::GCKind::PYTHON);
     BoxedInt* rtn = ::new (mem) BoxedInt(cls, 0);
     initUserAttrs(rtn, cls);
 

src/runtime/long.cpp

@@ -55,8 +55,8 @@ extern "C" Box* longNew(Box* _cls, Box* val) {
         raiseExcHelper(TypeError, "long.__new__(%s): %s is not a subtype of long", getNameOfClass(cls)->c_str(),
                        getNameOfClass(cls)->c_str());
 
-    assert(cls->instance_size >= sizeof(BoxedInt));
-    void* mem = gc_alloc(cls->instance_size, gc::GCKind::PYTHON);
+    assert(cls->tp_basicsize >= sizeof(BoxedInt));
+    void* mem = gc_alloc(cls->tp_basicsize, gc::GCKind::PYTHON);
     BoxedLong* rtn = ::new (mem) BoxedLong(cls);
     initUserAttrs(rtn, cls);
 

src/runtime/objmodel.cpp

@@ -361,8 +361,9 @@ extern "C" void checkUnpackingLength(i64 expected, i64 given) {
 
 BoxedClass::BoxedClass(BoxedClass* base, gcvisit_func gc_visit, int attrs_offset, int instance_size,
                        bool is_user_defined)
-    : Box(type_cls), base(base), gc_visit(gc_visit), attrs_offset(attrs_offset), instance_size(instance_size),
-      is_constant(false), is_user_defined(is_user_defined) {
+    : Box(type_cls), tp_basicsize(instance_size), tp_dealloc(NULL), base(base), gc_visit(gc_visit),
+      attrs_offset(attrs_offset), is_constant(false), is_user_defined(is_user_defined) {
+    assert(tp_dealloc == NULL);
 
     if (gc_visit == NULL) {
         assert(base);
@@ -378,15 +379,15 @@ BoxedClass::BoxedClass(BoxedClass* base, gcvisit_func gc_visit, int attrs_offset
         assert(object_cls);
         if (base->attrs_offset)
             RELEASE_ASSERT(attrs_offset == base->attrs_offset, "");
-        assert(instance_size >= base->instance_size);
+        assert(tp_basicsize >= base->tp_basicsize);
     }
 
     if (base && cls && str_cls)
         giveAttr("__base__", base);
 
-    assert(instance_size % sizeof(void*) == 0); // Not critical I suppose, but probably signals a bug
+    assert(tp_basicsize % sizeof(void*) == 0); // Not critical I suppose, but probably signals a bug
     if (attrs_offset) {
-        assert(instance_size >= attrs_offset + sizeof(HCAttrs));
+        assert(tp_basicsize >= attrs_offset + sizeof(HCAttrs));
         assert(attrs_offset % sizeof(void*) == 0); // Not critical I suppose, but probably signals a bug
     }
 
@@ -651,7 +652,7 @@ void Box::setattr(const std::string& attr, Box* val, SetattrRewriteArgs* rewrite
 }
 
 static Box* _handleClsAttr(Box* obj, Box* attr) {
-    if (attr->cls == function_cls) {
+    if (attr->cls == function_cls || attr->cls == method_cls) {
         Box* rtn = boxInstanceMethod(obj, attr);
         return rtn;
     }
@@ -1568,6 +1569,11 @@ extern "C" Box* callattrInternal(Box* obj, const std::string* attr, LookupScope
             r_clsattr.ensureDoneUsing();
         }
 
+        auto old_clsattr = clsattr;
+        clsattr = _handleClsAttr(obj, clsattr);
+        if (clsattr != old_clsattr)
+            rewrite_args = NULL;
+
         if (rewrite_args) {
             CallRewriteArgs srewrite_args(rewrite_args->rewriter, std::move(r_clsattr), rewrite_args->destination,
                                           rewrite_args->more_guards_after);

src/runtime/types.cpp

@@ -32,6 +32,7 @@
 #include "runtime/set.h"
 
 extern "C" void initerrno();
+extern "C" void init_sha();
 
 namespace pyston {
 
@@ -298,10 +299,10 @@ extern "C" Box* createUserClass(std::string* name, Box* _base, Box* _attr_dict)
     BoxedClass* made;
 
     if (base->instancesHaveAttrs()) {
-        made = new BoxedClass(base, NULL, base->attrs_offset, base->instance_size, true);
+        made = new BoxedClass(base, NULL, base->attrs_offset, base->tp_basicsize, true);
     } else {
-        assert(base->instance_size % sizeof(void*) == 0);
-        made = new BoxedClass(base, NULL, base->instance_size, base->instance_size + sizeof(HCAttrs), true);
+        assert(base->tp_basicsize % sizeof(void*) == 0);
+        made = new BoxedClass(base, NULL, base->tp_basicsize, base->tp_basicsize + sizeof(HCAttrs), true);
     }
 
     for (const auto& p : attr_dict->d) {
@@ -473,8 +474,8 @@ Box* objectNew(BoxedClass* cls, BoxedTuple* args) {
             raiseExcHelper(TypeError, "object.__new__() takes no parameters");
     }
 
-    assert(cls->instance_size >= sizeof(Box));
-    void* mem = gc::gc_alloc(cls->instance_size, gc::GCKind::PYTHON);
+    assert(cls->tp_basicsize >= sizeof(Box));
+    void* mem = gc::gc_alloc(cls->tp_basicsize, gc::GCKind::PYTHON);
 
     Box* rtn = ::new (mem) Box(cls);
     initUserAttrs(rtn, cls);
@@ -628,6 +629,7 @@ void setupRuntime() {
     setupCAPI();
 
     initerrno();
+    init_sha();
 
     setupSysEnd();
 

src/runtime/types.h

@@ -72,7 +72,7 @@ BoxedList* getSysPath();
 extern "C" {
 extern BoxedClass* object_cls, *type_cls, *bool_cls, *int_cls, *float_cls, *str_cls, *function_cls, *none_cls,
     *instancemethod_cls, *list_cls, *slice_cls, *module_cls, *dict_cls, *tuple_cls, *file_cls, *xrange_cls, *member_cls,
-    *closure_cls, *generator_cls;
+    *method_cls, *closure_cls, *generator_cls;
 }
 extern "C" { extern Box* None, *NotImplemented, *True, *False; }
 extern "C" {
@@ -88,7 +88,9 @@ extern "C" Box* boxFloat(double d);
 extern "C" Box* boxInstanceMethod(Box* obj, Box* func);
 extern "C" Box* boxStringPtr(const std::string* s);
 Box* boxString(const std::string& s);
+Box* boxString(std::string&& s);
 extern "C" BoxedString* boxStrConstant(const char* chars);
+extern "C" BoxedString* boxStrConstantSize(const char* chars, size_t n);
 extern "C" void listAppendInternal(Box* self, Box* v);
 extern "C" void listAppendArrayInternal(Box* self, Box** v, int nelts);
 extern "C" Box* boxCLFunction(CLFunction* f, BoxedClosure* closure, bool isGenerator,
@@ -199,6 +201,7 @@ public:
     // const std::basic_string<char, std::char_traits<char>, StlCompatAllocator<char> > s;
     const std::string s;
 
+    BoxedString(const char* s, size_t n) __attribute__((visibility("default"))) : Box(str_cls), s(s, n) {}
     BoxedString(const std::string&& s) __attribute__((visibility("default"))) : Box(str_cls), s(std::move(s)) {}
     BoxedString(const std::string& s) __attribute__((visibility("default"))) : Box(str_cls), s(s) {}
 };

test/tests/shatest.py

+# expected: fail
+# - warnings about PyString_AsString(), since that is allowed to be modified
+
+try:
+    import _sha as sha
+except ImportError:
+    import sha
+
+s = sha.new()
+print s.hexdigest()
+s.update("aoeu")
+print s.hexdigest()