Commit 1d396ecd authored by Kevin Modzelewski's avatar Kevin Modzelewski

Some work on the Grand CAPI Unification

ie trying to start running real C modules.  The goal has always been to
run them without modification and without a proxy layer, and this is the first step.

The first issue is conversion between PyObject and our internal representation,
"Box" (needs to be renamed).  Once they have the same layout, some hacks were
needed to make them seem like the same thing, even though we can attach C++
methods on the Box side, but use PyObject in C code.

The next major issue was PyTypeObject, since the structure is directly exposed to
extensions.  I just added the similar fields to BoxedClass, so they can be
used interchangeably; the "real" ones are the BoxedClass ones, but we'll migrate
them incrementally.

There's also the issue that PyTypeObject's are typically created statically,
so I added registerStaticRootMemory to the GC interface.

Also add in a smattering of other API functions, and the _sha module works
(who cares if it's normally disabled).

The C API is quite constraining about choices of data structure implementation; it's
in direct conflict with the std::string API, for example.  For now, close our eyes
and let the C API modify the internal bytes of std::string.
parent 89bbb329
...@@ -267,12 +267,15 @@ STDLIB_SRCS := $(wildcard runtime/inline/*.cpp) ...@@ -267,12 +267,15 @@ STDLIB_SRCS := $(wildcard runtime/inline/*.cpp)
SRCS := $(MAIN_SRCS) $(STDLIB_SRCS) SRCS := $(MAIN_SRCS) $(STDLIB_SRCS)
STDLIB_OBJS := stdlib.bc.o stdlib.stripped.bc.o STDLIB_OBJS := stdlib.bc.o stdlib.stripped.bc.o
STDLIB_RELEASE_OBJS := stdlib.release.bc.o STDLIB_RELEASE_OBJS := stdlib.release.bc.o
STDMODULES_SRCS := ../lib_python/2.7_Modules/errnomodule.c
STDMODULES_SRCS := errnomodule.c shamodule.c
STDMODULES_SRCS := $(addprefix ../lib_python/2.7_Modules/,$(STDMODULES_SRCS))
# The stdlib objects have slightly longer dependency chains, # The stdlib objects have slightly longer dependency chains,
# so put them first in the list: # so put them first in the list:
OBJS := $(STDLIB_OBJS) $(SRCS:.cpp=.o) $(STDMODULES_SRCS:.c=.o) OBJS := $(STDLIB_OBJS) $(SRCS:.cpp=.o) $(STDMODULES_SRCS:.c=.o)
PROFILE_OBJS := $(STDLIB_RELEASE_OBJS) $(MAIN_SRCS:.cpp=.prof.o) $(STDLIB_SRCS:.cpp=.release.o) $(STDMODULES_SRCS:.c=.o) PROFILE_OBJS := $(STDLIB_RELEASE_OBJS) $(MAIN_SRCS:.cpp=.prof.o) $(STDLIB_SRCS:.cpp=.release.o) $(STDMODULES_SRCS:.c=.release.o)
OPT_OBJS := $(STDLIB_RELEASE_OBJS) $(SRCS:.cpp=.release.o) $(STDMODULES_SRCS:.c=.o) OPT_OBJS := $(STDLIB_RELEASE_OBJS) $(SRCS:.cpp=.release.o) $(STDMODULES_SRCS:.c=.release.o)
OPTIONAL_SRCS := codegen/profiling/oprofile.cpp codegen/profiling/pprof.cpp OPTIONAL_SRCS := codegen/profiling/oprofile.cpp codegen/profiling/pprof.cpp
TOOL_SRCS := $(wildcard $(TOOLS_DIR)/*.cpp) TOOL_SRCS := $(wildcard $(TOOLS_DIR)/*.cpp)
...@@ -879,5 +882,9 @@ ext: ../test/test_extension/test.so ...@@ -879,5 +882,9 @@ ext: ../test/test_extension/test.so
$(CLANG_EXE) -O2 -fPIC -Wimplicit -I../include -c $< -o $@ -g $(CLANG_EXE) -O2 -fPIC -Wimplicit -I../include -c $< -o $@ -g
../lib_python/2.7_Modules/%.o: ../lib_python/2.7_Modules/%.c $(BUILD_SYSTEM_DEPS) ../lib_python/2.7_Modules/%.o: ../lib_python/2.7_Modules/%.c $(BUILD_SYSTEM_DEPS)
$(ECHO) Compiling extension file $@
$(VERB) $(CC) $(EXT_CFLAGS) -c $< -o $@ -g -MMD -MP -MF $<.d -O0
../lib_python/2.7_Modules/%.release.o: ../lib_python/2.7_Modules/%.c $(BUILD_SYSTEM_DEPS)
$(ECHO) Compiling extension file $@ $(ECHO) Compiling extension file $@
$(VERB) $(CC) $(EXT_CFLAGS) -c $< -o $@ -g -MMD -MP -MF $<.d $(VERB) $(CC) $(EXT_CFLAGS) -c $< -o $@ -g -MMD -MP -MF $<.d
...@@ -558,12 +558,15 @@ void Assembler::cmp(Indirect mem, Register reg) { ...@@ -558,12 +558,15 @@ void Assembler::cmp(Indirect mem, Register reg) {
emitRex(rex); emitRex(rex);
emitByte(0x3B); emitByte(0x3B);
assert(-0x80 <= mem.offset && mem.offset < 0x80);
if (mem.offset == 0) { if (mem.offset == 0) {
emitModRM(0b00, reg_idx, mem_idx); emitModRM(0b00, reg_idx, mem_idx);
} else { } else if (-0x80 <= mem.offset && mem.offset < 0x80) {
emitModRM(0b01, reg_idx, mem_idx); emitModRM(0b01, reg_idx, mem_idx);
emitByte(mem.offset); emitByte(mem.offset);
} else {
assert((-1L << 31) <= mem.offset && mem.offset < (1L << 31) - 1);
emitModRM(0b10, reg_idx, mem_idx);
emitInt(mem.offset, 4);
} }
} }
......
...@@ -124,9 +124,9 @@ private: ...@@ -124,9 +124,9 @@ private:
errs() << "attrs_offset; replacing with " << cls->attrs_offset << "\n"; errs() << "attrs_offset; replacing with " << cls->attrs_offset << "\n";
replaceUsesWithConstant(gep_load, cls->attrs_offset); replaceUsesWithConstant(gep_load, cls->attrs_offset);
changed = true; changed = true;
} else if (offset == offsetof(BoxedClass, instance_size)) { } else if (offset == offsetof(BoxedClass, tp_basicsize)) {
errs() << "instance_size; replacing with " << cls->instance_size << "\n"; errs() << "tp_basicsize; replacing with " << cls->tp_basicsize << "\n";
replaceUsesWithConstant(gep_load, cls->instance_size); replaceUsesWithConstant(gep_load, cls->tp_basicsize);
changed = true; changed = true;
} }
} }
......
...@@ -120,7 +120,7 @@ PatchpointSetupInfo* createGenericPatchpoint(CompiledFunction* parent_cf, TypeRe ...@@ -120,7 +120,7 @@ PatchpointSetupInfo* createGenericPatchpoint(CompiledFunction* parent_cf, TypeRe
} }
PatchpointSetupInfo* createGetattrPatchpoint(CompiledFunction* parent_cf, TypeRecorder* type_recorder) { PatchpointSetupInfo* createGetattrPatchpoint(CompiledFunction* parent_cf, TypeRecorder* type_recorder) {
return PatchpointSetupInfo::initialize(true, 1, 144, parent_cf, Getattr, type_recorder); return PatchpointSetupInfo::initialize(true, 1, 196, parent_cf, Getattr, type_recorder);
} }
PatchpointSetupInfo* createGetitemPatchpoint(CompiledFunction* parent_cf, TypeRecorder* type_recorder) { PatchpointSetupInfo* createGetitemPatchpoint(CompiledFunction* parent_cf, TypeRecorder* type_recorder) {
......
...@@ -19,7 +19,10 @@ ...@@ -19,7 +19,10 @@
// over having them spread randomly in different files, this should probably be split again // over having them spread randomly in different files, this should probably be split again
// but in a way that makes more sense. // but in a way that makes more sense.
#include <llvm/ADT/iterator_range.h> #include <stddef.h>
#include "llvm/ADT/iterator_range.h"
#include "Python.h"
#include "core/common.h" #include "core/common.h"
#include "core/stats.h" #include "core/stats.h"
...@@ -379,6 +382,9 @@ extern "C" const std::string* getTypeName(Box* o); ...@@ -379,6 +382,9 @@ extern "C" const std::string* getTypeName(Box* o);
class BoxedClass : public Box { class BoxedClass : public Box {
public: public:
Py_ssize_t ob_size; // CPython API compatibility
PyTypeObject_BODY;
HCAttrs attrs; HCAttrs attrs;
// If the user sets __getattribute__ or __getattr__, we will have to invalidate // If the user sets __getattribute__ or __getattr__, we will have to invalidate
...@@ -389,7 +395,7 @@ public: ...@@ -389,7 +395,7 @@ public:
// Only a single base supported for now. // Only a single base supported for now.
// Is NULL iff this is object_cls // Is NULL iff this is object_cls
BoxedClass* const base; BoxedClass* base;
typedef void (*gcvisit_func)(GCVisitor*, Box*); typedef void (*gcvisit_func)(GCVisitor*, Box*);
gcvisit_func gc_visit; gcvisit_func gc_visit;
...@@ -397,8 +403,6 @@ public: ...@@ -397,8 +403,6 @@ public:
// Offset of the HCAttrs object or 0 if there are no hcattrs. // Offset of the HCAttrs object or 0 if there are no hcattrs.
// Analogous to tp_dictoffset // Analogous to tp_dictoffset
const int attrs_offset; const int attrs_offset;
// Analogous to tp_basicsize
const int instance_size;
bool instancesHaveAttrs() { return attrs_offset != 0; } bool instancesHaveAttrs() { return attrs_offset != 0; }
...@@ -422,6 +426,17 @@ public: ...@@ -422,6 +426,17 @@ public:
} }
}; };
static_assert(sizeof(pyston::Box) == sizeof(struct _object), "");
static_assert(offsetof(pyston::Box, cls) == offsetof(struct _object, ob_type), "");
static_assert(offsetof(pyston::BoxedClass, cls) == offsetof(struct _typeobject, ob_type), "");
static_assert(offsetof(pyston::BoxedClass, tp_name) == offsetof(struct _typeobject, tp_name), "");
static_assert(offsetof(pyston::BoxedClass, attrs) == offsetof(struct _typeobject, _hcls), "");
static_assert(offsetof(pyston::BoxedClass, dependent_icgetattrs) == offsetof(struct _typeobject, _dep_getattrs), "");
static_assert(offsetof(pyston::BoxedClass, base) == offsetof(struct _typeobject, _base), "");
static_assert(offsetof(pyston::BoxedClass, gc_visit) == offsetof(struct _typeobject, _gcvisit_func), "");
static_assert(sizeof(pyston::BoxedClass) == sizeof(struct _typeobject), "");
// TODO these shouldn't be here // TODO these shouldn't be here
void setupRuntime(); void setupRuntime();
void teardownRuntime(); void teardownRuntime();
......
...@@ -42,6 +42,17 @@ void registerStaticRootObj(void* obj) { ...@@ -42,6 +42,17 @@ void registerStaticRootObj(void* obj) {
roots.push(obj); roots.push(obj);
} }
std::vector<std::pair<void*, void*> > static_root_memory;
void registerStaticRootMemory(void* start, void* end) {
assert(start < end);
// While these aren't necessary to work correctly, they are the anticipated use case:
assert(global_heap.getAllocationFromInteriorPointer(start) == NULL);
assert(global_heap.getAllocationFromInteriorPointer(end) == NULL);
static_root_memory.push_back(std::make_pair(start, end));
}
static std::unordered_set<StaticRootHandle*>* getRootHandles() { static std::unordered_set<StaticRootHandle*>* getRootHandles() {
static std::unordered_set<StaticRootHandle*> root_handles; static std::unordered_set<StaticRootHandle*> root_handles;
return &root_handles; return &root_handles;
...@@ -102,6 +113,10 @@ static void markPhase() { ...@@ -102,6 +113,10 @@ static void markPhase() {
TraceStackGCVisitor visitor(&stack); TraceStackGCVisitor visitor(&stack);
for (const auto& p : static_root_memory) {
visitor.visitPotentialRange((void**)p.first, (void**)p.second);
}
for (auto h : *getRootHandles()) { for (auto h : *getRootHandles()) {
visitor.visitPotential(h->value); visitor.visitPotential(h->value);
} }
......
...@@ -60,11 +60,13 @@ public: ...@@ -60,11 +60,13 @@ public:
void visitPotentialRange(void* const* start, void* const* end) override; void visitPotentialRange(void* const* start, void* const* end) override;
}; };
// Call it a "root obj" because this function takes the pointer to the object, not a pointer // Mark this gc-allocated object as being a root, even if there are no visible references to it.
// to a storage location where we might store different objects. // (Note: this marks the gc allocation itself, not the pointer that points to one. For that, use
// ie this only works for constant roots, and not out-of-gc-knowledge storage locations // a StaticRootHandle or registerStaticRootMemory)
// (that should be registerStaticRootPtr)
void registerStaticRootObj(void* root_obj); void registerStaticRootObj(void* root_obj);
// Register a non-gc region of memory (such as statically-allocated memory) as a source of potential
// GC roots.
void registerStaticRootMemory(void* start, void* end);
void runCollection(); void runCollection();
// If you want to have a static root "location" where multiple values could be stored, use this: // If you want to have a static root "location" where multiple values could be stored, use this:
......
...@@ -390,6 +390,16 @@ GCAllocation* Heap::getAllocationFromInteriorPointer(void* ptr) { ...@@ -390,6 +390,16 @@ GCAllocation* Heap::getAllocationFromInteriorPointer(void* ptr) {
return reinterpret_cast<GCAllocation*>(&b->atoms[atom_idx]); return reinterpret_cast<GCAllocation*>(&b->atoms[atom_idx]);
} }
static void _doFree(GCAllocation* al) {
if (VERBOSITY() >= 2)
printf("Freeing %p\n", al->user_data);
if (al->kind_id == GCKind::PYTHON) {
Box* b = (Box*)al->user_data;
ASSERT(b->cls->tp_dealloc == NULL, "%s", getTypeName(b)->c_str());
}
}
static Block** freeChain(Block** head) { static Block** freeChain(Block** head) {
while (Block* b = *head) { while (Block* b = *head) {
int num_objects = b->numObjects(); int num_objects = b->numObjects();
...@@ -411,8 +421,7 @@ static Block** freeChain(Block** head) { ...@@ -411,8 +421,7 @@ static Block** freeChain(Block** head) {
if (isMarked(al)) { if (isMarked(al)) {
clearMark(al); clearMark(al);
} else { } else {
if (VERBOSITY() >= 2) _doFree(al);
printf("Freeing %p\n", al->user_data);
// assert(p != (void*)0x127000d960); // the main module // assert(p != (void*)0x127000d960); // the main module
b->isfree[bitmap_idx] |= mask; b->isfree[bitmap_idx] |= mask;
...@@ -471,8 +480,7 @@ void Heap::freeUnmarked() { ...@@ -471,8 +480,7 @@ void Heap::freeUnmarked() {
if (isMarked(al)) { if (isMarked(al)) {
clearMark(al); clearMark(al);
} else { } else {
if (VERBOSITY() >= 2) _doFree(al);
printf("Freeing %p\n", al->user_data);
*cur->prev = cur->next; *cur->prev = cur->next;
if (cur->next) if (cur->next)
......
...@@ -410,7 +410,7 @@ public: ...@@ -410,7 +410,7 @@ public:
}; };
Box* exceptionNew2(BoxedClass* cls, Box* message) { Box* exceptionNew2(BoxedClass* cls, Box* message) {
assert(cls->instance_size == sizeof(BoxedException)); assert(cls->tp_basicsize == sizeof(BoxedException));
Box* r = new BoxedException(cls); Box* r = new BoxedException(cls);
r->giveAttr("message", message); r->giveAttr("message", message);
return r; return r;
......
...@@ -56,6 +56,39 @@ public: ...@@ -56,6 +56,39 @@ public:
} }
}; };
BoxedClass* method_cls;
class BoxedMethodDescriptor : public Box {
public:
PyMethodDef* method;
BoxedMethodDescriptor(PyMethodDef* method) : Box(method_cls), method(method) {}
static Box* __call__(BoxedMethodDescriptor* self, Box* obj, BoxedTuple* varargs, Box** _args) {
BoxedDict* kwargs = static_cast<BoxedDict*>(_args[0]);
assert(self->cls == method_cls);
assert(varargs->cls == tuple_cls);
assert(kwargs->cls == dict_cls);
threading::GLPromoteRegion _gil_lock;
int ml_flags = self->method->ml_flags;
Box* rtn;
if (ml_flags == METH_NOARGS) {
assert(varargs->elts.size() == 0);
assert(kwargs->d.size() == 0);
rtn = (Box*)self->method->ml_meth(obj, NULL);
} else if (ml_flags == METH_VARARGS) {
assert(kwargs->d.size() == 0);
rtn = (Box*)self->method->ml_meth(obj, varargs);
} else {
RELEASE_ASSERT(0, "0x%x", ml_flags);
}
assert(rtn);
return rtn;
}
};
extern "C" PyObject* PyModule_GetDict(PyObject* _m) { extern "C" PyObject* PyModule_GetDict(PyObject* _m) {
BoxedModule* m = static_cast<BoxedModule*>(_m); BoxedModule* m = static_cast<BoxedModule*>(_m);
assert(m->cls == module_cls); assert(m->cls == module_cls);
...@@ -63,6 +96,14 @@ extern "C" PyObject* PyModule_GetDict(PyObject* _m) { ...@@ -63,6 +96,14 @@ extern "C" PyObject* PyModule_GetDict(PyObject* _m) {
return new AttrWrapper(m); return new AttrWrapper(m);
} }
extern "C" int PyModule_AddIntConstant(PyObject* _m, const char* name, long value) {
BoxedModule* m = static_cast<BoxedModule*>(_m);
assert(m->cls == module_cls);
m->setattr(name, boxInt(value), NULL);
return 0;
}
extern "C" PyObject* PyDict_New() { extern "C" PyObject* PyDict_New() {
return new BoxedDict(); return new BoxedDict();
} }
...@@ -71,6 +112,28 @@ extern "C" PyObject* PyString_FromString(const char* s) { ...@@ -71,6 +112,28 @@ extern "C" PyObject* PyString_FromString(const char* s) {
return boxStrConstant(s); return boxStrConstant(s);
} }
extern "C" PyObject* PyString_FromStringAndSize(const char* s, ssize_t n) {
if (s == NULL)
return boxString(std::string(n, '\x00'));
return boxStrConstantSize(s, n);
}
extern "C" char* PyString_AsString(PyObject* o) {
assert(o->cls == str_cls);
// TODO this is very brittle, since
// - you are very much not supposed to change the data, and
// - the pointer doesn't have great longevity guarantees
// To satisfy this API we might have to change the string representation?
printf("Warning: PyString_AsString() currently has risky behavior\n");
return const_cast<char*>(static_cast<BoxedString*>(o)->s.data());
}
extern "C" Py_ssize_t PyString_Size(PyObject* s) {
RELEASE_ASSERT(s->cls == str_cls, "");
return static_cast<BoxedString*>(s)->s.size();
}
extern "C" PyObject* PyInt_FromLong(long n) { extern "C" PyObject* PyInt_FromLong(long n) {
return boxInt(n); return boxInt(n);
} }
...@@ -81,7 +144,14 @@ extern "C" int PyDict_SetItem(PyObject* mp, PyObject* _key, PyObject* _item) { ...@@ -81,7 +144,14 @@ extern "C" int PyDict_SetItem(PyObject* mp, PyObject* _key, PyObject* _item) {
Box* item = static_cast<Box*>(_item); Box* item = static_cast<Box*>(_item);
static std::string setitem_str("__setitem__"); static std::string setitem_str("__setitem__");
Box* r = callattrInternal(b, &setitem_str, CLASS_ONLY, NULL, ArgPassSpec(2), key, item, NULL, NULL, NULL); Box* r;
try {
// TODO should demote GIL?
r = callattrInternal(b, &setitem_str, CLASS_ONLY, NULL, ArgPassSpec(2), key, item, NULL, NULL, NULL);
} catch (Box* b) {
fprintf(stderr, "Error: uncaught error would be propagated to C code!\n");
abort();
}
RELEASE_ASSERT(r, ""); RELEASE_ASSERT(r, "");
return 0; return 0;
...@@ -95,26 +165,36 @@ extern "C" int PyDict_SetItemString(PyObject* mp, const char* key, PyObject* ite ...@@ -95,26 +165,36 @@ extern "C" int PyDict_SetItemString(PyObject* mp, const char* key, PyObject* ite
BoxedClass* capifunc_cls; BoxedClass* capifunc_cls;
class BoxedCApiFunction : public Box { class BoxedCApiFunction : public Box {
private: private:
int ml_flags;
Box* passthrough; Box* passthrough;
const char* name; const char* name;
PyCFunction func; PyCFunction func;
public: public:
BoxedCApiFunction(Box* passthrough, const char* name, PyCFunction func) BoxedCApiFunction(int ml_flags, Box* passthrough, const char* name, PyCFunction func)
: Box(capifunc_cls), passthrough(passthrough), name(name), func(func) {} : Box(capifunc_cls), ml_flags(ml_flags), passthrough(passthrough), name(name), func(func) {}
static BoxedString* __repr__(BoxedCApiFunction* self) { static BoxedString* __repr__(BoxedCApiFunction* self) {
assert(self->cls == capifunc_cls); assert(self->cls == capifunc_cls);
return boxStrConstant(self->name); return boxStrConstant(self->name);
} }
static Box* __call__(BoxedCApiFunction* self, BoxedTuple* varargs) { static Box* __call__(BoxedCApiFunction* self, BoxedTuple* varargs, BoxedDict* kwargs) {
assert(self->cls == capifunc_cls); assert(self->cls == capifunc_cls);
assert(varargs->cls == tuple_cls); assert(varargs->cls == tuple_cls);
assert(kwargs->cls == dict_cls);
threading::GLPromoteRegion _gil_lock; threading::GLPromoteRegion _gil_lock;
Box* rtn = (Box*)self->func(self->passthrough, varargs); Box* rtn;
if (self->ml_flags == METH_VARARGS) {
assert(kwargs->d.size() == 0);
rtn = (Box*)self->func(self->passthrough, varargs);
} else if (self->ml_flags == (METH_VARARGS | METH_KEYWORDS)) {
rtn = (Box*)((PyCFunctionWithKeywords)self->func)(self->passthrough, varargs, kwargs);
} else {
RELEASE_ASSERT(0, "0x%x", self->ml_flags);
}
assert(rtn); assert(rtn);
return rtn; return rtn;
} }
...@@ -131,9 +211,10 @@ extern "C" PyObject* Py_InitModule4(const char* name, PyMethodDef* methods, cons ...@@ -131,9 +211,10 @@ extern "C" PyObject* Py_InitModule4(const char* name, PyMethodDef* methods, cons
while (methods->ml_name) { while (methods->ml_name) {
if (VERBOSITY()) if (VERBOSITY())
printf("Loading method %s\n", methods->ml_name); printf("Loading method %s\n", methods->ml_name);
assert(methods->ml_flags == METH_VARARGS);
module->giveAttr(methods->ml_name, new BoxedCApiFunction(passthrough, methods->ml_name, methods->ml_meth)); assert((methods->ml_flags & (~(METH_VARARGS | METH_KEYWORDS))) == 0);
module->giveAttr(methods->ml_name,
new BoxedCApiFunction(methods->ml_flags, passthrough, methods->ml_name, methods->ml_meth));
methods++; methods++;
} }
...@@ -145,29 +226,238 @@ extern "C" PyObject* Py_InitModule4(const char* name, PyMethodDef* methods, cons ...@@ -145,29 +226,238 @@ extern "C" PyObject* Py_InitModule4(const char* name, PyMethodDef* methods, cons
return module; return module;
} }
extern "C" void conservativeGCHandler(GCVisitor* v, Box* b) {
v->visitPotentialRange((void* const*)b, (void* const*)((char*)b + b->cls->tp_basicsize));
}
extern "C" int PyType_Ready(PyTypeObject* cls) {
gc::registerStaticRootMemory(cls, cls + 1);
// unhandled fields:
RELEASE_ASSERT(cls->tp_print == NULL, "");
RELEASE_ASSERT(cls->tp_getattr == NULL, "");
RELEASE_ASSERT(cls->tp_setattr == NULL, "");
RELEASE_ASSERT(cls->tp_compare == NULL, "");
RELEASE_ASSERT(cls->tp_repr == NULL, "");
RELEASE_ASSERT(cls->tp_as_number == NULL, "");
RELEASE_ASSERT(cls->tp_as_sequence == NULL, "");
RELEASE_ASSERT(cls->tp_as_mapping == NULL, "");
RELEASE_ASSERT(cls->tp_hash == NULL, "");
RELEASE_ASSERT(cls->tp_call == NULL, "");
RELEASE_ASSERT(cls->tp_str == NULL, "");
RELEASE_ASSERT(cls->tp_getattro == NULL, "");
RELEASE_ASSERT(cls->tp_setattro == NULL, "");
RELEASE_ASSERT(cls->tp_as_buffer == NULL, "");
RELEASE_ASSERT(cls->tp_flags == Py_TPFLAGS_DEFAULT, "");
RELEASE_ASSERT(cls->tp_traverse == NULL, "");
RELEASE_ASSERT(cls->tp_clear == NULL, "");
RELEASE_ASSERT(cls->tp_richcompare == NULL, "");
RELEASE_ASSERT(cls->tp_weaklistoffset == 0, "");
RELEASE_ASSERT(cls->tp_iter == NULL, "");
RELEASE_ASSERT(cls->tp_iternext == NULL, "");
RELEASE_ASSERT(cls->tp_members == NULL, "");
RELEASE_ASSERT(cls->tp_base == NULL, "");
RELEASE_ASSERT(cls->tp_dict == NULL, "");
RELEASE_ASSERT(cls->tp_descr_get == NULL, "");
RELEASE_ASSERT(cls->tp_descr_set == NULL, "");
RELEASE_ASSERT(cls->tp_init == NULL, "");
RELEASE_ASSERT(cls->tp_alloc == NULL, "");
RELEASE_ASSERT(cls->tp_new == NULL, "");
RELEASE_ASSERT(cls->tp_free == NULL, "");
RELEASE_ASSERT(cls->tp_is_gc == NULL, "");
RELEASE_ASSERT(cls->tp_base == NULL, "");
RELEASE_ASSERT(cls->tp_mro == NULL, "");
RELEASE_ASSERT(cls->tp_cache == NULL, "");
RELEASE_ASSERT(cls->tp_subclasses == NULL, "");
RELEASE_ASSERT(cls->tp_weaklist == NULL, "");
RELEASE_ASSERT(cls->tp_del == NULL, "");
RELEASE_ASSERT(cls->tp_version_tag == 0, "");
#define INITIALIZE(a) new (&(a)) decltype(a)
INITIALIZE(cls->attrs);
INITIALIZE(cls->dependent_icgetattrs);
#undef INITIALIZE
assert(cls->tp_name);
cls->giveAttr("__name__", boxStrConstant(cls->tp_name));
// tp_name
// tp_basicsize, tp_itemsize
// tp_doc
if (cls->tp_methods) {
PyMethodDef* method = cls->tp_methods;
while (method->ml_name) {
auto desc = new BoxedMethodDescriptor(method);
cls->giveAttr(method->ml_name, desc);
method++;
}
}
if (cls->tp_getset) {
if (VERBOSITY())
printf("warning: ignoring tp_getset for now\n");
}
cls->base = object_cls;
cls->gc_visit = &conservativeGCHandler;
// TODO not sure how we can handle extension types that manually
// specify a dict...
RELEASE_ASSERT(cls->tp_dictoffset == 0, "");
// this should get automatically initialized to 0 on this path:
assert(cls->attrs_offset == 0);
return 0;
}
extern "C" PyObject* Py_BuildValue(const char* arg0, ...) { extern "C" PyObject* Py_BuildValue(const char* arg0, ...) {
assert(*arg0 == '\0'); assert(*arg0 == '\0');
return None; return None;
} }
extern "C" int PyArg_ParseTuple(PyObject* tuple, const char* fmt, ...) { // copied from CPython's getargs.c:
if (strcmp("", fmt) == 0) extern "C" int PyBuffer_FillInfo(Py_buffer* view, PyObject* obj, void* buf, Py_ssize_t len, int readonly, int flags) {
return 1; if (view == NULL)
return 0;
if (((flags & PyBUF_WRITABLE) == PyBUF_WRITABLE) && (readonly == 1)) {
// Don't support PyErr_SetString yet:
assert(0);
// PyErr_SetString(PyExc_BufferError, "Object is not writable.");
// return -1;
}
assert(strcmp("O", fmt) == 0); view->obj = obj;
if (obj)
Py_INCREF(obj);
view->buf = buf;
view->len = len;
view->readonly = readonly;
view->itemsize = 1;
view->format = NULL;
if ((flags & PyBUF_FORMAT) == PyBUF_FORMAT)
view->format = "B";
view->ndim = 1;
view->shape = NULL;
if ((flags & PyBUF_ND) == PyBUF_ND)
view->shape = &(view->len);
view->strides = NULL;
if ((flags & PyBUF_STRIDES) == PyBUF_STRIDES)
view->strides = &(view->itemsize);
view->suboffsets = NULL;
view->internal = NULL;
return 0;
}
extern "C" void PyBuffer_Release(Py_buffer* view) {
if (!view->buf) {
assert(!view->obj);
return;
}
PyObject* obj = view->obj;
assert(obj);
assert(obj->cls == str_cls);
if (obj && Py_TYPE(obj)->tp_as_buffer && Py_TYPE(obj)->tp_as_buffer->bf_releasebuffer)
Py_TYPE(obj)->tp_as_buffer->bf_releasebuffer(obj, view);
Py_XDECREF(obj);
view->obj = NULL;
}
BoxedTuple* varargs = (BoxedTuple*)tuple; int vPyArg_ParseTuple(PyObject* _tuple, const char* fmt, va_list ap) {
assert(varargs->elts.size() == 1); RELEASE_ASSERT(_tuple->cls == tuple_cls, "");
BoxedTuple* tuple = static_cast<BoxedTuple*>(_tuple);
bool now_optional = false;
int arg_idx = 0;
int tuple_size = tuple->elts.size();
while (char c = *fmt) {
fmt++;
if (c == ':') {
break;
} else if (c == '|') {
now_optional = true;
continue;
} else {
if (arg_idx >= tuple_size) {
RELEASE_ASSERT(now_optional, "");
break;
}
PyObject* arg = tuple->elts[arg_idx];
switch (c) {
case 's': {
if (*fmt == '*') {
Py_buffer* p = (Py_buffer*)va_arg(ap, Py_buffer*);
RELEASE_ASSERT(arg->cls == str_cls, "");
PyBuffer_FillInfo(p, arg, PyString_AS_STRING(arg), PyString_GET_SIZE(arg), 1, 0);
fmt++;
} else if (*fmt == ':') {
break;
} else {
RELEASE_ASSERT(0, "");
}
break;
}
case 'O': {
PyObject** p = (PyObject**)va_arg(ap, PyObject**);
*p = arg;
break;
}
default:
RELEASE_ASSERT(0, "Unhandled format character: '%c'", c);
}
}
}
return 1;
}
extern "C" int PyArg_ParseTuple(PyObject* _tuple, const char* fmt, ...) {
va_list ap; va_list ap;
va_start(ap, fmt); va_start(ap, fmt);
Box** arg0 = va_arg(ap, Box**); int r = vPyArg_ParseTuple(_tuple, fmt, ap);
*arg0 = varargs->elts[0];
va_end(ap); va_end(ap);
return 1; return r;
}
extern "C" int PyArg_ParseTupleAndKeywords(PyObject* args, PyObject* kwargs, const char* format, char** kwlist, ...) {
assert(kwargs->cls == dict_cls);
RELEASE_ASSERT(static_cast<BoxedDict*>(kwargs)->d.size() == 0, "");
va_list ap;
va_start(ap, kwlist);
int r = vPyArg_ParseTuple(args, format, ap);
va_end(ap);
return r;
}
extern "C" PyObject* _PyObject_New(PyTypeObject* cls) {
assert(cls->tp_itemsize == 0);
auto rtn = (PyObject*)gc_alloc(cls->tp_basicsize, gc::GCKind::PYTHON);
rtn->cls = cls;
return rtn;
}
extern "C" void PyObject_Free(void* p) {
gc::gc_free(p);
ASSERT(0, "I think this is good enough but I'm not sure; should test");
}
extern "C" PyObject* PyErr_Occurred() {
printf("need to hook exception handling -- make sure errors dont propagate into C code, and error codes get "
"checked coming out\n");
return NULL;
} }
BoxedModule* importTestExtension() { BoxedModule* importTestExtension() {
...@@ -211,7 +501,7 @@ void setupCAPI() { ...@@ -211,7 +501,7 @@ void setupCAPI() {
capifunc_cls->giveAttr("__str__", capifunc_cls->getattr("__repr__")); capifunc_cls->giveAttr("__str__", capifunc_cls->getattr("__repr__"));
capifunc_cls->giveAttr( capifunc_cls->giveAttr(
"__call__", new BoxedFunction(boxRTFunction((void*)BoxedCApiFunction::__call__, UNKNOWN, 1, 0, true, false))); "__call__", new BoxedFunction(boxRTFunction((void*)BoxedCApiFunction::__call__, UNKNOWN, 1, 0, true, true)));
capifunc_cls->freeze(); capifunc_cls->freeze();
...@@ -219,6 +509,12 @@ void setupCAPI() { ...@@ -219,6 +509,12 @@ void setupCAPI() {
attrwrapper_cls->giveAttr("__name__", boxStrConstant("attrwrapper")); attrwrapper_cls->giveAttr("__name__", boxStrConstant("attrwrapper"));
attrwrapper_cls->giveAttr("__setitem__", new BoxedFunction(boxRTFunction((void*)AttrWrapper::setitem, UNKNOWN, 3))); attrwrapper_cls->giveAttr("__setitem__", new BoxedFunction(boxRTFunction((void*)AttrWrapper::setitem, UNKNOWN, 3)));
attrwrapper_cls->freeze(); attrwrapper_cls->freeze();
method_cls = new BoxedClass(object_cls, NULL, 0, sizeof(BoxedMethodDescriptor), false);
method_cls->giveAttr("__name__", boxStrConstant("method"));
method_cls->giveAttr("__call__", new BoxedFunction(boxRTFunction((void*)BoxedMethodDescriptor::__call__, UNKNOWN, 2,
0, true, true)));
method_cls->freeze();
} }
void teardownCAPI() { void teardownCAPI() {
......
...@@ -32,6 +32,10 @@ extern "C" BoxedString* boxStrConstant(const char* chars) { ...@@ -32,6 +32,10 @@ extern "C" BoxedString* boxStrConstant(const char* chars) {
return new BoxedString(chars); return new BoxedString(chars);
} }
extern "C" BoxedString* boxStrConstantSize(const char* chars, size_t n) {
return new BoxedString(chars, n);
}
extern "C" Box* boxStringPtr(const std::string* s) { extern "C" Box* boxStringPtr(const std::string* s) {
return new BoxedString(*s); return new BoxedString(*s);
} }
...@@ -39,6 +43,9 @@ extern "C" Box* boxStringPtr(const std::string* s) { ...@@ -39,6 +43,9 @@ extern "C" Box* boxStringPtr(const std::string* s) {
Box* boxString(const std::string& s) { Box* boxString(const std::string& s) {
return new BoxedString(s); return new BoxedString(s);
} }
Box* boxString(std::string&& s) {
return new BoxedString(std::move(s));
}
extern "C" double unboxFloat(Box* b) { extern "C" double unboxFloat(Box* b) {
ASSERT(b->cls == float_cls, "%s", getTypeName(b)->c_str()); ASSERT(b->cls == float_cls, "%s", getTypeName(b)->c_str());
......
...@@ -31,6 +31,11 @@ ...@@ -31,6 +31,11 @@
namespace pyston { namespace pyston {
extern "C" long PyInt_AsLong(PyObject* obj) {
assert(obj->cls == int_cls);
return static_cast<BoxedInt*>(obj)->n;
}
BoxedInt* interned_ints[NUM_INTERNED_INTS]; BoxedInt* interned_ints[NUM_INTERNED_INTS];
// If we don't have fast overflow-checking builtins, provide some slow variants: // If we don't have fast overflow-checking builtins, provide some slow variants:
...@@ -588,8 +593,8 @@ extern "C" Box* intNew(Box* _cls, Box* val) { ...@@ -588,8 +593,8 @@ extern "C" Box* intNew(Box* _cls, Box* val) {
raiseExcHelper(TypeError, "int.__new__(%s): %s is not a subtype of int", getNameOfClass(cls)->c_str(), raiseExcHelper(TypeError, "int.__new__(%s): %s is not a subtype of int", getNameOfClass(cls)->c_str(),
getNameOfClass(cls)->c_str()); getNameOfClass(cls)->c_str());
assert(cls->instance_size >= sizeof(BoxedInt)); assert(cls->tp_basicsize >= sizeof(BoxedInt));
void* mem = gc_alloc(cls->instance_size, gc::GCKind::PYTHON); void* mem = gc_alloc(cls->tp_basicsize, gc::GCKind::PYTHON);
BoxedInt* rtn = ::new (mem) BoxedInt(cls, 0); BoxedInt* rtn = ::new (mem) BoxedInt(cls, 0);
initUserAttrs(rtn, cls); initUserAttrs(rtn, cls);
......
...@@ -55,8 +55,8 @@ extern "C" Box* longNew(Box* _cls, Box* val) { ...@@ -55,8 +55,8 @@ extern "C" Box* longNew(Box* _cls, Box* val) {
raiseExcHelper(TypeError, "long.__new__(%s): %s is not a subtype of long", getNameOfClass(cls)->c_str(), raiseExcHelper(TypeError, "long.__new__(%s): %s is not a subtype of long", getNameOfClass(cls)->c_str(),
getNameOfClass(cls)->c_str()); getNameOfClass(cls)->c_str());
assert(cls->instance_size >= sizeof(BoxedInt)); assert(cls->tp_basicsize >= sizeof(BoxedInt));
void* mem = gc_alloc(cls->instance_size, gc::GCKind::PYTHON); void* mem = gc_alloc(cls->tp_basicsize, gc::GCKind::PYTHON);
BoxedLong* rtn = ::new (mem) BoxedLong(cls); BoxedLong* rtn = ::new (mem) BoxedLong(cls);
initUserAttrs(rtn, cls); initUserAttrs(rtn, cls);
......
...@@ -361,8 +361,9 @@ extern "C" void checkUnpackingLength(i64 expected, i64 given) { ...@@ -361,8 +361,9 @@ extern "C" void checkUnpackingLength(i64 expected, i64 given) {
BoxedClass::BoxedClass(BoxedClass* base, gcvisit_func gc_visit, int attrs_offset, int instance_size, BoxedClass::BoxedClass(BoxedClass* base, gcvisit_func gc_visit, int attrs_offset, int instance_size,
bool is_user_defined) bool is_user_defined)
: Box(type_cls), base(base), gc_visit(gc_visit), attrs_offset(attrs_offset), instance_size(instance_size), : Box(type_cls), tp_basicsize(instance_size), tp_dealloc(NULL), base(base), gc_visit(gc_visit),
is_constant(false), is_user_defined(is_user_defined) { attrs_offset(attrs_offset), is_constant(false), is_user_defined(is_user_defined) {
assert(tp_dealloc == NULL);
if (gc_visit == NULL) { if (gc_visit == NULL) {
assert(base); assert(base);
...@@ -378,15 +379,15 @@ BoxedClass::BoxedClass(BoxedClass* base, gcvisit_func gc_visit, int attrs_offset ...@@ -378,15 +379,15 @@ BoxedClass::BoxedClass(BoxedClass* base, gcvisit_func gc_visit, int attrs_offset
assert(object_cls); assert(object_cls);
if (base->attrs_offset) if (base->attrs_offset)
RELEASE_ASSERT(attrs_offset == base->attrs_offset, ""); RELEASE_ASSERT(attrs_offset == base->attrs_offset, "");
assert(instance_size >= base->instance_size); assert(tp_basicsize >= base->tp_basicsize);
} }
if (base && cls && str_cls) if (base && cls && str_cls)
giveAttr("__base__", base); giveAttr("__base__", base);
assert(instance_size % sizeof(void*) == 0); // Not critical I suppose, but probably signals a bug assert(tp_basicsize % sizeof(void*) == 0); // Not critical I suppose, but probably signals a bug
if (attrs_offset) { if (attrs_offset) {
assert(instance_size >= attrs_offset + sizeof(HCAttrs)); assert(tp_basicsize >= attrs_offset + sizeof(HCAttrs));
assert(attrs_offset % sizeof(void*) == 0); // Not critical I suppose, but probably signals a bug assert(attrs_offset % sizeof(void*) == 0); // Not critical I suppose, but probably signals a bug
} }
...@@ -651,7 +652,7 @@ void Box::setattr(const std::string& attr, Box* val, SetattrRewriteArgs* rewrite ...@@ -651,7 +652,7 @@ void Box::setattr(const std::string& attr, Box* val, SetattrRewriteArgs* rewrite
} }
static Box* _handleClsAttr(Box* obj, Box* attr) { static Box* _handleClsAttr(Box* obj, Box* attr) {
if (attr->cls == function_cls) { if (attr->cls == function_cls || attr->cls == method_cls) {
Box* rtn = boxInstanceMethod(obj, attr); Box* rtn = boxInstanceMethod(obj, attr);
return rtn; return rtn;
} }
...@@ -1568,6 +1569,11 @@ extern "C" Box* callattrInternal(Box* obj, const std::string* attr, LookupScope ...@@ -1568,6 +1569,11 @@ extern "C" Box* callattrInternal(Box* obj, const std::string* attr, LookupScope
r_clsattr.ensureDoneUsing(); r_clsattr.ensureDoneUsing();
} }
auto old_clsattr = clsattr;
clsattr = _handleClsAttr(obj, clsattr);
if (clsattr != old_clsattr)
rewrite_args = NULL;
if (rewrite_args) { if (rewrite_args) {
CallRewriteArgs srewrite_args(rewrite_args->rewriter, std::move(r_clsattr), rewrite_args->destination, CallRewriteArgs srewrite_args(rewrite_args->rewriter, std::move(r_clsattr), rewrite_args->destination,
rewrite_args->more_guards_after); rewrite_args->more_guards_after);
......
...@@ -32,6 +32,7 @@ ...@@ -32,6 +32,7 @@
#include "runtime/set.h" #include "runtime/set.h"
extern "C" void initerrno(); extern "C" void initerrno();
extern "C" void init_sha();
namespace pyston { namespace pyston {
...@@ -298,10 +299,10 @@ extern "C" Box* createUserClass(std::string* name, Box* _base, Box* _attr_dict) ...@@ -298,10 +299,10 @@ extern "C" Box* createUserClass(std::string* name, Box* _base, Box* _attr_dict)
BoxedClass* made; BoxedClass* made;
if (base->instancesHaveAttrs()) { if (base->instancesHaveAttrs()) {
made = new BoxedClass(base, NULL, base->attrs_offset, base->instance_size, true); made = new BoxedClass(base, NULL, base->attrs_offset, base->tp_basicsize, true);
} else { } else {
assert(base->instance_size % sizeof(void*) == 0); assert(base->tp_basicsize % sizeof(void*) == 0);
made = new BoxedClass(base, NULL, base->instance_size, base->instance_size + sizeof(HCAttrs), true); made = new BoxedClass(base, NULL, base->tp_basicsize, base->tp_basicsize + sizeof(HCAttrs), true);
} }
for (const auto& p : attr_dict->d) { for (const auto& p : attr_dict->d) {
...@@ -473,8 +474,8 @@ Box* objectNew(BoxedClass* cls, BoxedTuple* args) { ...@@ -473,8 +474,8 @@ Box* objectNew(BoxedClass* cls, BoxedTuple* args) {
raiseExcHelper(TypeError, "object.__new__() takes no parameters"); raiseExcHelper(TypeError, "object.__new__() takes no parameters");
} }
assert(cls->instance_size >= sizeof(Box)); assert(cls->tp_basicsize >= sizeof(Box));
void* mem = gc::gc_alloc(cls->instance_size, gc::GCKind::PYTHON); void* mem = gc::gc_alloc(cls->tp_basicsize, gc::GCKind::PYTHON);
Box* rtn = ::new (mem) Box(cls); Box* rtn = ::new (mem) Box(cls);
initUserAttrs(rtn, cls); initUserAttrs(rtn, cls);
...@@ -628,6 +629,7 @@ void setupRuntime() { ...@@ -628,6 +629,7 @@ void setupRuntime() {
setupCAPI(); setupCAPI();
initerrno(); initerrno();
init_sha();
setupSysEnd(); setupSysEnd();
......
...@@ -72,7 +72,7 @@ BoxedList* getSysPath(); ...@@ -72,7 +72,7 @@ BoxedList* getSysPath();
extern "C" { extern "C" {
extern BoxedClass* object_cls, *type_cls, *bool_cls, *int_cls, *float_cls, *str_cls, *function_cls, *none_cls, extern BoxedClass* object_cls, *type_cls, *bool_cls, *int_cls, *float_cls, *str_cls, *function_cls, *none_cls,
*instancemethod_cls, *list_cls, *slice_cls, *module_cls, *dict_cls, *tuple_cls, *file_cls, *xrange_cls, *member_cls, *instancemethod_cls, *list_cls, *slice_cls, *module_cls, *dict_cls, *tuple_cls, *file_cls, *xrange_cls, *member_cls,
*closure_cls, *generator_cls; *method_cls, *closure_cls, *generator_cls;
} }
extern "C" { extern Box* None, *NotImplemented, *True, *False; } extern "C" { extern Box* None, *NotImplemented, *True, *False; }
extern "C" { extern "C" {
...@@ -88,7 +88,9 @@ extern "C" Box* boxFloat(double d); ...@@ -88,7 +88,9 @@ extern "C" Box* boxFloat(double d);
extern "C" Box* boxInstanceMethod(Box* obj, Box* func); extern "C" Box* boxInstanceMethod(Box* obj, Box* func);
extern "C" Box* boxStringPtr(const std::string* s); extern "C" Box* boxStringPtr(const std::string* s);
Box* boxString(const std::string& s); Box* boxString(const std::string& s);
Box* boxString(std::string&& s);
extern "C" BoxedString* boxStrConstant(const char* chars); extern "C" BoxedString* boxStrConstant(const char* chars);
extern "C" BoxedString* boxStrConstantSize(const char* chars, size_t n);
extern "C" void listAppendInternal(Box* self, Box* v); extern "C" void listAppendInternal(Box* self, Box* v);
extern "C" void listAppendArrayInternal(Box* self, Box** v, int nelts); extern "C" void listAppendArrayInternal(Box* self, Box** v, int nelts);
extern "C" Box* boxCLFunction(CLFunction* f, BoxedClosure* closure, bool isGenerator, extern "C" Box* boxCLFunction(CLFunction* f, BoxedClosure* closure, bool isGenerator,
...@@ -199,6 +201,7 @@ public: ...@@ -199,6 +201,7 @@ public:
// const std::basic_string<char, std::char_traits<char>, StlCompatAllocator<char> > s; // const std::basic_string<char, std::char_traits<char>, StlCompatAllocator<char> > s;
const std::string s; const std::string s;
BoxedString(const char* s, size_t n) __attribute__((visibility("default"))) : Box(str_cls), s(s, n) {}
BoxedString(const std::string&& s) __attribute__((visibility("default"))) : Box(str_cls), s(std::move(s)) {} BoxedString(const std::string&& s) __attribute__((visibility("default"))) : Box(str_cls), s(std::move(s)) {}
BoxedString(const std::string& s) __attribute__((visibility("default"))) : Box(str_cls), s(s) {} BoxedString(const std::string& s) __attribute__((visibility("default"))) : Box(str_cls), s(s) {}
}; };
......
# expected: fail
# - warnings about PyString_AsString(), since that is allowed to be modified
try:
import _sha as sha
except ImportError:
import sha
s = sha.new()
print s.hexdigest()
s.update("aoeu")
print s.hexdigest()
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment