Commit de5fde04 authored by Travis Hance's avatar Travis Hance

static closures

parent f69d1a99
......@@ -103,10 +103,14 @@ public:
bool usesNameLookup() override { return false; }
bool isPassedToViaClosure(InternedString name) override { return false; }
bool areLocalsFromModule() override { return true; }
DerefInfo getDerefInfo(InternedString) override { RELEASE_ASSERT(0, "This should never get called"); }
size_t getClosureOffset(InternedString) override { RELEASE_ASSERT(0, "This should never get called"); }
size_t getClosureSize() override { RELEASE_ASSERT(0, "This should never get called"); }
std::vector<std::pair<InternedString, DerefInfo>> v;
const std::vector<std::pair<InternedString, DerefInfo>>& getAllDerefVarsAndInfo() override { return v; }
InternedString mangleName(InternedString id) override { return id; }
InternedString internString(llvm::StringRef s) override { abort(); }
};
......@@ -165,10 +169,14 @@ public:
bool usesNameLookup() override { return true; }
bool isPassedToViaClosure(InternedString name) override { return false; }
bool areLocalsFromModule() override { return false; }
DerefInfo getDerefInfo(InternedString) override { RELEASE_ASSERT(0, "This should never get called"); }
size_t getClosureOffset(InternedString) override { RELEASE_ASSERT(0, "This should never get called"); }
size_t getClosureSize() override { RELEASE_ASSERT(0, "This should never get called"); }
std::vector<std::pair<InternedString, DerefInfo>> v;
const std::vector<std::pair<InternedString, DerefInfo>>& getAllDerefVarsAndInfo() override { return v; }
InternedString mangleName(InternedString id) override { return id; }
InternedString internString(llvm::StringRef s) override { abort(); }
};
......@@ -257,11 +265,22 @@ private:
AST* ast;
bool usesNameLookup_;
llvm::DenseMap<InternedString, size_t> closure_offsets;
std::vector<std::pair<InternedString, DerefInfo>> allDerefVarsAndInfo;
bool allDerefVarsAndInfoCached;
public:
ScopeInfoBase(ScopeInfo* parent, ScopingAnalysis::ScopeNameUsage* usage, AST* ast, bool usesNameLookup)
: parent(parent), usage(usage), ast(ast), usesNameLookup_(usesNameLookup) {
: parent(parent), usage(usage), ast(ast), usesNameLookup_(usesNameLookup), allDerefVarsAndInfoCached(false) {
assert(usage);
assert(ast);
int i = 0;
for (auto& p : usage->referenced_from_nested) {
closure_offsets[p] = i;
i++;
}
}
~ScopeInfoBase() override { delete this->usage; }
......@@ -300,20 +319,67 @@ public:
bool usesNameLookup() override { return usesNameLookup_; }
bool isPassedToViaClosure(InternedString name) override {
if (isCompilerCreatedName(name))
return false;
bool areLocalsFromModule() override { return false; }
DerefInfo getDerefInfo(InternedString name) override {
assert(getScopeTypeOfName(name) == VarScopeType::DEREF);
return usage->got_from_closure.count(name) > 0 || usage->passthrough_accesses.count(name) > 0;
// TODO pre-compute this?
size_t parentCounter = 0;
// Casting to a ScopeInfoBase* is okay because only a ScopeInfoBase can have a closure.
for (ScopeInfoBase* parent = static_cast<ScopeInfoBase*>(this->parent); parent != NULL;
parent = static_cast<ScopeInfoBase*>(parent->parent)) {
if (parent->createsClosure()) {
auto it = parent->closure_offsets.find(name);
if (it != parent->closure_offsets.end()) {
return DerefInfo{.num_parents_from_passed_closure = parentCounter, .offset = it->second };
}
parentCounter++;
}
}
bool areLocalsFromModule() override { return false; }
RELEASE_ASSERT(0, "Should not get here");
}
size_t getClosureOffset(InternedString name) override {
assert(getScopeTypeOfName(name) == VarScopeType::CLOSURE);
return closure_offsets[name];
}
size_t getClosureSize() override { return closure_offsets.size(); }
InternedString mangleName(const InternedString id) override {
return pyston::mangleName(id, usage->private_name, usage->scoping->getInternedStrings());
}
InternedString internString(llvm::StringRef s) override { return usage->scoping->getInternedStrings().get(s); }
const std::vector<std::pair<InternedString, DerefInfo>>& getAllDerefVarsAndInfo() override {
if (!allDerefVarsAndInfoCached) {
allDerefVarsAndInfoCached = true;
StrSet allDerefs = usage->got_from_closure;
for (InternedString name : usage->passthrough_accesses) {
if (allDerefs.find(name) != allDerefs.end()) {
allDerefs.insert(name);
}
}
for (InternedString name : allDerefs) {
allDerefVarsAndInfo.push_back({ name, getDerefInfo(name) });
}
std::sort(allDerefVarsAndInfo.begin(), allDerefVarsAndInfo.end(), derefComparator);
}
return allDerefVarsAndInfo;
}
private:
static bool derefComparator(const std::pair<InternedString, DerefInfo>& p1,
const std::pair<InternedString, DerefInfo>& p2) {
return p1.second.num_parents_from_passed_closure < p2.second.num_parents_from_passed_closure;
};
};
class NameCollectorVisitor : public ASTVisitor {
......
......@@ -25,6 +25,11 @@ class AST_Module;
class AST_Expression;
class AST_Suite;
struct DerefInfo {
size_t num_parents_from_passed_closure;
size_t offset;
};
class ScopeInfo {
public:
ScopeInfo() {}
......@@ -75,39 +80,6 @@ public:
};
virtual VarScopeType getScopeTypeOfName(InternedString name) = 0;
// Returns true if the variable should be passed via a closure to this scope.
// Note that:
// (a) This can be false even if there is an entry in the closure object
// passed to the scope, if the variable is not actually used in this
// scope or any child scopes. This can happen, because the variable
// could be in the closure to be accessed by a different function, e.g.
//
// def f();
// a = 0
// b = 0
// def g():
// print a
// def h():
// print b
// # locals() should not contain `a` even though `h` is
// # passed a closure object with `a` in it
// print locals()
//
// (b) This can be true even if it is not used in this scope, if it
// is used in a child scope. For example:
//
// def f():
// a = 0
// def g():
// def h():
// print a
// print locals() # should contain `a`
//
// This is useful because it determines whether a variable from a closure
// into the locals() dictionary.
virtual bool isPassedToViaClosure(InternedString name) = 0;
// Returns true if the scope may contain NAME variables.
// In particular, it returns true for ClassDef scope, for any scope
// with an `exec` statement or `import *` statement in it, or for any
......@@ -116,6 +88,11 @@ public:
virtual bool areLocalsFromModule() = 0;
virtual DerefInfo getDerefInfo(InternedString name) = 0;
virtual const std::vector<std::pair<InternedString, DerefInfo>>& getAllDerefVarsAndInfo() = 0;
virtual size_t getClosureOffset(InternedString name) = 0;
virtual size_t getClosureSize() = 0;
virtual InternedString mangleName(InternedString id) = 0;
virtual InternedString internString(llvm::StringRef) = 0;
};
......
......@@ -228,7 +228,7 @@ void ASTInterpreter::initArguments(int nargs, BoxedClosure* _closure, BoxedGener
generator = _generator;
if (scope_info->createsClosure())
created_closure = createClosure(passed_closure);
created_closure = createClosure(passed_closure, scope_info->getClosureSize());
std::vector<Box*, StlCompatAllocator<Box*>> argsArray{ arg1, arg2, arg3 };
for (int i = 3; i < nargs; ++i)
......@@ -354,8 +354,9 @@ void ASTInterpreter::doStore(InternedString name, Value value) {
setitem(frame_info.boxedLocals, boxString(name.str()), value.o);
} else {
sym_table[name] = value.o;
if (vst == ScopeInfo::VarScopeType::CLOSURE)
setattr(created_closure, name.c_str(), value.o);
if (vst == ScopeInfo::VarScopeType::CLOSURE) {
created_closure->elts[scope_info->getClosureOffset(name)] = value.o;
}
}
}
......@@ -1082,8 +1083,20 @@ Value ASTInterpreter::visit_name(AST_Name* node) {
switch (node->lookup_type) {
case ScopeInfo::VarScopeType::GLOBAL:
return getGlobal(source_info->parent_module, &node->id.str());
case ScopeInfo::VarScopeType::DEREF:
return getattr(passed_closure, node->id.c_str());
case ScopeInfo::VarScopeType::DEREF: {
DerefInfo deref_info = scope_info->getDerefInfo(node->id);
assert(passed_closure);
BoxedClosure* closure = passed_closure;
for (int i = 0; i < deref_info.num_parents_from_passed_closure; i++) {
closure = closure->parent;
}
Box* val = closure->elts[deref_info.offset];
if (val == NULL) {
raiseExcHelper(NameError, "free variable '%s' referenced before assignment in enclosing scope",
node->id.c_str());
}
return val;
}
case ScopeInfo::VarScopeType::FAST:
case ScopeInfo::VarScopeType::CLOSURE: {
SymMap::iterator it = sym_table.find(node->id);
......
......@@ -1781,15 +1781,17 @@ public:
CompilerVariable* getattr(IREmitter& emitter, const OpInfo& info, ConcreteCompilerVariable* var,
const std::string* attr, bool cls_only) override {
RELEASE_ASSERT(0, "should not be called\n");
/*
assert(!cls_only);
llvm::Value* bitcast = emitter.getBuilder()->CreateBitCast(var->getValue(), g.llvm_value_type_ptr);
return ConcreteCompilerVariable(UNKNOWN, bitcast, true).getattr(emitter, info, attr, cls_only);
*/
}
void setattr(IREmitter& emitter, const OpInfo& info, ConcreteCompilerVariable* var, const std::string* attr,
CompilerVariable* v) override {
llvm::Value* bitcast = emitter.getBuilder()->CreateBitCast(var->getValue(), g.llvm_value_type_ptr);
ConcreteCompilerVariable(UNKNOWN, bitcast, true).setattr(emitter, info, attr, v);
RELEASE_ASSERT(0, "should not be called\n");
}
ConcreteCompilerType* getConcreteType() override { return this; }
......
......@@ -898,10 +898,29 @@ private:
assert(!is_kill);
assert(scope_info->takesClosure());
CompilerVariable* closure = symbol_table[internString(PASSED_CLOSURE_NAME)];
assert(closure);
DerefInfo deref_info = scope_info->getDerefInfo(node->id);
static_assert(sizeof(Box) == offsetof(BoxedClosure, parent), "");
static_assert(offsetof(BoxedClosure, parent) + sizeof(BoxedClosure*) == offsetof(BoxedClosure, nelts), "");
static_assert(offsetof(BoxedClosure, nelts) + sizeof(size_t) == offsetof(BoxedClosure, elts), "");
return closure->getattr(emitter, getEmptyOpInfo(unw_info), &node->id.str(), false);
CompilerVariable* closure = symbol_table[internString(PASSED_CLOSURE_NAME)];
llvm::Value* closureValue = closure->makeConverted(emitter, CLOSURE)->getValue();
closure->decvref(emitter);
llvm::Value* gep;
for (int i = 0; i < deref_info.num_parents_from_passed_closure; i++) {
gep = emitter.getBuilder()->CreateConstInBoundsGEP2_32(closureValue, 0, 1);
closureValue = emitter.getBuilder()->CreateLoad(gep);
}
gep = emitter.getBuilder()->CreateGEP(closureValue,
{ llvm::ConstantInt::get(g.i32, 0), llvm::ConstantInt::get(g.i32, 3),
llvm::ConstantInt::get(g.i32, deref_info.offset) });
llvm::Value* lookupResult = emitter.getBuilder()->CreateLoad(gep);
emitter.createCall(unw_info, g.funcs.assertDerefNameDefined,
{ lookupResult, getStringConstantPtr(node->id.str() + '\0') });
return new ConcreteCompilerVariable(UNKNOWN, lookupResult, true);
} else if (vst == ScopeInfo::VarScopeType::NAME) {
llvm::Value* boxedLocals = irstate->getBoxedLocalsVar();
llvm::Value* attr = getStringConstantPtr(node->id.str() + '\0');
......@@ -1431,10 +1450,22 @@ private:
_popFake(defined_name, true);
if (vst == ScopeInfo::VarScopeType::CLOSURE) {
size_t offset = scope_info->getClosureOffset(name);
CompilerVariable* closure = symbol_table[internString(CREATED_CLOSURE_NAME)];
llvm::Value* closureValue = closure->makeConverted(emitter, CLOSURE)->getValue();
closure->decvref(emitter);
llvm::Value* gep = emitter.getBuilder()->CreateGEP(
closureValue, { llvm::ConstantInt::get(g.i32, 0), llvm::ConstantInt::get(g.i32, 3),
llvm::ConstantInt::get(g.i32, offset) });
emitter.getBuilder()->CreateStore(val->makeConverted(emitter, UNKNOWN)->getValue(), gep);
/*
CompilerVariable* closure = symbol_table[internString(CREATED_CLOSURE_NAME)];
assert(closure);
closure->setattr(emitter, getEmptyOpInfo(unw_info), &name.str(), val);
*/
}
}
}
......@@ -1893,7 +1924,8 @@ private:
assert(var->getType() != BOXED_FLOAT
&& "should probably unbox it, but why is it boxed in the first place?");
// This line can never get hit right now for the same reason that the variables must already be concrete,
// This line can never get hit right now for the same reason that the variables must already be
// concrete,
// because we're over-generating phis.
ASSERT(var->isGrabbed(), "%s", p.first.c_str());
// var->ensureGrabbed(emitter);
......@@ -2152,7 +2184,8 @@ private:
} else {
#ifndef NDEBUG
if (myblock->successors.size()) {
// TODO getTypeAtBlockEnd will automatically convert up to the concrete type, which we don't want
// TODO getTypeAtBlockEnd will automatically convert up to the concrete type, which we don't
// want
// here, but this is just for debugging so I guess let it happen for now:
ConcreteCompilerType* ending_type = types->getTypeAtBlockEnd(it->first, myblock);
ASSERT(it->second->canConvertTo(ending_type), "%s is supposed to be %s, but somehow is %s",
......@@ -2352,7 +2385,8 @@ public:
if (!passed_closure)
passed_closure = embedConstantPtr(nullptr, g.llvm_closure_type_ptr);
llvm::Value* new_closure = emitter.getBuilder()->CreateCall(g.funcs.createClosure, passed_closure);
llvm::Value* new_closure = emitter.getBuilder()->CreateCall2(
g.funcs.createClosure, passed_closure, getConstantInt(scope_info->getClosureSize(), g.i64));
symbol_table[internString(CREATED_CLOSURE_NAME)]
= new ConcreteCompilerVariable(getCreatedClosureType(), new_closure, true);
}
......
......@@ -216,6 +216,7 @@ void initGlobalFuncs(GlobalState& g) {
GET(raiseAttributeErrorStr);
GET(raiseNotIterableError);
GET(assertNameDefined);
GET(assertDerefNameDefined);
GET(assertFail);
GET(printFloat);
......
......@@ -41,7 +41,7 @@ struct GlobalFuncs {
*exceptionMatches, *yield, *getiterHelper, *hasnext;
llvm::Value* unpackIntoArray, *raiseAttributeError, *raiseAttributeErrorStr, *raiseNotIterableError,
*assertNameDefined, *assertFail;
*assertNameDefined, *assertFail, *assertDerefNameDefined;
llvm::Value* printFloat, *listAppendInternal, *getSysStdout;
llvm::Value* runtimeCall0, *runtimeCall1, *runtimeCall2, *runtimeCall3, *runtimeCall, *runtimeCallN;
llvm::Value* callattr0, *callattr1, *callattr2, *callattr3, *callattr, *callattrN;
......
......@@ -758,18 +758,21 @@ Box* fastLocalsToBoxedLocals() {
// Add the locals from the closure
// TODO in a ClassDef scope, we aren't supposed to add these
for (; closure != NULL; closure = closure->parent) {
assert(closure->cls == closure_cls);
for (auto& attr_offset : closure->attrs.hcls->getAttrOffsets()) {
const std::string& name = attr_offset.first();
int offset = attr_offset.second;
Box* val = closure->attrs.attr_list->attrs[offset];
if (val != NULL && scope_info->isPassedToViaClosure(scope_info->internString(name))) {
Box* boxedName = boxString(name);
if (d->d.count(boxedName) == 0) {
d->d[boxString(name)] = val;
}
}
size_t depth = 0;
for (auto& p : scope_info->getAllDerefVarsAndInfo()) {
InternedString name = p.first;
DerefInfo derefInfo = p.second;
while (depth < derefInfo.num_parents_from_passed_closure) {
depth++;
closure = closure->parent;
}
assert(closure != NULL);
Box* val = closure->elts[derefInfo.offset];
Box* boxedName = boxString(name.str());
if (val != NULL) {
d->d[boxedName] = val;
} else {
d->d.erase(boxedName);
}
}
......
......@@ -98,6 +98,7 @@ void force() {
FORCE(raiseAttributeErrorStr);
FORCE(raiseNotIterableError);
FORCE(assertNameDefined);
FORCE(assertDerefNameDefined);
FORCE(assertFail);
FORCE(printFloat);
......
......@@ -233,6 +233,12 @@ extern "C" void assertNameDefined(bool b, const char* name, BoxedClass* exc_cls,
}
}
extern "C" void assertDerefNameDefined(Box* b, const char* name) {
if (b == NULL) {
raiseExcHelper(NameError, "free variable '%s' referenced before assignment in enclosing scope", name);
}
}
extern "C" void raiseAttributeErrorStr(const char* typeName, const char* attr) {
raiseExcHelper(AttributeError, "'%s' object has no attribute '%s'", typeName, attr);
}
......@@ -1274,39 +1280,7 @@ Box* getattrInternalGeneric(Box* obj, const std::string& attr, GetattrRewriteArg
}
// TODO this should be a custom getattr
if (obj->cls == closure_cls) {
Box* val = NULL;
if (rewrite_args) {
GetattrRewriteArgs hrewrite_args(rewrite_args->rewriter, rewrite_args->obj, rewrite_args->destination);
val = obj->getattr(attr, &hrewrite_args);
if (!hrewrite_args.out_success) {
rewrite_args = NULL;
} else if (val) {
rewrite_args->out_rtn = hrewrite_args.out_rtn;
rewrite_args->out_success = true;
return val;
}
} else {
val = obj->getattr(attr, NULL);
if (val) {
return val;
}
}
// If val doesn't exist, then we move up to the parent closure
// TODO closures should get their own treatment, but now just piggy-back on the
// normal hidden-class IC logic.
// Can do better since we don't need to guard on the cls (always going to be closure)
BoxedClosure* closure = static_cast<BoxedClosure*>(obj);
if (closure->parent) {
if (rewrite_args) {
rewrite_args->obj = rewrite_args->obj->getAttr(offsetof(BoxedClosure, parent));
}
return getattrInternal(closure->parent, attr, rewrite_args);
}
raiseExcHelper(NameError, "free variable '%s' referenced before assignment in enclosing scope", attr.c_str());
}
assert(obj->cls != closure_cls);
// Handle descriptor logic here.
// A descriptor is either a data descriptor or a non-data descriptor.
......
......@@ -83,9 +83,10 @@ extern "C" Box* importFrom(Box* obj, const std::string* attr);
extern "C" Box* importStar(Box* from_module, BoxedModule* to_module);
extern "C" Box** unpackIntoArray(Box* obj, int64_t expected_size);
extern "C" void assertNameDefined(bool b, const char* name, BoxedClass* exc_cls, bool local_var_msg);
extern "C" void assertDerefNameDefined(Box* b, const char* name);
extern "C" void assertFail(BoxedModule* inModule, Box* msg);
extern "C" bool isSubclass(BoxedClass* child, BoxedClass* parent);
extern "C" BoxedClosure* createClosure(BoxedClosure* parent_closure);
extern "C" BoxedClosure* createClosure(BoxedClosure* parent_closure, size_t size);
Box* getiter(Box* o);
extern "C" Box* getPystonIter(Box* o);
......
......@@ -617,6 +617,11 @@ extern "C" void closureGCHandler(GCVisitor* v, Box* b) {
BoxedClosure* c = (BoxedClosure*)b;
if (c->parent)
v->visit(c->parent);
for (int i = 0; i < c->nelts; i++) {
if (c->elts[i])
v->visit(c->elts[i]);
}
}
extern "C" {
......@@ -830,10 +835,12 @@ extern "C" Box* createSlice(Box* start, Box* stop, Box* step) {
return rtn;
}
extern "C" BoxedClosure* createClosure(BoxedClosure* parent_closure) {
extern "C" BoxedClosure* createClosure(BoxedClosure* parent_closure, size_t n) {
if (parent_closure)
assert(parent_closure->cls == closure_cls);
return new BoxedClosure(parent_closure);
BoxedClosure* closure = new (n) BoxedClosure(parent_closure);
assert(closure->cls == closure_cls);
return closure;
}
extern "C" Box* sliceNew(Box* cls, Box* start, Box* stop, Box** args) {
......@@ -2002,8 +2009,8 @@ void setupRuntime() {
sizeof(BoxedSet), false, "frozenset");
capi_getset_cls
= BoxedHeapClass::create(type_cls, object_cls, NULL, 0, 0, sizeof(BoxedGetsetDescriptor), false, "getset");
closure_cls = BoxedHeapClass::create(type_cls, object_cls, &closureGCHandler, offsetof(BoxedClosure, attrs), 0,
sizeof(BoxedClosure), false, "closure");
closure_cls
= BoxedHeapClass::create(type_cls, object_cls, &closureGCHandler, 0, 0, sizeof(BoxedClosure), false, "closure");
property_cls = BoxedHeapClass::create(type_cls, object_cls, &propertyGCHandler, 0, 0, sizeof(BoxedProperty), false,
"property");
staticmethod_cls = BoxedHeapClass::create(type_cls, object_cls, &staticmethodGCHandler, 0, 0,
......
......@@ -618,15 +618,27 @@ public:
DEFAULT_CLASS_SIMPLE(classmethod_cls);
};
// TODO is there any particular reason to make this a Box, ie a python-level object?
// TODO is there any particular reason to make this a Box, i.e. a python-level object?
class BoxedClosure : public Box {
public:
HCAttrs attrs;
BoxedClosure* parent;
size_t nelts;
Box* elts[0];
BoxedClosure(BoxedClosure* parent) : parent(parent) {}
DEFAULT_CLASS(closure_cls);
void* operator new(size_t size, size_t nelts) __attribute__((visibility("default"))) {
/*
BoxedClosure* rtn
= static_cast<BoxedClosure*>(gc_alloc(_PyObject_VAR_SIZE(closure_cls, nelts), gc::GCKind::PYTHON));
*/
BoxedClosure* rtn
= static_cast<BoxedClosure*>(gc_alloc(sizeof(BoxedClosure) + nelts * sizeof(Box*), gc::GCKind::PYTHON));
rtn->nelts = nelts;
rtn->cls = closure_cls;
memset((void*)rtn->elts, 0, sizeof(Box*) * nelts);
return rtn;
}
};
class BoxedGenerator : public Box {
......
# should_error
# The use of c makes sure a closure gets passed through all 4 functions.
# The use of a in g makes sure that a is in f's closure.
# The a in j should refer to the a in h, thus throwing an exception since
# it is undefined (that is, it should *not* access the a from f even
# though it access via the closure).
def f():
c = 0
a = 0
def g():
print c
print a
def h():
print c
def j():
print c
print a
j()
a = 1
h()
g()
f()
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment