Skip to content

P
Pyston
Commit e7174486 authored by Marius Wachtler's avatar Marius Wachtler
Browse files
Options

Merge pull request #1227 from undingen/bjit_opt2 

major bjit improvements
parents 820dd0cd 9fd4924f
Show whitespace changes
Inline Side-by-side
Showing with 398 additions and 337 deletions
src/asm_writing/assembler.cpp
View file @ e7174486
......@@ -755,6 +755,7 @@ void Assembler::incq(Indirect mem) {
}
assert(src_idx >= 0 && src_idx < 8);
bool needssib = (src_idx == 0b100);
if (rex)
emitRex(rex);
......@@ -763,8 +764,12 @@ void Assembler::incq(Indirect mem) {
assert(-0x80 <= mem.offset && mem.offset < 0x80);
if (mem.offset == 0) {
emitModRM(0b00, 0, src_idx);
if (needssib)
emitSIB(0b00, 0b100, src_idx);
} else {
emitModRM(0b01, 0, src_idx);
if (needssib)
emitSIB(0b00, 0b100, src_idx);
emitByte(mem.offset);
}
}
......
src/asm_writing/rewriter.cpp
View file @ e7174486
......@@ -24,7 +24,7 @@
namespace pyston {
static const assembler::Register allocatable_regs[] = {
static const assembler::Register std_allocatable_regs[] = {
assembler::RAX, assembler::RCX, assembler::RDX,
// no RSP
// no RBP
......@@ -540,8 +540,8 @@ void Rewriter::_incref(RewriterVar* var, int num_refs) {
// this->_trap();
// this->_call(NULL, true, (void*)Helper::incref, llvm::ArrayRef<RewriterVar*>(&var, 1),
// llvm::ArrayRef<RewriterVar*>());
// this->_call(NULL, true, false /* can't throw */, (void*)Helper::incref, { var });
#ifdef Py_REF_DEBUG
// assembler->trap();
for (int i = 0; i < num_refs; ++i)
......@@ -569,16 +569,13 @@ void Rewriter::_decref(RewriterVar* var) {
assert(!var->nullable);
// assembler->trap();
// this->_call(NULL, true, (void*)Helper::decref, llvm::ArrayRef<RewriterVar*>(&var, 1),
// llvm::ArrayRef<RewriterVar*>(NULL, (int)0));
// this->_call(NULL, true, false /* can't throw */, (void*)Helper::decref, { var });
#ifdef Py_REF_DEBUG
// assembler->trap();
assembler->decq(assembler::Immediate(&_Py_RefTotal));
#endif
_setupCall(true, llvm::ArrayRef<RewriterVar*>(&var, 1), llvm::ArrayRef<RewriterVar*>(NULL, (size_t)0),
assembler::RAX);
_setupCall(true, { var }, {}, assembler::RAX);
#ifdef Py_REF_DEBUG
......@@ -612,8 +609,7 @@ void Rewriter::_xdecref(RewriterVar* var) {
assert(var->nullable);
// assembler->trap();
this->_call(NULL, true, (void*)Helper::xdecref, llvm::ArrayRef<RewriterVar*>(&var, 1),
llvm::ArrayRef<RewriterVar*>(NULL, (size_t)0));
this->_call(NULL, true, false /* can't throw */, (void*)Helper::xdecref, { var });
// Doesn't call bumpUse, since this function is designed to be callable from other emitting functions.
// (ie the caller should call bumpUse)
......@@ -632,14 +628,18 @@ void Rewriter::_cmp(RewriterVar* result, RewriterVar* v1, AST_TYPE::AST_TYPE cmp
if (LOG_IC_ASSEMBLY)
assembler->comment("_cmp");
assembler::Register v1_reg = v1->getInReg();
assembler::Register v2_reg = v2->getInReg();
assembler::Register v1_reg = v1->getInReg(Location::any(), false, dest);
assembler::Register v2_reg = v2->getInReg(Location::any(), false, dest);
assert(v1_reg != v2_reg); // TODO how do we ensure this?
v1->bumpUseEarlyIfPossible();
v2->bumpUseEarlyIfPossible();
assembler::Register newvar_reg = allocReg(dest);
// sete and setne has special register requirements (can't use r8-r15)
const assembler::Register valid_registers[] = {
assembler::RAX, assembler::RCX, assembler::RDX, assembler::RSI, assembler::RDI,
};
assembler::Register newvar_reg = allocReg(dest, Location::any(), valid_registers);
result->initializeInReg(newvar_reg);
assembler->cmp(v1_reg, v2_reg);
switch (cmp_type) {
......@@ -916,116 +916,76 @@ RewriterVar* Rewriter::loadConst(int64_t val, Location dest) {
return const_loader_var;
}
RewriterVar* Rewriter::call(bool has_side_effects, void* func_addr) {
RewriterVar* Rewriter::call(bool has_side_effects, void* func_addr, llvm::ArrayRef<RewriterVar*> args,
llvm::ArrayRef<RewriterVar*> args_xmm, llvm::ArrayRef<RewriterVar*> additional_uses) {
STAT_TIMER(t0, "us_timer_rewriter", 10);
RewriterVar* result = createNewVar();
RewriterVar::SmallVector args;
RewriterVar::SmallVector args_xmm;
return call(has_side_effects, func_addr, args, args_xmm);
}
RewriterVar* Rewriter::call(bool has_side_effects, void* func_addr, RewriterVar* arg0) {
STAT_TIMER(t0, "us_timer_rewriter", 10);
ActionType type;
if (has_side_effects)
type = ActionType::MUTATION;
else
type = ActionType::NORMAL;
RewriterVar::SmallVector args;
RewriterVar::SmallVector args_xmm;
args.push_back(arg0);
return call(has_side_effects, func_addr, args, args_xmm);
}
RewriterVar* Rewriter::call(bool has_side_effects, void* func_addr, RewriterVar* arg0, RewriterVar* arg1) {
STAT_TIMER(t0, "us_timer_rewriter", 10);
// TODO: we don't need to generate the decref info for calls which can't throw
bool can_throw = true;
auto args_array_ref = regionAllocArgs(args, args_xmm, additional_uses);
RewriterVar::SmallVector args;
RewriterVar::SmallVector args_xmm;
args.push_back(arg0);
args.push_back(arg1);
return call(has_side_effects, func_addr, args, args_xmm);
}
// Hack: explicitly order the closure arguments so they pad nicer
struct LambdaClosure {
RewriterVar** args_array;
struct {
unsigned int has_side_effects : 1;
unsigned int can_throw : 1;
unsigned int num_args : 16;
unsigned int num_args_xmm : 16;
unsigned int num_additional_uses : 16;
};
RewriterVar* Rewriter::call(bool has_side_effects, void* func_addr, RewriterVar* arg0, RewriterVar* arg1,
RewriterVar* arg2) {
STAT_TIMER(t0, "us_timer_rewriter", 10);
llvm::ArrayRef<RewriterVar*> allArgs() const {
return llvm::makeArrayRef(args_array, num_args + num_args_xmm + num_additional_uses);
}
RewriterVar::SmallVector args;
RewriterVar::SmallVector args_xmm;
args.push_back(arg0);
args.push_back(arg1);
args.push_back(arg2);
return call(has_side_effects, func_addr, args, args_xmm);
}
llvm::ArrayRef<RewriterVar*> args() const { return allArgs().slice(0, num_args); }
RewriterVar* Rewriter::call(bool has_side_effects, void* func_addr, RewriterVar* arg0, RewriterVar* arg1,
RewriterVar* arg2, RewriterVar* arg3) {
STAT_TIMER(t0, "us_timer_rewriter", 10);
llvm::ArrayRef<RewriterVar*> argsXmm() const { return allArgs().slice(num_args, num_args_xmm); }
RewriterVar::SmallVector args;
RewriterVar::SmallVector args_xmm;
args.push_back(arg0);
args.push_back(arg1);
args.push_back(arg2);
args.push_back(arg3);
return call(has_side_effects, func_addr, args, args_xmm);
}
RewriterVar* Rewriter::call(bool has_side_effects, void* func_addr, RewriterVar* arg0, RewriterVar* arg1,
RewriterVar* arg2, RewriterVar* arg3, RewriterVar* arg4) {
RewriterVar::SmallVector args;
RewriterVar::SmallVector args_xmm;
args.push_back(arg0);
args.push_back(arg1);
args.push_back(arg2);
args.push_back(arg3);
args.push_back(arg4);
return call(has_side_effects, func_addr, args, args_xmm);
}
RewriterVar* Rewriter::call(bool has_side_effects, void* func_addr, const RewriterVar::SmallVector& args,
const RewriterVar::SmallVector& args_xmm) {
RewriterVar* result = createNewVar();
RewriterVar::SmallVector uses;
for (RewriterVar* v : args) {
assert(v != NULL);
uses.push_back(v);
llvm::ArrayRef<RewriterVar*> additionalUses() const {
return allArgs().slice((int)num_args + (int)num_args_xmm, num_additional_uses);
}
for (RewriterVar* v : args_xmm) {
assert(v != NULL);
uses.push_back(v);
}
ActionType type;
if (has_side_effects)
type = ActionType::MUTATION;
else
type = ActionType::NORMAL;
// It's not nice to pass llvm::SmallVectors through a closure, especially with our SmallFunction
// optimization, so just regionAlloc them and copy the data in:
RewriterVar** _args = (RewriterVar**)this->regionAlloc(sizeof(RewriterVar*) * args.size());
memcpy(_args, args.begin(), sizeof(RewriterVar*) * args.size());
RewriterVar** _args_xmm = (RewriterVar**)this->regionAlloc(sizeof(RewriterVar*) * args_xmm.size());
memcpy(_args_xmm, args_xmm.begin(), sizeof(RewriterVar*) * args_xmm.size());
LambdaClosure(llvm::MutableArrayRef<RewriterVar*> args_array_ref, llvm::ArrayRef<RewriterVar*> _args,
llvm::ArrayRef<RewriterVar*> _args_xmm, llvm::ArrayRef<RewriterVar*> _addition_uses,
bool has_side_effects, bool can_throw)
: args_array(args_array_ref.data()),
has_side_effects(has_side_effects),
can_throw(can_throw),
num_args(_args.size()),
num_args_xmm(_args_xmm.size()),
num_additional_uses(_addition_uses.size()) {
assert(_args.size() < 1 << 16);
assert(_args_xmm.size() < 1 << 16);
assert(_addition_uses.size() < 1 << 16);
}
int args_size = args.size();
assert(args_xmm.size() <= 0x7fff);
// Hack: pack this into a short to make sure it fits in the closure
short xmm_args_size = args_xmm.size();
} lambda_closure(args_array_ref, args, args_xmm, additional_uses, has_side_effects, can_throw);
assert(lambda_closure.args().size() == args.size());
assert(lambda_closure.argsXmm().size() == args_xmm.size());
assert(lambda_closure.additionalUses().size() == additional_uses.size());
// Hack: explicitly order the closure arguments so they pad nicer
addAction([args_size, xmm_args_size, has_side_effects, this, result, func_addr, _args, _args_xmm]() {
this->_call(result, has_side_effects, func_addr, llvm::ArrayRef<RewriterVar*>(_args, args_size),
llvm::ArrayRef<RewriterVar*>(_args_xmm, xmm_args_size));
for (int i = 0; i < args_size; i++)
_args[i]->bumpUse();
for (int i = 0; i < xmm_args_size; i++)
_args_xmm[i]->bumpUse();
}, uses, type);
addAction([this, result, func_addr, lambda_closure]() {
this->_call(result, lambda_closure.has_side_effects, lambda_closure.can_throw, func_addr, lambda_closure.args(),
lambda_closure.argsXmm(), lambda_closure.allArgs());
}, lambda_closure.allArgs(), type);
return result;
}
void Rewriter::_setupCall(bool has_side_effects, llvm::ArrayRef<RewriterVar*> args,
llvm::ArrayRef<RewriterVar*> args_xmm, Location preserve) {
llvm::ArrayRef<RewriterVar*> args_xmm, Location preserve,
llvm::ArrayRef<RewriterVar*> bump_if_possible) {
if (has_side_effects)
assert(done_guarding);
......@@ -1112,6 +1072,10 @@ void Rewriter::_setupCall(bool has_side_effects, llvm::ArrayRef<RewriterVar*> ar
}
#endif
for (auto&& use : bump_if_possible) {
use->bumpUseEarlyIfPossible();
}
// Spill caller-saved registers:
for (auto check_reg : caller_save_registers) {
// check_reg.dump();
......@@ -1173,8 +1137,9 @@ void Rewriter::_setupCall(bool has_side_effects, llvm::ArrayRef<RewriterVar*> ar
#endif
}
void Rewriter::_call(RewriterVar* result, bool has_side_effects, void* func_addr, llvm::ArrayRef<RewriterVar*> args,
llvm::ArrayRef<RewriterVar*> args_xmm) {
void Rewriter::_call(RewriterVar* result, bool has_side_effects, bool can_throw, void* func_addr,
llvm::ArrayRef<RewriterVar*> args, llvm::ArrayRef<RewriterVar*> args_xmm,
llvm::ArrayRef<RewriterVar*> vars_to_bump) {
if (LOG_IC_ASSEMBLY)
assembler->comment("_call");
......@@ -1183,17 +1148,7 @@ void Rewriter::_call(RewriterVar* result, bool has_side_effects, void* func_addr
if (failed)
return;
_setupCall(has_side_effects, args, args_xmm, assembler::R11);
// This duty is now on the caller:
/*
for (RewriterVar* arg : args) {
arg->bumpUse();
}
for (RewriterVar* arg_xmm : args_xmm) {
arg_xmm->bumpUse();
}
*/
_setupCall(has_side_effects, args, args_xmm, assembler::R11, vars_to_bump);
assertConsistent();
......@@ -1211,7 +1166,7 @@ void Rewriter::_call(RewriterVar* result, bool has_side_effects, void* func_addr
assert(assembler->hasFailed() || asm_address == (uint64_t)assembler->curInstPointer());
}
// TODO: we don't need to generate the decref info for calls which can't throw
if (can_throw)
registerDecrefInfoHere();
if (!failed) {
......@@ -1224,24 +1179,39 @@ void Rewriter::_call(RewriterVar* result, bool has_side_effects, void* func_addr
if (result)
result->releaseIfNoUses();
for (RewriterVar* var : vars_to_bump) {
var->bumpUseLateIfNecessary();
}
}
std::vector<Location> Rewriter::getDecrefLocations() {
std::vector<Location> decref_infos;
for (RewriterVar& var : vars) {
if (var.locations.size() && var.needsDecref()) {
// TODO: add code to handle other location types and choose best location if there are several
Location l = *var.locations.begin();
if (var.locations.size() && var.needsDecref(current_action_idx)) {
bool found_location = false;
for (Location l : var.locations) {
if (l.type == Location::Scratch) {
// convert to stack based location because later on we may not know the offset of the scratch area from
// the SP.
// convert to stack based location because later on we may not know the offset of the scratch area
// from the SP.
decref_infos.emplace_back(Location::Stack, indirectFor(l).offset);
found_location = true;
break;
} else if (l.type == Location::Register) {
// CSRs shouldn't be getting allocated, and we should only be calling this at a callsite:
RELEASE_ASSERT(0, "we shouldn't be trying to decref anything in a register");
// we only allow registers which are not clobbered by a call
if (l.isClobberedByCall())
continue;
decref_infos.emplace_back(l);
found_location = true;
break;
} else
RELEASE_ASSERT(0, "not implemented");
}
if (!found_location) {
// this is very rare. just fail the rewrite for now
failed = true;
}
}
}
for (auto&& p : owned_attrs) {
......@@ -1335,16 +1305,25 @@ void RewriterVar::refConsumed(RewriterAction* action) {
last_refconsumed_numuses = uses.size();
if (!action)
action = rewriter->getLastAction();
action->consumed_refs.emplace_back(this);
action->consumed_refs.push_front(this);
}
void RewriterVar::refUsed() {
// TODO: This is a pretty silly implementation that might prevent other optimizations?
rewriter->addAction([=]() { this->bumpUse(); }, { this }, ActionType::NORMAL);
}
bool RewriterVar::needsDecref(int current_action_index) {
rewriter->assertPhaseEmitting();
if (reftype != RefType::OWNED)
return false;
// if nothing consumes this reference we need to create a decref entry
if (num_refs_consumed == 0)
return true;
// don't create decref entry if the currenty action hands off the ownership
int reference_handed_off_action_index = uses[last_refconsumed_numuses - 1];
if (reference_handed_off_action_index == current_action_index)
return false;
bool RewriterVar::needsDecref() {
return reftype == RefType::OWNED && !this->refHandedOff();
return true;
}
void RewriterVar::registerOwnedAttr(int byte_offset) {
......@@ -1497,6 +1476,7 @@ void Rewriter::commit() {
_incref(var, 1);
}
current_action_idx = i;
actions[i].action();
if (failed) {
......@@ -1926,7 +1906,7 @@ void Rewriter::_checkAndThrowCAPIException(RewriterVar* r, int64_t exc_val, asse
} else
assembler->cmp(var_reg, assembler::Immediate(exc_val), type);
_setupCall(false, RewriterVar::SmallVector(), RewriterVar::SmallVector());
_setupCall(false, {});
{
assembler::ForwardJump jnz(*assembler, assembler::COND_NOT_ZERO);
assembler->mov(assembler::Immediate((void*)throwCAPIException), assembler::R11);
......@@ -2004,6 +1984,11 @@ void Rewriter::spillRegister(assembler::XMMRegister reg) {
}
assembler::Register Rewriter::allocReg(Location dest, Location otherThan) {
return allocReg(dest, otherThan, allocatable_regs);
}
assembler::Register Rewriter::allocReg(Location dest, Location otherThan,
llvm::ArrayRef<assembler::Register> valid_registers) {
assertPhaseEmitting();
if (dest.type == Location::AnyReg) {
......@@ -2012,7 +1997,7 @@ assembler::Register Rewriter::allocReg(Location dest, Location otherThan) {
assembler::Register best_reg(0);
// TODO prioritize spilling a constant register?
for (assembler::Register reg : allocatable_regs) {
for (assembler::Register reg : valid_registers) {
if (Location(reg) != otherThan) {
if (vars_by_location.count(reg) == 0) {
return reg;
......@@ -2042,6 +2027,7 @@ assembler::Register Rewriter::allocReg(Location dest, Location otherThan) {
assert(failed || vars_by_location.count(best_reg) == 0);
return best_reg;
} else if (dest.type == Location::Register) {
assert(std::find(valid_registers.begin(), valid_registers.end(), dest.asRegister()) != valid_registers.end());
assembler::Register reg(dest.regnum);
if (vars_by_location.count(reg)) {
......@@ -2193,12 +2179,14 @@ Rewriter::Rewriter(std::unique_ptr<ICSlotRewrite> rewrite, int num_args, const L
return_location(this->rewrite->returnRegister()),
failed(false),
needs_invalidation_support(needs_invalidation_support),
current_action_idx(-1),
added_changing_action(false),
marked_inside_ic(false),
done_guarding(false),
last_guard_action(-1),
offset_eq_jmp_slowpath(-1),
offset_ne_jmp_slowpath(-1) {
offset_ne_jmp_slowpath(-1),
allocatable_regs(std_allocatable_regs) {
initPhaseCollecting();
finished = false;
......
src/asm_writing/rewriter.h
View file @ e7174486
......@@ -16,6 +16,7 @@
#define PYSTON_ASMWRITING_REWRITER_H
#include <deque>
#include <forward_list>
#include <list>
#include <map>
#include <memory>
......@@ -192,8 +193,6 @@ public:
// if no action is specified it will assume the last action consumed the reference
void refConsumed(RewriterAction* action = NULL);
void refUsed();
// registerOwnedAttr tells the refcounter that a certain memory location holds a pointer
// to an owned reference. This must be paired with a call to deregisterOwnedAttr
// Call these right before emitting the store (for register) or decref (for deregister).
......@@ -237,11 +236,11 @@ private:
// /* some code */
// bumpUseLateIfNecessary();
void bumpUseEarlyIfPossible() {
if (reftype != RefType::OWNED)
if (reftype != RefType::OWNED && !hasScratchAllocation())
bumpUse();
}
void bumpUseLateIfNecessary() {
if (reftype == RefType::OWNED)
if (reftype == RefType::OWNED || hasScratchAllocation())
bumpUse();
}
......@@ -254,7 +253,7 @@ private:
bool isDoneUsing() { return next_use == uses.size(); }
bool hasScratchAllocation() const { return scratch_allocation.second > 0; }
void resetHasScratchAllocation() { scratch_allocation = std::make_pair(0, 0); }
bool needsDecref();
bool needsDecref(int current_action_index);
// Indicates if this variable is an arg, and if so, what location the arg is from.
bool is_arg;
......@@ -339,8 +338,9 @@ public:
class RewriterAction {
public:
SmallFunction<56> action;
std::vector<RewriterVar*> consumed_refs;
SmallFunction<48> action;
std::forward_list<RewriterVar*> consumed_refs;
template <typename F> RewriterAction(F&& action) : action(std::forward<F>(action)) {}
......@@ -367,7 +367,33 @@ private:
protected:
// Allocates `bytes` bytes of data. The allocation will get freed when the rewriter gets freed.
void* regionAlloc(size_t bytes) { return allocator.Allocate(bytes, 16 /* alignment */); }
void* regionAlloc(size_t bytes, int alignment = 16) { return allocator.Allocate(bytes, alignment); }
template <typename T> llvm::MutableArrayRef<T> regionAlloc(size_t num_elements) {
return llvm::MutableArrayRef<T>(allocator.Allocate<T>(num_elements), num_elements);
}
// This takes a variable number of llvm::ArrayRef<RewriterVar*> and copies in all elements into a single contiguous
// memory location.
template <typename... Args>
llvm::MutableArrayRef<RewriterVar*> regionAllocArgs(llvm::ArrayRef<RewriterVar*> arg1, Args... args) {
size_t num_total_args = 0;
for (auto&& array : { arg1, args... }) {
num_total_args += array.size();
}
if (num_total_args == 0)
return llvm::MutableArrayRef<RewriterVar*>();
auto args_array_ref = regionAlloc<RewriterVar*>(num_total_args);
auto insert_point = args_array_ref;
for (auto&& array : { arg1, args... }) {
if (!array.empty()) {
memcpy(insert_point.data(), array.data(), array.size() * sizeof(RewriterVar*));
insert_point = insert_point.slice(array.size());
}
}
assert(insert_point.size() == 0);
return args_array_ref;
}
// Helps generating the best code for loading a const integer value.
// By keeping track of the last known value of every register and reusing it.
......@@ -432,6 +458,8 @@ protected:
bool needs_invalidation_support = true);
std::deque<RewriterAction> actions;
int current_action_idx; // in the emitting phase get's set to index of currently executed action
template <typename F> RewriterAction* addAction(F&& action, llvm::ArrayRef<RewriterVar*> vars, ActionType type) {
assertPhaseCollecting();
for (RewriterVar* var : vars) {
......@@ -483,6 +511,8 @@ protected:
// Allocates a register. dest must be of type Register or AnyReg
// If otherThan is a register, guaranteed to not use that register.
assembler::Register allocReg(Location dest, Location otherThan = Location::any());
assembler::Register allocReg(Location dest, Location otherThan,
llvm::ArrayRef<assembler::Register> valid_registers);
assembler::XMMRegister allocXMMReg(Location dest, Location otherThan = Location::any());
// Allocates an 8-byte region in the scratch space
Location allocScratch();
......@@ -507,11 +537,13 @@ protected:
void _slowpathJump(bool condition_eq);
void _trap();
void _loadConst(RewriterVar* result, int64_t val);
void _setupCall(bool has_side_effects, llvm::ArrayRef<RewriterVar*> args, llvm::ArrayRef<RewriterVar*> args_xmm,
Location preserve = Location::any());
void _setupCall(bool has_side_effects, llvm::ArrayRef<RewriterVar*> args = {},
llvm::ArrayRef<RewriterVar*> args_xmm = {}, Location preserve = Location::any(),
llvm::ArrayRef<RewriterVar*> bump_if_possible = {});
// _call does not call bumpUse on its arguments:
void _call(RewriterVar* result, bool has_side_effects, void* func_addr, llvm::ArrayRef<RewriterVar*> args,
llvm::ArrayRef<RewriterVar*> args_xmm);
void _call(RewriterVar* result, bool has_side_effects, bool can_throw, void* func_addr,
llvm::ArrayRef<RewriterVar*> args, llvm::ArrayRef<RewriterVar*> args_xmm = {},
llvm::ArrayRef<RewriterVar*> vars_to_bump = {});
void _add(RewriterVar* result, RewriterVar* a, int64_t b, Location dest);
int _allocate(RewriterVar* result, int n);
void _allocateAndCopy(RewriterVar* result, RewriterVar* array, int n);
......@@ -565,6 +597,8 @@ protected:
#endif
}
llvm::ArrayRef<assembler::Register> allocatable_regs;
public:
// This should be called exactly once for each argument
RewriterVar* getArg(int argnum);
......@@ -606,16 +640,13 @@ public:
// 2) does not have any side-effects that would be user-visible if we bailed out from the middle of the
// inline cache. (Extra allocations don't count even though they're potentially visible if you look
// hard enough.)
RewriterVar* call(bool has_side_effects, void* func_addr, const RewriterVar::SmallVector& args,
const RewriterVar::SmallVector& args_xmm = RewriterVar::SmallVector());
RewriterVar* call(bool has_side_effects, void* func_addr);
RewriterVar* call(bool has_side_effects, void* func_addr, RewriterVar* arg0);
RewriterVar* call(bool has_side_effects, void* func_addr, RewriterVar* arg0, RewriterVar* arg1);
RewriterVar* call(bool has_side_effects, void* func_addr, RewriterVar* arg0, RewriterVar* arg1, RewriterVar* arg2);
RewriterVar* call(bool has_side_effects, void* func_addr, RewriterVar* arg0, RewriterVar* arg1, RewriterVar* arg2,
RewriterVar* arg3);
RewriterVar* call(bool has_side_effects, void* func_addr, RewriterVar* arg0, RewriterVar* arg1, RewriterVar* arg2,
RewriterVar* arg3, RewriterVar* arg4);
RewriterVar* call(bool has_side_effects, void* func_addr, llvm::ArrayRef<RewriterVar*> args = {},
llvm::ArrayRef<RewriterVar*> args_xmm = {}, llvm::ArrayRef<RewriterVar*> additional_uses = {});
template <typename... Args>
RewriterVar* call(bool has_side_effects, void* func_addr, RewriterVar* arg1, Args... args) {
return call(has_side_effects, func_addr, llvm::ArrayRef<RewriterVar*>({ arg1, args... }), {});
}
RewriterVar* add(RewriterVar* a, int64_t b, Location dest);
// Allocates n pointer-sized stack slots:
RewriterVar* allocate(int n);
......
src/codegen/ast_interpreter.cpp
View file @ e7174486
......@@ -458,7 +458,7 @@ void ASTInterpreter::doStore(AST_Name* node, STOLEN(Value) value) {
ScopeInfo::VarScopeType vst = node->lookup_type;
if (vst == ScopeInfo::VarScopeType::GLOBAL) {
if (jit)
jit->emitSetGlobal(frame_info.globals, name.getBox(), value);
jit->emitSetGlobal(name.getBox(), value, getMD()->source->scoping->areGlobalsFromModule());
setGlobal(frame_info.globals, name.getBox(), value.o);
} else if (vst == ScopeInfo::VarScopeType::NAME) {
if (jit)
......@@ -471,13 +471,12 @@ void ASTInterpreter::doStore(AST_Name* node, STOLEN(Value) value) {
bool closure = vst == ScopeInfo::VarScopeType::CLOSURE;
if (jit) {
bool is_live = true;
// TODO: turn this optimization back on.
// if (!closure)
// is_live = source_info->getLiveness()->isLiveAtEnd(name, current_block);
if (!closure)
is_live = source_info->getLiveness()->isLiveAtEnd(name, current_block);
if (is_live)
jit->emitSetLocal(name, node->vreg, closure, value);
else
jit->emitSetBlockLocal(name, value);
jit->emitSetBlockLocal(name, node->vreg, value);
}
if (closure) {
......@@ -686,12 +685,12 @@ Value ASTInterpreter::visit_jump(AST_Jump* node) {
if (backedge)
++edgecount;
if (ENABLE_BASELINEJIT && backedge && edgecount == OSR_THRESHOLD_INTERPRETER && !jit && !node->target->code) {
if (ENABLE_BASELINEJIT && backedge && edgecount >= OSR_THRESHOLD_INTERPRETER && !jit && !node->target->code) {
should_jit = true;
startJITing(node->target);
}
if (backedge && edgecount == OSR_THRESHOLD_BASELINE) {
if (backedge && edgecount >= OSR_THRESHOLD_BASELINE) {
Box* rtn = doOSR(node);
if (rtn)
return Value(rtn, NULL);
......@@ -1173,12 +1172,9 @@ Value ASTInterpreter::createFunction(AST* node, AST_arguments* args, const std::
closure_var = jit->imm(0ul);
if (!passed_globals_var)
passed_globals_var = jit->imm(0ul);
rtn.var = jit->call(false, (void*)createFunctionFromMetadata, jit->imm(md), closure_var, passed_globals_var,
defaults_var, jit->imm(args->defaults.size()))->setType(RefType::OWNED);
for (auto d_var : defaults_vars) {
d_var->refUsed();
}
rtn.var = jit->call(false, (void*)createFunctionFromMetadata, { jit->imm(md), closure_var, passed_globals_var,
defaults_var, jit->imm(args->defaults.size()) },
{}, defaults_vars)->setType(RefType::OWNED);
}
rtn.o = createFunctionFromMetadata(md, closure, passed_globals, u.il);
......@@ -1661,7 +1657,7 @@ Value ASTInterpreter::visit_name(AST_Name* node) {
assert(!node->is_kill);
Value v;
if (jit)
v.var = jit->emitGetGlobal(frame_info.globals, node->id.getBox());
v.var = jit->emitGetGlobal(node->id.getBox());
v.o = getGlobal(frame_info.globals, node->id.getBox());
return v;
......@@ -1781,6 +1777,10 @@ int ASTInterpreterJitInterface::getBoxedLocalsOffset() {
return offsetof(ASTInterpreter, frame_info.boxedLocals);
}
int ASTInterpreterJitInterface::getCreatedClosureOffset() {
return offsetof(ASTInterpreter, created_closure);
}
int ASTInterpreterJitInterface::getCurrentBlockOffset() {
return offsetof(ASTInterpreter, current_block);
}
......
src/codegen/ast_interpreter.h
View file @ e7174486
......@@ -39,6 +39,7 @@ struct ASTInterpreterJitInterface {
static constexpr uint64_t osr_dummy_value = -1;
static int getBoxedLocalsOffset();
static int getCreatedClosureOffset();
static int getCurrentBlockOffset();
static int getCurrentInstOffset();
static int getEdgeCountOffset();
......
src/codegen/baseline_jit.cpp
View file @ e7174486
......@@ -16,6 +16,7 @@
#include <llvm/ADT/DenseMap.h>
#include <llvm/ADT/DenseSet.h>
#include <sys/mman.h>
#include "codegen/irgen/hooks.h"
#include "codegen/memmgr.h"
......@@ -38,30 +39,43 @@ static llvm::DenseMap<CFGBlock*, std::vector<void*>> block_patch_locations;
//
// long foo(char* c);
// void bjit() {
// asm volatile ("" ::: "r15");
// asm volatile ("" ::: "r14");
// asm volatile ("" ::: "r13");
// asm volatile ("" ::: "r12");
// char scratch[256+16];
// foo(scratch);
// }
//
// It omits the frame pointer but saves r13 and r14
// It omits the frame pointer but saves r12, r13, r14 and r15
// use 'objdump -s -j .eh_frame <obj.file>' to dump it
const unsigned char eh_info[]
= { 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x7a, 0x52, 0x00, 0x01, 0x78, 0x10,
0x01, 0x1b, 0x0c, 0x07, 0x08, 0x90, 0x01, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x1c, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00, 0x42, 0x0e, 0x10, 0x42,
0x0e, 0x18, 0x47, 0x0e, 0xb0, 0x02, 0x8d, 0x03, 0x8e, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00 };
= { 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x7a, 0x52, 0x00, 0x01, 0x78, 0x10, 0x01,
0x1b, 0x0c, 0x07, 0x08, 0x90, 0x01, 0x00, 0x00, 0x2c, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0x00, 0x42, 0x0e, 0x10, 0x42, 0x0e, 0x18, 0x42,
0x0e, 0x20, 0x42, 0x0e, 0x28, 0x47, 0x0e, 0xc0, 0x02, 0x8c, 0x05, 0x8d, 0x04, 0x8e, 0x03, 0x8f,
0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 };
static_assert(JitCodeBlock::num_stack_args == 2, "have to update EH table!");
static_assert(JitCodeBlock::scratch_size == 256, "have to update EH table!");
constexpr int code_size = JitCodeBlock::memory_size - sizeof(eh_info);
JitCodeBlock::MemoryManager::MemoryManager() {
int protection = PROT_READ | PROT_WRITE | PROT_EXEC;
int flags = MAP_PRIVATE | MAP_ANONYMOUS;
#if ENABLE_BASELINEJIT_MAP_32BIT
flags |= MAP_32BIT;
#endif
addr = (uint8_t*)mmap(NULL, JitCodeBlock::memory_size, protection, flags, -1, 0);
}
JitCodeBlock::MemoryManager::~MemoryManager() {
munmap(addr, JitCodeBlock::memory_size);
addr = NULL;
}
JitCodeBlock::JitCodeBlock(llvm::StringRef name)
: memory(new uint8_t[memory_size]),
entry_offset(0),
a(memory.get() + sizeof(eh_info), code_size),
is_currently_writing(false),
asm_failed(false) {
: entry_offset(0), a(memory.get() + sizeof(eh_info), code_size), is_currently_writing(false), asm_failed(false) {
static StatCounter num_jit_code_blocks("num_baselinejit_code_blocks");
num_jit_code_blocks.log();
static StatCounter num_jit_total_bytes("num_baselinejit_total_bytes");
......@@ -70,8 +84,10 @@ JitCodeBlock::JitCodeBlock(llvm::StringRef name)
uint8_t* code = a.curInstPointer();
// emit prolog
a.push(assembler::R15);
a.push(assembler::R14);
a.push(assembler::R13);
a.push(assembler::R12);
static_assert(sp_adjustment % 16 == 8, "stack isn't aligned");
a.sub(assembler::Immediate(sp_adjustment), assembler::RSP);
a.mov(assembler::RDI, assembler::R13); // interpreter pointer
......@@ -136,6 +152,12 @@ void JitCodeBlock::fragmentFinished(int bytes_written, int num_bytes_overlapping
ic_info.appendDecrefInfosTo(decref_infos);
}
static const assembler::Register bjit_allocatable_regs[]
= { assembler::RAX, assembler::RCX, assembler::RDX,
// no RSP
// no RBP
assembler::RDI, assembler::RSI, assembler::R8, assembler::R9,
assembler::R10, assembler::R11, assembler::R12, assembler::R15 };
JitFragmentWriter::JitFragmentWriter(CFGBlock* block, std::unique_ptr<ICInfo> ic_info,
std::unique_ptr<ICSlotRewrite> rewrite, int code_offset, int num_bytes_overlapping,
......@@ -149,6 +171,8 @@ JitFragmentWriter::JitFragmentWriter(CFGBlock* block, std::unique_ptr<ICInfo> ic
code_block(code_block),
interp(0),
ic_info(std::move(ic_info)) {
allocatable_regs = bjit_allocatable_regs;
if (LOG_BJIT_ASSEMBLY)
comment("BJIT: JitFragmentWriter() start");
interp = createNewVar();
......@@ -198,10 +222,10 @@ RewriterVar* JitFragmentWriter::emitCallattr(AST_expr* node, RewriterVar* obj, B
call_args.push_back(args.size() > 1 ? args[1] : imm(0ul));
call_args.push_back(args.size() > 2 ? args[2] : imm(0ul));
llvm::ArrayRef<RewriterVar*> additional_uses;
if (args.size() > 3) {
RewriterVar* scratch = allocate(args.size() - 3);
for (int i = 0; i < args.size() - 3; ++i)
scratch->setAttr(i * sizeof(void*), args[i + 3], RewriterVar::SetattrType::REF_USED);
additional_uses = args.slice(3);
RewriterVar* scratch = allocArgs(additional_uses, RewriterVar::SetattrType::REF_USED);
call_args.push_back(scratch);
} else if (keyword_names) {
call_args.push_back(imm(0ul));
......@@ -210,11 +234,8 @@ RewriterVar* JitFragmentWriter::emitCallattr(AST_expr* node, RewriterVar* obj, B
if (keyword_names)
call_args.push_back(imm(keyword_names));
auto r = emitPPCall((void*)callattr, call_args, 2, 640, node, type_recorder).first->setType(RefType::OWNED);
for (int i = 3; i < args.size(); i++) {
args[i]->refUsed();
}
return r;
return emitPPCall((void*)callattr, call_args, 2, 640, node, type_recorder, additional_uses)
.first->setType(RefType::OWNED);
#else
// We could make this faster but for now: keep it simple, stupid...
RewriterVar* attr_var = imm(attr);
......@@ -223,7 +244,7 @@ RewriterVar* JitFragmentWriter::emitCallattr(AST_expr* node, RewriterVar* obj, B
RewriterVar* args_array = nullptr;
if (args.size())
args_array = allocArgs(args);
args_array = allocArgs(args, RewriterVar::SetattrType::REF_USED);
else
RELEASE_ASSERT(!keyword_names_var, "0 args but keyword names are set");
......@@ -244,7 +265,10 @@ RewriterVar* JitFragmentWriter::emitCallattr(AST_expr* node, RewriterVar* obj, B
}
RewriterVar* JitFragmentWriter::emitCompare(AST_expr* node, RewriterVar* lhs, RewriterVar* rhs, int op_type) {
// TODO: can directly emit the assembly for Is/IsNot
if (op_type == AST_TYPE::Is || op_type == AST_TYPE::IsNot) {
RewriterVar* cmp_result = lhs->cmp(op_type == AST_TYPE::IsNot ? AST_TYPE::NotEq : AST_TYPE::Eq, rhs);
return call(false, (void*)boxBool, cmp_result)->setType(RefType::OWNED);
}
return emitPPCall((void*)compare, { lhs, rhs, imm(op_type) }, 2, 240, node).first->setType(RefType::OWNED);
}
......@@ -258,42 +282,9 @@ void JitFragmentWriter::emitDictSet(RewriterVar* dict, RewriterVar* k, RewriterV
v->refConsumed();
}
// TODO: merge this function's functionality with refUsed
RewriterVar* JitFragmentWriter::emitCallWithAllocatedArgs(void* func_addr, const llvm::ArrayRef<RewriterVar*> args,
const llvm::ArrayRef<RewriterVar*> additional_uses) {
RewriterVar* result = createNewVar();
RewriterVar::SmallVector uses;
for (RewriterVar* v : args) {
assert(v != NULL);
uses.push_back(v);
}
for (RewriterVar* v : additional_uses) {
assert(v != NULL);
uses.push_back(v);
}
// It's not nice to pass llvm::SmallVectors through a closure, especially with our SmallFunction
// optimization, so just regionAlloc them and copy the data in:
RewriterVar** _args = (RewriterVar**)this->regionAlloc(sizeof(RewriterVar*) * args.size());
memcpy(_args, args.begin(), sizeof(RewriterVar*) * args.size());
RewriterVar** _args_additional = (RewriterVar**)this->regionAlloc(sizeof(RewriterVar*) * additional_uses.size());
memcpy(_args_additional, additional_uses.begin(), sizeof(RewriterVar*) * additional_uses.size());
int args_size = args.size();
assert(additional_uses.size() <= 0x7fff);
// Hack: pack this into a short to make sure it fits in the closure
short args_additional_size = additional_uses.size();
// Hack: explicitly order the closure arguments so they pad nicer
addAction([args_size, args_additional_size, func_addr, this, result, _args, _args_additional]() {
this->_call(result, false, func_addr, llvm::ArrayRef<RewriterVar*>(_args, args_size),
llvm::ArrayRef<RewriterVar*>());
for (int i = 0; i < args_size; i++)
_args[i]->bumpUse();
for (int i = 0; i < args_additional_size; i++)
_args_additional[i]->bumpUse();
}, uses, ActionType::NORMAL);
return result;
return call(false, func_addr, args, {}, additional_uses);
}
RewriterVar* JitFragmentWriter::emitCreateList(const llvm::ArrayRef<RewriterVar*> values) {
......@@ -361,7 +352,7 @@ RewriterVar* JitFragmentWriter::emitExceptionMatches(RewriterVar* v, RewriterVar
}
RewriterVar* JitFragmentWriter::emitGetAttr(RewriterVar* obj, BoxedString* s, AST_expr* node) {
return emitPPCall((void*)getattr, { obj, imm(s) }, 2, 512, node, getTypeRecorderForNode(node))
return emitPPCall((void*)getattr, { obj, imm(s) }, 2, 256, node, getTypeRecorderForNode(node))
.first->setType(RefType::OWNED);
}
......@@ -369,16 +360,15 @@ RewriterVar* JitFragmentWriter::emitGetBlockLocal(InternedString s, int vreg) {
auto it = local_syms.find(s);
if (it == local_syms.end()) {
auto r = emitGetLocal(s, vreg);
// TODO: clear out the vreg?
// assert(r->reftype == RefType::OWNED);
// emitSetLocal(s, vreg, false, imm(nullptr));
// emitSetBlockLocal(s, r);
assert(r->reftype == RefType::OWNED);
emitSetBlockLocal(s, vreg, r);
return r;
}
return it->second;
}
void JitFragmentWriter::emitKillTemporary(InternedString s, int vreg) {
if (!local_syms.count(s))
emitSetLocal(s, vreg, false, imm(nullptr));
}
......@@ -393,23 +383,21 @@ RewriterVar* JitFragmentWriter::emitGetBoxedLocals() {
}
RewriterVar* JitFragmentWriter::emitGetClsAttr(RewriterVar* obj, BoxedString* s) {
return emitPPCall((void*)getclsattr, { obj, imm(s) }, 2, 512).first->setType(RefType::OWNED);
return emitPPCall((void*)getclsattr, { obj, imm(s) }, 2, 256).first->setType(RefType::OWNED);
}
RewriterVar* JitFragmentWriter::emitGetGlobal(Box* global, BoxedString* s) {
RewriterVar* JitFragmentWriter::emitGetGlobal(BoxedString* s) {
if (s->s() == "None") {
RewriterVar* r = imm(None)->setType(RefType::BORROWED);
return r;
}
RewriterVar* args[] = { NULL, NULL };
args[0] = imm(global);
args[1] = imm(s);
return emitPPCall((void*)getGlobal, args, 2, 512).first->setType(RefType::OWNED);
RewriterVar* globals = getInterp()->getAttr(ASTInterpreterJitInterface::getGlobalsOffset());
return emitPPCall((void*)getGlobal, { globals, imm(s) }, 1, 128).first->setType(RefType::OWNED);
}
RewriterVar* JitFragmentWriter::emitGetItem(AST_expr* node, RewriterVar* value, RewriterVar* slice) {
return emitPPCall((void*)getitem, { value, slice }, 2, 512, node).first->setType(RefType::OWNED);
return emitPPCall((void*)getitem, { value, slice }, 1, 256, node).first->setType(RefType::OWNED);
}
RewriterVar* JitFragmentWriter::emitGetLocal(InternedString s, int vreg) {
......@@ -453,11 +441,11 @@ RewriterVar* JitFragmentWriter::emitLandingpad() {
RewriterVar* JitFragmentWriter::emitNonzero(RewriterVar* v) {
// nonzeroHelper returns bool
return call(false, (void*)nonzeroHelper, v)->setType(RefType::OWNED);
return call(false, (void*)nonzeroHelper, v)->setType(RefType::BORROWED);
}
RewriterVar* JitFragmentWriter::emitNotNonzero(RewriterVar* v) {
return call(false, (void*)notHelper, v)->setType(RefType::OWNED);
return call(false, (void*)notHelper, v)->setType(RefType::BORROWED);
}
RewriterVar* JitFragmentWriter::emitRepr(RewriterVar* v) {
......@@ -478,21 +466,18 @@ RewriterVar* JitFragmentWriter::emitRuntimeCall(AST_expr* node, RewriterVar* obj
call_args.push_back(args.size() > 1 ? args[1] : imm(0ul));
call_args.push_back(args.size() > 2 ? args[2] : imm(0ul));
llvm::ArrayRef<RewriterVar*> additional_uses;
if (args.size() > 3) {
RewriterVar* scratch = allocate(args.size() - 3);
for (int i = 0; i < args.size() - 3; ++i)
scratch->setAttr(i * sizeof(void*), args[i + 3], RewriterVar::SetattrType::REF_USED);
additional_uses = args.slice(3);
RewriterVar* scratch = allocArgs(additional_uses, RewriterVar::SetattrType::REF_USED);
call_args.push_back(scratch);
} else
call_args.push_back(imm(0ul));
if (keyword_names)
call_args.push_back(imm(keyword_names));
auto r = emitPPCall((void*)runtimeCall, call_args, 2, 640, node, type_recorder).first->setType(RefType::OWNED);
for (int i = 3; i < args.size(); i++) {
args[i]->refUsed();
}
return r;
return emitPPCall((void*)runtimeCall, call_args, 2, 640, node, type_recorder, additional_uses)
.first->setType(RefType::OWNED);
#else
RewriterVar* argspec_var = imm(argspec.asInt());
RewriterVar* keyword_names_var = keyword_names ? imm(keyword_names) : nullptr;
......@@ -535,22 +520,40 @@ std::vector<RewriterVar*> JitFragmentWriter::emitUnpackIntoArray(RewriterVar* v,
}
RewriterVar* JitFragmentWriter::emitYield(RewriterVar* v) {
auto rtn = call(false, (void*)ASTInterpreterJitInterface::yieldHelper, getInterp(), v)->setType(RefType::OWNED);
llvm::SmallVector<RewriterVar*, 16> local_args;
local_args.push_back(interp->getAttr(ASTInterpreterJitInterface::getCreatedClosureOffset()));
// we have to pass all owned references which are not stored in the vregs to yield() so that the GC can traverse it
for (auto&& sym : local_syms) {
if (sym.second == v)
continue;
if (sym.second->reftype == RefType::OWNED)
local_args.push_back(sym.second);
}
// erase duplicate entries
std::sort(local_args.begin(), local_args.end());
local_args.erase(std::unique(local_args.begin(), local_args.end()), local_args.end());
auto&& args = allocArgs(local_args, RewriterVar::SetattrType::REF_USED);
RewriterVar* generator = interp->getAttr(ASTInterpreterJitInterface::getGeneratorOffset());
auto rtn = call(false, (void*)yield, { generator, v, args, imm(local_args.size()) }, {}, local_args)
->setType(RefType::OWNED);
v->refConsumed();
return rtn;
}
void JitFragmentWriter::emitDelAttr(RewriterVar* target, BoxedString* attr) {
emitPPCall((void*)delattr, { target, imm(attr) }, 1, 512).first;
emitPPCall((void*)delattr, { target, imm(attr) }, 1, 144).first;
}
void JitFragmentWriter::emitDelGlobal(BoxedString* name) {
RewriterVar* globals = getInterp()->getAttr(ASTInterpreterJitInterface::getGlobalsOffset());
emitPPCall((void*)delGlobal, { globals, imm(name) }, 1, 512).first;
// does not get rewriten yet
// emitPPCall((void*)delGlobal, { globals, imm(name) }, 1, 512).first;
call(false, (void*)delGlobal, globals, imm(name));
}
void JitFragmentWriter::emitDelItem(RewriterVar* target, RewriterVar* slice) {
emitPPCall((void*)delitem, { target, slice }, 1, 512).first;
emitPPCall((void*)delitem, { target, slice }, 1, 256).first;
}
void JitFragmentWriter::emitDelName(InternedString name) {
......@@ -623,14 +626,16 @@ void JitFragmentWriter::emitReturn(RewriterVar* v) {
}
void JitFragmentWriter::emitSetAttr(AST_expr* node, RewriterVar* obj, BoxedString* s, STOLEN(RewriterVar*) attr) {
auto rtn = emitPPCall((void*)setattr, { obj, imm(s), attr }, 2, 512, node);
auto rtn = emitPPCall((void*)setattr, { obj, imm(s), attr }, 2, 256, node);
attr->refConsumed(rtn.second);
}
void JitFragmentWriter::emitSetBlockLocal(InternedString s, STOLEN(RewriterVar*) v) {
void JitFragmentWriter::emitSetBlockLocal(InternedString s, int vreg, STOLEN(RewriterVar*) v) {
if (LOG_BJIT_ASSEMBLY)
comment("BJIT: emitSetBlockLocal() start");
RewriterVar* prev = local_syms[s];
if (!prev)
emitSetLocal(s, vreg, false, imm(nullptr)); // clear out the vreg
local_syms[s] = v;
if (LOG_BJIT_ASSEMBLY)
comment("BJIT: emitSetBlockLocal() end");
......@@ -647,8 +652,13 @@ void JitFragmentWriter::emitSetExcInfo(RewriterVar* type, RewriterVar* value, Re
traceback->refConsumed();
}
void JitFragmentWriter::emitSetGlobal(Box* global, BoxedString* s, STOLEN(RewriterVar*) v) {
auto rtn = emitPPCall((void*)setGlobal, { imm(global), imm(s), v }, 2, 512);
void JitFragmentWriter::emitSetGlobal(BoxedString* s, STOLEN(RewriterVar*) v, bool are_globals_from_module) {
RewriterVar* globals = getInterp()->getAttr(ASTInterpreterJitInterface::getGlobalsOffset());
std::pair<RewriterVar*, RewriterAction*> rtn;
if (are_globals_from_module)
rtn = emitPPCall((void*)setattr, { globals, imm(s), v }, 1, 256);
else
rtn = emitPPCall((void*)setGlobal, { globals, imm(s), v }, 1, 256);
v->refConsumed(rtn.second);
}
......@@ -853,31 +863,38 @@ uint64_t JitFragmentWriter::asUInt(InternedString s) {
}
#endif
std::pair<RewriterVar*, RewriterAction*> JitFragmentWriter::emitPPCall(void* func_addr,
llvm::ArrayRef<RewriterVar*> args, int num_slots,
int slot_size, AST* ast_node,
TypeRecorder* type_recorder) {
std::pair<RewriterVar*, RewriterAction*>
JitFragmentWriter::emitPPCall(void* func_addr, llvm::ArrayRef<RewriterVar*> args, unsigned char num_slots,
unsigned short slot_size, AST* ast_node, TypeRecorder* type_recorder,
llvm::ArrayRef<RewriterVar*> additional_uses) {
if (LOG_BJIT_ASSEMBLY)
comment("BJIT: emitPPCall() start");
RewriterVar::SmallVector args_vec(args.begin(), args.end());
#if ENABLE_BASELINEJIT_ICS
RewriterVar* result = createNewVar();
auto args_array_ref = regionAllocArgs(args, additional_uses);
assert(args.size() < 1ul << 32);
int args_size = args.size();
RewriterVar** _args = (RewriterVar**)regionAlloc(sizeof(RewriterVar*) * args_size);
memcpy(_args, args.begin(), sizeof(RewriterVar*) * args_size);
assert(additional_uses.size() < 1 << 8);
unsigned char num_additional = additional_uses.size();
RewriterVar** args_array = args_array_ref.data();
RewriterAction* call_action = addAction([=]() {
this->_emitPPCall(result, func_addr, llvm::ArrayRef<RewriterVar*>(_args, args_size), num_slots, slot_size,
ast_node);
}, args, ActionType::NORMAL);
RewriterAction* call_action
= addAction([this, result, func_addr, ast_node, args_array, args_size, slot_size, num_slots, num_additional]() {
auto all_args = llvm::makeArrayRef(args_array, args_size + num_additional);
auto args = all_args.slice(0, args_size);
this->_emitPPCall(result, func_addr, args, num_slots, slot_size, ast_node, all_args);
}, args_array_ref, ActionType::NORMAL);
if (type_recorder) {
RewriterVar* type_recorder_var = imm(type_recorder);
RewriterVar* obj_cls_var = result->getAttr(offsetof(Box, cls));
addAction([=]() { _emitRecordType(type_recorder_var, obj_cls_var); }, { type_recorder_var, obj_cls_var },
ActionType::NORMAL);
result->refUsed();
addAction([=]() {
_emitRecordType(type_recorder_var, obj_cls_var);
result->bumpUse();
}, { type_recorder_var, obj_cls_var, result }, ActionType::NORMAL);
emitPendingCallsCheck();
return std::make_pair(result, call_action);
......@@ -952,12 +969,12 @@ Box* JitFragmentWriter::hasnextHelper(Box* b) {
return boxBool(pyston::hasnext(b));
}
Box* JitFragmentWriter::nonzeroHelper(Box* b) {
return boxBool(b->nonzeroIC());
BORROWED(Box*) JitFragmentWriter::nonzeroHelper(Box* b) {
return b->nonzeroIC() ? True : False;
}
Box* JitFragmentWriter::notHelper(Box* b) {
return boxBool(!b->nonzeroIC());
BORROWED(Box*) JitFragmentWriter::notHelper(Box* b) {
return b->nonzeroIC() ? False : True;
}
Box* JitFragmentWriter::runtimeCallHelper(Box* obj, ArgPassSpec argspec, TypeRecorder* type_recorder, Box** args,
......@@ -972,7 +989,7 @@ void JitFragmentWriter::_emitGetLocal(RewriterVar* val_var, const char* name) {
assembler::Register var_reg = val_var->getInReg();
assembler->test(var_reg, var_reg);
_setupCall(false, RewriterVar::SmallVector(), RewriterVar::SmallVector());
_setupCall(false, {});
{
assembler::ForwardJump jnz(*assembler, assembler::COND_NOT_ZERO);
assembler->mov(assembler::Immediate((uint64_t)name), assembler::RDI);
......@@ -1001,8 +1018,10 @@ void JitFragmentWriter::_emitJump(CFGBlock* b, RewriterVar* block_next, ExitInfo
exit_info.exit_start = assembler->curInstPointer();
block_next->getInReg(assembler::RAX, true);
assembler->add(assembler::Immediate(JitCodeBlock::sp_adjustment), assembler::RSP);
assembler->pop(assembler::R12);
assembler->pop(assembler::R13);
assembler->pop(assembler::R14);
assembler->pop(assembler::R15);
assembler->retq();
// make sure we have at least 'min_patch_size' of bytes available.
......@@ -1034,8 +1053,10 @@ void JitFragmentWriter::_emitOSRPoint() {
assembler->clear_reg(assembler::RAX); // = next block to execute
assembler->mov(assembler::Immediate(ASTInterpreterJitInterface::osr_dummy_value), assembler::RDX);
assembler->add(assembler::Immediate(JitCodeBlock::sp_adjustment), assembler::RSP);
assembler->pop(assembler::R12);
assembler->pop(assembler::R13);
assembler->pop(assembler::R14);
assembler->pop(assembler::R15);
assembler->retq();
}
interp->bumpUse();
......@@ -1043,7 +1064,8 @@ void JitFragmentWriter::_emitOSRPoint() {
}
void JitFragmentWriter::_emitPPCall(RewriterVar* result, void* func_addr, llvm::ArrayRef<RewriterVar*> args,
int num_slots, int slot_size, AST* ast_node) {
int num_slots, int slot_size, AST* ast_node,
llvm::ArrayRef<RewriterVar*> vars_to_bump) {
assembler::Register r = allocReg(assembler::R11);
if (args.size() > 6) { // only 6 args can get passed in registers.
......@@ -1052,11 +1074,9 @@ void JitFragmentWriter::_emitPPCall(RewriterVar* result, void* func_addr, llvm::
assembler::Register reg = args[i]->getInReg(Location::any(), true);
assembler->mov(reg, assembler::Indirect(assembler::RSP, sizeof(void*) * (i - 6)));
}
RewriterVar::SmallVector reg_args(args.begin(), args.begin() + 6);
assert(reg_args.size() == 6);
_setupCall(false, reg_args, RewriterVar::SmallVector());
_setupCall(false, args.slice(0, 6), {}, Location::any(), vars_to_bump);
} else
_setupCall(false, args, RewriterVar::SmallVector());
_setupCall(false, args, {}, Location::any(), vars_to_bump);
if (failed)
return;
......@@ -1101,15 +1121,8 @@ void JitFragmentWriter::_emitPPCall(RewriterVar* result, void* func_addr, llvm::
result->releaseIfNoUses();
// TODO: it would be nice to be able to bumpUse on these earlier so that we could potentially avoid spilling
// the args across the call if we don't need to.
// This had to get moved to the very end of this function due to the fact that bumpUse can cause refcounting
// operations to happen.
// I'm not sure though that just moving this earlier is good enough though -- we also might need to teach setupCall
// that the args might not be needed afterwards?
// Anyway this feels like micro-optimizations at the moment and we can figure it out later.
for (RewriterVar* arg : args) {
arg->bumpUse();
for (RewriterVar* use : vars_to_bump) {
use->bumpUseLateIfNecessary();
}
}
......@@ -1137,8 +1150,10 @@ void JitFragmentWriter::_emitReturn(RewriterVar* return_val) {
return_val->getInReg(assembler::RDX, true);
assembler->clear_reg(assembler::RAX);
assembler->add(assembler::Immediate(JitCodeBlock::sp_adjustment), assembler::RSP);
assembler->pop(assembler::R12);
assembler->pop(assembler::R13);
assembler->pop(assembler::R14);
assembler->pop(assembler::R15);
assembler->retq();
return_val->bumpUse();
}
......@@ -1156,12 +1171,12 @@ void JitFragmentWriter::_emitSideExit(STOLEN(RewriterVar*) var, RewriterVar* val
// Really, we should probably do a decref on either side post-jump.
// But the automatic refcounter doesn't support that, and since the value is either True or False,
// we can get away with doing the decref early.
// TODO: better solution is to just make NONZERO return a borrowed ref, so we don't have to decref at all.
if (var->reftype == RefType::OWNED) {
_decref(var);
// Hax: override the automatic refcount system
assert(var->reftype == RefType::OWNED);
assert(var->num_refs_consumed == 0);
var->reftype = RefType::BORROWED;
}
assert(var->num_refs_consumed == 0);
assembler::Register var_reg = var->getInReg();
if (isLargeConstant(val)) {
......
src/codegen/baseline_jit.h
View file @ e7174486
......@@ -23,6 +23,9 @@
namespace pyston {
// passes MAP_32BIT to mmap when allocating the memory for the bjit code.
// it's nice for inspecting the generated asm because the debugger is able to show the name of called C/C++ functions
#define ENABLE_BASELINEJIT_MAP_32BIT 1
#define ENABLE_BASELINEJIT_ICS 1
class AST_stmt;
......@@ -70,6 +73,7 @@ class JitFragmentWriter;
// register or stack slot but we aren't if it outlives the block - we have to store it in the interpreter instance.
//
// We use the following callee-save regs to speed up the generated code:
// r12, r15: temporary values
// r13: pointer to ASTInterpreter instance
// r14: pointer to the vregs array
//
......@@ -90,8 +94,10 @@ class JitFragmentWriter;
//
// Basic layout of generated code block is:
// entry_code:
// push %r15 ; save r15
// push %r14 ; save r14
// push %r13 ; save r13
// push %r12 ; save r12
// sub $0x118,%rsp ; setup scratch, 0x118 = scratch_size + 16 = space for two func args passed on the
// stack + 8 byte for stack alignment
// mov %rdi,%r13 ; copy the pointer to ASTInterpreter instance into r13
......@@ -107,8 +113,10 @@ class JitFragmentWriter;
// jne end_side_exit
// movabs $0x215bb60,%rax ; rax = CFGBlock* to interpret next (rax is the 1. return reg)
// add $0x118,%rsp ; restore stack pointer
// pop %r12 ; restore r12
// pop %r13 ; restore r13
// pop %r14 ; restore r14
// pop %r15 ; restore r15
// ret ; exit to the interpreter which will interpret the specified CFGBLock*
// end_side_exit:
// ....
......@@ -120,8 +128,10 @@ class JitFragmentWriter;
// in this case 0 which means we are finished
// movabs $0x1270014108,%rdx ; rdx must contain the Box* value to return
// add $0x118,%rsp ; restore stack pointer
// pop %r12 ; restore r12
// pop %r13 ; restore r13
// pop %r14 ; restore r14
// pop %r15 ; restore r15
// ret
//
// nth_JitFragment:
......@@ -140,8 +150,18 @@ public:
static constexpr int sp_adjustment = scratch_size + num_stack_args * 8 + 8 /* = alignment */;
private:
struct MemoryManager {
private:
uint8_t* addr;
public:
MemoryManager();
~MemoryManager();
uint8_t* get() { return addr; }
};
// the memory block contains the EH frame directly followed by the generated machine code.
std::unique_ptr<uint8_t[]> memory;
MemoryManager memory;
int entry_offset;
assembler::Assembler a;
bool is_currently_writing;
......@@ -234,7 +254,7 @@ public:
RewriterVar* emitGetBoxedLocal(BoxedString* s);
RewriterVar* emitGetBoxedLocals();
RewriterVar* emitGetClsAttr(RewriterVar* obj, BoxedString* s);
RewriterVar* emitGetGlobal(Box* global, BoxedString* s);
RewriterVar* emitGetGlobal(BoxedString* s);
RewriterVar* emitGetItem(AST_expr* node, RewriterVar* value, RewriterVar* slice);
RewriterVar* emitGetLocal(InternedString s, int vreg);
RewriterVar* emitGetPystonIter(RewriterVar* v);
......@@ -265,10 +285,10 @@ public:
void emitRaise3(RewriterVar* arg0, RewriterVar* arg1, RewriterVar* arg2);
void emitReturn(RewriterVar* v);
void emitSetAttr(AST_expr* node, RewriterVar* obj, BoxedString* s, STOLEN(RewriterVar*) attr);
void emitSetBlockLocal(InternedString s, STOLEN(RewriterVar*) v);
void emitSetBlockLocal(InternedString s, int vreg, STOLEN(RewriterVar*) v);
void emitSetCurrentInst(AST_stmt* node);
void emitSetExcInfo(RewriterVar* type, RewriterVar* value, RewriterVar* traceback);
void emitSetGlobal(Box* global, BoxedString* s, STOLEN(RewriterVar*) v);
void emitSetGlobal(BoxedString* s, STOLEN(RewriterVar*) v, bool are_globals_from_module);
void emitSetItemName(BoxedString* s, RewriterVar* v);
void emitSetItem(RewriterVar* target, RewriterVar* slice, RewriterVar* value);
void emitSetLocal(InternedString s, int vreg, bool set_closure, STOLEN(RewriterVar*) v);
......@@ -296,8 +316,9 @@ private:
RewriterVar* emitCallWithAllocatedArgs(void* func_addr, const llvm::ArrayRef<RewriterVar*> args,
const llvm::ArrayRef<RewriterVar*> additional_uses);
std::pair<RewriterVar*, RewriterAction*> emitPPCall(void* func_addr, llvm::ArrayRef<RewriterVar*> args,
int num_slots, int slot_size, AST* ast_node = NULL,
TypeRecorder* type_recorder = NULL);
unsigned char num_slots, unsigned short slot_size,
AST* ast_node = NULL, TypeRecorder* type_recorder = NULL,
llvm::ArrayRef<RewriterVar*> additional_uses = {});
static void assertNameDefinedHelper(const char* id);
static Box* callattrHelper(Box* obj, BoxedString* attr, CallattrFlags flags, TypeRecorder* type_recorder,
......@@ -308,8 +329,8 @@ private:
static Box* createTupleHelper(uint64_t num, Box** data);
static Box* exceptionMatchesHelper(Box* obj, Box* cls);
static Box* hasnextHelper(Box* b);
static Box* nonzeroHelper(Box* b);
static Box* notHelper(Box* b);
static BORROWED(Box*) nonzeroHelper(Box* b);
static BORROWED(Box*) notHelper(Box* b);
static Box* runtimeCallHelper(Box* obj, ArgPassSpec argspec, TypeRecorder* type_recorder, Box** args,
std::vector<BoxedString*>* keyword_names);
......@@ -317,7 +338,7 @@ private:
void _emitJump(CFGBlock* b, RewriterVar* block_next, ExitInfo& exit_info);
void _emitOSRPoint();
void _emitPPCall(RewriterVar* result, void* func_addr, llvm::ArrayRef<RewriterVar*> args, int num_slots,
int slot_size, AST* ast_node);
int slot_size, AST* ast_node, llvm::ArrayRef<RewriterVar*> vars_to_bump);
void _emitRecordType(RewriterVar* type_recorder_var, RewriterVar* obj_cls_var);
void _emitReturn(RewriterVar* v);
void _emitSideExit(STOLEN(RewriterVar*) var, RewriterVar* val_constant, CFGBlock* next_block,
......
src/codegen/unwinding.cpp
View file @ e7174486
......@@ -593,11 +593,7 @@ public:
assert(l.stack_second_offset % 8 == 0);
b = b_ptr[l.stack_second_offset / 8];
} else if (l.type == Location::Register) {
RELEASE_ASSERT(0, "untested");
// This branch should never get hit since we shouldn't generate Register locations,
// since we don't allow allocating callee-save registers.
// If we did, this code might be right:
// b = (Box*)get_cursor_reg(cursor, l.regnum);
b = (Box*)get_cursor_reg(cursor, l.regnum);
} else {
RELEASE_ASSERT(0, "not implemented");
}
......
src/runtime/objmodel.cpp
View file @ e7174486
......@@ -4870,32 +4870,32 @@ Box* callCLFunc(FunctionMetadata* md, CallRewriteArgs* rewrite_args, int num_out
} else {
// Hacky workaround: the rewriter can only pass arguments in registers, so use this helper function
// to unpack some of the additional arguments:
llvm::SmallVector<RewriterVar*, 4> additional_uses;
RewriterVar* arg_array = rewrite_args->rewriter->allocate(4);
arg_vec.push_back(arg_array);
if (num_output_args >= 1)
if (num_output_args >= 1) {
arg_array->setAttr(0, rewrite_args->arg1, RewriterVar::SetattrType::REF_USED);
if (num_output_args >= 2)
additional_uses.push_back(rewrite_args->arg1);
}
if (num_output_args >= 2) {
arg_array->setAttr(8, rewrite_args->arg2, RewriterVar::SetattrType::REF_USED);
if (num_output_args >= 3)
additional_uses.push_back(rewrite_args->arg2);
}
if (num_output_args >= 3) {
arg_array->setAttr(16, rewrite_args->arg3, RewriterVar::SetattrType::REF_USED);
if (num_output_args >= 4)
additional_uses.push_back(rewrite_args->arg3);
}
if (num_output_args >= 4) {
arg_array->setAttr(24, rewrite_args->args, RewriterVar::SetattrType::REF_USED);
additional_uses.push_back(rewrite_args->args);
}
if (S == CXX)
rewrite_args->out_rtn = rewrite_args->rewriter->call(true, (void*)astInterpretHelper, arg_vec)
->setType(RefType::OWNED);
rewrite_args->out_rtn = rewrite_args->rewriter->call(true, (void*)astInterpretHelper, arg_vec, {},
additional_uses)->setType(RefType::OWNED);
else
rewrite_args->out_rtn = rewrite_args->rewriter->call(true, (void*)astInterpretHelperCapi, arg_vec)
->setType(RefType::OWNED);
if (num_output_args >= 1)
rewrite_args->arg1->refUsed();
if (num_output_args >= 2)
rewrite_args->arg2->refUsed();
if (num_output_args >= 3)
rewrite_args->arg3->refUsed();
if (num_output_args >= 4)
rewrite_args->args->refUsed();
rewrite_args->out_rtn = rewrite_args->rewriter->call(true, (void*)astInterpretHelperCapi, arg_vec,
{}, additional_uses)->setType(RefType::OWNED);
}
rewrite_args->out_success = true;
......@@ -5732,8 +5732,8 @@ Box* compareInternal(Box* lhs, Box* rhs, int op_type, CompareRewriteArgs* rewrit
bool neg = (op_type == AST_TYPE::IsNot);
if (rewrite_args) {
RewriterVar* cmpres = rewrite_args->lhs->cmp(neg ? AST_TYPE::NotEq : AST_TYPE::Eq, rewrite_args->rhs,
rewrite_args->destination);
RewriterVar* cmpres
= rewrite_args->lhs->cmp(neg ? AST_TYPE::NotEq : AST_TYPE::Eq, rewrite_args->rhs, assembler::RDI);
rewrite_args->out_rtn
= rewrite_args->rewriter->call(false, (void*)boxBool, cmpres)->setType(RefType::OWNED);
rewrite_args->out_success = true;
......@@ -7302,6 +7302,9 @@ extern "C" Box* getGlobal(Box* globals, BoxedString* name) {
}
extern "C" void setGlobal(Box* globals, BoxedString* name, STOLEN(Box*) value) {
static StatCounter slowpath_setglobal("slowpath_setglobal");
slowpath_setglobal.log();
if (globals->cls == attrwrapper_cls) {
globals = unwrapAttrWrapper(globals);
RELEASE_ASSERT(globals->cls == module_cls, "%s", globals->cls->tp_name);
......
test/tests/iter_del_time.py
View file @ e7174486
# fail-if: '-n' in EXTRA_JIT_ARGS or '-O' in EXTRA_JIT_ARGS
# expected: fail
# this only works in the interpreter and not in the bjit and llvm jit
class C(object):
def next(self):
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7