Commit 28c0c4b8 authored by Marius Wachtler's avatar Marius Wachtler

Add signal support

We check for signals on most calls to runtime functions in the llvm tier and the bjit and inside the interpreter on some additional places,
because checking on every bytecode is a too large slowdown (>10%).
parent f11cb876
# expected: fail
import unittest
from test import test_support
from contextlib import closing
......@@ -80,7 +79,8 @@ class InterProcessSignalTests(unittest.TestCase):
# don't worry about re-setting the default handlers.
signal.signal(signal.SIGHUP, self.handlerA)
signal.signal(signal.SIGUSR1, self.handlerB)
signal.signal(signal.SIGUSR2, signal.SIG_IGN)
# Pyston change: pyston uses SIGUSR2 internally
# signal.signal(signal.SIGUSR2, signal.SIG_IGN)
signal.signal(signal.SIGALRM, signal.default_int_handler)
# Variables the signals will modify:
......@@ -117,9 +117,11 @@ class InterProcessSignalTests(unittest.TestCase):
if test_support.verbose:
print "HandlerBCalled exception caught"
child = ignoring_eintr(subprocess.Popen, ['kill', '-USR2', str(pid)])
if child:
self.wait(child) # Nothing should happen.
# Pyston change: pyston uses SIGUSR2 internally
# child = ignoring_eintr(subprocess.Popen, ['kill', '-USR2', str(pid)])
# if child:
# self.wait(child) # Nothing should happen.
try:
signal.alarm(1)
......
......@@ -626,6 +626,9 @@ initsignal(void)
old_siginthandler = PyOS_setsig(SIGINT, signal_handler);
}
// Pyston change: let the GC scan the handlers
PyGC_AddPotentialRoot(Handlers, sizeof(Handlers));
#ifdef SIGHUP
x = PyInt_FromLong(SIGHUP);
PyDict_SetItemString(d, "SIGHUP", x);
......@@ -895,10 +898,6 @@ PyErr_CheckSignals(void)
if (!is_tripped)
return 0;
// Pyston change:
Py_FatalError("TODO");
#if 0
int i;
PyObject *f;
......@@ -943,7 +942,6 @@ PyErr_CheckSignals(void)
Py_DECREF(result);
}
}
#endif
return 0;
}
......
......@@ -922,43 +922,66 @@ Value ASTInterpreter::visit_stmt(AST_stmt* node) {
printf("\n");
}
Value rtn;
switch (node->type) {
case AST_TYPE::Assert:
return visit_assert((AST_Assert*)node);
rtn = visit_assert((AST_Assert*)node);
ASTInterpreterJitInterface::pendingCallsCheckHelper();
break;
case AST_TYPE::Assign:
return visit_assign((AST_Assign*)node);
rtn = visit_assign((AST_Assign*)node);
ASTInterpreterJitInterface::pendingCallsCheckHelper();
break;
case AST_TYPE::Delete:
return visit_delete((AST_Delete*)node);
rtn = visit_delete((AST_Delete*)node);
ASTInterpreterJitInterface::pendingCallsCheckHelper();
break;
case AST_TYPE::Exec:
return visit_exec((AST_Exec*)node);
rtn = visit_exec((AST_Exec*)node);
ASTInterpreterJitInterface::pendingCallsCheckHelper();
break;
case AST_TYPE::Expr:
// docstrings are str constant expression statements.
// ignore those while interpreting.
if ((((AST_Expr*)node)->value)->type != AST_TYPE::Str)
return visit_expr((AST_Expr*)node);
if ((((AST_Expr*)node)->value)->type != AST_TYPE::Str) {
rtn = visit_expr((AST_Expr*)node);
ASTInterpreterJitInterface::pendingCallsCheckHelper();
}
break;
case AST_TYPE::Pass:
return Value(); // nothing todo
ASTInterpreterJitInterface::pendingCallsCheckHelper();
break; // nothing todo
case AST_TYPE::Print:
return visit_print((AST_Print*)node);
rtn = visit_print((AST_Print*)node);
ASTInterpreterJitInterface::pendingCallsCheckHelper();
break;
case AST_TYPE::Raise:
return visit_raise((AST_Raise*)node);
rtn = visit_raise((AST_Raise*)node);
ASTInterpreterJitInterface::pendingCallsCheckHelper();
break;
case AST_TYPE::Return:
return visit_return((AST_Return*)node);
rtn = visit_return((AST_Return*)node);
ASTInterpreterJitInterface::pendingCallsCheckHelper();
break;
case AST_TYPE::Global:
return visit_global((AST_Global*)node);
rtn = visit_global((AST_Global*)node);
ASTInterpreterJitInterface::pendingCallsCheckHelper();
break;
// pseudo
case AST_TYPE::Branch:
return visit_branch((AST_Branch*)node);
rtn = visit_branch((AST_Branch*)node);
break;
case AST_TYPE::Jump:
return visit_jump((AST_Jump*)node);
rtn = visit_jump((AST_Jump*)node);
break;
case AST_TYPE::Invoke:
return visit_invoke((AST_Invoke*)node);
rtn = visit_invoke((AST_Invoke*)node);
break;
default:
RELEASE_ASSERT(0, "not implemented");
};
return Value();
return rtn;
}
Value ASTInterpreter::visit_return(AST_Return* node) {
......@@ -1652,6 +1675,11 @@ Box* ASTInterpreterJitInterface::landingpadHelper(void* _interpreter) {
return rtn;
}
void ASTInterpreterJitInterface::pendingCallsCheckHelper() {
if (unlikely(_pendingcalls_to_do))
makePendingCalls();
}
Box* ASTInterpreterJitInterface::setExcInfoHelper(void* _interpreter, Box* type, Box* value, Box* traceback) {
ASTInterpreter* interpreter = (ASTInterpreter*)_interpreter;
interpreter->getFrameInfo()->exc = ExcInfo(type, value, traceback);
......
......@@ -45,6 +45,7 @@ struct ASTInterpreterJitInterface {
static Box* derefHelper(void* interp, InternedString s);
static Box* doOSRHelper(void* interp, AST_Jump* node);
static Box* landingpadHelper(void* interp);
static void pendingCallsCheckHelper();
static Box* setExcInfoHelper(void* interp, Box* type, Box* value, Box* traceback);
static void setLocalClosureHelper(void* interp, long vreg, InternedString id, Box* v);
static Box* uncacheExcInfoHelper(void* interp);
......
......@@ -474,6 +474,10 @@ void JitFragmentWriter::emitOSRPoint(AST_Jump* node) {
addAction([=]() { _emitOSRPoint(result, node_var); }, { result, node_var, getInterp() }, ActionType::NORMAL);
}
void JitFragmentWriter::emitPendingCallsCheck() {
call(false, (void*)ASTInterpreterJitInterface::pendingCallsCheckHelper);
}
void JitFragmentWriter::emitPrint(RewriterVar* dest, RewriterVar* var, bool nl) {
if (!dest)
dest = call(false, (void*)getSysStdout);
......@@ -696,12 +700,17 @@ RewriterVar* JitFragmentWriter::emitPPCall(void* func_addr, llvm::ArrayRef<Rewri
RewriterVar* obj_cls_var = result->getAttr(offsetof(Box, cls));
addAction([=]() { _emitRecordType(type_recorder_var, obj_cls_var); }, { type_recorder_var, obj_cls_var },
ActionType::NORMAL);
emitPendingCallsCheck();
return result;
}
emitPendingCallsCheck();
return result;
#else
assert(args_vec.size() < 7);
return call(false, func_addr, args_vec);
RewriterVar* result = call(false, func_addr, args_vec);
emitPendingCallsCheck();
return result;
#endif
}
......
......@@ -246,6 +246,7 @@ public:
void emitExec(RewriterVar* code, RewriterVar* globals, RewriterVar* locals, FutureFlags flags);
void emitJump(CFGBlock* b);
void emitOSRPoint(AST_Jump* node);
void emitPendingCallsCheck();
void emitPrint(RewriterVar* dest, RewriterVar* var, bool nl);
void emitRaise0();
void emitRaise3(RewriterVar* arg0, RewriterVar* arg1, RewriterVar* arg2);
......
......@@ -378,16 +378,6 @@ static void handle_sigprof_investigate_stattimer(int signum) {
}
#endif
static void handle_sigint(int signum) {
assert(signum == SIGINT);
// TODO: this should set a flag saying a KeyboardInterrupt is pending.
// For now, just call abort(), so that we get a traceback at least.
fprintf(stderr, "SIGINT!\n");
joinRuntime();
Stats::dump(false);
abort();
}
void initCodegen() {
llvm::InitializeNativeTarget();
llvm::InitializeNativeTargetAsmPrinter();
......@@ -481,9 +471,8 @@ void initCodegen() {
setupRuntime();
// signal(SIGFPE, &handle_sigfpe);
signal(SIGUSR1, &handle_sigusr1);
signal(SIGINT, &handle_sigint);
// signal(SIGFPE, &handle_sigfpe);
// signal(SIGUSR1, &handle_sigusr1);
#if ENABLE_SAMPLING_PROFILER
struct itimerval prof_timer;
......
......@@ -347,11 +347,53 @@ private:
llvm::BasicBlock*& curblock;
IRGenerator* irgenerator;
void emitPendingCallsCheck(llvm::BasicBlock* exc_dest) {
auto&& builder = *getBuilder();
llvm::GlobalVariable* pendingcalls_to_do_gv = g.cur_module->getGlobalVariable("_pendingcalls_to_do");
if (!pendingcalls_to_do_gv) {
static_assert(sizeof(_pendingcalls_to_do) == 4, "");
pendingcalls_to_do_gv = new llvm::GlobalVariable(
*g.cur_module, g.i32, false, llvm::GlobalValue::ExternalLinkage, 0, "_pendingcalls_to_do");
pendingcalls_to_do_gv->setAlignment(4);
}
llvm::BasicBlock* cur_block = builder.GetInsertBlock();
llvm::BasicBlock* pendingcalls_set = createBasicBlock("_pendingcalls_set");
pendingcalls_set->moveAfter(cur_block);
llvm::BasicBlock* join_block = createBasicBlock("continue_after_pendingcalls_check");
join_block->moveAfter(pendingcalls_set);
llvm::Value* pendingcalls_to_do_val = builder.CreateLoad(pendingcalls_to_do_gv, true /* volatile */);
llvm::Value* is_zero
= builder.CreateICmpEQ(pendingcalls_to_do_val, getConstantInt(0, pendingcalls_to_do_val->getType()));
llvm::Metadata* md_vals[]
= { llvm::MDString::get(g.context, "branch_weights"), llvm::ConstantAsMetadata::get(getConstantInt(1000)),
llvm::ConstantAsMetadata::get(getConstantInt(1)) };
llvm::MDNode* branch_weights = llvm::MDNode::get(g.context, llvm::ArrayRef<llvm::Metadata*>(md_vals));
builder.CreateCondBr(is_zero, join_block, pendingcalls_set, branch_weights);
{
setCurrentBasicBlock(pendingcalls_set);
if (exc_dest) {
builder.CreateInvoke(g.funcs.makePendingCalls, join_block, exc_dest);
} else {
builder.CreateCall(g.funcs.makePendingCalls);
builder.CreateBr(join_block);
}
}
cur_block = join_block;
setCurrentBasicBlock(join_block);
}
llvm::CallSite emitCall(const UnwindInfo& unw_info, llvm::Value* callee, const std::vector<llvm::Value*>& args,
ExceptionStyle target_exception_style) {
llvm::Value* stmt = unw_info.current_stmt ? embedRelocatablePtr(unw_info.current_stmt, g.llvm_aststmt_type_ptr)
: getNullPtr(g.llvm_aststmt_type_ptr);
getBuilder()->CreateStore(stmt, irstate->getStmtVar());
emitSetCurrentStmt(unw_info.current_stmt);
if (target_exception_style == CXX && (unw_info.hasHandler() || irstate->getExceptionStyle() == CAPI)) {
// Create the invoke:
......@@ -375,9 +417,13 @@ private:
// Normal case:
getBuilder()->SetInsertPoint(normal_dest);
curblock = normal_dest;
emitPendingCallsCheck(exc_dest);
return rtn;
} else {
llvm::CallInst* cs = getBuilder()->CreateCall(callee, args);
if (target_exception_style == CXX)
emitPendingCallsCheck(NULL);
return cs;
}
}
......@@ -479,6 +525,15 @@ public:
return llvm::BasicBlock::Create(g.context, name, irstate->getLLVMFunction());
}
// Our current frame introspection approach requires that we update the currently executed stmt before doing a call
// to a function which could throw an exception, inspect the python call frame,...
// Only patchpoint don't need to set the current statement because the stmt will be inluded in the stackmap args.
void emitSetCurrentStmt(AST_stmt* stmt) {
getBuilder()->CreateStore(stmt ? embedRelocatablePtr(stmt, g.llvm_aststmt_type_ptr)
: getNullPtr(g.llvm_aststmt_type_ptr),
irstate->getStmtVar());
}
llvm::Value* createCall(const UnwindInfo& unw_info, llvm::Value* callee, const std::vector<llvm::Value*>& args,
ExceptionStyle target_exception_style = CXX) override {
#ifndef NDEBUG
......@@ -2137,7 +2192,7 @@ private:
// Don't call deinitFrame when this is a OSR function because the interpreter will call it
if (!irstate->getCurFunction()->entry_descriptor)
emitter.createCall(unw_info, g.funcs.deinitFrame, irstate->getFrameInfoVar());
emitter.getBuilder()->CreateCall(g.funcs.deinitFrame, irstate->getFrameInfoVar());
for (auto& p : symbol_table) {
p.second->decvref(emitter);
......@@ -2878,9 +2933,10 @@ public:
}
void doSafePoint(AST_stmt* next_statement) override {
// Unwind info is always needed in allowGLReadPreemption if it has any chance of
// running arbitrary code like finalizers.
emitter.createCall(UnwindInfo(next_statement, NULL), g.funcs.allowGLReadPreemption);
// We need to setup frame introspection by updating the current stmt because we can run can run arbitrary code
// like finalizers inside allowGLReadPreemption.
emitter.emitSetCurrentStmt(next_statement);
emitter.getBuilder()->CreateCall(g.funcs.allowGLReadPreemption);
}
// Create a (or reuse an existing) block that will catch a CAPI exception, and then forward
......@@ -2902,8 +2958,9 @@ public:
assert(!phi_node);
phi_node = emitter.getBuilder()->CreatePHI(g.llvm_aststmt_type_ptr, 0);
emitter.createCall(UnwindInfo(current_stmt, NULL), g.funcs.caughtCapiException,
{ phi_node, embedRelocatablePtr(irstate->getSourceInfo(), g.i8_ptr) });
emitter.emitSetCurrentStmt(current_stmt);
emitter.getBuilder()->CreateCall(g.funcs.caughtCapiException,
{ phi_node, embedRelocatablePtr(irstate->getSourceInfo(), g.i8_ptr) });
if (!final_dest) {
// Propagate the exception out of the function:
......@@ -2911,7 +2968,7 @@ public:
emitter.getBuilder()->CreateCall(g.funcs.reraiseCapiExcAsCxx);
emitter.getBuilder()->CreateUnreachable();
} else {
emitter.createCall(UnwindInfo(current_stmt, NULL), g.funcs.deinitFrame, irstate->getFrameInfoVar());
emitter.getBuilder()->CreateCall(g.funcs.deinitFrame, irstate->getFrameInfoVar());
emitter.getBuilder()->CreateRet(getNullPtr(g.llvm_value_type_ptr));
}
} else {
......
......@@ -200,6 +200,7 @@ void initGlobalFuncs(GlobalState& g) {
GET(createSet);
GET(initFrame);
GET(deinitFrame);
GET(makePendingCalls);
GET(getattr);
GET(getattr_capi);
......
......@@ -34,7 +34,7 @@ struct GlobalFuncs {
llvm::Value* boxInt, *unboxInt, *boxFloat, *unboxFloat, *createFunctionFromMetadata, *getFunctionMetadata,
*boxInstanceMethod, *boxBool, *unboxBool, *createTuple, *createDict, *createList, *createSlice,
*createUserClass, *createClosure, *createGenerator, *createSet, *initFrame, *deinitFrame;
*createUserClass, *createClosure, *createGenerator, *createSet, *initFrame, *deinitFrame, *makePendingCalls;
llvm::Value* getattr, *getattr_capi, *setattr, *delattr, *delitem, *delGlobal, *nonzero, *binop, *compare,
*augbinop, *unboxedLen, *getitem, *getitem_capi, *getclsattr, *getGlobal, *setitem, *unaryop, *import,
*importFrom, *importStar, *repr, *exceptionMatches, *yield, *getiterHelper, *hasnext, *setGlobal, *apply_slice;
......
......@@ -497,9 +497,13 @@ static void* find_stack() {
return NULL; /* not found =^P */
}
static long main_thread_id;
void registerMainThread() {
LOCK_REGION(&threading_lock);
main_thread_id = pthread_self();
assert(!current_internal_thread_state);
current_internal_thread_state = new ThreadStateInternal(find_stack(), pthread_self(), &cur_thread_state);
current_threads[pthread_self()] = current_internal_thread_state;
......@@ -524,6 +528,10 @@ void finishMainThread() {
// TODO maybe this is the place to wait for non-daemon threads?
}
bool isMainThread() {
return pthread_self() == main_thread_id;
}
// For the "AllowThreads" regions, let's save the thread state at the beginning of the region.
// This means that the thread won't get interrupted by the signals we would otherwise need to
......
......@@ -51,6 +51,8 @@ intptr_t start_thread(void* (*start_func)(Box*, Box*, Box*), Box* arg1, Box* arg
void registerMainThread();
void finishMainThread();
bool isMainThread();
// Hook for the GC; will visit all the threads (including the current one), visiting their
// stacks and thread-local PyThreadState objects
void visitAllStacks(gc::GCVisitor* v);
......
......@@ -17,6 +17,7 @@
#include <string.h>
#include "Python.h"
#include "pythread.h"
#include "codegen/cpython_ast.h"
#include "grammar.h"
......@@ -1547,9 +1548,126 @@ extern "C" PyOS_sighandler_t PyOS_setsig(int sig, PyOS_sighandler_t handler) noe
#endif
}
static PyThread_type_lock pending_lock = 0; /* for pending calls */
/* The WITH_THREAD implementation is thread-safe. It allows
scheduling to be made from any thread, and even from an executing
callback.
*/
#define NPENDINGCALLS 32
static struct {
int (*func)(void*);
void* arg;
} pendingcalls[NPENDINGCALLS];
static int pendingfirst = 0;
static int pendinglast = 0;
// Pyston change
// static volatile int pendingcalls_to_do = 1; /* trigger initialization of lock */
extern "C" {
volatile int _pendingcalls_to_do = 1;
}
static char pendingbusy = 0;
extern "C" int Py_AddPendingCall(int (*func)(void*), void* arg) noexcept {
fatalOrError(PyExc_NotImplementedError, "unimplemented");
return -1;
int i, j, result = 0;
PyThread_type_lock lock = pending_lock;
/* try a few times for the lock. Since this mechanism is used
* for signal handling (on the main thread), there is a (slim)
* chance that a signal is delivered on the same thread while we
* hold the lock during the Py_MakePendingCalls() function.
* This avoids a deadlock in that case.
* Note that signals can be delivered on any thread. In particular,
* on Windows, a SIGINT is delivered on a system-created worker
* thread.
* We also check for lock being NULL, in the unlikely case that
* this function is called before any bytecode evaluation takes place.
*/
if (lock != NULL) {
for (i = 0; i < 100; i++) {
if (PyThread_acquire_lock(lock, NOWAIT_LOCK))
break;
}
if (i == 100)
return -1;
}
i = pendinglast;
j = (i + 1) % NPENDINGCALLS;
if (j == pendingfirst) {
result = -1; /* Queue full */
} else {
pendingcalls[i].func = func;
pendingcalls[i].arg = arg;
pendinglast = j;
}
/* signal main loop */
// Pyston change: we don't have a _Py_Ticker
// _Py_Ticker = 0;
_pendingcalls_to_do = 1;
if (lock != NULL)
PyThread_release_lock(lock);
return result;
}
extern "C" int Py_MakePendingCalls(void) noexcept {
int i;
int r = 0;
if (!pending_lock) {
/* initial allocation of the lock */
pending_lock = PyThread_allocate_lock();
if (pending_lock == NULL)
return -1;
// Pyston change: we could potentialy store a python object inside the arg field
PyGC_AddPotentialRoot(pendingcalls, sizeof(pendingcalls));
}
/* only service pending calls on main thread */
// Pyston change:
// if (main_thread && PyThread_get_thread_ident() != main_thread)
if (!threading::isMainThread())
return 0;
/* don't perform recursive pending calls */
if (pendingbusy)
return 0;
pendingbusy = 1;
/* perform a bounded number of calls, in case of recursion */
for (i = 0; i < NPENDINGCALLS; i++) {
int j;
int (*func)(void*);
void* arg = NULL;
/* pop one item off the queue while holding the lock */
PyThread_acquire_lock(pending_lock, WAIT_LOCK);
j = pendingfirst;
if (j == pendinglast) {
func = NULL; /* Queue empty */
} else {
func = pendingcalls[j].func;
arg = pendingcalls[j].arg;
pendingfirst = (j + 1) % NPENDINGCALLS;
}
_pendingcalls_to_do = pendingfirst != pendinglast;
PyThread_release_lock(pending_lock);
/* having released the lock, perform the callback */
if (func == NULL)
break;
r = func(arg);
if (r)
break;
}
pendingbusy = 0;
return r;
}
extern "C" void makePendingCalls() {
int ret = Py_MakePendingCalls();
if (ret != 0)
throwCAPIException();
}
extern "C" PyObject* _PyImport_FixupExtension(char* name, char* filename) noexcept {
......@@ -1716,6 +1834,16 @@ extern "C" void PyEval_RestoreThread(PyThreadState* tstate) noexcept {
endAllowThreads();
}
extern "C" struct _frame* PyEval_GetFrame(void) noexcept {
Box* frame = NULL;
try {
frame = getFrame(0);
} catch (ExcInfo) {
RELEASE_ASSERT(0, "untested");
}
return (struct _frame*)frame;
}
extern "C" char* PyModule_GetName(PyObject* m) noexcept {
PyObject* d;
PyObject* nameobj;
......
......@@ -74,6 +74,7 @@ void force() {
FORCE(decodeUTF8StringPtr);
FORCE(initFrame);
FORCE(deinitFrame);
FORCE(makePendingCalls);
FORCE(getattr);
FORCE(getattr_capi);
......
......@@ -170,6 +170,7 @@ extern "C" Box* createDict();
extern "C" Box* createList();
extern "C" Box* createSlice(Box* start, Box* stop, Box* step);
extern "C" Box* createTuple(int64_t nelts, Box** elts);
extern "C" void makePendingCalls();
Box* objectStr(Box*);
Box* objectRepr(Box*);
......@@ -1181,6 +1182,8 @@ inline Box*& getArg(int idx, Box*& arg1, Box*& arg2, Box*& arg3, Box** args) {
return arg3;
return args[idx - 3];
}
extern "C" volatile int _pendingcalls_to_do;
}
#endif
......@@ -183,7 +183,6 @@ test_scope eval of code object from existing function (not currentl
test_scriptpackages [unknown]
test_shelve [unknown]
test_shlex [unknown]
test_signal [unknown]
test_site [unknown]
test_smtpnet [unknown]
test_socketserver [unknown]
......
......@@ -6,3 +6,23 @@ for k in sorted(dir(signal)):
print k, getattr(signal, k)
print hasattr(signal, "alarm")
import time
import signal
def sig_handler(signum, stack):
print "inside sig_handler"
import sys, traceback
traceback.print_stack(stack)
sys.exit(0)
def f(lst):
signal.signal(signal.SIGALRM, sig_handler)
signal.setitimer(signal.ITIMER_REAL, 2, 1)
for x in lst:
time.sleep(x) #1
time.sleep(x) #2
f([0] * 100 + [10])
assert False, "shuld not get executed"
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment