Skip to content

G
go
Commit 9cd57068 authored by Keith Randall's avatar Keith Randall
Browse files
Options

runtime: reimplement reflect.call to not use stack splitting. 

R=golang-dev, r, khr, rsc
CC=golang-dev
https://golang.org/cl/12053043
parent b8c8cb85
Hide whitespace changes
Inline Side-by-side
Showing with 321 additions and 24 deletions
misc/cgo/test/callback.go
View file @ 9cd57068
......@@ -151,6 +151,7 @@ func testCallbackCallers(t *testing.T) {
n := 0
name := []string{
"test.goCallback",
"runtime.call16",
"runtime.cgocallbackg1",
"runtime.cgocallbackg",
"runtime.cgocallback_gofunc",
......
src/cmd/ld/lib.c
View file @ 9cd57068
......@@ -1522,7 +1522,7 @@ pctospadj(Sym *sym, int32 oldval, Prog *p, int32 phase, int32 arg)
oldval = 0;
if(phase == 0)
return oldval;
if(oldval + p->spadj < -10000 || oldval + p->spadj > 1000000000) {
if(oldval + p->spadj < -10000 || oldval + p->spadj > 1100000000) {
diag("overflow in spadj: %d + %d = %d", oldval, p->spadj, oldval + p->spadj);
errorexit();
}
......
src/pkg/reflect/all_test.go
View file @ 9cd57068
......@@ -3509,3 +3509,14 @@ func (x *exhaustive) Choose(max int) int {
func (x *exhaustive) Maybe() bool {
return x.Choose(2) == 1
}
func bigArgFunc(v [(1<<30)+64]byte) {
}
func TestBigArgs(t *testing.T) {
if !testing.Short() && ^uint(0)>>32 != 0 { // test on 64-bit only
v := new([(1<<30)+64]byte)
bigArgFunc(*v) // regular calls are ok
shouldPanic(func() {ValueOf(bigArgFunc).Call([]Value{ValueOf(*v)})}) // ... just not reflect calls
}
}
src/pkg/runtime/asm_386.s
View file @ 9cd57068
......@@ -244,12 +244,12 @@ TEXT runtime·morestack(SB),7,$0-0
MOVL $0, 0x1003 // crash if newstack returns
RET
// Called from reflection library. Mimics morestack,
// Called from panic. Mimics morestack,
// reuses stack growth code to create a frame
// with the desired args running the desired function.
//
// func call(fn *byte, arg *byte, argsize uint32).
TEXT reflect·call(SB), 7, $0-12
TEXT runtime·newstackcall(SB), 7, $0-12
get_tls(CX)
MOVL m(CX), BX
......@@ -264,12 +264,12 @@ TEXT reflect·call(SB), 7, $0-12
// Save our own state as the PC and SP to restore
// if this goroutine needs to be restarted.
MOVL $reflect·call(SB), (g_sched+gobuf_pc)(AX)
MOVL $runtime·newstackcall(SB), (g_sched+gobuf_pc)(AX)
MOVL SP, (g_sched+gobuf_sp)(AX)
// Set up morestack arguments to call f on a new stack.
// We set f's frame size to 1, as a hint to newstack
// that this is a call from reflect·call.
// that this is a call from runtime·newstackcall.
// If it turns out that f needs a larger frame than
// the default stack, f's usual stack growth prolog will
// allocate a new segment (and recopy the arguments).
......@@ -291,6 +291,95 @@ TEXT reflect·call(SB), 7, $0-12
MOVL $0, 0x1103 // crash if newstack returns
RET
// reflect·call: call a function with the given argument list
// func call(f *FuncVal, arg *byte, argsize uint32).
// we don't have variable-sized frames, so we use a small number
// of constant-sized-frame functions to encode a few bits of size in the pc.
// Caution: ugly multiline assembly macros in your future!
#define DISPATCH(NAME,MAXSIZE) \
CMPL CX, $MAXSIZE; \
JA 3(PC); \
MOVL $runtime·NAME(SB), AX; \
JMP AX
// Note: can't just "JMP runtime·NAME(SB)" - bad inlining results.
TEXT reflect·call(SB), 7, $0-12
MOVL argsize+8(FP), CX
DISPATCH(call16, 16)
DISPATCH(call32, 32)
DISPATCH(call64, 64)
DISPATCH(call128, 128)
DISPATCH(call256, 256)
DISPATCH(call512, 512)
DISPATCH(call1024, 1024)
DISPATCH(call2048, 2048)
DISPATCH(call4096, 4096)
DISPATCH(call8192, 8192)
DISPATCH(call16384, 16384)
DISPATCH(call32768, 32768)
DISPATCH(call65536, 65536)
DISPATCH(call131072, 131072)
DISPATCH(call262144, 262144)
DISPATCH(call524288, 524288)
DISPATCH(call1048576, 1048576)
DISPATCH(call2097152, 2097152)
DISPATCH(call4194304, 4194304)
DISPATCH(call8388608, 8388608)
DISPATCH(call16777216, 16777216)
DISPATCH(call33554432, 33554432)
DISPATCH(call67108864, 67108864)
DISPATCH(call134217728, 134217728)
DISPATCH(call268435456, 268435456)
DISPATCH(call536870912, 536870912)
DISPATCH(call1073741824, 1073741824)
MOVL $runtime·badreflectcall(SB), AX
JMP AX
#define CALLFN(NAME,MAXSIZE,FLAGS) \
TEXT runtime·NAME(SB), FLAGS, $MAXSIZE-12; \
/* copy arguments to stack */ \
MOVL argptr+4(FP), SI; \
MOVL argsize+8(FP), CX; \
MOVL SP, DI; \
REP;MOVSB; \
/* call function */ \
MOVL f+0(FP), DX; \
CALL (DX); \
/* copy return values back */ \
MOVL argptr+4(FP), DI; \
MOVL argsize+8(FP), CX; \
MOVL SP, SI; \
REP;MOVSB; \
RET
CALLFN(call16, 16, 7)
CALLFN(call32, 32, 7)
CALLFN(call64, 64, 7)
CALLFN(call128, 128, 0)
CALLFN(call256, 256, 0)
CALLFN(call512, 512, 0)
CALLFN(call1024, 1024, 0)
CALLFN(call2048, 2048, 0)
CALLFN(call4096, 4096, 0)
CALLFN(call8192, 8192, 0)
CALLFN(call16384, 16384, 0)
CALLFN(call32768, 32768, 0)
CALLFN(call65536, 65536, 0)
CALLFN(call131072, 131072, 0)
CALLFN(call262144, 262144, 0)
CALLFN(call524288, 524288, 0)
CALLFN(call1048576, 1048576, 0)
CALLFN(call2097152, 2097152, 0)
CALLFN(call4194304, 4194304, 0)
CALLFN(call8388608, 8388608, 0)
CALLFN(call16777216, 16777216, 0)
CALLFN(call33554432, 33554432, 0)
CALLFN(call67108864, 67108864, 0)
CALLFN(call134217728, 134217728, 0)
CALLFN(call268435456, 268435456, 0)
CALLFN(call536870912, 536870912, 0)
CALLFN(call1073741824, 1073741824, 0)
// Return point when leaving stack.
//
......
src/pkg/runtime/asm_amd64.s
View file @ 9cd57068
......@@ -225,12 +225,12 @@ TEXT runtime·morestack(SB),7,$0-0
MOVQ $0, 0x1003 // crash if newstack returns
RET
// Called from reflection library. Mimics morestack,
// Called from panic. Mimics morestack,
// reuses stack growth code to create a frame
// with the desired args running the desired function.
//
// func call(fn *byte, arg *byte, argsize uint32).
TEXT reflect·call(SB), 7, $0-20
TEXT runtime·newstackcall(SB), 7, $0-20
get_tls(CX)
MOVQ m(CX), BX
......@@ -245,12 +245,12 @@ TEXT reflect·call(SB), 7, $0-20
// Save our own state as the PC and SP to restore
// if this goroutine needs to be restarted.
MOVQ $reflect·call(SB), (g_sched+gobuf_pc)(AX)
MOVQ $runtime·newstackcall(SB), (g_sched+gobuf_pc)(AX)
MOVQ SP, (g_sched+gobuf_sp)(AX)
// Set up morestack arguments to call f on a new stack.
// We set f's frame size to 1, as a hint to newstack
// that this is a call from reflect·call.
// that this is a call from runtime·newstackcall.
// If it turns out that f needs a larger frame than
// the default stack, f's usual stack growth prolog will
// allocate a new segment (and recopy the arguments).
......@@ -272,6 +272,96 @@ TEXT reflect·call(SB), 7, $0-20
MOVQ $0, 0x1103 // crash if newstack returns
RET
// reflect·call: call a function with the given argument list
// func call(f *FuncVal, arg *byte, argsize uint32).
// we don't have variable-sized frames, so we use a small number
// of constant-sized-frame functions to encode a few bits of size in the pc.
// Caution: ugly multiline assembly macros in your future!
#define DISPATCH(NAME,MAXSIZE) \
CMPQ CX, $MAXSIZE; \
JA 3(PC); \
MOVQ $runtime·NAME(SB), AX; \
JMP AX
// Note: can't just "JMP runtime·NAME(SB)" - bad inlining results.
TEXT reflect·call(SB), 7, $0-20
MOVLQZX argsize+16(FP), CX
DISPATCH(call16, 16)
DISPATCH(call32, 32)
DISPATCH(call64, 64)
DISPATCH(call128, 128)
DISPATCH(call256, 256)
DISPATCH(call512, 512)
DISPATCH(call1024, 1024)
DISPATCH(call2048, 2048)
DISPATCH(call4096, 4096)
DISPATCH(call8192, 8192)
DISPATCH(call16384, 16384)
DISPATCH(call32768, 32768)
DISPATCH(call65536, 65536)
DISPATCH(call131072, 131072)
DISPATCH(call262144, 262144)
DISPATCH(call524288, 524288)
DISPATCH(call1048576, 1048576)
DISPATCH(call2097152, 2097152)
DISPATCH(call4194304, 4194304)
DISPATCH(call8388608, 8388608)
DISPATCH(call16777216, 16777216)
DISPATCH(call33554432, 33554432)
DISPATCH(call67108864, 67108864)
DISPATCH(call134217728, 134217728)
DISPATCH(call268435456, 268435456)
DISPATCH(call536870912, 536870912)
DISPATCH(call1073741824, 1073741824)
MOVQ $runtime·badreflectcall(SB), AX
JMP AX
#define CALLFN(NAME,MAXSIZE,FLAGS) \
TEXT runtime·NAME(SB), FLAGS, $MAXSIZE-20; \
/* copy arguments to stack */ \
MOVQ argptr+8(FP), SI; \
MOVLQZX argsize+16(FP), CX; \
MOVQ SP, DI; \
REP;MOVSB; \
/* call function */ \
MOVQ f+0(FP), DX; \
CALL (DX); \
/* copy return values back */ \
MOVQ argptr+8(FP), DI; \
MOVLQZX argsize+16(FP), CX; \
MOVQ SP, SI; \
REP;MOVSB; \
RET
CALLFN(call16, 16, 7)
CALLFN(call32, 32, 7)
CALLFN(call64, 64, 7)
CALLFN(call128, 128, 0)
CALLFN(call256, 256, 0)
CALLFN(call512, 512, 0)
CALLFN(call1024, 1024, 0)
CALLFN(call2048, 2048, 0)
CALLFN(call4096, 4096, 0)
CALLFN(call8192, 8192, 0)
CALLFN(call16384, 16384, 0)
CALLFN(call32768, 32768, 0)
CALLFN(call65536, 65536, 0)
CALLFN(call131072, 131072, 0)
CALLFN(call262144, 262144, 0)
CALLFN(call524288, 524288, 0)
CALLFN(call1048576, 1048576, 0)
CALLFN(call2097152, 2097152, 0)
CALLFN(call4194304, 4194304, 0)
CALLFN(call8388608, 8388608, 0)
CALLFN(call16777216, 16777216, 0)
CALLFN(call33554432, 33554432, 0)
CALLFN(call67108864, 67108864, 0)
CALLFN(call134217728, 134217728, 0)
CALLFN(call268435456, 268435456, 0)
CALLFN(call536870912, 536870912, 0)
CALLFN(call1073741824, 1073741824, 0)
// Return point when leaving stack.
//
// Lessstack can appear in stack traces for the same reason
......
src/pkg/runtime/asm_arm.s
View file @ 9cd57068
......@@ -208,12 +208,12 @@ TEXT runtime·morestack(SB),7,$-4-0
// is still in this function, and not the beginning of the next.
RET
// Called from reflection library. Mimics morestack,
// Called from panic. Mimics morestack,
// reuses stack growth code to create a frame
// with the desired args running the desired function.
//
// func call(fn *byte, arg *byte, argsize uint32).
TEXT reflect·call(SB), 7, $-4-12
TEXT runtime·newstackcall(SB), 7, $-4-12
// Save our caller's state as the PC and SP to
// restore when returning from f.
MOVW LR, (m_morebuf+gobuf_pc)(m) // our caller's PC
......@@ -222,14 +222,14 @@ TEXT reflect·call(SB), 7, $-4-12
// Save our own state as the PC and SP to restore
// if this goroutine needs to be restarted.
MOVW $reflect·call(SB), R11
MOVW $runtime·newstackcall(SB), R11
MOVW R11, (g_sched+gobuf_pc)(g)
MOVW LR, (g_sched+gobuf_lr)(g)
MOVW SP, (g_sched+gobuf_sp)(g)
// Set up morestack arguments to call f on a new stack.
// We set f's frame size to 1, as a hint to newstack
// that this is a call from reflect·call.
// that this is a call from runtime·newstackcall.
// If it turns out that f needs a larger frame than
// the default stack, f's usual stack growth prolog will
// allocate a new segment (and recopy the arguments).
......@@ -248,6 +248,105 @@ TEXT reflect·call(SB), 7, $-4-12
MOVW (g_sched+gobuf_sp)(g), SP
B runtime·newstack(SB)
// reflect·call: call a function with the given argument list
// func call(f *FuncVal, arg *byte, argsize uint32).
// we don't have variable-sized frames, so we use a small number
// of constant-sized-frame functions to encode a few bits of size in the pc.
// Caution: ugly multiline assembly macros in your future!
#define DISPATCH(NAME,MAXSIZE) \
CMP $MAXSIZE, R0; \
B.HI 3(PC); \
MOVW $runtime·NAME(SB), R1; \
B (R1)
TEXT reflect·call(SB), 7, $-4-12
MOVW argsize+8(FP), R0
DISPATCH(call16, 16)
DISPATCH(call32, 32)
DISPATCH(call64, 64)
DISPATCH(call128, 128)
DISPATCH(call256, 256)
DISPATCH(call512, 512)
DISPATCH(call1024, 1024)
DISPATCH(call2048, 2048)
DISPATCH(call4096, 4096)
DISPATCH(call8192, 8192)
DISPATCH(call16384, 16384)
DISPATCH(call32768, 32768)
DISPATCH(call65536, 65536)
DISPATCH(call131072, 131072)
DISPATCH(call262144, 262144)
DISPATCH(call524288, 524288)
DISPATCH(call1048576, 1048576)
DISPATCH(call2097152, 2097152)
DISPATCH(call4194304, 4194304)
DISPATCH(call8388608, 8388608)
DISPATCH(call16777216, 16777216)
DISPATCH(call33554432, 33554432)
DISPATCH(call67108864, 67108864)
DISPATCH(call134217728, 134217728)
DISPATCH(call268435456, 268435456)
DISPATCH(call536870912, 536870912)
DISPATCH(call1073741824, 1073741824)
MOVW $runtime·badreflectcall(SB), R1
B (R1)
#define CALLFN(NAME,MAXSIZE,FLAGS) \
TEXT runtime·NAME(SB), FLAGS, $MAXSIZE-12; \
/* copy arguments to stack */ \
MOVW argptr+4(FP), R0; \
MOVW argsize+8(FP), R2; \
ADD $4, SP, R1; \
CMP $0, R2; \
B.EQ 5(PC); \
MOVBU.P 1(R0), R5; \
MOVBU.P R5, 1(R1); \
SUB $1, R2, R2; \
B -5(PC); \
/* call function */ \
MOVW f+0(FP), R7; \
MOVW (R7), R0; \
BL (R0); \
/* copy return values back */ \
MOVW argptr+4(FP), R0; \
MOVW argsize+8(FP), R2; \
ADD $4, SP, R1; \
CMP $0, R2; \
RET.EQ ; \
MOVBU.P 1(R1), R5; \
MOVBU.P R5, 1(R0); \
SUB $1, R2, R2; \
B -5(PC) \
CALLFN(call16, 16, 7)
CALLFN(call32, 32, 7)
CALLFN(call64, 64, 7)
CALLFN(call128, 128, 0)
CALLFN(call256, 256, 0)
CALLFN(call512, 512, 0)
CALLFN(call1024, 1024, 0)
CALLFN(call2048, 2048, 0)
CALLFN(call4096, 4096, 0)
CALLFN(call8192, 8192, 0)
CALLFN(call16384, 16384, 0)
CALLFN(call32768, 32768, 0)
CALLFN(call65536, 65536, 0)
CALLFN(call131072, 131072, 0)
CALLFN(call262144, 262144, 0)
CALLFN(call524288, 524288, 0)
CALLFN(call1048576, 1048576, 0)
CALLFN(call2097152, 2097152, 0)
CALLFN(call4194304, 4194304, 0)
CALLFN(call8388608, 8388608, 0)
CALLFN(call16777216, 16777216, 0)
CALLFN(call33554432, 33554432, 0)
CALLFN(call67108864, 67108864, 0)
CALLFN(call134217728, 134217728, 0)
CALLFN(call268435456, 268435456, 0)
CALLFN(call536870912, 536870912, 0)
CALLFN(call1073741824, 1073741824, 0)
// Return point when leaving stack.
// using frame size $-4 means do not save LR on stack.
//
......
src/pkg/runtime/panic.c
View file @ 9cd57068
......@@ -241,10 +241,10 @@ runtime·panic(Eface e)
break;
// take defer off list in case of recursive panic
popdefer();
g->ispanic = true; // rock for newstack, where reflect.call ends up
g->ispanic = true; // rock for newstack, where reflect.newstackcall ends up
argp = d->argp;
pc = d->pc;
reflect·call(d->fn, (byte*)d->args, d->siz);
runtime·newstackcall(d->fn, (byte*)d->args, d->siz);
freedefer(d);
if(p->recovered) {
g->panic = p->link;
......
src/pkg/runtime/proc.c
View file @ 9cd57068
......@@ -1926,6 +1926,12 @@ runtime·badmcall2(void) // called from assembly
runtime·throw("runtime: mcall function returned");
}
void
runtime·badreflectcall(void) // called from assembly
{
runtime·panicstring("runtime: arg size to reflect.call more than 1GB");
}
static struct {
Lock;
void (*fn)(uintptr*, int32);
......
src/pkg/runtime/runtime.h
View file @ 9cd57068
......@@ -985,6 +985,7 @@ void runtime·printuint(uint64);
void runtime·printhex(uint64);
void runtime·printslice(Slice);
void runtime·printcomplex(Complex128);
void runtime·newstackcall(FuncVal*, byte*, uint32);
void reflect·call(FuncVal*, byte*, uint32);
void runtime·panic(Eface);
void runtime·panicindex(void);
......
src/pkg/runtime/stack.c
View file @ 9cd57068
......@@ -183,7 +183,7 @@ runtime·oldstack(void)
runtime·gogo(&gp->sched);
}
// Called from reflect·call or from runtime·morestack when a new
// Called from runtime·newstackcall or from runtime·morestack when a new
// stack segment is needed. Allocate a new stack big enough for
// m->moreframesize bytes, copy m->moreargsize bytes to the new frame,
// and then act as though runtime·lessstack called the function at
......@@ -198,7 +198,7 @@ runtime·newstack(void)
uintptr *src, *dst, *dstend;
G *gp;
Gobuf label;
bool reflectcall;
bool newstackcall;
uintptr free;
if(m->morebuf.g != m->curg) {
......@@ -217,12 +217,12 @@ runtime·newstack(void)
argsize = m->moreargsize;
gp->status = Gwaiting;
gp->waitreason = "stack split";
reflectcall = framesize==1;
if(reflectcall)
newstackcall = framesize==1;
if(newstackcall)
framesize = 0;
// For reflectcall the context already points to beginning of reflect·call.
if(!reflectcall)
// For newstackcall the context already points to beginning of runtime·newstackcall.
if(!newstackcall)
runtime·rewindmorestack(&gp->sched);
sp = gp->sched.sp;
......@@ -269,8 +269,8 @@ runtime·newstack(void)
runtime·gosched0(gp); // never return
}
if(reflectcall && m->morebuf.sp - sizeof(Stktop) - argsize - 32 > gp->stackguard) {
// special case: called from reflect.call (framesize==1)
if(newstackcall && m->morebuf.sp - sizeof(Stktop) - argsize - 32 > gp->stackguard) {
// special case: called from runtime.newstackcall (framesize==1)
// to call code with an arbitrary argument size,
// and we have enough space on the current stack.
// the new Stktop* is necessary to unwind, but
......@@ -334,7 +334,7 @@ runtime·newstack(void)
label.sp = sp;
label.pc = (uintptr)runtime·lessstack;
label.g = m->curg;
if(reflectcall)
if(newstackcall)
runtime·gostartcallfn(&label, (FuncVal*)m->cret);
else {
runtime·gostartcall(&label, (void(*)(void))gp->sched.pc, gp->sched.ctxt);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7