Skip to content

G
go
Commit df027ace authored by Russ Cox's avatar Russ Cox
Browse files
Options

reflect: add write barriers 

Use typedmemmove, typedslicecopy, and adjust reflect.call
to execute the necessary write barriers.

Found with GODEBUG=wbshadow=2 mode.
Eventually that will run automatically, but right now
it still detects other missing write barriers.

Change-Id: Iec5b5b0c1be5589295e28e5228e37f1a92e07742
Reviewed-on: https://go-review.googlesource.com/2312Reviewed-by: default avatarKeith Randall <khr@golang.org>
parent 813386f2
Hide whitespace changes
Inline Side-by-side
Showing with 263 additions and 93 deletions
misc/cgo/test/callback.go
View file @ df027ace
...@@ -167,6 +167,9 @@ func testCallbackCallers(t *testing.T) { ...@@ -167,6 +167,9 @@ func testCallbackCallers(t *testing.T) {
"testing.tRunner", "testing.tRunner",
"runtime.goexit", "runtime.goexit",
} }
if unsafe.Sizeof((*byte)(nil)) == 8 {
name[1] = "runtime.call32"
}
nestedCall(func() { nestedCall(func() {
n = runtime.Callers(2, pc) n = runtime.Callers(2, pc)
}) })
......
src/reflect/value.go
View file @ df027ace
...@@ -107,7 +107,7 @@ func packEface(v Value) interface{} { ...@@ -107,7 +107,7 @@ func packEface(v Value) interface{} {
// TODO: pass safe boolean from valueInterface so // TODO: pass safe boolean from valueInterface so
// we don't need to copy if safe==true? // we don't need to copy if safe==true?
c := unsafe_New(t) c := unsafe_New(t)
memmove(c, ptr, t.size) typedmemmove(t, c, ptr)
ptr = c ptr = c
} }
e.word = ptr e.word = ptr
...@@ -412,7 +412,7 @@ func (v Value) call(op string, in []Value) []Value { ...@@ -412,7 +412,7 @@ func (v Value) call(op string, in []Value) []Value {
addr := unsafe.Pointer(uintptr(args) + off) addr := unsafe.Pointer(uintptr(args) + off)
v = v.assignTo("reflect.Value.Call", targ, addr) v = v.assignTo("reflect.Value.Call", targ, addr)
if v.flag&flagIndir != 0 { if v.flag&flagIndir != 0 {
memmove(addr, v.ptr, n) typedmemmove(targ, addr, v.ptr)
} else { } else {
*(*unsafe.Pointer)(addr) = v.ptr *(*unsafe.Pointer)(addr) = v.ptr
} }
...@@ -420,7 +420,7 @@ func (v Value) call(op string, in []Value) []Value { ...@@ -420,7 +420,7 @@ func (v Value) call(op string, in []Value) []Value {
} }
// Call. // Call.
call(fn, args, uint32(frametype.size), uint32(retOffset)) call(frametype, fn, args, uint32(frametype.size), uint32(retOffset))
// For testing; see TestCallMethodJump. // For testing; see TestCallMethodJump.
if callGC { if callGC {
...@@ -473,7 +473,7 @@ func callReflect(ctxt *makeFuncImpl, frame unsafe.Pointer) { ...@@ -473,7 +473,7 @@ func callReflect(ctxt *makeFuncImpl, frame unsafe.Pointer) {
// and we cannot let f keep a reference to the stack frame // and we cannot let f keep a reference to the stack frame
// after this function returns, not even a read-only reference. // after this function returns, not even a read-only reference.
v.ptr = unsafe_New(typ) v.ptr = unsafe_New(typ)
memmove(v.ptr, addr, typ.size) typedmemmove(typ, v.ptr, addr)
v.flag |= flagIndir v.flag |= flagIndir
} else { } else {
v.ptr = *(*unsafe.Pointer)(addr) v.ptr = *(*unsafe.Pointer)(addr)
...@@ -509,7 +509,7 @@ func callReflect(ctxt *makeFuncImpl, frame unsafe.Pointer) { ...@@ -509,7 +509,7 @@ func callReflect(ctxt *makeFuncImpl, frame unsafe.Pointer) {
off += -off & uintptr(typ.align-1) off += -off & uintptr(typ.align-1)
addr := unsafe.Pointer(uintptr(ptr) + off) addr := unsafe.Pointer(uintptr(ptr) + off)
if v.flag&flagIndir != 0 { if v.flag&flagIndir != 0 {
memmove(addr, v.ptr, typ.size) typedmemmove(typ, addr, v.ptr)
} else { } else {
*(*unsafe.Pointer)(addr) = v.ptr *(*unsafe.Pointer)(addr) = v.ptr
} }
...@@ -603,10 +603,10 @@ func callMethod(ctxt *methodValue, frame unsafe.Pointer) { ...@@ -603,10 +603,10 @@ func callMethod(ctxt *methodValue, frame unsafe.Pointer) {
// Copy in receiver and rest of args. // Copy in receiver and rest of args.
storeRcvr(rcvr, args) storeRcvr(rcvr, args)
memmove(unsafe.Pointer(uintptr(args)+ptrSize), frame, argSize-ptrSize) typedmemmovepartial(frametype, unsafe.Pointer(uintptr(args)+ptrSize), frame, ptrSize, argSize-ptrSize)
// Call. // Call.
call(fn, args, uint32(frametype.size), uint32(retOffset)) call(frametype, fn, args, uint32(frametype.size), uint32(retOffset))
// Copy return values. On amd64p32, the beginning of return values // Copy return values. On amd64p32, the beginning of return values
// is 64-bit aligned, so the caller's frame layout (which doesn't have // is 64-bit aligned, so the caller's frame layout (which doesn't have
...@@ -617,8 +617,11 @@ func callMethod(ctxt *methodValue, frame unsafe.Pointer) { ...@@ -617,8 +617,11 @@ func callMethod(ctxt *methodValue, frame unsafe.Pointer) {
if runtime.GOARCH == "amd64p32" { if runtime.GOARCH == "amd64p32" {
callerRetOffset = align(argSize-ptrSize, 8) callerRetOffset = align(argSize-ptrSize, 8)
} }
memmove(unsafe.Pointer(uintptr(frame)+callerRetOffset), typedmemmovepartial(frametype,
unsafe.Pointer(uintptr(args)+retOffset), frametype.size-retOffset) unsafe.Pointer(uintptr(frame)+callerRetOffset),
unsafe.Pointer(uintptr(args)+retOffset),
retOffset,
frametype.size-retOffset)
} }
// funcName returns the name of f, for use in error messages. // funcName returns the name of f, for use in error messages.
...@@ -1017,7 +1020,7 @@ func (v Value) MapIndex(key Value) Value { ...@@ -1017,7 +1020,7 @@ func (v Value) MapIndex(key Value) Value {
// Copy result so future changes to the map // Copy result so future changes to the map
// won't change the underlying value. // won't change the underlying value.
c := unsafe_New(typ) c := unsafe_New(typ)
memmove(c, e, typ.size) typedmemmove(typ, c, e)
return Value{typ, c, fl | flagIndir} return Value{typ, c, fl | flagIndir}
} else { } else {
return Value{typ, *(*unsafe.Pointer)(e), fl} return Value{typ, *(*unsafe.Pointer)(e), fl}
...@@ -1055,7 +1058,7 @@ func (v Value) MapKeys() []Value { ...@@ -1055,7 +1058,7 @@ func (v Value) MapKeys() []Value {
// Copy result so future changes to the map // Copy result so future changes to the map
// won't change the underlying value. // won't change the underlying value.
c := unsafe_New(keyType) c := unsafe_New(keyType)
memmove(c, key, keyType.size) typedmemmove(keyType, c, key)
a[i] = Value{keyType, c, fl | flagIndir} a[i] = Value{keyType, c, fl | flagIndir}
} else { } else {
a[i] = Value{keyType, *(*unsafe.Pointer)(key), fl} a[i] = Value{keyType, *(*unsafe.Pointer)(key), fl}
...@@ -1301,7 +1304,7 @@ func (v Value) Set(x Value) { ...@@ -1301,7 +1304,7 @@ func (v Value) Set(x Value) {
} }
x = x.assignTo("reflect.Set", v.typ, target) x = x.assignTo("reflect.Set", v.typ, target)
if x.flag&flagIndir != 0 { if x.flag&flagIndir != 0 {
memmove(v.ptr, x.ptr, v.typ.size) typedmemmove(v.typ, v.ptr, x.ptr)
} else { } else {
*(*unsafe.Pointer)(v.ptr) = x.ptr *(*unsafe.Pointer)(v.ptr) = x.ptr
} }
...@@ -1815,27 +1818,23 @@ func Copy(dst, src Value) int { ...@@ -1815,27 +1818,23 @@ func Copy(dst, src Value) int {
se := src.typ.Elem() se := src.typ.Elem()
typesMustMatch("reflect.Copy", de, se) typesMustMatch("reflect.Copy", de, se)
n := dst.Len() var ds, ss sliceHeader
if sn := src.Len(); n > sn {
n = sn
}
// Copy via memmove.
var da, sa unsafe.Pointer
if dk == Array { if dk == Array {
da = dst.ptr ds.Data = dst.ptr
ds.Len = dst.Len()
ds.Cap = ds.Len
} else { } else {
da = (*sliceHeader)(dst.ptr).Data ds = *(*sliceHeader)(dst.ptr)
} }
if src.flag&flagIndir == 0 { if sk == Array {
sa = unsafe.Pointer(&src.ptr) ss.Data = src.ptr
} else if sk == Array { ss.Len = src.Len()
sa = src.ptr ss.Cap = ss.Len
} else { } else {
sa = (*sliceHeader)(src.ptr).Data ss = *(*sliceHeader)(src.ptr)
} }
memmove(da, sa, uintptr(n)*de.Size())
return n return typedslicecopy(de.common(), ds, ss)
} }
// A runtimeSelect is a single case passed to rselect. // A runtimeSelect is a single case passed to rselect.
...@@ -2376,7 +2375,7 @@ func cvtDirect(v Value, typ Type) Value { ...@@ -2376,7 +2375,7 @@ func cvtDirect(v Value, typ Type) Value {
if f&flagAddr != 0 { if f&flagAddr != 0 {
// indirect, mutable word - make a copy // indirect, mutable word - make a copy
c := unsafe_New(t) c := unsafe_New(t)
memmove(c, ptr, t.size) typedmemmove(t, c, ptr)
ptr = c ptr = c
f &^= flagAddr f &^= flagAddr
} }
...@@ -2425,12 +2424,29 @@ func mapiterinit(t *rtype, m unsafe.Pointer) unsafe.Pointer ...@@ -2425,12 +2424,29 @@ func mapiterinit(t *rtype, m unsafe.Pointer) unsafe.Pointer
func mapiterkey(it unsafe.Pointer) (key unsafe.Pointer) func mapiterkey(it unsafe.Pointer) (key unsafe.Pointer)
func mapiternext(it unsafe.Pointer) func mapiternext(it unsafe.Pointer)
func maplen(m unsafe.Pointer) int func maplen(m unsafe.Pointer) int
func call(fn, arg unsafe.Pointer, n uint32, retoffset uint32)
// call calls fn with a copy of the n argument bytes pointed at by arg.
// After fn returns, reflectcall copies n-retoffset result bytes
// back into arg+retoffset before returning. If copying result bytes back,
// the caller must pass the argument frame type as argtype, so that
// call can execute appropriate write barriers during the copy.
func call(argtype *rtype, fn, arg unsafe.Pointer, n uint32, retoffset uint32)
func ifaceE2I(t *rtype, src interface{}, dst unsafe.Pointer) func ifaceE2I(t *rtype, src interface{}, dst unsafe.Pointer)
// typedmemmove copies a value of type t to dst from src.
//go:noescape
func typedmemmove(t *rtype, dst, src unsafe.Pointer)
// typedmemmovepartial is like typedmemmove but assumes that
// dst and src point off bytes into the value and only copies size bytes.
//go:noescape
func typedmemmovepartial(t *rtype, dst, src unsafe.Pointer, off, size uintptr)
// typedslicecopy copies a slice of elemType values from src to dst,
// returning the number of elements copied.
//go:noescape //go:noescape
func memmove(adst, asrc unsafe.Pointer, n uintptr) func typedslicecopy(elemType *rtype, dst, src sliceHeader) int
// Dummy annotation marking that the value x escapes, // Dummy annotation marking that the value x escapes,
// for use in cases where the reflect code is so clever that // for use in cases where the reflect code is so clever that
......
src/runtime/asm_386.s
View file @ df027ace
...@@ -329,7 +329,7 @@ TEXT runtime·morestack_noctxt(SB),NOSPLIT,$0-0 ...@@ -329,7 +329,7 @@ TEXT runtime·morestack_noctxt(SB),NOSPLIT,$0-0
JMP runtime·morestack(SB) JMP runtime·morestack(SB)
// reflectcall: call a function with the given argument list // reflectcall: call a function with the given argument list
// func call(f *FuncVal, arg *byte, argsize, retoffset uint32). // func call(argtype *_type, f *FuncVal, arg *byte, argsize, retoffset uint32).
// we don't have variable-sized frames, so we use a small number // we don't have variable-sized frames, so we use a small number
// of constant-sized-frame functions to encode a few bits of size in the pc. // of constant-sized-frame functions to encode a few bits of size in the pc.
// Caution: ugly multiline assembly macros in your future! // Caution: ugly multiline assembly macros in your future!
...@@ -344,8 +344,8 @@ TEXT runtime·morestack_noctxt(SB),NOSPLIT,$0-0 ...@@ -344,8 +344,8 @@ TEXT runtime·morestack_noctxt(SB),NOSPLIT,$0-0
TEXT reflect·call(SB), NOSPLIT, $0-0 TEXT reflect·call(SB), NOSPLIT, $0-0
JMP ·reflectcall(SB) JMP ·reflectcall(SB)
TEXT ·reflectcall(SB), NOSPLIT, $0-16 TEXT ·reflectcall(SB), NOSPLIT, $0-20
MOVL argsize+8(FP), CX MOVL argsize+12(FP), CX
DISPATCH(runtime·call16, 16) DISPATCH(runtime·call16, 16)
DISPATCH(runtime·call32, 32) DISPATCH(runtime·call32, 32)
DISPATCH(runtime·call64, 64) DISPATCH(runtime·call64, 64)
...@@ -377,27 +377,37 @@ TEXT ·reflectcall(SB), NOSPLIT, $0-16 ...@@ -377,27 +377,37 @@ TEXT ·reflectcall(SB), NOSPLIT, $0-16
JMP AX JMP AX
#define CALLFN(NAME,MAXSIZE) \ #define CALLFN(NAME,MAXSIZE) \
TEXT NAME(SB), WRAPPER, $MAXSIZE-16; \ TEXT NAME(SB), WRAPPER, $MAXSIZE-20; \
NO_LOCAL_POINTERS; \ NO_LOCAL_POINTERS; \
/* copy arguments to stack */ \ /* copy arguments to stack */ \
MOVL argptr+4(FP), SI; \ MOVL argptr+8(FP), SI; \
MOVL argsize+8(FP), CX; \ MOVL argsize+12(FP), CX; \
MOVL SP, DI; \ MOVL SP, DI; \
REP;MOVSB; \ REP;MOVSB; \
/* call function */ \ /* call function */ \
MOVL f+0(FP), DX; \ MOVL f+4(FP), DX; \
MOVL (DX), AX; \ MOVL (DX), AX; \
PCDATA $PCDATA_StackMapIndex, $0; \ PCDATA $PCDATA_StackMapIndex, $0; \
CALL AX; \ CALL AX; \
/* copy return values back */ \ /* copy return values back */ \
MOVL argptr+4(FP), DI; \ MOVL argptr+8(FP), DI; \
MOVL argsize+8(FP), CX; \ MOVL argsize+12(FP), CX; \
MOVL retoffset+12(FP), BX; \ MOVL retoffset+16(FP), BX; \
MOVL SP, SI; \ MOVL SP, SI; \
ADDL BX, DI; \ ADDL BX, DI; \
ADDL BX, SI; \ ADDL BX, SI; \
SUBL BX, CX; \ SUBL BX, CX; \
REP;MOVSB; \ REP;MOVSB; \
/* execute write barrier updates */ \
MOVL argtype+0(FP), DX; \
MOVL argptr+8(FP), DI; \
MOVL argsize+12(FP), CX; \
MOVL retoffset+16(FP), BX; \
MOVL DX, 0(SP); \
MOVL DI, 4(SP); \
MOVL CX, 8(SP); \
MOVL BX, 12(SP); \
CALL runtime·callwritebarrier(SB); \
RET RET
CALLFN(·call16, 16) CALLFN(·call16, 16)
......
src/runtime/asm_amd64.s
View file @ df027ace
...@@ -318,7 +318,7 @@ TEXT runtime·morestack_noctxt(SB),NOSPLIT,$0 ...@@ -318,7 +318,7 @@ TEXT runtime·morestack_noctxt(SB),NOSPLIT,$0
JMP runtime·morestack(SB) JMP runtime·morestack(SB)
// reflectcall: call a function with the given argument list // reflectcall: call a function with the given argument list
// func call(f *FuncVal, arg *byte, argsize, retoffset uint32). // func call(argtype *_type, f *FuncVal, arg *byte, argsize, retoffset uint32).
// we don't have variable-sized frames, so we use a small number // we don't have variable-sized frames, so we use a small number
// of constant-sized-frame functions to encode a few bits of size in the pc. // of constant-sized-frame functions to encode a few bits of size in the pc.
// Caution: ugly multiline assembly macros in your future! // Caution: ugly multiline assembly macros in your future!
...@@ -333,9 +333,10 @@ TEXT runtime·morestack_noctxt(SB),NOSPLIT,$0 ...@@ -333,9 +333,10 @@ TEXT runtime·morestack_noctxt(SB),NOSPLIT,$0
TEXT reflect·call(SB), NOSPLIT, $0-0 TEXT reflect·call(SB), NOSPLIT, $0-0
JMP ·reflectcall(SB) JMP ·reflectcall(SB)
TEXT ·reflectcall(SB), NOSPLIT, $0-24 TEXT ·reflectcall(SB), NOSPLIT, $0-32
MOVLQZX argsize+16(FP), CX MOVLQZX argsize+24(FP), CX
DISPATCH(runtime·call16, 16) // NOTE(rsc): No call16, because CALLFN needs four words
// of argument space to invoke callwritebarrier.
DISPATCH(runtime·call32, 32) DISPATCH(runtime·call32, 32)
DISPATCH(runtime·call64, 64) DISPATCH(runtime·call64, 64)
DISPATCH(runtime·call128, 128) DISPATCH(runtime·call128, 128)
...@@ -366,29 +367,38 @@ TEXT ·reflectcall(SB), NOSPLIT, $0-24 ...@@ -366,29 +367,38 @@ TEXT ·reflectcall(SB), NOSPLIT, $0-24
JMP AX JMP AX
#define CALLFN(NAME,MAXSIZE) \ #define CALLFN(NAME,MAXSIZE) \
TEXT NAME(SB), WRAPPER, $MAXSIZE-24; \ TEXT NAME(SB), WRAPPER, $MAXSIZE-32; \
NO_LOCAL_POINTERS; \ NO_LOCAL_POINTERS; \
/* copy arguments to stack */ \ /* copy arguments to stack */ \
MOVQ argptr+8(FP), SI; \ MOVQ argptr+16(FP), SI; \
MOVLQZX argsize+16(FP), CX; \ MOVLQZX argsize+24(FP), CX; \
MOVQ SP, DI; \ MOVQ SP, DI; \
REP;MOVSB; \ REP;MOVSB; \
/* call function */ \ /* call function */ \
MOVQ f+0(FP), DX; \ MOVQ f+8(FP), DX; \
PCDATA $PCDATA_StackMapIndex, $0; \ PCDATA $PCDATA_StackMapIndex, $0; \
CALL (DX); \ CALL (DX); \
/* copy return values back */ \ /* copy return values back */ \
MOVQ argptr+8(FP), DI; \ MOVQ argptr+16(FP), DI; \
MOVLQZX argsize+16(FP), CX; \ MOVLQZX argsize+24(FP), CX; \
MOVLQZX retoffset+20(FP), BX; \ MOVLQZX retoffset+28(FP), BX; \
MOVQ SP, SI; \ MOVQ SP, SI; \
ADDQ BX, DI; \ ADDQ BX, DI; \
ADDQ BX, SI; \ ADDQ BX, SI; \
SUBQ BX, CX; \ SUBQ BX, CX; \
REP;MOVSB; \ REP;MOVSB; \
/* execute write barrier updates */ \
MOVQ argtype+0(FP), DX; \
MOVQ argptr+16(FP), DI; \
MOVLQZX argsize+24(FP), CX; \
MOVLQZX retoffset+28(FP), BX; \
MOVQ DX, 0(SP); \
MOVQ DI, 8(SP); \
MOVQ CX, 16(SP); \
MOVQ BX, 24(SP); \
CALL runtime·callwritebarrier(SB); \
RET RET
CALLFN(·call16, 16)
CALLFN(·call32, 32) CALLFN(·call32, 32)
CALLFN(·call64, 64) CALLFN(·call64, 64)
CALLFN(·call128, 128) CALLFN(·call128, 128)
......
src/runtime/asm_amd64p32.s
View file @ df027ace
...@@ -290,7 +290,7 @@ TEXT runtime·morestack_noctxt(SB),NOSPLIT,$0 ...@@ -290,7 +290,7 @@ TEXT runtime·morestack_noctxt(SB),NOSPLIT,$0
JMP runtime·morestack(SB) JMP runtime·morestack(SB)
// reflectcall: call a function with the given argument list // reflectcall: call a function with the given argument list
// func call(f *FuncVal, arg *byte, argsize, retoffset uint32). // func call(argtype *_type, f *FuncVal, arg *byte, argsize, retoffset uint32).
// we don't have variable-sized frames, so we use a small number // we don't have variable-sized frames, so we use a small number
// of constant-sized-frame functions to encode a few bits of size in the pc. // of constant-sized-frame functions to encode a few bits of size in the pc.
// Caution: ugly multiline assembly macros in your future! // Caution: ugly multiline assembly macros in your future!
...@@ -305,8 +305,8 @@ TEXT runtime·morestack_noctxt(SB),NOSPLIT,$0 ...@@ -305,8 +305,8 @@ TEXT runtime·morestack_noctxt(SB),NOSPLIT,$0
TEXT reflect·call(SB), NOSPLIT, $0-0 TEXT reflect·call(SB), NOSPLIT, $0-0
JMP ·reflectcall(SB) JMP ·reflectcall(SB)
TEXT ·reflectcall(SB), NOSPLIT, $0-16 TEXT ·reflectcall(SB), NOSPLIT, $0-20
MOVLQZX argsize+8(FP), CX MOVLQZX argsize+12(FP), CX
DISPATCH(runtime·call16, 16) DISPATCH(runtime·call16, 16)
DISPATCH(runtime·call32, 32) DISPATCH(runtime·call32, 32)
DISPATCH(runtime·call64, 64) DISPATCH(runtime·call64, 64)
...@@ -338,26 +338,36 @@ TEXT ·reflectcall(SB), NOSPLIT, $0-16 ...@@ -338,26 +338,36 @@ TEXT ·reflectcall(SB), NOSPLIT, $0-16
JMP AX JMP AX
#define CALLFN(NAME,MAXSIZE) \ #define CALLFN(NAME,MAXSIZE) \
TEXT NAME(SB), WRAPPER, $MAXSIZE-16; \ TEXT NAME(SB), WRAPPER, $MAXSIZE-20; \
NO_LOCAL_POINTERS; \ NO_LOCAL_POINTERS; \
/* copy arguments to stack */ \ /* copy arguments to stack */ \
MOVL argptr+4(FP), SI; \ MOVL argptr+8(FP), SI; \
MOVL argsize+8(FP), CX; \ MOVL argsize+12(FP), CX; \
MOVL SP, DI; \ MOVL SP, DI; \
REP;MOVSB; \ REP;MOVSB; \
/* call function */ \ /* call function */ \
MOVL f+0(FP), DX; \ MOVL f+4(FP), DX; \
MOVL (DX), AX; \ MOVL (DX), AX; \
CALL AX; \ CALL AX; \
/* copy return values back */ \ /* copy return values back */ \
MOVL argptr+4(FP), DI; \ MOVL argptr+8(FP), DI; \
MOVL argsize+8(FP), CX; \ MOVL argsize+12(FP), CX; \
MOVL retoffset+12(FP), BX; \ MOVL retoffset+16(FP), BX; \
MOVL SP, SI; \ MOVL SP, SI; \
ADDL BX, DI; \ ADDL BX, DI; \
ADDL BX, SI; \ ADDL BX, SI; \
SUBL BX, CX; \ SUBL BX, CX; \
REP;MOVSB; \ REP;MOVSB; \
/* execute write barrier updates */ \
MOVL argtype+0(FP), DX; \
MOVL argptr+8(FP), DI; \
MOVL argsize+12(FP), CX; \
MOVL retoffset+16(FP), BX; \
MOVL DX, 0(SP); \
MOVL DI, 4(SP); \
MOVL CX, 8(SP); \
MOVL BX, 12(SP); \
CALL runtime·callwritebarrier(SB); \
RET RET
CALLFN(·call16, 16) CALLFN(·call16, 16)
......
src/runtime/asm_arm.s
View file @ df027ace
...@@ -322,7 +322,7 @@ TEXT runtime·morestack_noctxt(SB),NOSPLIT,$-4-0 ...@@ -322,7 +322,7 @@ TEXT runtime·morestack_noctxt(SB),NOSPLIT,$-4-0
B runtime·morestack(SB) B runtime·morestack(SB)
// reflectcall: call a function with the given argument list // reflectcall: call a function with the given argument list
// func call(f *FuncVal, arg *byte, argsize, retoffset uint32). // func call(argtype *_type, f *FuncVal, arg *byte, argsize, retoffset uint32).
// we don't have variable-sized frames, so we use a small number // we don't have variable-sized frames, so we use a small number
// of constant-sized-frame functions to encode a few bits of size in the pc. // of constant-sized-frame functions to encode a few bits of size in the pc.
// Caution: ugly multiline assembly macros in your future! // Caution: ugly multiline assembly macros in your future!
...@@ -336,8 +336,8 @@ TEXT runtime·morestack_noctxt(SB),NOSPLIT,$-4-0 ...@@ -336,8 +336,8 @@ TEXT runtime·morestack_noctxt(SB),NOSPLIT,$-4-0
TEXT reflect·call(SB), NOSPLIT, $0-0 TEXT reflect·call(SB), NOSPLIT, $0-0
B ·reflectcall(SB) B ·reflectcall(SB)
TEXT ·reflectcall(SB),NOSPLIT,$-4-16 TEXT ·reflectcall(SB),NOSPLIT,$-4-20
MOVW argsize+8(FP), R0 MOVW argsize+12(FP), R0
DISPATCH(runtime·call16, 16) DISPATCH(runtime·call16, 16)
DISPATCH(runtime·call32, 32) DISPATCH(runtime·call32, 32)
DISPATCH(runtime·call64, 64) DISPATCH(runtime·call64, 64)
...@@ -369,11 +369,11 @@ TEXT ·reflectcall(SB),NOSPLIT,$-4-16 ...@@ -369,11 +369,11 @@ TEXT ·reflectcall(SB),NOSPLIT,$-4-16
B (R1) B (R1)
#define CALLFN(NAME,MAXSIZE) \ #define CALLFN(NAME,MAXSIZE) \
TEXT NAME(SB), WRAPPER, $MAXSIZE-16; \ TEXT NAME(SB), WRAPPER, $MAXSIZE-20; \
NO_LOCAL_POINTERS; \ NO_LOCAL_POINTERS; \
/* copy arguments to stack */ \ /* copy arguments to stack */ \
MOVW argptr+4(FP), R0; \ MOVW argptr+8(FP), R0; \
MOVW argsize+8(FP), R2; \ MOVW argsize+12(FP), R2; \
ADD $4, SP, R1; \ ADD $4, SP, R1; \
CMP $0, R2; \ CMP $0, R2; \
B.EQ 5(PC); \ B.EQ 5(PC); \
...@@ -382,24 +382,37 @@ TEXT NAME(SB), WRAPPER, $MAXSIZE-16; \ ...@@ -382,24 +382,37 @@ TEXT NAME(SB), WRAPPER, $MAXSIZE-16; \
SUB $1, R2, R2; \ SUB $1, R2, R2; \
B -5(PC); \ B -5(PC); \
/* call function */ \ /* call function */ \
MOVW f+0(FP), R7; \ MOVW f+4(FP), R7; \
MOVW (R7), R0; \ MOVW (R7), R0; \
PCDATA $PCDATA_StackMapIndex, $0; \ PCDATA $PCDATA_StackMapIndex, $0; \
BL (R0); \ BL (R0); \
/* copy return values back */ \ /* copy return values back */ \
MOVW argptr+4(FP), R0; \ MOVW argptr+8(FP), R0; \
MOVW argsize+8(FP), R2; \ MOVW argsize+12(FP), R2; \
MOVW retoffset+12(FP), R3; \ MOVW retoffset+16(FP), R3; \
ADD $4, SP, R1; \ ADD $4, SP, R1; \
ADD R3, R1; \ ADD R3, R1; \
ADD R3, R0; \ ADD R3, R0; \
SUB R3, R2; \ SUB R3, R2; \
loop:
CMP $0, R2; \ CMP $0, R2; \
RET.EQ ; \ B.EQ end; \
MOVBU.P 1(R1), R5; \ MOVBU.P 1(R1), R5; \
MOVBU.P R5, 1(R0); \ MOVBU.P R5, 1(R0); \
SUB $1, R2, R2; \ SUB $1, R2, R2; \
B -5(PC) \ B loop; \
end: \
/* execute write barrier updates */ \
MOVW argtype+0(FP), R1; \
MOVW argptr+8(FP), R0; \
MOVW argsize+12(FP), R2; \
MOVW retoffset+16(FP), R3; \
MOVW R1, 4(R13); \
MOVW R0, 8(R13); \
MOVW R2, 12(R13); \
MOVW R3, 16(R13); \
BL runtime·callwritebarrier(SB); \
RET
CALLFN(·call16, 16) CALLFN(·call16, 16)
CALLFN(·call32, 32) CALLFN(·call32, 32)
......
src/runtime/asm_ppc64x.s
View file @ df027ace
...@@ -271,7 +271,7 @@ TEXT runtime·morestack_noctxt(SB),NOSPLIT,$-8-0 ...@@ -271,7 +271,7 @@ TEXT runtime·morestack_noctxt(SB),NOSPLIT,$-8-0
BR runtime·morestack(SB) BR runtime·morestack(SB)
// reflectcall: call a function with the given argument list // reflectcall: call a function with the given argument list
// func call(f *FuncVal, arg *byte, argsize, retoffset uint32). // func call(argtype *_type, f *FuncVal, arg *byte, argsize, retoffset uint32).
// we don't have variable-sized frames, so we use a small number // we don't have variable-sized frames, so we use a small number
// of constant-sized-frame functions to encode a few bits of size in the pc. // of constant-sized-frame functions to encode a few bits of size in the pc.
// Caution: ugly multiline assembly macros in your future! // Caution: ugly multiline assembly macros in your future!
...@@ -288,9 +288,10 @@ TEXT runtime·morestack_noctxt(SB),NOSPLIT,$-8-0 ...@@ -288,9 +288,10 @@ TEXT runtime·morestack_noctxt(SB),NOSPLIT,$-8-0
TEXT reflect·call(SB), NOSPLIT, $0-0 TEXT reflect·call(SB), NOSPLIT, $0-0
BR ·reflectcall(SB) BR ·reflectcall(SB)
TEXT ·reflectcall(SB), NOSPLIT, $-8-24 TEXT ·reflectcall(SB), NOSPLIT, $-8-32
MOVWZ n+16(FP), R3 MOVWZ argsize+24(FP), R3
DISPATCH(runtime·call16, 16) // NOTE(rsc): No call16, because CALLFN needs four words
// of argument space to invoke callwritebarrier.
DISPATCH(runtime·call32, 32) DISPATCH(runtime·call32, 32)
DISPATCH(runtime·call64, 64) DISPATCH(runtime·call64, 64)
DISPATCH(runtime·call128, 128) DISPATCH(runtime·call128, 128)
...@@ -325,8 +326,8 @@ TEXT ·reflectcall(SB), NOSPLIT, $-8-24 ...@@ -325,8 +326,8 @@ TEXT ·reflectcall(SB), NOSPLIT, $-8-24
TEXT NAME(SB), WRAPPER, $MAXSIZE-24; \ TEXT NAME(SB), WRAPPER, $MAXSIZE-24; \
NO_LOCAL_POINTERS; \ NO_LOCAL_POINTERS; \
/* copy arguments to stack */ \ /* copy arguments to stack */ \
MOVD arg+8(FP), R3; \ MOVD arg+16(FP), R3; \
MOVWZ n+16(FP), R4; \ MOVWZ argsize+24(FP), R4; \
MOVD R1, R5; \ MOVD R1, R5; \
ADD $(8-1), R5; \ ADD $(8-1), R5; \
SUB $1, R3; \ SUB $1, R3; \
...@@ -337,15 +338,15 @@ TEXT NAME(SB), WRAPPER, $MAXSIZE-24; \ ...@@ -337,15 +338,15 @@ TEXT NAME(SB), WRAPPER, $MAXSIZE-24; \
MOVBZU R6, 1(R5); \ MOVBZU R6, 1(R5); \
BR -4(PC); \ BR -4(PC); \
/* call function */ \ /* call function */ \
MOVD f+0(FP), R11; \ MOVD f+8(FP), R11; \
MOVD (R11), R31; \ MOVD (R11), R31; \
MOVD R31, CTR; \ MOVD R31, CTR; \
PCDATA $PCDATA_StackMapIndex, $0; \ PCDATA $PCDATA_StackMapIndex, $0; \
BL (CTR); \ BL (CTR); \
/* copy return values back */ \ /* copy return values back */ \
MOVD arg+8(FP), R3; \ MOVD arg+16(FP), R3; \
MOVWZ n+16(FP), R4; \ MOVWZ n+24(FP), R4; \
MOVWZ retoffset+20(FP), R6; \ MOVWZ retoffset+28(FP), R6; \
MOVD R1, R5; \ MOVD R1, R5; \
ADD R6, R5; \ ADD R6, R5; \
ADD R6, R3; \ ADD R6, R3; \
...@@ -353,11 +354,23 @@ TEXT NAME(SB), WRAPPER, $MAXSIZE-24; \ ...@@ -353,11 +354,23 @@ TEXT NAME(SB), WRAPPER, $MAXSIZE-24; \
ADD $(8-1), R5; \ ADD $(8-1), R5; \
SUB $1, R3; \ SUB $1, R3; \
ADD R5, R4; \ ADD R5, R4; \
loop: \
CMP R5, R4; \ CMP R5, R4; \
BEQ 4(PC); \ BEQ end; \
MOVBZU 1(R5), R6; \ MOVBZU 1(R5), R6; \
MOVBZU R6, 1(R3); \ MOVBZU R6, 1(R3); \
BR -4(PC); \ BR loop; \
end: \
/* execute write barrier updates */ \
MOVD argtype+0(FP), R7; \
MOVD arg+16(FP), R3; \
MOVWZ n+24(FP), R4; \
MOVWZ retoffset+28(FP), R6; \
MOVD R7, 8(R1); \
MOVD R3, 16(R1); \
MOVD R4, 24(R1); \
MOVD R6, 32(R1); \
BL runtime·callwritebarrier(SB); \
RETURN RETURN
CALLFN(·call16, 16) CALLFN(·call16, 16)
......
src/runtime/cgocall.go
View file @ df027ace
...@@ -232,7 +232,12 @@ func cgocallbackg1() { ...@@ -232,7 +232,12 @@ func cgocallbackg1() {
} }
// Invoke callback. // Invoke callback.
reflectcall(unsafe.Pointer(cb.fn), unsafe.Pointer(cb.arg), uint32(cb.argsize), 0) // NOTE(rsc): passing nil for argtype means that the copying of the
// results back into cb.arg happens without any corresponding write barriers.
// For cgo, cb.arg points into a C stack frame and therefore doesn't
// hold any pointers that the GC can find anyway - the write barrier
// would be a no-op.
reflectcall(nil, unsafe.Pointer(cb.fn), unsafe.Pointer(cb.arg), uint32(cb.argsize), 0)
if raceenabled { if raceenabled {
racereleasemerge(unsafe.Pointer(&racecgosync)) racereleasemerge(unsafe.Pointer(&racecgosync))
......
src/runtime/malloc.go
View file @ df027ace
...@@ -942,7 +942,7 @@ func runfinq() { ...@@ -942,7 +942,7 @@ func runfinq() {
default: default:
throw("bad kind in runfinq") throw("bad kind in runfinq")
} }
reflectcall(unsafe.Pointer(f.fn), frame, uint32(framesz), uint32(framesz)) reflectcall(nil, unsafe.Pointer(f.fn), frame, uint32(framesz), uint32(framesz))
// drop finalizer queue references to finalized object // drop finalizer queue references to finalized object
f.fn = nil f.fn = nil
......
src/runtime/mgc0.go
View file @ df027ace
...@@ -104,6 +104,7 @@ const ( ...@@ -104,6 +104,7 @@ const (
_PoisonStack = 0x6868686868686868 & (1<<(8*ptrSize) - 1) _PoisonStack = 0x6868686868686868 & (1<<(8*ptrSize) - 1)
) )
//go:nosplit
func needwb() bool { func needwb() bool {
return gcphase == _GCmark || gcphase == _GCmarktermination || mheap_.shadow_enabled return gcphase == _GCmark || gcphase == _GCmarktermination || mheap_.shadow_enabled
} }
...@@ -232,7 +233,7 @@ func writebarrierptr_nostore(dst *uintptr, src uintptr) { ...@@ -232,7 +233,7 @@ func writebarrierptr_nostore(dst *uintptr, src uintptr) {
// Apply changes to shadow. // Apply changes to shadow.
// Since *dst has been overwritten already, we cannot check // Since *dst has been overwritten already, we cannot check
// whether there were any missed updates, but writebarrierptr_nostore // whether there were any missed updates, but writebarrierptr_nostore
// is only rarely used (right now there is just one call, in newstack). // is only rarely used.
if mheap_.shadow_enabled { if mheap_.shadow_enabled {
systemstack(func() { systemstack(func() {
addr := uintptr(unsafe.Pointer(dst)) addr := uintptr(unsafe.Pointer(dst))
...@@ -287,6 +288,12 @@ func writebarrieriface(dst *[2]uintptr, src [2]uintptr) { ...@@ -287,6 +288,12 @@ func writebarrieriface(dst *[2]uintptr, src [2]uintptr) {
// all the combinations of ptr+scalar up to four words. // all the combinations of ptr+scalar up to four words.
// The implementations are written to wbfat.go. // The implementations are written to wbfat.go.
//go:linkname reflect_typedmemmove reflect.typedmemmove
func reflect_typedmemmove(typ *_type, dst, src unsafe.Pointer) {
typedmemmove(typ, dst, src)
}
// typedmemmove copies a value of type t to dst from src.
//go:nosplit //go:nosplit
func typedmemmove(typ *_type, dst, src unsafe.Pointer) { func typedmemmove(typ *_type, dst, src unsafe.Pointer) {
if !needwb() || (typ.kind&kindNoPointers) != 0 { if !needwb() || (typ.kind&kindNoPointers) != 0 {
...@@ -322,6 +329,79 @@ func typedmemmove(typ *_type, dst, src unsafe.Pointer) { ...@@ -322,6 +329,79 @@ func typedmemmove(typ *_type, dst, src unsafe.Pointer) {
}) })
} }
// typedmemmovepartial is like typedmemmove but assumes that
// dst and src point off bytes into the value and only copies size bytes.
//go:linkname reflect_typedmemmovepartial reflect.typedmemmovepartial
func reflect_typedmemmovepartial(typ *_type, dst, src unsafe.Pointer, off, size uintptr) {
if !needwb() || (typ.kind&kindNoPointers) != 0 || size < ptrSize {
memmove(dst, src, size)
return
}
if off&(ptrSize-1) != 0 {
frag := -off & (ptrSize - 1)
// frag < size, because size >= ptrSize, checked above.
memmove(dst, src, frag)
size -= frag
dst = add(noescape(dst), frag)
src = add(noescape(src), frag)
off += frag
}
mask := loadPtrMask(typ)
nptr := (off + size) / ptrSize
for i := uintptr(off / ptrSize); i < nptr; i++ {
bits := mask[i/2] >> ((i & 1) << 2)
if (bits>>2)&_BitsMask == _BitsPointer {
writebarrierptr((*uintptr)(dst), *(*uintptr)(src))
} else {
*(*uintptr)(dst) = *(*uintptr)(src)
}
// TODO(rsc): The noescape calls should be unnecessary.
dst = add(noescape(dst), ptrSize)
src = add(noescape(src), ptrSize)
}
size &= ptrSize - 1
if size > 0 {
memmove(dst, src, size)
}
}
// callwritebarrier is invoked at the end of reflectcall, to execute
// write barrier operations to record the fact that a call's return
// values have just been copied to frame, starting at retoffset
// and continuing to framesize. The entire frame (not just the return
// values) is described by typ. Because the copy has already
// happened, we call writebarrierptr_nostore, and we must be careful
// not to be preempted before the write barriers have been run.
//go:nosplit
func callwritebarrier(typ *_type, frame unsafe.Pointer, framesize, retoffset uintptr) {
if !needwb() || typ == nil || (typ.kind&kindNoPointers) != 0 || framesize-retoffset < ptrSize {
return
}
systemstack(func() {
mask := loadPtrMask(typ)
// retoffset is known to be pointer-aligned (at least).
// TODO(rsc): The noescape call should be unnecessary.
dst := add(noescape(frame), retoffset)
nptr := framesize / ptrSize
for i := uintptr(retoffset / ptrSize); i < nptr; i++ {
bits := mask[i/2] >> ((i & 1) << 2)
if (bits>>2)&_BitsMask == _BitsPointer {
writebarrierptr_nostore((*uintptr)(dst), *(*uintptr)(dst))
}
// TODO(rsc): The noescape call should be unnecessary.
dst = add(noescape(dst), ptrSize)
}
})
}
//go:linkname reflect_typedslicecopy reflect.typedslicecopy
func reflect_typedslicecopy(elemType *_type, dst, src slice) int {
return typedslicecopy(elemType, dst, src)
}
//go:nosplit //go:nosplit
func typedslicecopy(typ *_type, dst, src slice) int { func typedslicecopy(typ *_type, dst, src slice) int {
n := dst.len n := dst.len
......
src/runtime/panic.go
View file @ df027ace
...@@ -301,7 +301,7 @@ func Goexit() { ...@@ -301,7 +301,7 @@ func Goexit() {
continue continue
} }
d.started = true d.started = true
reflectcall(unsafe.Pointer(d.fn), deferArgs(d), uint32(d.siz), uint32(d.siz)) reflectcall(nil, unsafe.Pointer(d.fn), deferArgs(d), uint32(d.siz), uint32(d.siz))
if gp._defer != d { if gp._defer != d {
throw("bad defer entry in Goexit") throw("bad defer entry in Goexit")
} }
...@@ -401,7 +401,7 @@ func gopanic(e interface{}) { ...@@ -401,7 +401,7 @@ func gopanic(e interface{}) {
d._panic = (*_panic)(noescape((unsafe.Pointer)(&p))) d._panic = (*_panic)(noescape((unsafe.Pointer)(&p)))
p.argp = unsafe.Pointer(getargp(0)) p.argp = unsafe.Pointer(getargp(0))
reflectcall(unsafe.Pointer(d.fn), deferArgs(d), uint32(d.siz), uint32(d.siz)) reflectcall(nil, unsafe.Pointer(d.fn), deferArgs(d), uint32(d.siz), uint32(d.siz))
p.argp = nil p.argp = nil
// reflectcall did not panic. Remove d. // reflectcall did not panic. Remove d.
......
src/runtime/stubs.go
View file @ df027ace
...@@ -110,7 +110,17 @@ func asminit() ...@@ -110,7 +110,17 @@ func asminit()
func setg(gg *g) func setg(gg *g)
func breakpoint() func breakpoint()
func reflectcall(fn, arg unsafe.Pointer, n uint32, retoffset uint32) // reflectcall calls fn with a copy of the n argument bytes pointed at by arg.
// After fn returns, reflectcall copies n-retoffset result bytes
// back into arg+retoffset before returning. If copying result bytes back,
// the caller should pass the argument frame type as argtype, so that
// call can execute appropriate write barriers during the copy.
// Package reflect passes a frame type. In package runtime, there is only
// one call that copies results back, in cgocallbackg1, and it does NOT pass a
// frame type, meaning there are no write barriers invoked. See that call
// site for justification.
func reflectcall(argtype *_type, fn, arg unsafe.Pointer, argsize uint32, retoffset uint32)
func procyield(cycles uint32) func procyield(cycles uint32)
func cgocallback_gofunc(fv *funcval, frame unsafe.Pointer, framesize uintptr) func cgocallback_gofunc(fv *funcval, frame unsafe.Pointer, framesize uintptr)
func goexit() func goexit()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7