Skip to content

G
go
Commit 31457cef authored by Russ Cox's avatar Russ Cox
Browse files
Options

all: merge dev.garbage (d1238958d4ae) into default branch 

When we start work on Gerrit, ppc64 and garbage collection
work will continue in the master branch, not the dev branches.

(We may still use dev branches for other things later, but
these are ready to be merged, and doing it now, before moving
to Git means we don't have to have dev branches working
in the Gerrit workflow on day one.)

TBR=rlh
CC=golang-codereviews
https://golang.org/cl/183140043
parents 09d92b6b db406241
Hide whitespace changes
Inline Side-by-side
Showing with 1890 additions and 554 deletions
lib/codereview/codereview.py
View file @ 31457cef
......@@ -2024,13 +2024,13 @@ def submit(ui, repo, *pats, **opts):
# push to remote; if it fails for any reason, roll back
try:
new_heads = len(hg_heads(ui, repo).split())
if old_heads != new_heads and not (old_heads == 0 and new_heads == 1):
if cl.desc.find("create new branch") < 0 and old_heads != new_heads and not (old_heads == 0 and new_heads == 1):
# Created new head, so we weren't up to date.
need_sync()
# Push changes to remote. If it works, we're committed. If not, roll back.
try:
if hg_push(ui, repo):
if hg_push(ui, repo, new_branch=cl.desc.find("create new branch")>=0):
raise hg_util.Abort("push error")
except hg_error.Abort, e:
if e.message.find("push creates new heads") >= 0:
......
src/cmd/gc/builtin.c
View file @ 31457cef
......@@ -24,6 +24,8 @@ char *runtimeimport =
"func @\"\".printslice (? any)\n"
"func @\"\".printnl ()\n"
"func @\"\".printsp ()\n"
"func @\"\".printlock ()\n"
"func @\"\".printunlock ()\n"
"func @\"\".concatstring2 (? string, ? string) (? string)\n"
"func @\"\".concatstring3 (? string, ? string, ? string) (? string)\n"
"func @\"\".concatstring4 (? string, ? string, ? string, ? string) (? string)\n"
......@@ -86,10 +88,33 @@ char *runtimeimport =
"func @\"\".writebarrierstring (@\"\".dst·1 *any, @\"\".src·2 any)\n"
"func @\"\".writebarrierslice (@\"\".dst·1 *any, @\"\".src·2 any)\n"
"func @\"\".writebarrieriface (@\"\".dst·1 *any, @\"\".src·2 any)\n"
"func @\"\".writebarrierfat2 (@\"\".dst·1 *any, _ *byte, @\"\".src·3 any)\n"
"func @\"\".writebarrierfat3 (@\"\".dst·1 *any, _ *byte, @\"\".src·3 any)\n"
"func @\"\".writebarrierfat4 (@\"\".dst·1 *any, _ *byte, @\"\".src·3 any)\n"
"func @\"\".writebarrierfat01 (@\"\".dst·1 *any, _ *byte, @\"\".src·3 any)\n"
"func @\"\".writebarrierfat10 (@\"\".dst·1 *any, _ *byte, @\"\".src·3 any)\n"
"func @\"\".writebarrierfat11 (@\"\".dst·1 *any, _ *byte, @\"\".src·3 any)\n"
"func @\"\".writebarrierfat001 (@\"\".dst·1 *any, _ *byte, @\"\".src·3 any)\n"
"func @\"\".writebarrierfat010 (@\"\".dst·1 *any, _ *byte, @\"\".src·3 any)\n"
"func @\"\".writebarrierfat011 (@\"\".dst·1 *any, _ *byte, @\"\".src·3 any)\n"
"func @\"\".writebarrierfat100 (@\"\".dst·1 *any, _ *byte, @\"\".src·3 any)\n"
"func @\"\".writebarrierfat101 (@\"\".dst·1 *any, _ *byte, @\"\".src·3 any)\n"
"func @\"\".writebarrierfat110 (@\"\".dst·1 *any, _ *byte, @\"\".src·3 any)\n"
"func @\"\".writebarrierfat111 (@\"\".dst·1 *any, _ *byte, @\"\".src·3 any)\n"
"func @\"\".writebarrierfat0001 (@\"\".dst·1 *any, _ *byte, @\"\".src·3 any)\n"
"func @\"\".writebarrierfat0010 (@\"\".dst·1 *any, _ *byte, @\"\".src·3 any)\n"
"func @\"\".writebarrierfat0011 (@\"\".dst·1 *any, _ *byte, @\"\".src·3 any)\n"
"func @\"\".writebarrierfat0100 (@\"\".dst·1 *any, _ *byte, @\"\".src·3 any)\n"
"func @\"\".writebarrierfat0101 (@\"\".dst·1 *any, _ *byte, @\"\".src·3 any)\n"
"func @\"\".writebarrierfat0110 (@\"\".dst·1 *any, _ *byte, @\"\".src·3 any)\n"
"func @\"\".writebarrierfat0111 (@\"\".dst·1 *any, _ *byte, @\"\".src·3 any)\n"
"func @\"\".writebarrierfat1000 (@\"\".dst·1 *any, _ *byte, @\"\".src·3 any)\n"
"func @\"\".writebarrierfat1001 (@\"\".dst·1 *any, _ *byte, @\"\".src·3 any)\n"
"func @\"\".writebarrierfat1010 (@\"\".dst·1 *any, _ *byte, @\"\".src·3 any)\n"
"func @\"\".writebarrierfat1011 (@\"\".dst·1 *any, _ *byte, @\"\".src·3 any)\n"
"func @\"\".writebarrierfat1100 (@\"\".dst·1 *any, _ *byte, @\"\".src·3 any)\n"
"func @\"\".writebarrierfat1101 (@\"\".dst·1 *any, _ *byte, @\"\".src·3 any)\n"
"func @\"\".writebarrierfat1110 (@\"\".dst·1 *any, _ *byte, @\"\".src·3 any)\n"
"func @\"\".writebarrierfat1111 (@\"\".dst·1 *any, _ *byte, @\"\".src·3 any)\n"
"func @\"\".writebarrierfat (@\"\".typ·1 *byte, @\"\".dst·2 *any, @\"\".src·3 *any)\n"
"func @\"\".writebarriercopy (@\"\".typ·2 *byte, @\"\".dst·3 any, @\"\".src·4 any) (? int)\n"
"func @\"\".selectnbsend (@\"\".chanType·2 *byte, @\"\".hchan·3 chan<- any, @\"\".elem·4 *any) (? bool)\n"
"func @\"\".selectnbrecv (@\"\".chanType·2 *byte, @\"\".elem·3 *any, @\"\".hchan·4 <-chan any) (? bool)\n"
"func @\"\".selectnbrecv2 (@\"\".chanType·2 *byte, @\"\".elem·3 *any, @\"\".received·4 *bool, @\"\".hchan·5 <-chan any) (? bool)\n"
......
src/cmd/gc/go.h
View file @ 31457cef
......@@ -1473,6 +1473,7 @@ void walk(Node *fn);
void walkexpr(Node **np, NodeList **init);
void walkexprlist(NodeList *l, NodeList **init);
void walkexprlistsafe(NodeList *l, NodeList **init);
void walkexprlistcheap(NodeList *l, NodeList **init);
void walkstmt(Node **np);
void walkstmtlist(NodeList *l);
Node* conv(Node*, Type*);
......
src/cmd/gc/runtime.go
View file @ 31457cef
......@@ -36,6 +36,8 @@ func printeface(any)
func printslice(any)
func printnl()
func printsp()
func printlock()
func printunlock()
func concatstring2(string, string) string
func concatstring3(string, string, string) string
......@@ -115,10 +117,35 @@ func writebarrieriface(dst *any, src any)
// The unused *byte argument makes sure that src is 2-pointer-aligned,
// which is the maximum alignment on NaCl amd64p32
// (and possibly on 32-bit systems if we start 64-bit aligning uint64s).
func writebarrierfat2(dst *any, _ *byte, src any)
func writebarrierfat3(dst *any, _ *byte, src any)
func writebarrierfat4(dst *any, _ *byte, src any)
// The bitmap in the name tells which words being copied are pointers.
func writebarrierfat01(dst *any, _ *byte, src any)
func writebarrierfat10(dst *any, _ *byte, src any)
func writebarrierfat11(dst *any, _ *byte, src any)
func writebarrierfat001(dst *any, _ *byte, src any)
func writebarrierfat010(dst *any, _ *byte, src any)
func writebarrierfat011(dst *any, _ *byte, src any)
func writebarrierfat100(dst *any, _ *byte, src any)
func writebarrierfat101(dst *any, _ *byte, src any)
func writebarrierfat110(dst *any, _ *byte, src any)
func writebarrierfat111(dst *any, _ *byte, src any)
func writebarrierfat0001(dst *any, _ *byte, src any)
func writebarrierfat0010(dst *any, _ *byte, src any)
func writebarrierfat0011(dst *any, _ *byte, src any)
func writebarrierfat0100(dst *any, _ *byte, src any)
func writebarrierfat0101(dst *any, _ *byte, src any)
func writebarrierfat0110(dst *any, _ *byte, src any)
func writebarrierfat0111(dst *any, _ *byte, src any)
func writebarrierfat1000(dst *any, _ *byte, src any)
func writebarrierfat1001(dst *any, _ *byte, src any)
func writebarrierfat1010(dst *any, _ *byte, src any)
func writebarrierfat1011(dst *any, _ *byte, src any)
func writebarrierfat1100(dst *any, _ *byte, src any)
func writebarrierfat1101(dst *any, _ *byte, src any)
func writebarrierfat1110(dst *any, _ *byte, src any)
func writebarrierfat1111(dst *any, _ *byte, src any)
func writebarrierfat(typ *byte, dst *any, src *any)
func writebarriercopy(typ *byte, dst any, src any) int
func selectnbsend(chanType *byte, hchan chan<- any, elem *any) bool
func selectnbrecv(chanType *byte, elem *any, hchan <-chan any) bool
......
src/cmd/gc/typecheck.c
View file @ 31457cef
......@@ -2891,7 +2891,8 @@ typecheckas(Node *n)
case OSLICE3:
case OSLICESTR:
// For x = x[0:y], x can be updated in place, without touching pointer.
if(samesafeexpr(n->left, n->right->left) && (n->right->right->left == N || iszero(n->right->right->left)))
// TODO(rsc): Reenable once it is actually updated in place without touching the pointer.
if(0 && samesafeexpr(n->left, n->right->left) && (n->right->right->left == N || iszero(n->right->right->left)))
n->right->reslice = 1;
break;
......@@ -2899,7 +2900,8 @@ typecheckas(Node *n)
// For x = append(x, ...), x can be updated in place when there is capacity,
// without touching the pointer; otherwise the emitted code to growslice
// can take care of updating the pointer, and only in that case.
if(n->right->list != nil && samesafeexpr(n->left, n->right->list->n))
// TODO(rsc): Reenable once the emitted code does update the pointer.
if(0 && n->right->list != nil && samesafeexpr(n->left, n->right->list->n))
n->right->reslice = 1;
break;
}
......
src/cmd/gc/walk.c
View file @ 31457cef
......@@ -6,6 +6,7 @@
#include <libc.h>
#include "go.h"
#include "../ld/textflag.h"
#include "../../runtime/mgc0.h"
static Node* walkprint(Node*, NodeList**);
static Node* writebarrierfn(char*, Type*, Type*);
......@@ -362,6 +363,15 @@ walkexprlistsafe(NodeList *l, NodeList **init)
}
}
void
walkexprlistcheap(NodeList *l, NodeList **init)
{
for(; l; l=l->next) {
l->n = cheapexpr(l->n, init);
walkexpr(&l->n, init);
}
}
void
walkexpr(Node **np, NodeList **init)
{
......@@ -1771,6 +1781,11 @@ walkprint(Node *nn, NodeList **init)
calls = nil;
notfirst = 0;
// Hoist all the argument evaluation up before the lock.
walkexprlistcheap(all, init);
calls = list(calls, mkcall("printlock", T, init));
for(l=all; l; l=l->next) {
if(notfirst) {
calls = list(calls, mkcall("printsp", T, init));
......@@ -1851,6 +1866,9 @@ walkprint(Node *nn, NodeList **init)
if(op == OPRINTN)
calls = list(calls, mkcall("printnl", T, nil));
calls = list(calls, mkcall("printunlock", T, init));
typechecklist(calls, Etop);
walkexprlist(calls, init);
......@@ -1987,6 +2005,9 @@ applywritebarrier(Node *n, NodeList **init)
{
Node *l, *r;
Type *t;
vlong x;
static Bvec *bv;
char name[32];
if(n->left && n->right && needwritebarrier(n->left, n->right)) {
t = n->left->type;
......@@ -2004,14 +2025,35 @@ applywritebarrier(Node *n, NodeList **init)
} else if(isinter(t)) {
n = mkcall1(writebarrierfn("writebarrieriface", t, n->right->type), T, init,
l, n->right);
} else if(t->width == 2*widthptr) {
n = mkcall1(writebarrierfn("writebarrierfat2", t, n->right->type), T, init,
l, nodnil(), n->right);
} else if(t->width == 3*widthptr) {
n = mkcall1(writebarrierfn("writebarrierfat3", t, n->right->type), T, init,
l, nodnil(), n->right);
} else if(t->width == 4*widthptr) {
n = mkcall1(writebarrierfn("writebarrierfat4", t, n->right->type), T, init,
} else if(t->width <= 4*widthptr) {
x = 0;
if(bv == nil)
bv = bvalloc(BitsPerPointer*4);
bvresetall(bv);
twobitwalktype1(t, &x, bv);
// The bvgets are looking for BitsPointer in successive slots.
enum {
PtrBit = 1,
};
if(BitsPointer != (1<<PtrBit))
fatal("wrong PtrBit");
switch(t->width/widthptr) {
default:
fatal("found writebarrierfat for %d-byte object of type %T", (int)t->width, t);
case 2:
snprint(name, sizeof name, "writebarrierfat%d%d",
bvget(bv, PtrBit), bvget(bv, BitsPerPointer+PtrBit));
break;
case 3:
snprint(name, sizeof name, "writebarrierfat%d%d%d",
bvget(bv, PtrBit), bvget(bv, BitsPerPointer+PtrBit), bvget(bv, 2*BitsPerPointer+PtrBit));
break;
case 4:
snprint(name, sizeof name, "writebarrierfat%d%d%d%d",
bvget(bv, PtrBit), bvget(bv, BitsPerPointer+PtrBit), bvget(bv, 2*BitsPerPointer+PtrBit), bvget(bv, 3*BitsPerPointer+PtrBit));
break;
}
n = mkcall1(writebarrierfn(name, t, n->right->type), T, init,
l, nodnil(), n->right);
} else {
r = n->right;
......@@ -2873,6 +2915,11 @@ copyany(Node *n, NodeList **init, int runtimecall)
{
Node *nl, *nr, *nfrm, *nto, *nif, *nlen, *nwid, *fn;
NodeList *l;
if(haspointers(n->left->type->type)) {
fn = writebarrierfn("writebarriercopy", n->left->type, n->right->type);
return mkcall1(fn, n->type, init, typename(n->left->type->type), n->left, n->right);
}
if(runtimecall) {
if(n->right->type->etype == TSTRING)
......
src/runtime/asm_386.s
View file @ 31457cef
......@@ -2285,3 +2285,23 @@ TEXT runtime·getg(SB),NOSPLIT,$0-4
MOVL AX, ret+0(FP)
RET
TEXT runtime·prefetcht0(SB),NOSPLIT,$0-4
MOVL addr+0(FP), AX
PREFETCHT0 (AX)
RET
TEXT runtime·prefetcht1(SB),NOSPLIT,$0-4
MOVL addr+0(FP), AX
PREFETCHT1 (AX)
RET
TEXT runtime·prefetcht2(SB),NOSPLIT,$0-4
MOVL addr+0(FP), AX
PREFETCHT2 (AX)
RET
TEXT runtime·prefetchnta(SB),NOSPLIT,$0-4
MOVL addr+0(FP), AX
PREFETCHNTA (AX)
RET
src/runtime/asm_amd64.s
View file @ 31457cef
......@@ -2228,3 +2228,23 @@ TEXT runtime·getg(SB),NOSPLIT,$0-8
MOVQ g(CX), AX
MOVQ AX, ret+0(FP)
RET
TEXT runtime·prefetcht0(SB),NOSPLIT,$0-8
MOVQ addr+0(FP), AX
PREFETCHT0 (AX)
RET
TEXT runtime·prefetcht1(SB),NOSPLIT,$0-8
MOVQ addr+0(FP), AX
PREFETCHT1 (AX)
RET
TEXT runtime·prefetcht2(SB),NOSPLIT,$0-8
MOVQ addr+0(FP), AX
PREFETCHT2 (AX)
RET
TEXT runtime·prefetchnta(SB),NOSPLIT,$0-8
MOVQ addr+0(FP), AX
PREFETCHNTA (AX)
RET
src/runtime/asm_amd64p32.s
View file @ 31457cef
......@@ -1079,3 +1079,24 @@ TEXT runtime·getg(SB),NOSPLIT,$0-4
MOVL g(CX), AX
MOVL AX, ret+0(FP)
RET
TEXT runtime·prefetcht0(SB),NOSPLIT,$0-4
MOVL addr+0(FP), AX
PREFETCHT0 (AX)
RET
TEXT runtime·prefetcht1(SB),NOSPLIT,$0-4
MOVL addr+0(FP), AX
PREFETCHT1 (AX)
RET
TEXT runtime·prefetcht2(SB),NOSPLIT,$0-4
MOVL addr+0(FP), AX
PREFETCHT2 (AX)
RET
TEXT runtime·prefetchnta(SB),NOSPLIT,$0-4
MOVL addr+0(FP), AX
PREFETCHNTA (AX)
RET
src/runtime/asm_arm.s
View file @ 31457cef
......@@ -1320,3 +1320,15 @@ TEXT runtime·goexit(SB),NOSPLIT,$-4-0
TEXT runtime·getg(SB),NOSPLIT,$-4-4
MOVW g, ret+0(FP)
RET
TEXT runtime·prefetcht0(SB),NOSPLIT,$0-4
RET
TEXT runtime·prefetcht1(SB),NOSPLIT,$0-4
RET
TEXT runtime·prefetcht2(SB),NOSPLIT,$0-4
RET
TEXT runtime·prefetchnta(SB),NOSPLIT,$0-4
RET
src/runtime/asm_ppc64x.s
View file @ 31457cef
......@@ -977,3 +977,15 @@ TEXT runtime·goexit(SB),NOSPLIT,$-8-0
TEXT runtime·getg(SB),NOSPLIT,$-8-8
MOVD g, ret+0(FP)
RETURN
TEXT runtime·prefetcht0(SB),NOSPLIT,$0-8
RETURN
TEXT runtime·prefetcht1(SB),NOSPLIT,$0-8
RETURN
TEXT runtime·prefetcht2(SB),NOSPLIT,$0-8
RETURN
TEXT runtime·prefetchnta(SB),NOSPLIT,$0-8
RETURN
src/runtime/export_test.go
View file @ 31457cef
......@@ -26,7 +26,7 @@ var Exitsyscall = exitsyscall
var LockedOSThread = lockedOSThread
type LFNode struct {
Next *LFNode
Next uint64
Pushcnt uintptr
}
......
src/runtime/heapdump.go
View file @ 31457cef
......@@ -464,8 +464,8 @@ func dumpobjs() {
if n > uintptr(len(freemark)) {
gothrow("freemark array doesn't have enough entries")
}
for l := s.freelist; l != nil; l = l.next {
freemark[(uintptr(unsafe.Pointer(l))-p)/size] = true
for l := s.freelist; l.ptr() != nil; l = l.ptr().next {
freemark[(uintptr(l)-p)/size] = true
}
for j := uintptr(0); j < n; j, p = j+1, p+size {
if freemark[j] {
......
src/runtime/lfstack.go
View file @ 31457cef
......@@ -18,7 +18,7 @@ func lfstackpush(head *uint64, node *lfnode) {
}
for {
old := atomicload64(head)
node.next, _ = lfstackUnpack(old)
node.next = old
if cas64(head, old, new) {
break
}
......@@ -32,12 +32,8 @@ func lfstackpop(head *uint64) unsafe.Pointer {
return nil
}
node, _ := lfstackUnpack(old)
node2 := (*lfnode)(atomicloadp(unsafe.Pointer(&node.next)))
new := uint64(0)
if node2 != nil {
new = lfstackPack(node2, node2.pushcnt)
}
if cas64(head, old, new) {
next := atomicload64(&node.next)
if cas64(head, old, next) {
return unsafe.Pointer(node)
}
}
......
src/runtime/lfstack_test.go
View file @ 31457cef
......@@ -121,7 +121,7 @@ func TestLFStackStress(t *testing.T) {
}
cnt++
sum2 += node.data
node.Next = nil
node.Next = 0
}
}
if cnt != K {
......
src/runtime/malloc.go
View file @ 31457cef
......@@ -140,14 +140,14 @@ func mallocgc(size uintptr, typ *_type, flags uint32) unsafe.Pointer {
// Allocate a new maxTinySize block.
s = c.alloc[tinySizeClass]
v := s.freelist
if v == nil {
if v.ptr() == nil {
systemstack(func() {
mCache_Refill(c, tinySizeClass)
})
s = c.alloc[tinySizeClass]
v = s.freelist
}
s.freelist = v.next
s.freelist = v.ptr().next
s.ref++
//TODO: prefetch v.next
x = unsafe.Pointer(v)
......@@ -170,19 +170,19 @@ func mallocgc(size uintptr, typ *_type, flags uint32) unsafe.Pointer {
size = uintptr(class_to_size[sizeclass])
s = c.alloc[sizeclass]
v := s.freelist
if v == nil {
if v.ptr() == nil {
systemstack(func() {
mCache_Refill(c, int32(sizeclass))
})
s = c.alloc[sizeclass]
v = s.freelist
}
s.freelist = v.next
s.freelist = v.ptr().next
s.ref++
//TODO: prefetch
x = unsafe.Pointer(v)
if flags&flagNoZero == 0 {
v.next = nil
v.ptr().next = 0
if size > 2*ptrSize && ((*[2]uintptr)(x))[1] != 0 {
memclr(unsafe.Pointer(v), size)
}
......@@ -241,6 +241,8 @@ func mallocgc(size uintptr, typ *_type, flags uint32) unsafe.Pointer {
masksize = masksize * pointersPerByte / 8 // 4 bits per word
masksize++ // unroll flag in the beginning
if masksize > maxGCMask && typ.gc[1] != 0 {
// write barriers have not been updated to deal with this case yet.
gothrow("maxGCMask too small for now")
// If the mask is too large, unroll the program directly
// into the GC bitmap. It's 7 times slower than copying
// from the pre-unrolled mask, but saves 1/16 of type size
......@@ -295,6 +297,17 @@ func mallocgc(size uintptr, typ *_type, flags uint32) unsafe.Pointer {
}
}
marked:
// GCmarkterminate allocates black
// All slots hold nil so no scanning is needed.
// This may be racing with GC so do it atomically if there can be
// a race marking the bit.
if gcphase == _GCmarktermination {
systemstack(func() {
gcmarknewobject_m(uintptr(x))
})
}
if raceenabled {
racemalloc(x, size)
}
......@@ -328,13 +341,43 @@ marked:
}
}
if memstats.heap_alloc >= memstats.next_gc {
if memstats.heap_alloc >= memstats.next_gc/2 {
gogc(0)
}
return x
}
func loadPtrMask(typ *_type) []uint8 {
var ptrmask *uint8
nptr := (uintptr(typ.size) + ptrSize - 1) / ptrSize
if typ.kind&kindGCProg != 0 {
masksize := nptr
if masksize%2 != 0 {
masksize *= 2 // repeated
}
masksize = masksize * pointersPerByte / 8 // 4 bits per word
masksize++ // unroll flag in the beginning
if masksize > maxGCMask && typ.gc[1] != 0 {
// write barriers have not been updated to deal with this case yet.
gothrow("maxGCMask too small for now")
}
ptrmask = (*uint8)(unsafe.Pointer(uintptr(typ.gc[0])))
// Check whether the program is already unrolled
// by checking if the unroll flag byte is set
maskword := uintptr(atomicloadp(unsafe.Pointer(ptrmask)))
if *(*uint8)(unsafe.Pointer(&maskword)) == 0 {
systemstack(func() {
unrollgcprog_m(typ)
})
}
ptrmask = (*uint8)(add(unsafe.Pointer(ptrmask), 1)) // skip the unroll flag byte
} else {
ptrmask = (*uint8)(unsafe.Pointer(typ.gc[0])) // pointer to unrolled mask
}
return (*[1 << 30]byte)(unsafe.Pointer(ptrmask))[:(nptr+1)/2]
}
// implementation of new builtin
func newobject(typ *_type) unsafe.Pointer {
flags := uint32(0)
......@@ -429,7 +472,21 @@ func gogc(force int32) {
mp = acquirem()
mp.gcing = 1
releasem(mp)
systemstack(stoptheworld)
systemstack(finishsweep_m) // finish sweep before we start concurrent scan.
if true { // To turn on concurrent scan and mark set to true...
systemstack(starttheworld)
// Do a concurrent heap scan before we stop the world.
systemstack(gcscan_m)
systemstack(stoptheworld)
systemstack(gcinstallmarkwb_m)
systemstack(starttheworld)
systemstack(gcmark_m)
systemstack(stoptheworld)
systemstack(gcinstalloffwb_m)
}
if mp != acquirem() {
gothrow("gogc: rescheduled")
}
......@@ -445,17 +502,21 @@ func gogc(force int32) {
if debug.gctrace > 1 {
n = 2
}
eagersweep := force >= 2
for i := 0; i < n; i++ {
if i > 0 {
startTime = nanotime()
}
// switch to g0, call gc, then switch back
eagersweep := force >= 2
systemstack(func() {
gc_m(startTime, eagersweep)
})
}
systemstack(func() {
gccheckmark_m(startTime, eagersweep)
})
// all done
mp.gcing = 0
semrelease(&worldsema)
......@@ -470,6 +531,14 @@ func gogc(force int32) {
}
}
func GCcheckmarkenable() {
systemstack(gccheckmarkenable_m)
}
func GCcheckmarkdisable() {
systemstack(gccheckmarkdisable_m)
}
// GC runs a garbage collection.
func GC() {
gogc(2)
......
src/runtime/malloc2.go
View file @ 31457cef
......@@ -139,10 +139,35 @@ const (
)
// A generic linked list of blocks. (Typically the block is bigger than sizeof(MLink).)
// Since assignments to mlink.next will result in a write barrier being preformed
// this can not be used by some of the internal GC structures. For example when
// the sweeper is placing an unmarked object on the free list it does not want the
// write barrier to be called since that could result in the object being reachable.
type mlink struct {
next *mlink
}
// A gclink is a node in a linked list of blocks, like mlink,
// but it is opaque to the garbage collector.
// The GC does not trace the pointers during collection,
// and the compiler does not emit write barriers for assignments
// of gclinkptr values. Code should store references to gclinks
// as gclinkptr, not as *gclink.
type gclink struct {
next gclinkptr
}
// A gclinkptr is a pointer to a gclink, but it is opaque
// to the garbage collector.
type gclinkptr uintptr
// ptr returns the *gclink form of p.
// The result should be used for accessing fields, not stored
// in other data structures.
func (p gclinkptr) ptr() *gclink {
return (*gclink)(unsafe.Pointer(p))
}
// sysAlloc obtains a large chunk of zeroed memory from the
// operating system, typically on the order of a hundred kilobytes
// or a megabyte.
......@@ -275,8 +300,8 @@ type mcachelist struct {
}
type stackfreelist struct {
list *mlink // linked list of free stacks
size uintptr // total size of stacks in list
list gclinkptr // linked list of free stacks
size uintptr // total size of stacks in list
}
// Per-thread (in Go, per-P) cache for small objects.
......@@ -299,8 +324,6 @@ type mcache struct {
sudogcache *sudog
gcworkbuf unsafe.Pointer
// Local allocator stats, flushed during GC.
local_nlookup uintptr // number of pointer lookups
local_largefree uintptr // bytes freed for large objects (>maxsmallsize)
......@@ -348,11 +371,11 @@ const (
)
type mspan struct {
next *mspan // in a span linked list
prev *mspan // in a span linked list
start pageID // starting page number
npages uintptr // number of pages in span
freelist *mlink // list of free objects
next *mspan // in a span linked list
prev *mspan // in a span linked list
start pageID // starting page number
npages uintptr // number of pages in span
freelist gclinkptr // list of free objects
// sweep generation:
// if sweepgen == h->sweepgen - 2, the span needs sweeping
// if sweepgen == h->sweepgen - 1, the span is currently being swept
......
src/runtime/mcache.go
View file @ 31457cef
......@@ -38,7 +38,12 @@ func freemcache(c *mcache) {
systemstack(func() {
mCache_ReleaseAll(c)
stackcache_clear(c)
gcworkbuffree(c.gcworkbuf)
// NOTE(rsc,rlh): If gcworkbuffree comes back, we need to coordinate
// with the stealing of gcworkbufs during garbage collection to avoid
// a race where the workbuf is double-freed.
// gcworkbuffree(c.gcworkbuf)
lock(&mheap_.lock)
purgecachedstats(c)
fixAlloc_Free(&mheap_.cachealloc, unsafe.Pointer(c))
......@@ -54,7 +59,7 @@ func mCache_Refill(c *mcache, sizeclass int32) *mspan {
_g_.m.locks++
// Return the current cached span to the central lists.
s := c.alloc[sizeclass]
if s.freelist != nil {
if s.freelist.ptr() != nil {
gothrow("refill on a nonempty span")
}
if s != &emptymspan {
......@@ -66,7 +71,7 @@ func mCache_Refill(c *mcache, sizeclass int32) *mspan {
if s == nil {
gothrow("out of memory")
}
if s.freelist == nil {
if s.freelist.ptr() == nil {
println(s.ref, (s.npages<<_PageShift)/s.elemsize)
gothrow("empty span")
}
......
src/runtime/mcentral.go
View file @ 31457cef
......@@ -55,7 +55,7 @@ retry:
mSpanList_InsertBack(&c.empty, s)
unlock(&c.lock)
mSpan_Sweep(s, true)
if s.freelist != nil {
if s.freelist.ptr() != nil {
goto havespan
}
lock(&c.lock)
......@@ -90,7 +90,7 @@ havespan:
if n == 0 {
gothrow("empty span")
}
if s.freelist == nil {
if s.freelist.ptr() == nil {
gothrow("freelist empty")
}
s.incache = true
......@@ -122,14 +122,14 @@ func mCentral_UncacheSpan(c *mcentral, s *mspan) {
// the latest generation.
// If preserve=true, don't return the span to heap nor relink in MCentral lists;
// caller takes care of it.
func mCentral_FreeSpan(c *mcentral, s *mspan, n int32, start *mlink, end *mlink, preserve bool) bool {
func mCentral_FreeSpan(c *mcentral, s *mspan, n int32, start gclinkptr, end gclinkptr, preserve bool) bool {
if s.incache {
gothrow("freespan into cached span")
}
// Add the objects back to s's free list.
wasempty := s.freelist == nil
end.next = s.freelist
wasempty := s.freelist.ptr() == nil
end.ptr().next = s.freelist
s.freelist = start
s.ref -= uint16(n)
......@@ -165,7 +165,7 @@ func mCentral_FreeSpan(c *mcentral, s *mspan, n int32, start *mlink, end *mlink,
// s is completely freed, return it to the heap.
mSpanList_Remove(s)
s.needzero = 1
s.freelist = nil
s.freelist = 0
unlock(&c.lock)
unmarkspan(uintptr(s.start)<<_PageShift, s.npages<<_PageShift)
mHeap_Free(&mheap_, s, 0)
......@@ -183,17 +183,21 @@ func mCentral_Grow(c *mcentral) *mspan {
return nil
}
// Carve span into sequence of blocks.
tailp := &s.freelist
p := uintptr(s.start << _PageShift)
s.limit = p + size*n
for i := uintptr(0); i < n; i++ {
v := (*mlink)(unsafe.Pointer(p))
*tailp = v
tailp = &v.next
head := gclinkptr(p)
tail := gclinkptr(p)
// i==0 iteration already done
for i := uintptr(1); i < n; i++ {
p += size
tail.ptr().next = gclinkptr(p)
tail = gclinkptr(p)
}
*tailp = nil
if s.freelist.ptr() != nil {
gothrow("freelist not empty")
}
tail.ptr().next = 0
s.freelist = head
markspan(unsafe.Pointer(uintptr(s.start)<<_PageShift), size, n, size*n < s.npages<<_PageShift)
return s
}
src/runtime/mgc.go
View file @ 31457cef
......@@ -7,22 +7,72 @@
// Garbage collector (GC).
//
// GC is:
// - mark&sweep
// - mostly precise (with the exception of some C-allocated objects, assembly frames/arguments, etc)
// - parallel (up to MaxGcproc threads)
// - partially concurrent (mark is stop-the-world, while sweep is concurrent)
// - non-moving/non-compacting
// - full (non-partial)
// The GC runs concurrently with mutator threads, is type accurate (aka precise), allows multiple GC
// thread to run in parallel. It is a concurrent mark and sweep that uses a write barrier. It is
// non-generational and non-compacting. Allocation is done using size segregated per P allocation
// areas to minimize fragmentation while eliminating locks in the common case.
//
// GC rate.
// Next GC is after we've allocated an extra amount of memory proportional to
// the amount already in use. The proportion is controlled by GOGC environment variable
// (100 by default). If GOGC=100 and we're using 4M, we'll GC again when we get to 8M
// (this mark is tracked in next_gc variable). This keeps the GC cost in linear
// proportion to the allocation cost. Adjusting GOGC just changes the linear constant
// (and also the amount of extra memory used).
// The algorithm decomposes into several steps.
// This is a high level description of the algorithm being used. For an overview of GC a good
// place to start is Richard Jones' gchandbook.org.
//
// The algorithm's intellectual heritage includes Dijkstra's on-the-fly algorithm, see
// Edsger W. Dijkstra, Leslie Lamport, A. J. Martin, C. S. Scholten, and E. F. M. Steffens. 1978.
// On-the-fly garbage collection: an exercise in cooperation. Commun. ACM 21, 11 (November 1978), 966-975.
// For journal quality proofs that these steps are complete, correct, and terminate see
// Hudson, R., and Moss, J.E.B. Copying Garbage Collection without stopping the world.
// Concurrency and Computation: Practice and Experience 15(3-5), 2003.
//
// 0. Set phase = GCscan from GCoff.
// 1. Wait for all P's to acknowledge phase change.
// At this point all goroutines have passed through a GC safepoint and
// know we are in the GCscan phase.
// 2. GC scans all goroutine stacks, mark and enqueues all encountered pointers
// (marking avoids most duplicate enqueuing but races may produce duplication which is benign).
// Preempted goroutines are scanned before P schedules next goroutine.
// 3. Set phase = GCmark.
// 4. Wait for all P's to acknowledge phase change.
// 5. Now write barrier marks and enqueues black, grey, or white to white pointers.
// Malloc still allocates white (non-marked) objects.
// 6. Meanwhile GC transitively walks the heap marking reachable objects.
// 7. When GC finishes marking heap, it preempts P's one-by-one and
// retakes partial wbufs (filled by write barrier or during a stack scan of the goroutine
// currently scheduled on the P).
// 8. Once the GC has exhausted all available marking work it sets phase = marktermination.
// 9. Wait for all P's to acknowledge phase change.
// 10. Malloc now allocates black objects, so number of unmarked reachable objects
// monotonically decreases.
// 11. GC preempts P's one-by-one taking partial wbufs and marks all unmarked yet reachable objects.
// 12. When GC completes a full cycle over P's and discovers no new grey
// objects, (which means all reachable objects are marked) set phase = GCsweep.
// 13. Wait for all P's to acknowledge phase change.
// 14. Now malloc allocates white (but sweeps spans before use).
// Write barrier becomes nop.
// 15. GC does background sweeping, see description below.
// 16. When sweeping is complete set phase to GCoff.
// 17. When sufficient allocation has taken place replay the sequence starting at 0 above,
// see discussion of GC rate below.
// Changing phases.
// Phases are changed by setting the gcphase to the next phase and possibly calling ackgcphase.
// All phase action must be benign in the presence of a change.
// Starting with GCoff
// GCoff to GCscan
// GSscan scans stacks and globals greying them and never marks an object black.
// Once all the P's are aware of the new phase they will scan gs on preemption.
// This means that the scanning of preempted gs can't start until all the Ps
// have acknowledged.
// GCscan to GCmark
// GCMark turns on the write barrier which also only greys objects. No scanning
// of objects (making them black) can happen until all the Ps have acknowledged
// the phase change.
// GCmark to GCmarktermination
// The only change here is that we start allocating black so the Ps must acknowledge
// the change before we begin the termination algorithm
// GCmarktermination to GSsweep
// Object currently on the freelist must be marked black for this to work.
// Are things on the free lists black or white? How does the sweep phase work?
// Concurrent sweep.
// The sweep phase proceeds concurrently with normal program execution.
// The heap is swept span-by-span both lazily (when a goroutine needs another span)
......@@ -53,6 +103,14 @@
// The finalizer goroutine is kicked off only when all spans are swept.
// When the next GC starts, it sweeps all not-yet-swept spans (if any).
// GC rate.
// Next GC is after we've allocated an extra amount of memory proportional to
// the amount already in use. The proportion is controlled by GOGC environment variable
// (100 by default). If GOGC=100 and we're using 4M, we'll GC again when we get to 8M
// (this mark is tracked in next_gc variable). This keeps the GC cost in linear
// proportion to the allocation cost. Adjusting GOGC just changes the linear constant
// (and also the amount of extra memory used).
package runtime
import "unsafe"
......@@ -75,7 +133,7 @@ const (
// ptrmask for an allocation containing a single pointer.
var oneptr = [...]uint8{bitsPointer}
// Initialized from $GOGC. GOGC=off means no gc.
// Initialized from $GOGC. GOGC=off means no GC.
var gcpercent int32
// Holding worldsema grants an M the right to try to stop the world.
......@@ -93,6 +151,17 @@ var gcpercent int32
//
var worldsema uint32 = 1
// It is a bug if bits does not have bitBoundary set but
// there are still some cases where this happens related
// to stack spans.
type markbits struct {
bitp *byte // pointer to the byte holding xbits
shift uintptr // bits xbits needs to be shifted to get bits
xbits byte // byte holding all the bits from *bitp
bits byte // mark and boundary bits relevant to corresponding slot.
tbits byte // pointer||scalar bits relevant to corresponding slot.
}
type workbuf struct {
node lfnode // must be first
nobj uintptr
......@@ -121,6 +190,7 @@ var nbadblock int32
type workdata struct {
full uint64 // lock-free list of full blocks
empty uint64 // lock-free list of empty blocks
partial uint64 // lock-free list of partially filled blocks
pad0 [_CacheLineSize]uint8 // prevents false-sharing between full/empty and nproc/nwait
nproc uint32
tstart int64
......@@ -143,293 +213,405 @@ func have_cgo_allocate() bool {
return &weak_cgo_allocate != nil
}
// scanblock scans a block of n bytes starting at pointer b for references
// to other objects, scanning any it finds recursively until there are no
// unscanned objects left. Instead of using an explicit recursion, it keeps
// a work list in the Workbuf* structures and loops in the main function
// body. Keeping an explicit work list is easier on the stack allocator and
// more efficient.
func scanblock(b, n uintptr, ptrmask *uint8) {
// Cache memory arena parameters in local vars.
arena_start := mheap_.arena_start
arena_used := mheap_.arena_used
wbuf := getempty(nil)
nobj := wbuf.nobj
wp := &wbuf.obj[nobj]
keepworking := b == 0
// To help debug the concurrent GC we remark with the world
// stopped ensuring that any object encountered has their normal
// mark bit set. To do this we use an orthogonal bit
// pattern to indicate the object is marked. The following pattern
// uses the upper two bits in the object's bounday nibble.
// 01: scalar not marked
// 10: pointer not marked
// 11: pointer marked
// 00: scalar marked
// Xoring with 01 will flip the pattern from marked to unmarked and vica versa.
// The higher bit is 1 for pointers and 0 for scalars, whether the object
// is marked or not.
// The first nibble no longer holds the bitsDead pattern indicating that the
// there are no more pointers in the object. This information is held
// in the second nibble.
// When marking an object if the bool checkmark is true one uses the above
// encoding, otherwise one uses the bitMarked bit in the lower two bits
// of the nibble.
var (
checkmark = false
gccheckmarkenable = true
)
var ptrbitp unsafe.Pointer
// Is address b in the known heap. If it doesn't have a valid gcmap
// returns false. For example pointers into stacks will return false.
func inheap(b uintptr) bool {
if b == 0 || b < mheap_.arena_start || b >= mheap_.arena_used {
return false
}
// Not a beginning of a block, consult span table to find the block beginning.
k := b >> _PageShift
x := k
x -= mheap_.arena_start >> _PageShift
s := h_spans[x]
if s == nil || pageID(k) < s.start || b >= s.limit || s.state != mSpanInUse {
return false
}
return true
}
// ptrmask can have 2 possible values:
// 1. nil - obtain pointer mask from GC bitmap.
// 2. pointer to a compact mask (for stacks and data).
goto_scanobj := b != 0
// Given an address in the heap return the relevant byte from the gcmap. This routine
// can be used on addresses to the start of an object or to the interior of the an object.
func slottombits(obj uintptr, mbits *markbits) {
off := (obj&^(ptrSize-1) - mheap_.arena_start) / ptrSize
mbits.bitp = (*byte)(unsafe.Pointer(mheap_.arena_start - off/wordsPerBitmapByte - 1))
mbits.shift = off % wordsPerBitmapByte * gcBits
mbits.xbits = *mbits.bitp
mbits.bits = (mbits.xbits >> mbits.shift) & bitMask
mbits.tbits = ((mbits.xbits >> mbits.shift) & bitPtrMask) >> 2
}
// b is a pointer into the heap.
// Find the start of the object refered to by b.
// Set mbits to the associated bits from the bit map.
// If b is not a valid heap object return nil and
// undefined values in mbits.
func objectstart(b uintptr, mbits *markbits) uintptr {
obj := b &^ (ptrSize - 1)
for {
if goto_scanobj {
goto_scanobj = false
} else {
if nobj == 0 {
// Out of work in workbuf.
if !keepworking {
putempty(wbuf)
return
}
slottombits(obj, mbits)
if mbits.bits&bitBoundary == bitBoundary {
break
}
// Refill workbuf from global queue.
wbuf = getfull(wbuf)
if wbuf == nil {
return
}
nobj = wbuf.nobj
if nobj < uintptr(len(wbuf.obj)) {
wp = &wbuf.obj[nobj]
} else {
wp = nil
}
// Not a beginning of a block, consult span table to find the block beginning.
k := b >> _PageShift
x := k
x -= mheap_.arena_start >> _PageShift
s := h_spans[x]
if s == nil || pageID(k) < s.start || b >= s.limit || s.state != mSpanInUse {
if s != nil && s.state == _MSpanStack {
return 0 // This is legit.
}
// If another proc wants a pointer, give it some.
if work.nwait > 0 && nobj > 4 && work.full == 0 {
wbuf.nobj = nobj
wbuf = handoff(wbuf)
nobj = wbuf.nobj
if nobj < uintptr(len(wbuf.obj)) {
wp = &wbuf.obj[nobj]
// The following ensures that we are rigorous about what data
// structures hold valid pointers
if false {
// Still happens sometimes. We don't know why.
printlock()
print("runtime:objectstart Span weird: obj=", hex(obj), " k=", hex(k))
if s == nil {
print(" s=nil\n")
} else {
wp = nil
print(" s.start=", hex(s.start<<_PageShift), " s.limit=", hex(s.limit), " s.state=", s.state, "\n")
}
printunlock()
gothrow("objectstart: bad pointer in unexpected span")
}
nobj--
wp = &wbuf.obj[nobj]
b = *wp
n = arena_used - uintptr(b)
ptrmask = nil // use GC bitmap for pointer info
return 0
}
if _DebugGCPtrs {
print("scanblock ", b, " +", hex(n), " ", ptrmask, "\n")
p := uintptr(s.start) << _PageShift
if s.sizeclass != 0 {
size := s.elemsize
idx := (obj - p) / size
p = p + idx*size
}
// Find bits of the beginning of the object.
if ptrmask == nil {
off := (uintptr(b) - arena_start) / ptrSize
ptrbitp = unsafe.Pointer(arena_start - off/wordsPerBitmapByte - 1)
if p == obj {
print("runtime: failed to find block beginning for ", hex(p), " s=", hex(s.start*_PageSize), " s.limit=", hex(s.limit), "\n")
gothrow("failed to find block beginning")
}
obj = p
}
var i uintptr
for i = 0; i < n; i += ptrSize {
// Find bits for this word.
var bits uintptr
if ptrmask == nil {
// Check if we have reached end of span.
if (uintptr(b)+i)%_PageSize == 0 &&
h_spans[(uintptr(b)-arena_start)>>_PageShift] != h_spans[(uintptr(b)+i-arena_start)>>_PageShift] {
break
}
// if size(obj.firstfield) < PtrSize, the &obj.secondfield could map to the boundary bit
// Clear any low bits to get to the start of the object.
// greyobject depends on this.
return obj
}
// Consult GC bitmap.
bits = uintptr(*(*byte)(ptrbitp))
// Slow for now as we serialize this, since this is on a debug path
// speed is not critical at this point.
var andlock mutex
if wordsPerBitmapByte != 2 {
gothrow("alg doesn't work for wordsPerBitmapByte != 2")
}
j := (uintptr(b) + i) / ptrSize & 1
ptrbitp = add(ptrbitp, -j)
bits >>= gcBits * j
func atomicand8(src *byte, val byte) {
lock(&andlock)
*src &= val
unlock(&andlock)
}
if bits&bitBoundary != 0 && i != 0 {
break // reached beginning of the next object
}
bits = (bits >> 2) & bitsMask
if bits == bitsDead {
break // reached no-scan part of the object
}
} else {
// dense mask (stack or data)
bits = (uintptr(*(*byte)(add(unsafe.Pointer(ptrmask), (i/ptrSize)/4))) >> (((i / ptrSize) % 4) * bitsPerPointer)) & bitsMask
}
// Mark using the checkmark scheme.
func docheckmark(mbits *markbits) {
// xor 01 moves 01(scalar unmarked) to 00(scalar marked)
// and 10(pointer unmarked) to 11(pointer marked)
if mbits.tbits == _BitsScalar {
atomicand8(mbits.bitp, ^byte(_BitsCheckMarkXor<<mbits.shift<<2))
} else if mbits.tbits == _BitsPointer {
atomicor8(mbits.bitp, byte(_BitsCheckMarkXor<<mbits.shift<<2))
}
if bits <= _BitsScalar { // BitsScalar || BitsDead
continue
}
// reload bits for ischeckmarked
mbits.xbits = *mbits.bitp
mbits.bits = (mbits.xbits >> mbits.shift) & bitMask
mbits.tbits = ((mbits.xbits >> mbits.shift) & bitPtrMask) >> 2
}
if bits != _BitsPointer {
gothrow("unexpected garbage collection bits")
}
// In the default scheme does mbits refer to a marked object.
func ismarked(mbits *markbits) bool {
if mbits.bits&bitBoundary != bitBoundary {
gothrow("ismarked: bits should have boundary bit set")
}
return mbits.bits&bitMarked == bitMarked
}
obj := *(*uintptr)(unsafe.Pointer(b + i))
obj0 := obj
// In the checkmark scheme does mbits refer to a marked object.
func ischeckmarked(mbits *markbits) bool {
if mbits.bits&bitBoundary != bitBoundary {
gothrow("ischeckmarked: bits should have boundary bit set")
}
return mbits.tbits == _BitsScalarMarked || mbits.tbits == _BitsPointerMarked
}
markobj:
var s *mspan
var off, bitp, shift, xbits uintptr
// When in GCmarkterminate phase we allocate black.
func gcmarknewobject_m(obj uintptr) {
if gcphase != _GCmarktermination {
gothrow("marking new object while not in mark termination phase")
}
if checkmark { // The world should be stopped so this should not happen.
gothrow("gcmarknewobject called while doing checkmark")
}
// At this point we have extracted the next potential pointer.
// Check if it points into heap.
if obj == 0 {
continue
}
if obj < arena_start || arena_used <= obj {
if uintptr(obj) < _PhysPageSize && invalidptr != 0 {
s = nil
goto badobj
}
continue
}
var mbits markbits
slottombits(obj, &mbits)
if mbits.bits&bitMarked != 0 {
return
}
// Mark the object.
obj &^= ptrSize - 1
off = (obj - arena_start) / ptrSize
bitp = arena_start - off/wordsPerBitmapByte - 1
shift = (off % wordsPerBitmapByte) * gcBits
xbits = uintptr(*(*byte)(unsafe.Pointer(bitp)))
bits = (xbits >> shift) & bitMask
if (bits & bitBoundary) == 0 {
// Not a beginning of a block, consult span table to find the block beginning.
k := pageID(obj >> _PageShift)
x := k
x -= pageID(arena_start >> _PageShift)
s = h_spans[x]
if s == nil || k < s.start || s.limit <= obj || s.state != mSpanInUse {
// Stack pointers lie within the arena bounds but are not part of the GC heap.
// Ignore them.
if s != nil && s.state == _MSpanStack {
continue
}
goto badobj
}
p := uintptr(s.start) << _PageShift
if s.sizeclass != 0 {
size := s.elemsize
idx := (obj - p) / size
p = p + idx*size
}
if p == obj {
print("runtime: failed to find block beginning for ", hex(p), " s=", hex(s.start*_PageSize), " s.limit=", hex(s.limit), "\n")
gothrow("failed to find block beginning")
// Each byte of GC bitmap holds info for two words.
// If the current object is larger than two words, or if the object is one word
// but the object it shares the byte with is already marked,
// then all the possible concurrent updates are trying to set the same bit,
// so we can use a non-atomic update.
if mbits.xbits&(bitMask|(bitMask<<gcBits)) != bitBoundary|bitBoundary<<gcBits || work.nproc == 1 {
*mbits.bitp = mbits.xbits | bitMarked<<mbits.shift
} else {
atomicor8(mbits.bitp, bitMarked<<mbits.shift)
}
}
// obj is the start of an object with mark mbits.
// If it isn't already marked, mark it and enqueue into workbuf.
// Return possibly new workbuf to use.
func greyobject(obj uintptr, mbits *markbits, wbuf *workbuf) *workbuf {
// obj should be start of allocation, and so must be at least pointer-aligned.
if obj&(ptrSize-1) != 0 {
gothrow("greyobject: obj not pointer-aligned")
}
if checkmark {
if !ismarked(mbits) {
print("runtime:greyobject: checkmarks finds unexpected unmarked object obj=", hex(obj), ", mbits->bits=", hex(mbits.bits), " *mbits->bitp=", hex(*mbits.bitp), "\n")
k := obj >> _PageShift
x := k
x -= mheap_.arena_start >> _PageShift
s := h_spans[x]
printlock()
print("runtime:greyobject Span: obj=", hex(obj), " k=", hex(k))
if s == nil {
print(" s=nil\n")
} else {
print(" s.start=", hex(s.start*_PageSize), " s.limit=", hex(s.limit), " s.sizeclass=", s.sizeclass, " s.elemsize=", s.elemsize, "\n")
// NOTE(rsc): This code is using s.sizeclass as an approximation of the
// number of pointer-sized words in an object. Perhaps not what was intended.
for i := 0; i < int(s.sizeclass); i++ {
print(" *(obj+", i*ptrSize, ") = ", hex(*(*uintptr)(unsafe.Pointer(obj + uintptr(i)*ptrSize))), "\n")
}
obj = p
goto markobj
}
gothrow("checkmark found unmarked object")
}
if ischeckmarked(mbits) {
return wbuf
}
docheckmark(mbits)
if !ischeckmarked(mbits) {
print("mbits xbits=", hex(mbits.xbits), " bits=", hex(mbits.bits), " tbits=", hex(mbits.tbits), " shift=", mbits.shift, "\n")
gothrow("docheckmark and ischeckmarked disagree")
}
} else {
// If marked we have nothing to do.
if mbits.bits&bitMarked != 0 {
return wbuf
}
if _DebugGCPtrs {
print("scan *", hex(b+i), " = ", hex(obj0), " => base ", hex(obj), "\n")
}
// Each byte of GC bitmap holds info for two words.
// If the current object is larger than two words, or if the object is one word
// but the object it shares the byte with is already marked,
// then all the possible concurrent updates are trying to set the same bit,
// so we can use a non-atomic update.
if mbits.xbits&(bitMask|bitMask<<gcBits) != bitBoundary|bitBoundary<<gcBits || work.nproc == 1 {
*mbits.bitp = mbits.xbits | bitMarked<<mbits.shift
} else {
atomicor8(mbits.bitp, bitMarked<<mbits.shift)
}
}
if nbadblock > 0 && obj == badblock[nbadblock-1] {
// Running garbage collection again because
// we want to find the path from a root to a bad pointer.
// Found possible next step; extend or finish path.
for j := int32(0); j < nbadblock; j++ {
if badblock[j] == b {
goto AlreadyBad
}
}
print("runtime: found *(", hex(b), "+", hex(i), ") = ", hex(obj0), "+", hex(obj-obj0), "\n")
if ptrmask != nil {
gothrow("bad pointer")
}
if nbadblock >= int32(len(badblock)) {
gothrow("badblock trace too long")
}
badblock[nbadblock] = uintptr(b)
nbadblock++
AlreadyBad:
if !checkmark && (mbits.xbits>>(mbits.shift+2))&_BitsMask == _BitsDead {
return wbuf // noscan object
}
// Queue the obj for scanning. The PREFETCH(obj) logic has been removed but
// seems like a nice optimization that can be added back in.
// There needs to be time between the PREFETCH and the use.
// Previously we put the obj in an 8 element buffer that is drained at a rate
// to give the PREFETCH time to do its work.
// Use of PREFETCHNTA might be more appropriate than PREFETCH
// If workbuf is full, obtain an empty one.
if wbuf.nobj >= uintptr(len(wbuf.obj)) {
wbuf = getempty(wbuf)
}
wbuf.obj[wbuf.nobj] = obj
wbuf.nobj++
return wbuf
}
// Scan the object b of size n, adding pointers to wbuf.
// Return possibly new wbuf to use.
// If ptrmask != nil, it specifies where pointers are in b.
// If ptrmask == nil, the GC bitmap should be consulted.
// In this case, n may be an overestimate of the size; the GC bitmap
// must also be used to make sure the scan stops at the end of b.
func scanobject(b, n uintptr, ptrmask *uint8, wbuf *workbuf) *workbuf {
arena_start := mheap_.arena_start
arena_used := mheap_.arena_used
// Find bits of the beginning of the object.
var ptrbitp unsafe.Pointer
var mbits markbits
if ptrmask == nil {
b = objectstart(b, &mbits)
if b == 0 {
return wbuf
}
ptrbitp = unsafe.Pointer(mbits.bitp)
}
for i := uintptr(0); i < n; i += ptrSize {
// Find bits for this word.
var bits uintptr
if ptrmask != nil {
// dense mask (stack or data)
bits = (uintptr(*(*byte)(add(unsafe.Pointer(ptrmask), (i/ptrSize)/4))) >> (((i / ptrSize) % 4) * bitsPerPointer)) & bitsMask
} else {
// Check if we have reached end of span.
// n is an overestimate of the size of the object.
if (b+i)%_PageSize == 0 && h_spans[(b-arena_start)>>_PageShift] != h_spans[(b+i-arena_start)>>_PageShift] {
break
}
// Now we have bits, bitp, and shift correct for
// obj pointing at the base of the object.
// Only care about not marked objects.
if bits&bitMarked != 0 {
continue
// Consult GC bitmap.
bits = uintptr(*(*byte)(ptrbitp))
if wordsPerBitmapByte != 2 {
gothrow("alg doesn't work for wordsPerBitmapByte != 2")
}
j := (uintptr(b) + i) / ptrSize & 1 // j indicates upper nibble or lower nibble
bits >>= gcBits * j
if i == 0 {
bits &^= bitBoundary
}
ptrbitp = add(ptrbitp, -j)
// If obj size is greater than 8, then each byte of GC bitmap
// contains info for at most one object. In such case we use
// non-atomic byte store to mark the object. This can lead
// to double enqueue of the object for scanning, but scanning
// is an idempotent operation, so it is OK. This cannot lead
// to bitmap corruption because the single marked bit is the
// only thing that can change in the byte.
// For 8-byte objects we use non-atomic store, if the other
// quadruple is already marked. Otherwise we resort to CAS
// loop for marking.
if xbits&(bitMask|bitMask<<gcBits) != bitBoundary|bitBoundary<<gcBits || work.nproc == 1 {
*(*byte)(unsafe.Pointer(bitp)) = uint8(xbits | bitMarked<<shift)
} else {
atomicor8((*byte)(unsafe.Pointer(bitp)), bitMarked<<shift)
if bits&bitBoundary != 0 && i != 0 {
break // reached beginning of the next object
}
bits = (bits & bitPtrMask) >> 2 // bits refer to the type bits.
if (xbits>>(shift+2))&bitsMask == bitsDead {
continue // noscan object
if i != 0 && bits == bitsDead { // BitsDead in first nibble not valid during checkmark
break // reached no-scan part of the object
}
}
// Queue the obj for scanning.
// TODO: PREFETCH here.
if bits <= _BitsScalar { // _BitsScalar, _BitsDead, _BitsScalarMarked
continue
}
// If workbuf is full, obtain an empty one.
if nobj >= uintptr(len(wbuf.obj)) {
wbuf.nobj = nobj
wbuf = getempty(wbuf)
nobj = wbuf.nobj
wp = &wbuf.obj[nobj]
}
*wp = obj
nobj++
if nobj < uintptr(len(wbuf.obj)) {
wp = &wbuf.obj[nobj]
} else {
wp = nil
}
if bits&_BitsPointer != _BitsPointer {
print("gc checkmark=", checkmark, " b=", hex(b), " ptrmask=", ptrmask, " mbits.bitp=", mbits.bitp, " mbits.xbits=", hex(mbits.xbits), " bits=", hex(bits), "\n")
gothrow("unexpected garbage collection bits")
}
obj := *(*uintptr)(unsafe.Pointer(b + i))
// At this point we have extracted the next potential pointer.
// Check if it points into heap.
if obj == 0 || obj < arena_start || obj >= arena_used {
continue
}
// Mark the object. return some important bits.
// We we combine the following two rotines we don't have to pass mbits or obj around.
var mbits markbits
obj = objectstart(obj, &mbits)
if obj == 0 {
continue
}
wbuf = greyobject(obj, &mbits, wbuf)
}
return wbuf
}
badobj:
// If cgo_allocate is linked into the binary, it can allocate
// memory as []unsafe.Pointer that may not contain actual
// pointers and must be scanned conservatively.
// In this case alone, allow the bad pointer.
if have_cgo_allocate() && ptrmask == nil {
continue
// scanblock starts by scanning b as scanobject would.
// If the gcphase is GCscan, that's all scanblock does.
// Otherwise it traverses some fraction of the pointers it found in b, recursively.
// As a special case, scanblock(nil, 0, nil) means to scan previously queued work,
// stopping only when no work is left in the system.
func scanblock(b, n uintptr, ptrmask *uint8) {
wbuf := getpartialorempty()
if b != 0 {
wbuf = scanobject(b, n, ptrmask, wbuf)
if gcphase == _GCscan {
if inheap(b) && ptrmask == nil {
// b is in heap, we are in GCscan so there should be a ptrmask.
gothrow("scanblock: In GCscan phase and inheap is true.")
}
// GCscan only goes one level deep since mark wb not turned on.
putpartial(wbuf)
return
}
}
if gcphase == _GCscan {
gothrow("scanblock: In GCscan phase but no b passed in.")
}
// Anything else indicates a bug somewhere.
// If we're in the middle of chasing down a different bad pointer,
// don't confuse the trace by printing about this one.
if nbadblock > 0 {
continue
}
keepworking := b == 0
print("runtime: garbage collector found invalid heap pointer *(", hex(b), "+", hex(i), ")=", hex(obj))
if s == nil {
print(" s=nil\n")
} else {
print(" span=", uintptr(s.start)<<_PageShift, "-", s.limit, "-", (uintptr(s.start)+s.npages)<<_PageShift, " state=", s.state, "\n")
// ptrmask can have 2 possible values:
// 1. nil - obtain pointer mask from GC bitmap.
// 2. pointer to a compact mask (for stacks and data).
for {
if wbuf.nobj == 0 {
if !keepworking {
putempty(wbuf)
return
}
if ptrmask != nil {
gothrow("invalid heap pointer")
// Refill workbuf from global queue.
wbuf = getfull(wbuf)
if wbuf == nil { // nil means out of work barrier reached
return
}
// Add to badblock list, which will cause the garbage collection
// to keep repeating until it has traced the chain of pointers
// leading to obj all the way back to a root.
if nbadblock == 0 {
badblock[nbadblock] = uintptr(b)
nbadblock++
if wbuf.nobj <= 0 {
gothrow("runtime:scanblock getfull returns empty buffer")
}
}
if _DebugGCPtrs {
print("end scanblock ", hex(b), " +", hex(n), " ", ptrmask, "\n")
}
if _DebugGC > 0 && ptrmask == nil {
// For heap objects ensure that we did not overscan.
var p, n uintptr
if mlookup(b, &p, &n, nil) == 0 || b != p || i > n {
print("runtime: scanned (", hex(b), "+", hex(i), "), heap object (", hex(p), "+", hex(n), ")\n")
gothrow("scanblock: scanned invalid object")
}
// If another proc wants a pointer, give it some.
if work.nwait > 0 && wbuf.nobj > 4 && work.full == 0 {
wbuf = handoff(wbuf)
}
// This might be a good place to add prefetch code...
// if(wbuf->nobj > 4) {
// PREFETCH(wbuf->obj[wbuf->nobj - 3];
// }
wbuf.nobj--
b = wbuf.obj[wbuf.nobj]
wbuf = scanobject(b, mheap_.arena_used-b, nil, wbuf)
}
}
......@@ -455,7 +637,8 @@ func markroot(desc *parfor, i uint32) {
if s.state != mSpanInUse {
continue
}
if s.sweepgen != sg {
if !checkmark && s.sweepgen != sg {
// sweepgen was updated (+2) during non-checkmark GC pass
print("sweep ", s.sweepgen, " ", sg, "\n")
gothrow("gc: unswept span")
}
......@@ -468,13 +651,17 @@ func markroot(desc *parfor, i uint32) {
spf := (*specialfinalizer)(unsafe.Pointer(sp))
// A finalizer can be set for an inner byte of an object, find object beginning.
p := uintptr(s.start<<_PageShift) + uintptr(spf.special.offset)/s.elemsize*s.elemsize
scanblock(p, s.elemsize, nil)
if gcphase != _GCscan {
scanblock(p, s.elemsize, nil) // scanned during mark phase
}
scanblock(uintptr(unsafe.Pointer(&spf.fn)), ptrSize, &oneptr[0])
}
}
case _RootFlushCaches:
flushallmcaches()
if gcphase != _GCscan { // Do not flush mcaches during GCscan phase.
flushallmcaches()
}
default:
// the rest is scanning goroutine stacks
......@@ -482,21 +669,44 @@ func markroot(desc *parfor, i uint32) {
gothrow("markroot: bad index")
}
gp := allgs[i-_RootCount]
// remember when we've first observed the G blocked
// needed only to output in traceback
status := readgstatus(gp)
status := readgstatus(gp) // We are not in a scan state
if (status == _Gwaiting || status == _Gsyscall) && gp.waitsince == 0 {
gp.waitsince = work.tstart
}
// Shrink a stack if not much of it is being used.
shrinkstack(gp)
// Shrink a stack if not much of it is being used but not in the scan phase.
if gcphase != _GCscan { // Do not shrink during GCscan phase.
shrinkstack(gp)
}
if readgstatus(gp) == _Gdead {
gp.gcworkdone = true
} else {
gp.gcworkdone = false
}
restart := stopg(gp)
scanstack(gp)
// goroutine will scan its own stack when it stops running.
// Wait until it has.
for readgstatus(gp) == _Grunning && !gp.gcworkdone {
}
// scanstack(gp) is done as part of gcphasework
// But to make sure we finished we need to make sure that
// the stack traps have all responded so drop into
// this while loop until they respond.
for !gp.gcworkdone {
status = readgstatus(gp)
if status == _Gdead {
gp.gcworkdone = true // scan is a noop
break
}
if status == _Gwaiting || status == _Grunnable {
restart = stopg(gp)
}
}
if restart {
restartg(gp)
}
......@@ -506,48 +716,83 @@ func markroot(desc *parfor, i uint32) {
// Get an empty work buffer off the work.empty list,
// allocating new buffers as needed.
func getempty(b *workbuf) *workbuf {
_g_ := getg()
if b != nil {
lfstackpush(&work.full, &b.node)
putfull(b)
b = nil
}
b = nil
c := _g_.m.mcache
if c.gcworkbuf != nil {
b = (*workbuf)(c.gcworkbuf)
c.gcworkbuf = nil
}
if b == nil {
if work.empty != 0 {
b = (*workbuf)(lfstackpop(&work.empty))
}
if b != nil && b.nobj != 0 {
_g_ := getg()
print("m", _g_.m.id, ": getempty: popped b=", b, " with non-zero b.nobj=", b.nobj, "\n")
gothrow("getempty: workbuffer not empty, b->nobj not 0")
}
if b == nil {
b = (*workbuf)(persistentalloc(unsafe.Sizeof(*b), _CacheLineSize, &memstats.gc_sys))
b.nobj = 0
}
b.nobj = 0
return b
}
func putempty(b *workbuf) {
_g_ := getg()
c := _g_.m.mcache
if c.gcworkbuf == nil {
c.gcworkbuf = (unsafe.Pointer)(b)
return
if b.nobj != 0 {
gothrow("putempty: b->nobj not 0")
}
lfstackpush(&work.empty, &b.node)
}
func gcworkbuffree(b unsafe.Pointer) {
if b != nil {
putempty((*workbuf)(b))
func putfull(b *workbuf) {
if b.nobj <= 0 {
gothrow("putfull: b->nobj <= 0")
}
lfstackpush(&work.full, &b.node)
}
// Get an partially empty work buffer
// if none are available get an empty one.
func getpartialorempty() *workbuf {
b := (*workbuf)(lfstackpop(&work.partial))
if b == nil {
b = getempty(nil)
}
return b
}
func putpartial(b *workbuf) {
if b.nobj == 0 {
lfstackpush(&work.empty, &b.node)
} else if b.nobj < uintptr(len(b.obj)) {
lfstackpush(&work.partial, &b.node)
} else if b.nobj == uintptr(len(b.obj)) {
lfstackpush(&work.full, &b.node)
} else {
print("b=", b, " b.nobj=", b.nobj, " len(b.obj)=", len(b.obj), "\n")
gothrow("putpartial: bad Workbuf b.nobj")
}
}
// Get a full work buffer off the work.full list, or return nil.
// Get a full work buffer off the work.full or a partially
// filled one off the work.partial list. If nothing is available
// wait until all the other gc helpers have finished and then
// return nil.
// getfull acts as a barrier for work.nproc helpers. As long as one
// gchelper is actively marking objects it
// may create a workbuffer that the other helpers can work on.
// The for loop either exits when a work buffer is found
// or when _all_ of the work.nproc GC helpers are in the loop
// looking for work and thus not capable of creating new work.
// This is in fact the termination condition for the STW mark
// phase.
func getfull(b *workbuf) *workbuf {
if b != nil {
lfstackpush(&work.empty, &b.node)
putempty(b)
}
b = (*workbuf)(lfstackpop(&work.full))
if b == nil {
b = (*workbuf)(lfstackpop(&work.partial))
}
if b != nil || work.nproc == 1 {
return b
}
......@@ -557,6 +802,9 @@ func getfull(b *workbuf) *workbuf {
if work.full != 0 {
xadd(&work.nwait, -1)
b = (*workbuf)(lfstackpop(&work.full))
if b == nil {
b = (*workbuf)(lfstackpop(&work.partial))
}
if b != nil {
return b
}
......@@ -675,14 +923,11 @@ func scanframe(frame *stkframe, unused unsafe.Pointer) bool {
}
func scanstack(gp *g) {
// TODO(rsc): Due to a precedence error, this was never checked in the original C version.
// If you enable the check, the gothrow happens.
/*
if readgstatus(gp)&_Gscan == 0 {
print("runtime: gp=", gp, ", goid=", gp.goid, ", gp->atomicstatus=", readgstatus(gp), "\n")
gothrow("mark - bad status")
}
*/
if readgstatus(gp)&_Gscan == 0 {
print("runtime:scanstack: gp=", gp, ", goid=", gp.goid, ", gp->atomicstatus=", hex(readgstatus(gp)), "\n")
gothrow("scanstack - bad status")
}
switch readgstatus(gp) &^ _Gscan {
default:
......@@ -692,7 +937,7 @@ func scanstack(gp *g) {
return
case _Grunning:
print("runtime: gp=", gp, ", goid=", gp.goid, ", gp->atomicstatus=", readgstatus(gp), "\n")
gothrow("mark - world not stopped")
gothrow("scanstack: goroutine not stopped")
case _Grunnable, _Gsyscall, _Gwaiting:
// ok
}
......@@ -709,18 +954,114 @@ func scanstack(gp *g) {
tracebackdefers(gp, scanframe, nil)
}
// The gp has been moved to a gc safepoint. If there is gcphase specific
// work it is done here.
// If the slot is grey or black return true, if white return false.
// If the slot is not in the known heap and thus does not have a valid GC bitmap then
// it is considered grey. Globals and stacks can hold such slots.
// The slot is grey if its mark bit is set and it is enqueued to be scanned.
// The slot is black if it has already been scanned.
// It is white if it has a valid mark bit and the bit is not set.
func shaded(slot uintptr) bool {
if !inheap(slot) { // non-heap slots considered grey
return true
}
var mbits markbits
valid := objectstart(slot, &mbits)
if valid == 0 {
return true
}
if checkmark {
return ischeckmarked(&mbits)
}
return mbits.bits&bitMarked != 0
}
// Shade the object if it isn't already.
// The object is not nil and known to be in the heap.
func shade(b uintptr) {
if !inheap(b) {
gothrow("shade: passed an address not in the heap")
}
wbuf := getpartialorempty()
// Mark the object, return some important bits.
// If we combine the following two rotines we don't have to pass mbits or obj around.
var mbits markbits
obj := objectstart(b, &mbits)
if obj != 0 {
wbuf = greyobject(obj, &mbits, wbuf) // augments the wbuf
}
putpartial(wbuf)
}
// This is the Dijkstra barrier coarsened to always shade the ptr (dst) object.
// The original Dijkstra barrier only shaded ptrs being placed in black slots.
//
// Shade indicates that it has seen a white pointer by adding the referent
// to wbuf as well as marking it.
//
// slot is the destination (dst) in go code
// ptr is the value that goes into the slot (src) in the go code
//
// Dijkstra pointed out that maintaining the no black to white
// pointers means that white to white pointers not need
// to be noted by the write barrier. Furthermore if either
// white object dies before it is reached by the
// GC then the object can be collected during this GC cycle
// instead of waiting for the next cycle. Unfortunately the cost of
// ensure that the object holding the slot doesn't concurrently
// change to black without the mutator noticing seems prohibitive.
//
// Consider the following example where the mutator writes into
// a slot and then loads the slot's mark bit while the GC thread
// writes to the slot's mark bit and then as part of scanning reads
// the slot.
//
// Initially both [slot] and [slotmark] are 0 (nil)
// Mutator thread GC thread
// st [slot], ptr st [slotmark], 1
//
// ld r1, [slotmark] ld r2, [slot]
//
// This is a classic example of independent reads of independent writes,
// aka IRIW. The question is if r1==r2==0 is allowed and for most HW the
// answer is yes without inserting a memory barriers between the st and the ld.
// These barriers are expensive so we have decided that we will
// always grey the ptr object regardless of the slot's color.
func gcmarkwb_m(slot *uintptr, ptr uintptr) {
switch gcphase {
default:
gothrow("gcphasework in bad gcphase")
case _GCoff, _GCquiesce, _GCstw, _GCsweep, _GCscan:
// ok
case _GCmark, _GCmarktermination:
if ptr != 0 && inheap(ptr) {
shade(ptr)
}
}
}
// The gp has been moved to a GC safepoint. GC phase specific
// work is done here.
func gcphasework(gp *g) {
switch gcphase {
default:
gothrow("gcphasework in bad gcphase")
case _GCoff, _GCquiesce, _GCstw, _GCsweep:
// No work for now.
// No work.
case _GCscan:
// scan the stack, mark the objects, put pointers in work buffers
// hanging off the P where this is being run.
scanstack(gp)
case _GCmark:
// Disabled until concurrent GC is implemented
// but indicate the scan has been done.
// scanstack(gp);
// No work.
case _GCmarktermination:
scanstack(gp)
// All available mark work will be emptied before returning.
}
gp.gcworkdone = true
}
......@@ -797,6 +1138,7 @@ func iterate_finq(callback func(*funcval, unsafe.Pointer, uintptr, *_type, *ptrt
}
}
// Returns only when span s has been swept.
func mSpan_EnsureSwept(s *mspan) {
// Caller must disable preemption.
// Otherwise when this function returns the span can become unswept again
......@@ -810,6 +1152,7 @@ func mSpan_EnsureSwept(s *mspan) {
if atomicload(&s.sweepgen) == sg {
return
}
// The caller must be sure that the span is a MSpanInUse span.
if cas(&s.sweepgen, sg-2, sg-1) {
mSpan_Sweep(s, false)
return
......@@ -826,6 +1169,10 @@ func mSpan_EnsureSwept(s *mspan) {
// If preserve=true, don't return it to heap nor relink in MCentral lists;
// caller takes care of it.
func mSpan_Sweep(s *mspan, preserve bool) bool {
if checkmark {
gothrow("MSpan_Sweep: checkmark only runs in STW and after the sweep")
}
// It's critical that we enter this function with preemption disabled,
// GC must not start while we are in the middle of this function.
_g_ := getg()
......@@ -851,13 +1198,14 @@ func mSpan_Sweep(s *mspan, preserve bool) bool {
}
res := false
nfree := 0
var head mlink
end := &head
var head, end gclinkptr
c := _g_.m.mcache
sweepgenset := false
// Mark any free objects in this span so we don't collect them.
for link := s.freelist; link != nil; link = link.next {
for link := s.freelist; link.ptr() != nil; link = link.ptr().next {
off := (uintptr(unsafe.Pointer(link)) - arena_start) / ptrSize
bitp := arena_start - off/wordsPerBitmapByte - 1
shift := (off % wordsPerBitmapByte) * gcBits
......@@ -978,8 +1326,13 @@ func mSpan_Sweep(s *mspan, preserve bool) bool {
} else if size > ptrSize {
*(*uintptr)(unsafe.Pointer(p + ptrSize)) = 0
}
end.next = (*mlink)(unsafe.Pointer(p))
end = end.next
if head.ptr() == nil {
head = gclinkptr(p)
} else {
end.ptr().next = gclinkptr(p)
}
end = gclinkptr(p)
end.ptr().next = gclinkptr(0x0bade5)
nfree++
}
}
......@@ -1002,7 +1355,7 @@ func mSpan_Sweep(s *mspan, preserve bool) bool {
c.local_nsmallfree[cl] += uintptr(nfree)
c.local_cachealloc -= intptr(uintptr(nfree) * size)
xadd64(&memstats.next_gc, -int64(nfree)*int64(size)*int64(gcpercent+100)/100)
res = mCentral_FreeSpan(&mheap_.central[cl].mcentral, s, int32(nfree), head.next, end, preserve)
res = mCentral_FreeSpan(&mheap_.central[cl].mcentral, s, int32(nfree), head, end, preserve)
// MCentral_FreeSpan updates sweepgen
}
return res
......@@ -1073,11 +1426,11 @@ func gchelper() {
_g_.m.traceback = 2
gchelperstart()
// parallel mark for over gc roots
// parallel mark for over GC roots
parfordo(work.markfor)
// help other threads scan secondary blocks
scanblock(0, 0, nil)
if gcphase != _GCscan {
scanblock(0, 0, nil) // blocks in getfull
}
nproc := work.nproc // work.nproc can change right after we increment work.ndone
if xadd(&work.ndone, +1) == nproc-1 {
......@@ -1204,6 +1557,7 @@ func gcinit() {
gcbssmask = unrollglobgcprog((*byte)(unsafe.Pointer(&gcbss)), uintptr(unsafe.Pointer(&ebss))-uintptr(unsafe.Pointer(&bss)))
}
// Called from malloc.go using onM, stopping and starting the world handled in caller.
func gc_m(start_time int64, eagersweep bool) {
_g_ := getg()
gp := _g_.m.curg
......@@ -1211,18 +1565,266 @@ func gc_m(start_time int64, eagersweep bool) {
gp.waitreason = "garbage collection"
gc(start_time, eagersweep)
casgstatus(gp, _Gwaiting, _Grunning)
}
// Similar to clearcheckmarkbits but works on a single span.
// It preforms two tasks.
// 1. When used before the checkmark phase it converts BitsDead (00) to bitsScalar (01)
// for nibbles with the BoundaryBit set.
// 2. When used after the checkmark phase it converts BitsPointerMark (11) to BitsPointer 10 and
// BitsScalarMark (00) to BitsScalar (01), thus clearing the checkmark mark encoding.
// For the second case it is possible to restore the BitsDead pattern but since
// clearmark is a debug tool performance has a lower priority than simplicity.
// The span is MSpanInUse and the world is stopped.
func clearcheckmarkbitsspan(s *mspan) {
if s.state != _MSpanInUse {
print("runtime:clearcheckmarkbitsspan: state=", s.state, "\n")
gothrow("clearcheckmarkbitsspan: bad span state")
}
if nbadblock > 0 {
// Work out path from root to bad block.
for {
gc(start_time, eagersweep)
if nbadblock >= int32(len(badblock)) {
gothrow("cannot find path to bad pointer")
arena_start := mheap_.arena_start
cl := s.sizeclass
size := s.elemsize
var n int32
if cl == 0 {
n = 1
} else {
// Chunk full of small blocks
npages := class_to_allocnpages[cl]
n = npages << _PageShift / int32(size)
}
// MSpan_Sweep has similar code but instead of overloading and
// complicating that routine we do a simpler walk here.
// Sweep through n objects of given size starting at p.
// This thread owns the span now, so it can manipulate
// the block bitmap without atomic operations.
p := uintptr(s.start) << _PageShift
// Find bits for the beginning of the span.
off := (p - arena_start) / ptrSize
bitp := (*byte)(unsafe.Pointer(arena_start - off/wordsPerBitmapByte - 1))
step := size / (ptrSize * wordsPerBitmapByte)
// The type bit values are:
// 00 - BitsDead, for us BitsScalarMarked
// 01 - BitsScalar
// 10 - BitsPointer
// 11 - unused, for us BitsPointerMarked
//
// When called to prepare for the checkmark phase (checkmark==1),
// we change BitsDead to BitsScalar, so that there are no BitsScalarMarked
// type bits anywhere.
//
// The checkmark phase marks by changing BitsScalar to BitsScalarMarked
// and BitsPointer to BitsPointerMarked.
//
// When called to clean up after the checkmark phase (checkmark==0),
// we unmark by changing BitsScalarMarked back to BitsScalar and
// BitsPointerMarked back to BitsPointer.
//
// There are two problems with the scheme as just described.
// First, the setup rewrites BitsDead to BitsScalar, but the type bits
// following a BitsDead are uninitialized and must not be used.
// Second, objects that are free are expected to have their type
// bits zeroed (BitsDead), so in the cleanup we need to restore
// any BitsDeads that were there originally.
//
// In a one-word object (8-byte allocation on 64-bit system),
// there is no difference between BitsScalar and BitsDead, because
// neither is a pointer and there are no more words in the object,
// so using BitsScalar during the checkmark is safe and mapping
// both back to BitsDead during cleanup is also safe.
//
// In a larger object, we need to be more careful. During setup,
// if the type of the first word is BitsDead, we change it to BitsScalar
// (as we must) but also initialize the type of the second
// word to BitsDead, so that a scan during the checkmark phase
// will still stop before seeing the uninitialized type bits in the
// rest of the object. The sequence 'BitsScalar BitsDead' never
// happens in real type bitmaps - BitsDead is always as early
// as possible, so immediately after the last BitsPointer.
// During cleanup, if we see a BitsScalar, we can check to see if it
// is followed by BitsDead. If so, it was originally BitsDead and
// we can change it back.
if step == 0 {
// updating top and bottom nibbles, all boundaries
for i := int32(0); i < n/2; i, bitp = i+1, addb(bitp, uintptrMask&-1) {
if *bitp&bitBoundary == 0 {
gothrow("missing bitBoundary")
}
b := (*bitp & bitPtrMask) >> 2
if !checkmark && (b == _BitsScalar || b == _BitsScalarMarked) {
*bitp &^= 0x0c // convert to _BitsDead
} else if b == _BitsScalarMarked || b == _BitsPointerMarked {
*bitp &^= _BitsCheckMarkXor << 2
}
if (*bitp>>gcBits)&bitBoundary == 0 {
gothrow("missing bitBoundary")
}
b = ((*bitp >> gcBits) & bitPtrMask) >> 2
if !checkmark && (b == _BitsScalar || b == _BitsScalarMarked) {
*bitp &^= 0xc0 // convert to _BitsDead
} else if b == _BitsScalarMarked || b == _BitsPointerMarked {
*bitp &^= _BitsCheckMarkXor << (2 + gcBits)
}
}
} else {
// updating bottom nibble for first word of each object
for i := int32(0); i < n; i, bitp = i+1, addb(bitp, -step) {
if *bitp&bitBoundary == 0 {
gothrow("missing bitBoundary")
}
b := (*bitp & bitPtrMask) >> 2
if checkmark && b == _BitsDead {
// move BitsDead into second word.
// set bits to BitsScalar in preparation for checkmark phase.
*bitp &^= 0xc0
*bitp |= _BitsScalar << 2
} else if !checkmark && (b == _BitsScalar || b == _BitsScalarMarked) && *bitp&0xc0 == 0 {
// Cleaning up after checkmark phase.
// First word is scalar or dead (we forgot)
// and second word is dead.
// First word might as well be dead too.
*bitp &^= 0x0c
} else if b == _BitsScalarMarked || b == _BitsPointerMarked {
*bitp ^= _BitsCheckMarkXor << 2
}
}
}
}
casgstatus(gp, _Gwaiting, _Grunning)
// clearcheckmarkbits preforms two tasks.
// 1. When used before the checkmark phase it converts BitsDead (00) to bitsScalar (01)
// for nibbles with the BoundaryBit set.
// 2. When used after the checkmark phase it converts BitsPointerMark (11) to BitsPointer 10 and
// BitsScalarMark (00) to BitsScalar (01), thus clearing the checkmark mark encoding.
// This is a bit expensive but preserves the BitsDead encoding during the normal marking.
// BitsDead remains valid for every nibble except the ones with BitsBoundary set.
func clearcheckmarkbits() {
for _, s := range work.spans {
if s.state == _MSpanInUse {
clearcheckmarkbitsspan(s)
}
}
}
// Called from malloc.go using onM.
// The world is stopped. Rerun the scan and mark phases
// using the bitMarkedCheck bit instead of the
// bitMarked bit. If the marking encounters an
// bitMarked bit that is not set then we throw.
func gccheckmark_m(startTime int64, eagersweep bool) {
if !gccheckmarkenable {
return
}
if checkmark {
gothrow("gccheckmark_m, entered with checkmark already true")
}
checkmark = true
clearcheckmarkbits() // Converts BitsDead to BitsScalar.
gc_m(startTime, eagersweep) // turns off checkmark
// Work done, fixed up the GC bitmap to remove the checkmark bits.
clearcheckmarkbits()
}
func gccheckmarkenable_m() {
gccheckmarkenable = true
}
func gccheckmarkdisable_m() {
gccheckmarkenable = false
}
func finishsweep_m() {
// The world is stopped so we should be able to complete the sweeps
// quickly.
for sweepone() != ^uintptr(0) {
sweep.npausesweep++
}
// There may be some other spans being swept concurrently that
// we need to wait for. If finishsweep_m is done with the world stopped
// this code is not required.
sg := mheap_.sweepgen
for _, s := range work.spans {
if s.sweepgen != sg && s.state == _MSpanInUse {
mSpan_EnsureSwept(s)
}
}
}
// Scan all of the stacks, greying (or graying if in America) the referents
// but not blackening them since the mark write barrier isn't installed.
func gcscan_m() {
_g_ := getg()
// Grab the g that called us and potentially allow rescheduling.
// This allows it to be scanned like other goroutines.
mastergp := _g_.m.curg
casgstatus(mastergp, _Grunning, _Gwaiting)
mastergp.waitreason = "garbage collection scan"
// Span sweeping has been done by finishsweep_m.
// Long term we will want to make this goroutine runnable
// by placing it onto a scanenqueue state and then calling
// runtime·restartg(mastergp) to make it Grunnable.
// At the bottom we will want to return this p back to the scheduler.
oldphase := gcphase
// Prepare flag indicating that the scan has not been completed.
lock(&allglock)
local_allglen := allglen
for i := uintptr(0); i < local_allglen; i++ {
gp := allgs[i]
gp.gcworkdone = false // set to true in gcphasework
}
unlock(&allglock)
work.nwait = 0
work.ndone = 0
work.nproc = 1 // For now do not do this in parallel.
gcphase = _GCscan
// ackgcphase is not needed since we are not scanning running goroutines.
parforsetup(work.markfor, work.nproc, uint32(_RootCount+local_allglen), nil, false, markroot)
parfordo(work.markfor)
lock(&allglock)
// Check that gc work is done.
for i := uintptr(0); i < local_allglen; i++ {
gp := allgs[i]
if !gp.gcworkdone {
gothrow("scan missed a g")
}
}
unlock(&allglock)
gcphase = oldphase
casgstatus(mastergp, _Gwaiting, _Grunning)
// Let the g that called us continue to run.
}
// Mark all objects that are known about.
func gcmark_m() {
scanblock(0, 0, nil)
}
// For now this must be bracketed with a stoptheworld and a starttheworld to ensure
// all go routines see the new barrier.
func gcinstallmarkwb_m() {
gcphase = _GCmark
}
// For now this must be bracketed with a stoptheworld and a starttheworld to ensure
// all go routines see the new barrier.
func gcinstalloffwb_m() {
gcphase = _GCoff
}
func gc(start_time int64, eagersweep bool) {
......@@ -1244,9 +1846,8 @@ func gc(start_time int64, eagersweep bool) {
t1 = nanotime()
}
// Sweep what is not sweeped by bgsweep.
for sweepone() != ^uintptr(0) {
sweep.npausesweep++
if !checkmark {
finishsweep_m() // skip during checkmark debug phase.
}
// Cache runtime.mheap_.allspans in work.spans to avoid conflicts with
......@@ -1266,10 +1867,19 @@ func gc(start_time int64, eagersweep bool) {
mheap_.gcspans = mheap_.allspans
work.spans = h_allspans
unlock(&mheap_.lock)
oldphase := gcphase
work.nwait = 0
work.ndone = 0
work.nproc = uint32(gcprocs())
gcphase = _GCmarktermination
// World is stopped so allglen will not change.
for i := uintptr(0); i < allglen; i++ {
gp := allgs[i]
gp.gcworkdone = false // set to true in gcphasework
}
parforsetup(work.markfor, work.nproc, uint32(_RootCount+allglen), nil, false, markroot)
if work.nproc > 1 {
noteclear(&work.alldone)
......@@ -1285,6 +1895,14 @@ func gc(start_time int64, eagersweep bool) {
parfordo(work.markfor)
scanblock(0, 0, nil)
if work.full != 0 {
gothrow("work.full != 0")
}
if work.partial != 0 {
gothrow("work.partial != 0")
}
gcphase = oldphase
var t3 int64
if debug.gctrace > 0 {
t3 = nanotime()
......@@ -1349,6 +1967,15 @@ func gc(start_time int64, eagersweep bool) {
sysFree(unsafe.Pointer(&work.spans[0]), uintptr(len(work.spans))*unsafe.Sizeof(work.spans[0]), &memstats.other_sys)
}
if gccheckmarkenable {
if !checkmark {
// first half of two-pass; don't set up sweep
unlock(&mheap_.lock)
return
}
checkmark = false // done checking marks
}
// Cache the current array for sweeping.
mheap_.gcspans = mheap_.allspans
mheap_.sweepgen += 2
......
src/runtime/mgc0.go
View file @ 31457cef
......@@ -97,54 +97,155 @@ func bgsweep() {
}
}
const (
_PoisonGC = 0xf969696969696969 & (1<<(8*ptrSize) - 1)
_PoisonStack = 0x6868686868686868 & (1<<(8*ptrSize) - 1)
)
func needwb() bool {
return gcphase == _GCmark || gcphase == _GCmarktermination
}
// NOTE: Really dst *unsafe.Pointer, src unsafe.Pointer,
// but if we do that, Go inserts a write barrier on *dst = src.
//go:nosplit
func writebarrierptr(dst *uintptr, src uintptr) {
*dst = src
if needwb() {
writebarrierptr_nostore(dst, src)
}
}
// Like writebarrierptr, but the store has already been applied.
// Do not reapply.
//go:nosplit
func writebarrierptr_nostore(dst *uintptr, src uintptr) {
if getg() == nil || !needwb() { // very low-level startup
return
}
if src != 0 && (src < _PageSize || src == _PoisonGC || src == _PoisonStack) {
systemstack(func() { gothrow("bad pointer in write barrier") })
}
mp := acquirem()
if mp.inwb || mp.dying > 0 {
releasem(mp)
return
}
mp.inwb = true
systemstack(func() {
gcmarkwb_m(dst, src)
})
mp.inwb = false
releasem(mp)
}
//go:nosplit
func writebarrierstring(dst *[2]uintptr, src [2]uintptr) {
dst[0] = src[0]
writebarrierptr(&dst[0], src[0])
dst[1] = src[1]
}
//go:nosplit
func writebarrierslice(dst *[3]uintptr, src [3]uintptr) {
dst[0] = src[0]
writebarrierptr(&dst[0], src[0])
dst[1] = src[1]
dst[2] = src[2]
}
//go:nosplit
func writebarrieriface(dst *[2]uintptr, src [2]uintptr) {
dst[0] = src[0]
dst[1] = src[1]
writebarrierptr(&dst[0], src[0])
writebarrierptr(&dst[1], src[1])
}
//go:nosplit
func writebarrierfat2(dst *[2]uintptr, _ *byte, src [2]uintptr) {
dst[0] = src[0]
dst[1] = src[1]
}
//go:generate go run wbfat_gen.go -- wbfat.go
//
// The above line generates multiword write barriers for
// all the combinations of ptr+scalar up to four words.
// The implementations are written to wbfat.go.
//go:nosplit
func writebarrierfat3(dst *[3]uintptr, _ *byte, src [3]uintptr) {
dst[0] = src[0]
dst[1] = src[1]
dst[2] = src[2]
}
func writebarrierfat(typ *_type, dst, src unsafe.Pointer) {
if !needwb() {
memmove(dst, src, typ.size)
return
}
//go:nosplit
func writebarrierfat4(dst *[4]uintptr, _ *byte, src [4]uintptr) {
dst[0] = src[0]
dst[1] = src[1]
dst[2] = src[2]
dst[3] = src[3]
systemstack(func() {
mask := loadPtrMask(typ)
nptr := typ.size / ptrSize
for i := uintptr(0); i < nptr; i += 2 {
bits := mask[i/2]
if (bits>>2)&_BitsMask == _BitsPointer {
writebarrierptr((*uintptr)(dst), *(*uintptr)(src))
} else {
*(*uintptr)(dst) = *(*uintptr)(src)
}
dst = add(dst, ptrSize)
src = add(src, ptrSize)
if i+1 == nptr {
break
}
bits >>= 4
if (bits>>2)&_BitsMask == _BitsPointer {
writebarrierptr((*uintptr)(dst), *(*uintptr)(src))
} else {
*(*uintptr)(dst) = *(*uintptr)(src)
}
dst = add(dst, ptrSize)
src = add(src, ptrSize)
}
})
}
//go:nosplit
func writebarrierfat(typ *_type, dst, src unsafe.Pointer) {
memmove(dst, src, typ.size)
func writebarriercopy(typ *_type, dst, src slice) int {
n := dst.len
if n > src.len {
n = src.len
}
if n == 0 {
return 0
}
dstp := unsafe.Pointer(dst.array)
srcp := unsafe.Pointer(src.array)
if !needwb() {
memmove(dstp, srcp, uintptr(n)*typ.size)
return int(n)
}
systemstack(func() {
if uintptr(srcp) < uintptr(dstp) && uintptr(srcp)+uintptr(n)*typ.size > uintptr(dstp) {
// Overlap with src before dst.
// Copy backward, being careful not to move dstp/srcp
// out of the array they point into.
dstp = add(dstp, uintptr(n-1)*typ.size)
srcp = add(srcp, uintptr(n-1)*typ.size)
i := uint(0)
for {
writebarrierfat(typ, dstp, srcp)
if i++; i >= n {
break
}
dstp = add(dstp, -typ.size)
srcp = add(srcp, -typ.size)
}
} else {
// Copy forward, being careful not to move dstp/srcp
// out of the array they point into.
i := uint(0)
for {
writebarrierfat(typ, dstp, srcp)
if i++; i >= n {
break
}
dstp = add(dstp, typ.size)
srcp = add(srcp, typ.size)
}
}
})
return int(n)
}
src/runtime/mgc0.h
View file @ 31457cef
......@@ -12,9 +12,11 @@ enum {
BitsPointer = 2,
BitsMask = 3,
PointersPerByte = 8/BitsPerPointer,
MaxGCMask = 64,
insData = 1,
insArray,
insArrayEnd,
insEnd,
// 64 bytes cover objects of size 1024/512 on 64/32 bits, respectively.
MaxGCMask = 65536, // TODO(rsc): change back to 64
};
src/runtime/mgc1.go
View file @ 31457cef
......@@ -50,12 +50,15 @@ const (
// If you change these, also change scanblock.
// scanblock does "if(bits == BitsScalar || bits == BitsDead)" as "if(bits <= BitsScalar)".
_BitsDead = 0
_BitsScalar = 1
_BitsPointer = 2
_BitsDead = 0
_BitsScalar = 1 // 01
_BitsPointer = 2 // 10
_BitsCheckMarkXor = 1 // 10
_BitsScalarMarked = _BitsScalar ^ _BitsCheckMarkXor // 00
_BitsPointerMarked = _BitsPointer ^ _BitsCheckMarkXor // 11
// 64 bytes cover objects of size 1024/512 on 64/32 bits, respectively.
_MaxGCMask = 64
_MaxGCMask = 65536 // TODO(rsc): change back to 64
)
// Bits in per-word bitmap.
......
src/runtime/mheap.go
View file @ 31457cef
......@@ -196,7 +196,7 @@ func mHeap_Alloc_m(h *mheap, npage uintptr, sizeclass int32, large bool) *mspan
// able to map interior pointer to containing span.
atomicstore(&s.sweepgen, h.sweepgen)
s.state = _MSpanInUse
s.freelist = nil
s.freelist = 0
s.ref = 0
s.sizeclass = uint8(sizeclass)
if sizeclass == 0 {
......@@ -248,7 +248,7 @@ func mHeap_AllocStack(h *mheap, npage uintptr) *mspan {
s := mHeap_AllocSpanLocked(h, npage)
if s != nil {
s.state = _MSpanStack
s.freelist = nil
s.freelist = 0
s.ref = 0
memstats.stacks_inuse += uint64(s.npages << _PageShift)
}
......@@ -571,7 +571,7 @@ func mSpan_Init(span *mspan, start pageID, npages uintptr) {
span.prev = nil
span.start = start
span.npages = npages
span.freelist = nil
span.freelist = 0
span.ref = 0
span.sizeclass = 0
span.incache = false
......
src/runtime/os_linux_386.go
View file @ 31457cef
......@@ -14,8 +14,7 @@ const (
var _vdso uint32
//go:nosplit
func linux_setup_vdso(argc int32, argv **byte) {
func sysargs(argc int32, argv **byte) {
// skip over argv, envv to get to auxv
n := argc + 1
for argv_index(argv, n) != nil {
......
src/runtime/print1.go
View file @ 31457cef
......@@ -41,7 +41,31 @@ func snprintf(dst *byte, n int32, s *byte) {
gp.writebuf = nil
}
//var debuglock mutex
var debuglock mutex
// The compiler emits calls to printlock and printunlock around
// the multiple calls that implement a single Go print or println
// statement. Some of the print helpers (printsp, for example)
// call print recursively. There is also the problem of a crash
// happening during the print routines and needing to acquire
// the print lock to print information about the crash.
// For both these reasons, let a thread acquire the printlock 'recursively'.
func printlock() {
mp := getg().m
mp.printlock++
if mp.printlock == 1 {
lock(&debuglock)
}
}
func printunlock() {
mp := getg().m
mp.printlock--
if mp.printlock == 0 {
unlock(&debuglock)
}
}
// write to goroutine-local buffer if diverting output,
// or else standard error.
......@@ -80,7 +104,7 @@ func printnl() {
// Very simple printf. Only for debugging prints.
// Do not add to this without checking with Rob.
func vprintf(str string, arg unsafe.Pointer) {
//lock(&debuglock);
printlock()
s := bytes(str)
start := 0
......@@ -160,7 +184,7 @@ func vprintf(str string, arg unsafe.Pointer) {
gwrite(s[start:i])
}
//unlock(&debuglock);
printunlock()
}
func printpc(p unsafe.Pointer) {
......
src/runtime/proc.go
View file @ 31457cef
......@@ -181,6 +181,9 @@ func acquireSudog() *sudog {
// which keeps the garbage collector from being invoked.
mp := acquirem()
p := new(sudog)
if p.elem != nil {
gothrow("acquireSudog: found p.elem != nil after new")
}
releasem(mp)
return p
}
......
src/runtime/proc1.go
View file @ 31457cef
......@@ -316,6 +316,10 @@ func casfrom_Gscanstatus(gp *g, oldval, newval uint32) {
// Check that transition is valid.
switch oldval {
default:
print("runtime: casfrom_Gscanstatus bad oldval gp=", gp, ", oldval=", hex(oldval), ", newval=", hex(newval), "\n")
dumpgstatus(gp)
gothrow("casfrom_Gscanstatus:top gp->status is not in scan state")
case _Gscanrunnable,
_Gscanwaiting,
_Gscanrunning,
......@@ -377,12 +381,12 @@ func casgstatus(gp *g, oldval, newval uint32) {
})
}
// Help GC if needed.
if gp.preemptscan && !gp.gcworkdone && (oldval == _Grunning || oldval == _Gsyscall) {
gp.preemptscan = false
systemstack(func() {
gcphasework(gp)
})
}
// if gp.preemptscan && !gp.gcworkdone && (oldval == _Grunning || oldval == _Gsyscall) {
// gp.preemptscan = false
// systemstack(func() {
// gcphasework(gp)
// })
// }
}
}
......@@ -512,9 +516,10 @@ func stopscanstart(gp *g) {
// Runs on g0 and does the actual work after putting the g back on the run queue.
func mquiesce(gpmaster *g) {
activeglen := len(allgs)
// enqueue the calling goroutine.
restartg(gpmaster)
activeglen := len(allgs)
for i := 0; i < activeglen; i++ {
gp := allgs[i]
if readgstatus(gp) == _Gdead {
......@@ -1579,7 +1584,8 @@ func save(pc, sp uintptr) {
_g_.sched.lr = 0
_g_.sched.ret = 0
_g_.sched.ctxt = nil
_g_.sched.g = _g_
// write as uintptr to avoid write barrier, which will smash _g_.sched.
*(*uintptr)(unsafe.Pointer(&_g_.sched.g)) = uintptr(unsafe.Pointer(_g_))
}
// The goroutine g is about to enter a system call.
......@@ -1625,7 +1631,10 @@ func reentersyscall(pc, sp uintptr) {
_g_.syscallpc = pc
casgstatus(_g_, _Grunning, _Gsyscall)
if _g_.syscallsp < _g_.stack.lo || _g_.stack.hi < _g_.syscallsp {
systemstack(entersyscall_bad)
systemstack(func() {
print("entersyscall inconsistent ", hex(_g_.syscallsp), " [", hex(_g_.stack.lo), ",", hex(_g_.stack.hi), "]\n")
gothrow("entersyscall")
})
}
if atomicload(&sched.sysmonwait) != 0 { // TODO: fast atomic
......@@ -1654,13 +1663,6 @@ func entersyscall(dummy int32) {
reentersyscall(getcallerpc(unsafe.Pointer(&dummy)), getcallersp(unsafe.Pointer(&dummy)))
}
func entersyscall_bad() {
var gp *g
gp = getg().m.curg
print("entersyscall inconsistent ", hex(gp.syscallsp), " [", hex(gp.stack.lo), ",", hex(gp.stack.hi), "]\n")
gothrow("entersyscall")
}
func entersyscall_sysmon() {
lock(&sched.lock)
if atomicload(&sched.sysmonwait) != 0 {
......@@ -1692,12 +1694,26 @@ func entersyscallblock(dummy int32) {
_g_.stackguard0 = stackPreempt // see comment in entersyscall
// Leave SP around for GC and traceback.
save(getcallerpc(unsafe.Pointer(&dummy)), getcallersp(unsafe.Pointer(&dummy)))
pc := getcallerpc(unsafe.Pointer(&dummy))
sp := getcallersp(unsafe.Pointer(&dummy))
save(pc, sp)
_g_.syscallsp = _g_.sched.sp
_g_.syscallpc = _g_.sched.pc
if _g_.syscallsp < _g_.stack.lo || _g_.stack.hi < _g_.syscallsp {
sp1 := sp
sp2 := _g_.sched.sp
sp3 := _g_.syscallsp
systemstack(func() {
print("entersyscallblock inconsistent ", hex(sp1), " ", hex(sp2), " ", hex(sp3), " [", hex(_g_.stack.lo), ",", hex(_g_.stack.hi), "]\n")
gothrow("entersyscallblock")
})
}
casgstatus(_g_, _Grunning, _Gsyscall)
if _g_.syscallsp < _g_.stack.lo || _g_.stack.hi < _g_.syscallsp {
systemstack(entersyscall_bad)
systemstack(func() {
print("entersyscallblock inconsistent ", hex(sp), " ", hex(_g_.sched.sp), " ", hex(_g_.syscallsp), " [", hex(_g_.stack.lo), ",", hex(_g_.stack.hi), "]\n")
gothrow("entersyscallblock")
})
}
systemstack(entersyscallblock_handoff)
......@@ -1776,6 +1792,7 @@ func exitsyscallfast() bool {
// Freezetheworld sets stopwait but does not retake P's.
if sched.stopwait != 0 {
_g_.m.mcache = nil
_g_.m.p = nil
return false
}
......@@ -1789,6 +1806,7 @@ func exitsyscallfast() bool {
}
// Try to get any other idle P.
_g_.m.mcache = nil
_g_.m.p = nil
if sched.pidle != nil {
var ok bool
......@@ -2363,6 +2381,8 @@ func setcpuprofilerate_m(hz int32) {
}
// Change number of processors. The world is stopped, sched is locked.
// gcworkbufs are not being modified by either the GC or
// the write barrier code.
func procresize(new int32) {
old := gomaxprocs
if old < 0 || old > _MaxGomaxprocs || new <= 0 || new > _MaxGomaxprocs {
......
src/runtime/rt0_linux_386.s
View file @ 31457cef
......@@ -9,7 +9,6 @@ TEXT _rt0_386_linux(SB),NOSPLIT,$8
LEAL 12(SP), BX
MOVL AX, 0(SP)
MOVL BX, 4(SP)
CALL runtime·linux_setup_vdso(SB)
CALL main(SB)
INT $3
......
src/runtime/runtime1.go
View file @ 31457cef
......@@ -97,7 +97,10 @@ func testAtomic64() {
z64 = 42
x64 = 0
// TODO: PREFETCH((unsafe.Pointer)(&z64))
prefetcht0(uintptr(unsafe.Pointer(&z64)))
prefetcht1(uintptr(unsafe.Pointer(&z64)))
prefetcht2(uintptr(unsafe.Pointer(&z64)))
prefetchnta(uintptr(unsafe.Pointer(&z64)))
if cas64(&z64, x64, 1) {
gothrow("cas64 failed")
}
......
src/runtime/runtime2.go
View file @ 31457cef
......@@ -227,6 +227,8 @@ type m struct {
helpgc int32
spinning bool // m is out of work and is actively looking for work
blocked bool // m is blocked on a note
inwb bool // m is executing a write barrier
printlock int8
fastrand uint32
ncgocall uint64 // number of cgo calls in total
ncgo int32 // number of cgo calls currently in progress
......@@ -402,8 +404,9 @@ type itab struct {
}
// Lock-free stack node.
// // Also known to export_test.go.
type lfnode struct {
next *lfnode
next uint64
pushcnt uintptr
}
......@@ -448,11 +451,13 @@ type debugvars struct {
// Indicates to write barrier and sychronization task to preform.
const (
_GCoff = iota // stop and start nop
_GCquiesce // stop and start nop
_GCstw // stop the ps nop
_GCmark // scan the stacks and start no white to black
_GCsweep // stop and start nop
_GCoff = iota // GC not running, write barrier disabled
_GCquiesce // unused state
_GCstw // unused state
_GCscan // GC collecting roots into workbufs, write barrier disabled
_GCmark // GC marking from workbufs, write barrier ENABLED
_GCmarktermination // GC mark termination: allocate black, P's help GC, write barrier ENABLED
_GCsweep // GC mark completed; sweeping in background, write barrier disabled
)
type forcegcstate struct {
......
src/runtime/select.go
View file @ 31457cef
......@@ -377,12 +377,7 @@ loop:
// iterating through the linked list they are in reverse order.
cas = nil
sglist = gp.waiting
// Clear all selectdone and elem before unlinking from gp.waiting.
// They must be cleared before being put back into the sudog cache.
// Clear before unlinking, because if a stack copy happens after the unlink,
// they will not be updated, they will be left pointing to the old stack,
// which creates dangling pointers, which may be detected by the
// garbage collector.
// Clear all elem before unlinking from gp.waiting.
for sg1 := gp.waiting; sg1 != nil; sg1 = sg1.waitlink {
sg1.selectdone = nil
sg1.elem = nil
......
src/runtime/stack.h
View file @ 31457cef
......@@ -17,7 +17,7 @@ enum {
#endif // Windows
StackBig = 4096,
StackGuard = 512 + StackSystem,
StackGuard = 640 + StackSystem,
StackSmall = 128,
StackLimit = StackGuard - StackSystem - StackSmall,
};
......
src/runtime/stack1.go
View file @ 31457cef
......@@ -58,7 +58,7 @@ func stackinit() {
// Allocates a stack from the free pool. Must be called with
// stackpoolmu held.
func stackpoolalloc(order uint8) *mlink {
func stackpoolalloc(order uint8) gclinkptr {
list := &stackpool[order]
s := list.next
if s == list {
......@@ -70,23 +70,23 @@ func stackpoolalloc(order uint8) *mlink {
if s.ref != 0 {
gothrow("bad ref")
}
if s.freelist != nil {
if s.freelist.ptr() != nil {
gothrow("bad freelist")
}
for i := uintptr(0); i < _StackCacheSize; i += _FixedStack << order {
x := (*mlink)(unsafe.Pointer(uintptr(s.start)<<_PageShift + i))
x.next = s.freelist
x := gclinkptr(uintptr(s.start)<<_PageShift + i)
x.ptr().next = s.freelist
s.freelist = x
}
mSpanList_Insert(list, s)
}
x := s.freelist
if x == nil {
if x.ptr() == nil {
gothrow("span has no free stacks")
}
s.freelist = x.next
s.freelist = x.ptr().next
s.ref++
if s.freelist == nil {
if s.freelist.ptr() == nil {
// all stacks in s are allocated.
mSpanList_Remove(s)
}
......@@ -94,22 +94,22 @@ func stackpoolalloc(order uint8) *mlink {
}
// Adds stack x to the free pool. Must be called with stackpoolmu held.
func stackpoolfree(x *mlink, order uint8) {
func stackpoolfree(x gclinkptr, order uint8) {
s := mHeap_Lookup(&mheap_, (unsafe.Pointer)(x))
if s.state != _MSpanStack {
gothrow("freeing stack not in a stack span")
}
if s.freelist == nil {
if s.freelist.ptr() == nil {
// s will now have a free stack
mSpanList_Insert(&stackpool[order], s)
}
x.next = s.freelist
x.ptr().next = s.freelist
s.freelist = x
s.ref--
if s.ref == 0 {
// span is completely free - return to heap
mSpanList_Remove(s)
s.freelist = nil
s.freelist = 0
mHeap_FreeStack(&mheap_, s)
}
}
......@@ -123,12 +123,12 @@ func stackcacherefill(c *mcache, order uint8) {
// Grab some stacks from the global cache.
// Grab half of the allowed capacity (to prevent thrashing).
var list *mlink
var list gclinkptr
var size uintptr
lock(&stackpoolmu)
for size < _StackCacheSize/2 {
x := stackpoolalloc(order)
x.next = list
x.ptr().next = list
list = x
size += _FixedStack << order
}
......@@ -145,7 +145,7 @@ func stackcacherelease(c *mcache, order uint8) {
size := c.stackcache[order].size
lock(&stackpoolmu)
for size > _StackCacheSize/2 {
y := x.next
y := x.ptr().next
stackpoolfree(x, order)
x = y
size -= _FixedStack << order
......@@ -162,12 +162,12 @@ func stackcache_clear(c *mcache) {
lock(&stackpoolmu)
for order := uint8(0); order < _NumStackOrders; order++ {
x := c.stackcache[order].list
for x != nil {
y := x.next
for x.ptr() != nil {
y := x.ptr().next
stackpoolfree(x, order)
x = y
}
c.stackcache[order].list = nil
c.stackcache[order].list = 0
c.stackcache[order].size = 0
}
unlock(&stackpoolmu)
......@@ -207,7 +207,7 @@ func stackalloc(n uint32) stack {
order++
n2 >>= 1
}
var x *mlink
var x gclinkptr
c := thisg.m.mcache
if c == nil || thisg.m.gcing != 0 || thisg.m.helpgc != 0 {
// c == nil can happen in the guts of exitsyscall or
......@@ -219,11 +219,11 @@ func stackalloc(n uint32) stack {
unlock(&stackpoolmu)
} else {
x = c.stackcache[order].list
if x == nil {
if x.ptr() == nil {
stackcacherefill(c, order)
x = c.stackcache[order].list
}
c.stackcache[order].list = x.next
c.stackcache[order].list = x.ptr().next
c.stackcache[order].size -= uintptr(n)
}
v = (unsafe.Pointer)(x)
......@@ -270,7 +270,7 @@ func stackfree(stk stack) {
order++
n2 >>= 1
}
x := (*mlink)(v)
x := gclinkptr(v)
c := gp.m.mcache
if c == nil || gp.m.gcing != 0 || gp.m.helpgc != 0 {
lock(&stackpoolmu)
......@@ -280,7 +280,7 @@ func stackfree(stk stack) {
if c.stackcache[order].size >= _StackCacheSize {
stackcacherelease(c, order)
}
x.next = c.stackcache[order].list
x.ptr().next = c.stackcache[order].list
c.stackcache[order].list = x
c.stackcache[order].size += n
}
......@@ -526,6 +526,7 @@ func fillstack(stk stack, b byte) {
}
// Copies gp's stack to a new stack of a different size.
// Caller must have changed gp status to Gcopystack.
func copystack(gp *g, newsize uintptr) {
if gp.syscallsp != 0 {
gothrow("stack growth not allowed in system call")
......@@ -563,15 +564,11 @@ func copystack(gp *g, newsize uintptr) {
}
memmove(unsafe.Pointer(new.hi-used), unsafe.Pointer(old.hi-used), used)
oldstatus := casgcopystack(gp) // cas from Gwaiting or Grunnable to Gcopystack, return old status
// Swap out old stack for new one
gp.stack = new
gp.stackguard0 = new.lo + _StackGuard // NOTE: might clobber a preempt request
gp.sched.sp = new.hi - used
casgstatus(gp, _Gcopystack, oldstatus) // oldstatus is Gwaiting or Grunnable
// free old stack
if stackPoisonCopy != 0 {
fillstack(old, 0xfc)
......@@ -669,6 +666,14 @@ func newstack() {
gothrow("runtime: split stack overflow")
}
if gp.sched.ctxt != nil {
// morestack wrote sched.ctxt on its way in here,
// without a write barrier. Run the write barrier now.
// It is not possible to be preempted between then
// and now, so it's okay.
writebarrierptr_nostore((*uintptr)(unsafe.Pointer(&gp.sched.ctxt)), uintptr(gp.sched.ctxt))
}
if gp.stackguard0 == stackPreempt {
if gp == thisg.m.g0 {
gothrow("runtime: preempt g0")
......@@ -677,7 +682,12 @@ func newstack() {
gothrow("runtime: g is running but p is not")
}
if gp.preemptscan {
for !castogscanstatus(gp, _Gwaiting, _Gscanwaiting) {
// Likely to be racing with the GC as it sees a _Gwaiting and does the stack scan.
// If so this stack will be scanned twice which does not change correctness.
}
gcphasework(gp)
casfrom_Gscanstatus(gp, _Gscanwaiting, _Gwaiting)
casgstatus(gp, _Gwaiting, _Grunning)
gp.stackguard0 = gp.stack.lo + _StackGuard
gp.preempt = false
......@@ -708,13 +718,15 @@ func newstack() {
gothrow("stack overflow")
}
// Note that the concurrent GC might be scanning the stack as we try to replace it.
// copystack takes care of the appropriate coordination with the stack scanner.
casgstatus(gp, _Gwaiting, _Gcopystack)
// The concurrent GC will not scan the stack while we are doing the copy since
// the gp is in a Gcopystack status.
copystack(gp, uintptr(newsize))
if stackDebug >= 1 {
print("stack grow done\n")
}
casgstatus(gp, _Gwaiting, _Grunning)
casgstatus(gp, _Gcopystack, _Grunning)
gogo(&gp.sched)
}
......@@ -767,17 +779,17 @@ func shrinkstack(gp *g) {
if gp.syscallsp != 0 {
return
}
/* TODO
if goos_windows && gp.m != nil && gp.m.libcallsp != 0 {
if goos_windows != 0 && gp.m != nil && gp.m.libcallsp != 0 {
return
}
*/
if stackDebug > 0 {
print("shrinking stack ", oldsize, "->", newsize, "\n")
}
oldstatus := casgcopystack(gp)
copystack(gp, newsize)
casgstatus(gp, _Gcopystack, oldstatus)
}
// Do any delayed stack freeing that was queued up during GC.
......
src/runtime/stack2.go
View file @ 31457cef
......@@ -84,7 +84,7 @@ const (
// The stack guard is a pointer this many bytes above the
// bottom of the stack.
_StackGuard = 512 + _StackSystem
_StackGuard = 640 + _StackSystem
// After a stack split check the SP is allowed to be this
// many bytes below the stack guard. This saves an instruction
......
src/runtime/stubs.go
View file @ 31457cef
......@@ -231,3 +231,8 @@ func call536870912(fn, arg unsafe.Pointer, n, retoffset uint32)
func call1073741824(fn, arg unsafe.Pointer, n, retoffset uint32)
func systemstack_switch()
func prefetcht0(addr uintptr)
func prefetcht1(addr uintptr)
func prefetcht2(addr uintptr)
func prefetchnta(addr uintptr)
src/runtime/vdso_none.go
View file @ 31457cef
......@@ -3,6 +3,7 @@
// license that can be found in the LICENSE file.
// +build !linux !amd64
// +build !linux !386
package runtime
......
src/runtime/wbfat.go 0 → 100644
View file @ 31457cef
// generated by wbfat_gen.go; use go generate
package runtime
//go:nosplit
func writebarrierfat01(dst *[2]uintptr, _ *byte, src [2]uintptr) {
dst[0] = src[0]
writebarrierptr(&dst[1], src[1])
}
//go:nosplit
func writebarrierfat10(dst *[2]uintptr, _ *byte, src [2]uintptr) {
writebarrierptr(&dst[0], src[0])
dst[1] = src[1]
}
//go:nosplit
func writebarrierfat11(dst *[2]uintptr, _ *byte, src [2]uintptr) {
writebarrierptr(&dst[0], src[0])
writebarrierptr(&dst[1], src[1])
}
//go:nosplit
func writebarrierfat001(dst *[3]uintptr, _ *byte, src [3]uintptr) {
dst[0] = src[0]
dst[1] = src[1]
writebarrierptr(&dst[2], src[2])
}
//go:nosplit
func writebarrierfat010(dst *[3]uintptr, _ *byte, src [3]uintptr) {
dst[0] = src[0]
writebarrierptr(&dst[1], src[1])
dst[2] = src[2]
}
//go:nosplit
func writebarrierfat011(dst *[3]uintptr, _ *byte, src [3]uintptr) {
dst[0] = src[0]
writebarrierptr(&dst[1], src[1])
writebarrierptr(&dst[2], src[2])
}
//go:nosplit
func writebarrierfat100(dst *[3]uintptr, _ *byte, src [3]uintptr) {
writebarrierptr(&dst[0], src[0])
dst[1] = src[1]
dst[2] = src[2]
}
//go:nosplit
func writebarrierfat101(dst *[3]uintptr, _ *byte, src [3]uintptr) {
writebarrierptr(&dst[0], src[0])
dst[1] = src[1]
writebarrierptr(&dst[2], src[2])
}
//go:nosplit
func writebarrierfat110(dst *[3]uintptr, _ *byte, src [3]uintptr) {
writebarrierptr(&dst[0], src[0])
writebarrierptr(&dst[1], src[1])
dst[2] = src[2]
}
//go:nosplit
func writebarrierfat111(dst *[3]uintptr, _ *byte, src [3]uintptr) {
writebarrierptr(&dst[0], src[0])
writebarrierptr(&dst[1], src[1])
writebarrierptr(&dst[2], src[2])
}
//go:nosplit
func writebarrierfat0001(dst *[4]uintptr, _ *byte, src [4]uintptr) {
dst[0] = src[0]
dst[1] = src[1]
dst[2] = src[2]
writebarrierptr(&dst[3], src[3])
}
//go:nosplit
func writebarrierfat0010(dst *[4]uintptr, _ *byte, src [4]uintptr) {
dst[0] = src[0]
dst[1] = src[1]
writebarrierptr(&dst[2], src[2])
dst[3] = src[3]
}
//go:nosplit
func writebarrierfat0011(dst *[4]uintptr, _ *byte, src [4]uintptr) {
dst[0] = src[0]
dst[1] = src[1]
writebarrierptr(&dst[2], src[2])
writebarrierptr(&dst[3], src[3])
}
//go:nosplit
func writebarrierfat0100(dst *[4]uintptr, _ *byte, src [4]uintptr) {
dst[0] = src[0]
writebarrierptr(&dst[1], src[1])
dst[2] = src[2]
dst[3] = src[3]
}
//go:nosplit
func writebarrierfat0101(dst *[4]uintptr, _ *byte, src [4]uintptr) {
dst[0] = src[0]
writebarrierptr(&dst[1], src[1])
dst[2] = src[2]
writebarrierptr(&dst[3], src[3])
}
//go:nosplit
func writebarrierfat0110(dst *[4]uintptr, _ *byte, src [4]uintptr) {
dst[0] = src[0]
writebarrierptr(&dst[1], src[1])
writebarrierptr(&dst[2], src[2])
dst[3] = src[3]
}
//go:nosplit
func writebarrierfat0111(dst *[4]uintptr, _ *byte, src [4]uintptr) {
dst[0] = src[0]
writebarrierptr(&dst[1], src[1])
writebarrierptr(&dst[2], src[2])
writebarrierptr(&dst[3], src[3])
}
//go:nosplit
func writebarrierfat1000(dst *[4]uintptr, _ *byte, src [4]uintptr) {
writebarrierptr(&dst[0], src[0])
dst[1] = src[1]
dst[2] = src[2]
dst[3] = src[3]
}
//go:nosplit
func writebarrierfat1001(dst *[4]uintptr, _ *byte, src [4]uintptr) {
writebarrierptr(&dst[0], src[0])
dst[1] = src[1]
dst[2] = src[2]
writebarrierptr(&dst[3], src[3])
}
//go:nosplit
func writebarrierfat1010(dst *[4]uintptr, _ *byte, src [4]uintptr) {
writebarrierptr(&dst[0], src[0])
dst[1] = src[1]
writebarrierptr(&dst[2], src[2])
dst[3] = src[3]
}
//go:nosplit
func writebarrierfat1011(dst *[4]uintptr, _ *byte, src [4]uintptr) {
writebarrierptr(&dst[0], src[0])
dst[1] = src[1]
writebarrierptr(&dst[2], src[2])
writebarrierptr(&dst[3], src[3])
}
//go:nosplit
func writebarrierfat1100(dst *[4]uintptr, _ *byte, src [4]uintptr) {
writebarrierptr(&dst[0], src[0])
writebarrierptr(&dst[1], src[1])
dst[2] = src[2]
dst[3] = src[3]
}
//go:nosplit
func writebarrierfat1101(dst *[4]uintptr, _ *byte, src [4]uintptr) {
writebarrierptr(&dst[0], src[0])
writebarrierptr(&dst[1], src[1])
dst[2] = src[2]
writebarrierptr(&dst[3], src[3])
}
//go:nosplit
func writebarrierfat1110(dst *[4]uintptr, _ *byte, src [4]uintptr) {
writebarrierptr(&dst[0], src[0])
writebarrierptr(&dst[1], src[1])
writebarrierptr(&dst[2], src[2])
dst[3] = src[3]
}
//go:nosplit
func writebarrierfat1111(dst *[4]uintptr, _ *byte, src [4]uintptr) {
writebarrierptr(&dst[0], src[0])
writebarrierptr(&dst[1], src[1])
writebarrierptr(&dst[2], src[2])
writebarrierptr(&dst[3], src[3])
}
src/runtime/wbfat_gen.go 0 → 100644
View file @ 31457cef
// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build ignore
package main
import (
"flag"
"fmt"
"log"
"os"
)
func main() {
flag.Parse()
if flag.NArg() > 0 {
f, err := os.Create(flag.Arg(0))
if err != nil {
log.Fatal(err)
}
os.Stdout = f
}
fmt.Printf("// generated by wbfat_gen.go; use go generate\n\n")
fmt.Printf("package runtime\n")
for i := uint(2); i <= 4; i++ {
for j := 1; j < 1<<i; j++ {
fmt.Printf("\n//go:nosplit\n")
fmt.Printf("func writebarrierfat%0*b(dst *[%d]uintptr, _ *byte, src [%d]uintptr) {\n", int(i), j, i, i)
for k := uint(0); k < i; k++ {
if j&(1<<(i-1-k)) != 0 {
fmt.Printf("\twritebarrierptr(&dst[%d], src[%d])\n", k, k)
} else {
fmt.Printf("\tdst[%d] = src[%d]\n", k, k)
}
}
fmt.Printf("}\n")
}
}
}
src/sync/atomic/atomic_test.go
View file @ 31457cef
......@@ -164,7 +164,7 @@ func TestSwapPointer(t *testing.T) {
x.before = magicptr
x.after = magicptr
var j uintptr
for delta := uintptr(1); delta+delta > delta; delta += delta {
for delta := uintptr(1 << 16); delta+delta > delta; delta += delta {
k := SwapPointer(&x.i, unsafe.Pointer(delta))
if uintptr(x.i) != delta || uintptr(k) != j {
t.Fatalf("delta=%d i=%d j=%d k=%d", delta, x.i, j, k)
......@@ -456,7 +456,7 @@ func TestCompareAndSwapPointer(t *testing.T) {
magicptr := uintptr(m)
x.before = magicptr
x.after = magicptr
for val := uintptr(1); val+val > val; val += val {
for val := uintptr(1 << 16); val+val > val; val += val {
x.i = unsafe.Pointer(val)
if !CompareAndSwapPointer(&x.i, unsafe.Pointer(val), unsafe.Pointer(val+1)) {
t.Fatalf("should have swapped %#x %#x", val, val+1)
......@@ -595,7 +595,7 @@ func TestLoadPointer(t *testing.T) {
magicptr := uintptr(m)
x.before = magicptr
x.after = magicptr
for delta := uintptr(1); delta+delta > delta; delta += delta {
for delta := uintptr(1 << 16); delta+delta > delta; delta += delta {
k := LoadPointer(&x.i)
if k != x.i {
t.Fatalf("delta=%d i=%d k=%d", delta, x.i, k)
......@@ -731,7 +731,7 @@ func TestStorePointer(t *testing.T) {
x.before = magicptr
x.after = magicptr
v := unsafe.Pointer(uintptr(0))
for delta := uintptr(1); delta+delta > delta; delta += delta {
for delta := uintptr(1 << 16); delta+delta > delta; delta += delta {
StorePointer(&x.i, unsafe.Pointer(v))
if x.i != v {
t.Fatalf("delta=%d i=%d v=%d", delta, x.i, v)
......
test/live.go
View file @ 31457cef
......@@ -9,20 +9,39 @@
package main
func printnl()
//go:noescape
func printpointer(**int)
//go:noescape
func printintpointer(*int)
//go:noescape
func printstringpointer(*string)
//go:noescape
func printstring(string)
//go:noescape
func printbytepointer(*byte)
func printint(int)
func f1() {
var x *int
print(&x) // ERROR "live at call to printpointer: x$"
print(&x) // ERROR "live at call to printpointer: x$"
printpointer(&x) // ERROR "live at call to printpointer: x$"
printpointer(&x) // ERROR "live at call to printpointer: x$"
}
func f2(b bool) {
if b {
print(0) // nothing live here
printint(0) // nothing live here
return
}
var x *int
print(&x) // ERROR "live at call to printpointer: x$"
print(&x) // ERROR "live at call to printpointer: x$"
printpointer(&x) // ERROR "live at call to printpointer: x$"
printpointer(&x) // ERROR "live at call to printpointer: x$"
}
func f3(b bool) {
......@@ -30,22 +49,22 @@ func f3(b bool) {
// live throughout the function, to avoid being poisoned
// in GODEBUG=gcdead=1 mode.
print(0) // ERROR "live at call to printint: x y$"
printint(0) // ERROR "live at call to printint: x y$"
if b == false {
print(0) // ERROR "live at call to printint: x y$"
printint(0) // ERROR "live at call to printint: x y$"
return
}
if b {
var x *int
print(&x) // ERROR "live at call to printpointer: x y$"
print(&x) // ERROR "live at call to printpointer: x y$"
printpointer(&x) // ERROR "live at call to printpointer: x y$"
printpointer(&x) // ERROR "live at call to printpointer: x y$"
} else {
var y *int
print(&y) // ERROR "live at call to printpointer: x y$"
print(&y) // ERROR "live at call to printpointer: x y$"
printpointer(&y) // ERROR "live at call to printpointer: x y$"
printpointer(&y) // ERROR "live at call to printpointer: x y$"
}
print(0) // ERROR "live at call to printint: x y$" "x \(type \*int\) is ambiguously live" "y \(type \*int\) is ambiguously live"
printint(0) // ERROR "live at call to printint: x y$" "x \(type \*int\) is ambiguously live" "y \(type \*int\) is ambiguously live"
}
// The old algorithm treated x as live on all code that
......@@ -56,20 +75,20 @@ func f3(b bool) {
func f4(b1, b2 bool) { // x not live here
if b2 {
print(0) // x not live here
printint(0) // x not live here
return
}
var z **int
x := new(int)
*x = 42
z = &x
print(**z) // ERROR "live at call to printint: x z$"
printint(**z) // ERROR "live at call to printint: x z$"
if b2 {
print(1) // ERROR "live at call to printint: x$"
printint(1) // ERROR "live at call to printint: x$"
return
}
for {
print(**z) // ERROR "live at call to printint: x z$"
printint(**z) // ERROR "live at call to printint: x z$"
}
}
......@@ -84,7 +103,7 @@ func f5(b1 bool) {
*y = 54
z = &y
}
print(**z) // ERROR "live at call to printint: x y$" "x \(type \*int\) is ambiguously live" "y \(type \*int\) is ambiguously live"
printint(**z) // ERROR "live at call to printint: x y$" "x \(type \*int\) is ambiguously live" "y \(type \*int\) is ambiguously live"
}
// confusion about the _ result used to cause spurious "live at entry to f6: _".
......@@ -155,8 +174,8 @@ func f11b() *int {
// At this point p is dead: the code here cannot
// get to the bottom of the function.
// This used to have a spurious "live at call to printint: p".
print(1) // nothing live here!
select { // ERROR "live at call to newselect: autotmp" "live at call to selectgo: autotmp"
printint(1) // nothing live here!
select { // ERROR "live at call to newselect: autotmp" "live at call to selectgo: autotmp"
case <-c: // ERROR "live at call to selectrecv: autotmp"
return nil
case <-c: // ERROR "live at call to selectrecv: autotmp"
......@@ -172,8 +191,8 @@ func f11c() *int {
if b {
// Unlike previous, the cases in this select fall through,
// so we can get to the println, so p is not dead.
print(1) // ERROR "live at call to printint: p"
select { // ERROR "live at call to newselect: autotmp.* p" "live at call to selectgo: autotmp.* p"
printint(1) // ERROR "live at call to printint: p"
select { // ERROR "live at call to newselect: autotmp.* p" "live at call to selectgo: autotmp.* p"
case <-c: // ERROR "live at call to selectrecv: autotmp.* p"
case <-c: // ERROR "live at call to selectrecv: autotmp.* p"
}
......@@ -209,7 +228,7 @@ func h13(string, string) string
func f14() {
x := g14()
print(&x) // ERROR "live at call to printpointer: x"
printstringpointer(&x) // ERROR "live at call to printstringpointer: x"
}
func g14() string
......@@ -217,8 +236,8 @@ func g14() string
func f15() {
var x string
_ = &x
x = g15() // ERROR "live at call to g15: x"
print(x) // ERROR "live at call to printstring: x"
x = g15() // ERROR "live at call to g15: x"
printstring(x) // ERROR "live at call to printstring: x"
}
func g15() string
......@@ -282,7 +301,7 @@ func f18() {
}
z = m2[g18()] // ERROR "live at call to mapaccess1: autotmp_[0-9]+$"
z = m2[g18()] // ERROR "live at call to mapaccess1: autotmp_[0-9]+$"
print(z)
printbytepointer(z)
}
var ch chan *byte
......@@ -296,7 +315,7 @@ func f19() {
}
z = <-ch // ERROR "live at call to chanrecv1: autotmp_[0-9]+$"
z = <-ch // ERROR "live at call to chanrecv1: autotmp_[0-9]+$"
print(z)
printbytepointer(z)
}
func f20() {
......@@ -316,7 +335,7 @@ func f21() {
}
z = m2[[2]string{"x", "y"}] // ERROR "live at call to mapaccess1: autotmp_[0-9]+$"
z = m2[[2]string{"x", "y"}] // ERROR "live at call to mapaccess1: autotmp_[0-9]+$"
print(z)
printbytepointer(z)
}
func f23() {
......@@ -328,7 +347,8 @@ func f23() {
}
z, ok = m2[[2]string{"x", "y"}] // ERROR "live at call to mapaccess2: autotmp_[0-9]+$"
z, ok = m2[[2]string{"x", "y"}] // ERROR "live at call to mapaccess2: autotmp_[0-9]+$"
print(z, ok)
printbytepointer(z)
print(ok)
}
func f24() {
......@@ -350,8 +370,8 @@ func f25(b bool) {
}
var x string
_ = &x
x = g15() // ERROR "live at call to g15: x"
print(x) // ERROR "live at call to printstring: x"
x = g15() // ERROR "live at call to g15: x"
printstring(x) // ERROR "live at call to printstring: x"
} // ERROR "live at call to deferreturn: x"
func g25()
......@@ -366,7 +386,7 @@ func f26(b bool) {
}
print26((*int)(nil), (*int)(nil), (*int)(nil)) // ERROR "live at call to print26: autotmp_[0-9]+$"
print26((*int)(nil), (*int)(nil), (*int)(nil)) // ERROR "live at call to print26: autotmp_[0-9]+$"
println()
printnl()
}
//go:noescape
......@@ -381,7 +401,7 @@ func f27(b bool) {
}
call27(func() { x++ }) // ERROR "live at call to call27: autotmp_[0-9]+$"
call27(func() { x++ }) // ERROR "live at call to call27: autotmp_[0-9]+$"
println()
printnl()
}
// but defer does escape to later execution in the function
......@@ -392,7 +412,7 @@ func f27defer(b bool) {
defer call27(func() { x++ }) // ERROR "live at call to deferproc: autotmp_[0-9]+$" "live at call to deferreturn: autotmp_[0-9]+$"
}
defer call27(func() { x++ }) // ERROR "live at call to deferproc: autotmp_[0-9]+ autotmp_[0-9]+$" "live at call to deferreturn: autotmp_[0-9]+ autotmp_[0-9]+$" "ambiguously live"
println() // ERROR "live at call to printnl: autotmp_[0-9]+ autotmp_[0-9]+$"
printnl() // ERROR "live at call to printnl: autotmp_[0-9]+ autotmp_[0-9]+$"
} // ERROR "live at call to deferreturn: autotmp_[0-9]+ autotmp_[0-9]+$"
// and newproc (go) escapes to the heap
......@@ -403,7 +423,7 @@ func f27go(b bool) {
go call27(func() { x++ }) // ERROR "live at call to newobject: &x" "live at call to newproc: &x$"
}
go call27(func() { x++ }) // ERROR "live at call to newobject: &x"
println()
printnl()
}
//go:noescape
......@@ -415,10 +435,10 @@ var s1, s2, s3, s4, s5, s6, s7, s8, s9, s10 string
func f28(b bool) {
if b {
print(s1 + s2 + s3 + s4 + s5 + s6 + s7 + s8 + s9 + s10) // ERROR "live at call to concatstrings: autotmp_[0-9]+$" "live at call to printstring: autotmp_[0-9]+$"
printstring(s1 + s2 + s3 + s4 + s5 + s6 + s7 + s8 + s9 + s10) // ERROR "live at call to concatstrings: autotmp_[0-9]+$" "live at call to printstring: autotmp_[0-9]+$"
}
print(s1 + s2 + s3 + s4 + s5 + s6 + s7 + s8 + s9 + s10) // ERROR "live at call to concatstrings: autotmp_[0-9]+$" "live at call to printstring: autotmp_[0-9]+$"
print(s1 + s2 + s3 + s4 + s5 + s6 + s7 + s8 + s9 + s10) // ERROR "live at call to concatstrings: autotmp_[0-9]+$" "live at call to printstring: autotmp_[0-9]+$"
printstring(s1 + s2 + s3 + s4 + s5 + s6 + s7 + s8 + s9 + s10) // ERROR "live at call to concatstrings: autotmp_[0-9]+$" "live at call to printstring: autotmp_[0-9]+$"
printstring(s1 + s2 + s3 + s4 + s5 + s6 + s7 + s8 + s9 + s10) // ERROR "live at call to concatstrings: autotmp_[0-9]+$" "live at call to printstring: autotmp_[0-9]+$"
}
// map iterator should die on end of range loop
......@@ -426,14 +446,14 @@ func f28(b bool) {
func f29(b bool) {
if b {
for k := range m { // ERROR "live at call to mapiterinit: autotmp_[0-9]+$" "live at call to mapiternext: autotmp_[0-9]+$"
print(k) // ERROR "live at call to printstring: autotmp_[0-9]+$"
printstring(k) // ERROR "live at call to printstring: autotmp_[0-9]+$"
}
}
for k := range m { // ERROR "live at call to mapiterinit: autotmp_[0-9]+$" "live at call to mapiternext: autotmp_[0-9]+$"
print(k) // ERROR "live at call to printstring: autotmp_[0-9]+$"
printstring(k) // ERROR "live at call to printstring: autotmp_[0-9]+$"
}
for k := range m { // ERROR "live at call to mapiterinit: autotmp_[0-9]+$" "live at call to mapiternext: autotmp_[0-9]+$"
print(k) // ERROR "live at call to printstring: autotmp_[0-9]+$"
printstring(k) // ERROR "live at call to printstring: autotmp_[0-9]+$"
}
}
......@@ -446,14 +466,14 @@ func f30(b bool) {
// the copy of ptrarr and the internal iterator pointer.
if b {
for _, p := range ptrarr {
print(p) // ERROR "live at call to printpointer: autotmp_[0-9]+ autotmp_[0-9]+$"
printintpointer(p) // ERROR "live at call to printintpointer: autotmp_[0-9]+ autotmp_[0-9]+$"
}
}
for _, p := range ptrarr {
print(p) // ERROR "live at call to printpointer: autotmp_[0-9]+ autotmp_[0-9]+$"
printintpointer(p) // ERROR "live at call to printintpointer: autotmp_[0-9]+ autotmp_[0-9]+$"
}
for _, p := range ptrarr {
print(p) // ERROR "live at call to printpointer: autotmp_[0-9]+ autotmp_[0-9]+$"
printintpointer(p) // ERROR "live at call to printintpointer: autotmp_[0-9]+ autotmp_[0-9]+$"
}
}
......@@ -503,44 +523,44 @@ var m33 map[interface{}]int
func f33() {
if m33[nil] == 0 { // ERROR "live at call to mapaccess1: autotmp_[0-9]+$"
println()
printnl()
return
} else {
println()
printnl()
}
println()
printnl()
}
func f34() {
if m33[nil] == 0 { // ERROR "live at call to mapaccess1: autotmp_[0-9]+$"
println()
printnl()
return
}
println()
printnl()
}
func f35() {
if m33[nil] == 0 && m33[nil] == 0 { // ERROR "live at call to mapaccess1: autotmp_[0-9]+$"
println()
printnl()
return
}
println()
printnl()
}
func f36() {
if m33[nil] == 0 || m33[nil] == 0 { // ERROR "live at call to mapaccess1: autotmp_[0-9]+$"
println()
printnl()
return
}
println()
printnl()
}
func f37() {
if (m33[nil] == 0 || m33[nil] == 0) && m33[nil] == 0 { // ERROR "live at call to mapaccess1: autotmp_[0-9]+$"
println()
printnl()
return
}
println()
printnl()
}
// select temps should disappear in the case bodies
......@@ -558,44 +578,44 @@ func f38(b bool) {
if b {
select { // ERROR "live at call"
case <-fc38(): // ERROR "live at call"
println()
printnl()
case fc38() <- *fi38(1): // ERROR "live at call"
println()
printnl()
case *fi38(2) = <-fc38(): // ERROR "live at call"
println()
printnl()
case *fi38(3), *fb38() = <-fc38(): // ERROR "live at call"
println()
printnl()
}
println()
printnl()
}
println()
printnl()
}
// issue 8097: mishandling of x = x during return.
func f39() (x []int) {
x = []int{1}
println() // ERROR "live at call to printnl: x"
printnl() // ERROR "live at call to printnl: x"
return x
}
func f39a() (x []int) {
x = []int{1}
println() // ERROR "live at call to printnl: x"
printnl() // ERROR "live at call to printnl: x"
return
}
func f39b() (x [10]*int) {
x = [10]*int{}
x[0] = new(int) // ERROR "live at call to newobject: x"
println() // ERROR "live at call to printnl: x"
printnl() // ERROR "live at call to printnl: x"
return x
}
func f39c() (x [10]*int) {
x = [10]*int{}
x[0] = new(int) // ERROR "live at call to newobject: x"
println() // ERROR "live at call to printnl: x"
printnl() // ERROR "live at call to printnl: x"
return
}
......@@ -615,13 +635,13 @@ func newT40() *T40 {
func bad40() {
t := newT40()
_ = t
println()
printnl()
}
func good40() {
ret := T40{}
ret.m = make(map[int]int) // ERROR "live at call to makemap: ret"
t := &ret
println() // ERROR "live at call to printnl: ret"
printnl() // ERROR "live at call to printnl: ret"
_ = t
}
test/live2.go
View file @ 31457cef
......@@ -12,6 +12,8 @@ package main
// issue 8142: lost 'addrtaken' bit on inlined variables.
// no inlining in this test, so just checking that non-inlined works.
func printnl()
type T40 struct {
m map[int]int
}
......@@ -24,7 +26,7 @@ func newT40() *T40 {
func bad40() {
t := newT40() // ERROR "live at call to makemap: ret"
println() // ERROR "live at call to printnl: ret"
printnl() // ERROR "live at call to printnl: ret"
_ = t
}
......@@ -32,6 +34,6 @@ func good40() {
ret := T40{}
ret.m = make(map[int]int) // ERROR "live at call to makemap: ret"
t := &ret
println() // ERROR "live at call to printnl: ret"
printnl() // ERROR "live at call to printnl: ret"
_ = t
}
test/nosplit.go
View file @ 31457cef
......@@ -268,11 +268,11 @@ TestCases:
name := m[1]
size, _ := strconv.Atoi(m[2])
// The limit was originally 128 but is now 384.
// The limit was originally 128 but is now 512.
// Instead of rewriting the test cases above, adjust
// the first stack frame to use up the extra 32 bytes.
if i == 0 {
size += 384 - 128
size += 512 - 128
}
if size%ptrSize == 4 {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7