Commit 479501c1 authored by Austin Clements's avatar Austin Clements

runtime: count black allocations toward scan work

Currently we count black allocations toward the scannable heap size,
but not toward the scan work we've done so far. This is clearly
inconsistent (we have, in effect, scanned these allocations and since
they're already black, we're not going to scan them again). Worse, it
means we don't count black allocations toward the scannable heap size
as of the *next* GC because this is based on the amount of scan work
we did in this cycle.

Fix this by counting black allocations as scan work. Currently the GC
spends very little time in allocate-black mode, so this probably
hasn't been a problem, but this will become important when we switch
to always allocating black.

Change-Id: If6ff693b070c385b65b6ecbbbbf76283a0f9d990
Reviewed-on: https://go-review.googlesource.com/22119Reviewed-by: default avatarRick Hudson <rlh@golang.org>
Run-TryBot: Austin Clements <austin@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
parent a683c385
......@@ -655,6 +655,7 @@ func mallocgc(size uintptr, typ *_type, needzero bool) unsafe.Pointer {
size = s.elemsize
}
var scanSize uintptr
if noscan {
// All objects are pre-marked as noscan. Nothing to do.
} else {
......@@ -673,11 +674,12 @@ func mallocgc(size uintptr, typ *_type, needzero bool) unsafe.Pointer {
// pointers, GC has to scan to the last
// element.
if typ.ptrdata != 0 {
c.local_scan += dataSize - typ.size + typ.ptrdata
scanSize = dataSize - typ.size + typ.ptrdata
}
} else {
c.local_scan += typ.ptrdata
scanSize = typ.ptrdata
}
c.local_scan += scanSize
// Ensure that the stores above that initialize x to
// type-safe memory and set the heap bits occur before
......@@ -694,7 +696,7 @@ func mallocgc(size uintptr, typ *_type, needzero bool) unsafe.Pointer {
// a race marking the bit.
if gcphase == _GCmarktermination || gcBlackenPromptly {
systemstack(func() {
gcmarknewobject_m(uintptr(x), size)
gcmarknewobject_m(uintptr(x), size, scanSize)
})
}
......
......@@ -304,7 +304,8 @@ type gcControllerState struct {
// scanWork is the total scan work performed this cycle. This
// is updated atomically during the cycle. Updates occur in
// bounded batches, since it is both written and read
// throughout the cycle.
// throughout the cycle. At the end of the cycle, this is how
// much of the retained heap is scannable.
//
// Currently this is the bytes of heap scanned. For most uses,
// this is an opaque unit of work, but for estimation the
......@@ -1578,9 +1579,13 @@ func gcMark(start_time int64) {
work.markrootDone = true
for i := 0; i < int(gomaxprocs); i++ {
if !allp[i].gcw.empty() {
gcw := &allp[i].gcw
if !gcw.empty() {
throw("P has cached GC work at end of mark termination")
}
if gcw.scanWork != 0 || gcw.bytesMarked != 0 {
throw("P has unflushed stats at end of mark termination")
}
}
if trace.enabled {
......
......@@ -1134,12 +1134,14 @@ func gcDumpObject(label string, obj, off uintptr) {
// If gcBlackenPromptly is true we are in the second mark phase phase so we allocate black.
//go:nowritebarrier
func gcmarknewobject_m(obj, size uintptr) {
func gcmarknewobject_m(obj, size, scanSize uintptr) {
if useCheckmark && !gcBlackenPromptly { // The world should be stopped so this should not happen.
throw("gcmarknewobject called while doing checkmark")
}
heapBitsForAddr(obj).setMarked()
atomic.Xadd64(&work.bytesMarked, int64(size))
gcw := &getg().m.p.ptr().gcw
gcw.scanWork += int64(scanSize)
}
// Checkmarking
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment