runtime: reset mark state before checkmark and gctrace=2 mark

Currently we fail to reset the live heap accounting state before the
checkmark mark and before the gctrace=2 extra mark. As a result, if
either are enabled, at the end of GC it thinks there are 0 bytes of
live heap, which causes the GC controller to initiate a new GC
immediately, regardless of the true heap size.

Fix this by factoring this state reset into a function and calling it
before all three possible marks.

This function should be merged with gcResetGState, but doing so
requires some additional cleanup, so it will wait for after the
freeze. Filed #11427 for this cleanup.

Fixes #10492.

Change-Id: Ibe46348916fc8368fac6f086e142815c970a6f4d
Reviewed-on: https://go-review.googlesource.com/11561Reviewed-by: Russ Cox <rsc@golang.org>

src/runtime/mgc.go

@@ -895,8 +895,7 @@ func gc(mode int) {
 	// reclaimed until the next GC cycle.
 	clearpools()
 
-	work.bytesMarked = 0
-	work.initialHeapLive = memstats.heap_live
+	gcResetMarkState()
 
 	if mode == gcBackgroundMode { // Do as much work concurrently as possible
 		gcController.startCycle()
@@ -1048,6 +1047,7 @@ func gc(mode int) {
 			// to check that we didn't forget to mark anything during
 			// the concurrent mark process.
 			gcResetGState() // Rescan stacks
+			gcResetMarkState()
 			initCheckmarks()
 			gcMark(startTime)
 			clearCheckmarks()
@@ -1063,6 +1063,7 @@ func gc(mode int) {
 			// they have gcscanvalid==true and gcworkdone==true.
 			// Reset these so that all stacks will be rescanned.
 			gcResetGState()
+			gcResetMarkState()
 			finishsweep_m()
 
 			// Still in STW but gcphase is _GCoff, reset to _GCmarktermination
@@ -1537,6 +1538,14 @@ func gcResetGState() (numgs int) {
 	return
 }
 
+// gcResetMarkState resets state prior to marking (concurrent or STW).
+//
+// TODO(austin): Merge with gcResetGState. See issue #11427.
+func gcResetMarkState() {
+	work.bytesMarked = 0
+	work.initialHeapLive = memstats.heap_live
+}
+
 // Hooks for other packages
 
 var poolcleanup func()