runtime: decentralize concurrent sweep termination

This moves concurrent sweep termination from the coordinator to the
off->mark transition. This allows it to be performed by all Gs
attempting to start the GC.

Updates #11970.

Change-Id: I24428e8599a759398c2ef7ec996ba755a448f947
Reviewed-on: https://go-review.googlesource.com/16356Reviewed-by: Rick Hudson <rlh@golang.org>
Run-TryBot: Austin Clements <austin@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>

src/runtime/mgc.go

@@ -999,6 +999,20 @@ func gcStart(mode gcMode, forceTrigger bool) {
 	releasem(mp)
 	mp = nil
 
+	// Pick up the remaining unswept/not being swept spans concurrently
+	//
+	// This shouldn't happen if we're being invoked in background
+	// mode since proportional sweep should have just finished
+	// sweeping everything, but rounding errors, etc, may leave a
+	// few spans unswept. In forced mode, this is necessary since
+	// GC can be forced at any point in the sweeping cycle.
+	//
+	// We check the transition condition continuously here in case
+	// this G gets delayed in to the next GC cycle.
+	for (mode != gcBackgroundMode || gcShouldStart(forceTrigger)) && gosweepone() != ^uintptr(0) {
+		sweep.nbgsweep++
+	}
+
 	// Perform GC initialization and the sweep termination
 	// transition.
 	//
@@ -1041,17 +1055,6 @@ func gc(mode gcMode) {
 	// Ok, we're doing it!  Stop everybody else
 	semacquire(&worldsema, false)
 
-	// Pick up the remaining unswept/not being swept spans concurrently
-	//
-	// This shouldn't happen if we're being invoked in background
-	// mode since proportional sweep should have just finished
-	// sweeping everything, but rounding errors, etc, may leave a
-	// few spans unswept. In forced mode, this is necessary since
-	// GC can be forced at any point in the sweeping cycle.
-	for gosweepone() != ^uintptr(0) {
-		sweep.nbgsweep++
-	}
-
 	if trace.enabled {
 		traceGCStart()
 	}