runtime: make treap iteration more efficient

This change introduces a treapIterFilter type which represents the
power set of states described by a treapIterType.

This change then adds a treapIterFilter field to each treap node
indicating the types of spans that live in that subtree. The field is
maintained via the same mechanism used to maintain maxPages. This allows
pred, succ, start, and end to be judicious about which subtrees it will
visit, ensuring that iteration avoids traversing irrelevant territory.

Without this change, repeated scavenging attempts can end up being N^2
as the scavenger walks over what it already scavenged before finding new
spans available for scavenging.

Finally, this change also only scavenges a span once it is removed from
the treap. There was always an invariant that spans owned by the treap
may not be mutated in-place, but with this change violating that
invariant can cause issues with scavenging.

For #30333.

Change-Id: I8040b997e21c94a8d3d9c8c6accfe23cebe0c3d3
Reviewed-on: https://go-review.googlesource.com/c/go/+/174878
Run-TryBot: Michael Knyszek <mknyszek@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Austin Clements <austin@google.com>

src/runtime/export_test.go

@@ -546,15 +546,21 @@ func (s Span) Pages() uintptr {
 	return s.mspan.npages
 }
 
-type TreapIterType int
+type TreapIterType treapIterType
 
 const (
 	TreapIterScav TreapIterType = TreapIterType(treapIterScav)
 	TreapIterBits               = treapIterBits
 )
 
+type TreapIterFilter treapIterFilter
+
+func TreapFilter(mask, match TreapIterType) TreapIterFilter {
+	return TreapIterFilter(treapFilter(treapIterType(mask), treapIterType(match)))
+}
+
 func (s Span) MatchesIter(mask, match TreapIterType) bool {
-	return s.mspan.matchesIter(treapIterType(mask), treapIterType(match))
+	return treapFilter(treapIterType(mask), treapIterType(match)).matches(s.treapFilter())
 }
 
 type TreapIter struct {
@@ -639,5 +645,5 @@ func (t *Treap) Size() int {
 
 func (t *Treap) CheckInvariants() {
 	t.mTreap.treap.walkTreap(checkTreapNode)
-	t.mTreap.treap.validateMaxPages()
+	t.mTreap.treap.validateInvariants()
 }

src/runtime/mheap.go

@@ -1330,21 +1330,21 @@ func (h *mheap) scavengeLocked(nbytes uintptr) {
 	released := uintptr(0)
 	for t := h.free.end(treapIterScav, 0); released < nbytes && t.valid(); {
 		s := t.span()
-		r := s.scavenge()
-		if r == 0 {
+		start, end := s.physPageBounds()
+		if start >= end {
 			// This span doesn't cover at least one physical page, so skip it.
 			t = t.prev()
 			continue
 		}
 		n := t.prev()
 		h.free.erase(t)
+		released += s.scavenge()
 		// Now that s is scavenged, we must eagerly coalesce it
 		// with its neighbors to prevent having two spans with
 		// the same scavenged state adjacent to each other.
 		h.coalesce(s)
 		t = n
 		h.free.insert(s)
-		released += r
 	}
 	// If we over-scavenged, turn that extra amount into credit.
 	if released > nbytes {
@@ -1363,13 +1363,13 @@ func (h *mheap) scavengeAllLocked(now, limit uint64) uintptr {
 		s := t.span()
 		n := t.next()
 		if (now - uint64(s.unusedsince)) > limit {
-			r := s.scavenge()
-			if r != 0 {
+			start, end := s.physPageBounds()
+			if start < end {
 				h.free.erase(t)
-				// See (*mheap).scavenge.
+				released += s.scavenge()
+				// See (*mheap).scavengeLocked.
 				h.coalesce(s)
 				h.free.insert(s)
-				released += r
 			}
 		}
 		t = n

src/runtime/treap_test.go

@@ -36,6 +36,25 @@ func maskMatchName(mask, match runtime.TreapIterType) string {
 	return fmt.Sprintf("%0*b-%0*b", runtime.TreapIterBits, uint8(mask), runtime.TreapIterBits, uint8(match))
 }
 
+func TestTreapFilter(t *testing.T) {
+	var iterTypes = [...]struct {
+		mask, match runtime.TreapIterType
+		filter      runtime.TreapIterFilter // expected filter
+	}{
+		{0, 0, 0x3},
+		{runtime.TreapIterScav, 0, 0x1},
+		{runtime.TreapIterScav, runtime.TreapIterScav, 0x2},
+		{0, runtime.TreapIterScav, 0x0},
+	}
+	for _, it := range iterTypes {
+		t.Run(maskMatchName(it.mask, it.match), func(t *testing.T) {
+			if f := runtime.TreapFilter(it.mask, it.match); f != it.filter {
+				t.Fatalf("got %#x, want %#x", f, it.filter)
+			}
+		})
+	}
+}
+
 // This test ensures that the treap implementation in the runtime
 // maintains all stated invariants after different sequences of
 // insert, removeSpan, find, and erase. Invariants specific to the