Commit fa8470a8 authored by Michael Anthony Knyszek's avatar Michael Anthony Knyszek Committed by Michael Knyszek

runtime: make treap iteration more efficient

This change introduces a treapIterFilter type which represents the
power set of states described by a treapIterType.

This change then adds a treapIterFilter field to each treap node
indicating the types of spans that live in that subtree. The field is
maintained via the same mechanism used to maintain maxPages. This allows
pred, succ, start, and end to be judicious about which subtrees it will
visit, ensuring that iteration avoids traversing irrelevant territory.

Without this change, repeated scavenging attempts can end up being N^2
as the scavenger walks over what it already scavenged before finding new
spans available for scavenging.

Finally, this change also only scavenges a span once it is removed from
the treap. There was always an invariant that spans owned by the treap
may not be mutated in-place, but with this change violating that
invariant can cause issues with scavenging.

For #30333.

Change-Id: I8040b997e21c94a8d3d9c8c6accfe23cebe0c3d3
Reviewed-on: https://go-review.googlesource.com/c/go/+/174878
Run-TryBot: Michael Knyszek <mknyszek@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: default avatarAustin Clements <austin@google.com>
parent 9baa4301
...@@ -546,15 +546,21 @@ func (s Span) Pages() uintptr { ...@@ -546,15 +546,21 @@ func (s Span) Pages() uintptr {
return s.mspan.npages return s.mspan.npages
} }
type TreapIterType int type TreapIterType treapIterType
const ( const (
TreapIterScav TreapIterType = TreapIterType(treapIterScav) TreapIterScav TreapIterType = TreapIterType(treapIterScav)
TreapIterBits = treapIterBits TreapIterBits = treapIterBits
) )
type TreapIterFilter treapIterFilter
func TreapFilter(mask, match TreapIterType) TreapIterFilter {
return TreapIterFilter(treapFilter(treapIterType(mask), treapIterType(match)))
}
func (s Span) MatchesIter(mask, match TreapIterType) bool { func (s Span) MatchesIter(mask, match TreapIterType) bool {
return s.mspan.matchesIter(treapIterType(mask), treapIterType(match)) return treapFilter(treapIterType(mask), treapIterType(match)).matches(s.treapFilter())
} }
type TreapIter struct { type TreapIter struct {
...@@ -639,5 +645,5 @@ func (t *Treap) Size() int { ...@@ -639,5 +645,5 @@ func (t *Treap) Size() int {
func (t *Treap) CheckInvariants() { func (t *Treap) CheckInvariants() {
t.mTreap.treap.walkTreap(checkTreapNode) t.mTreap.treap.walkTreap(checkTreapNode)
t.mTreap.treap.validateMaxPages() t.mTreap.treap.validateInvariants()
} }
This diff is collapsed.
...@@ -1330,21 +1330,21 @@ func (h *mheap) scavengeLocked(nbytes uintptr) { ...@@ -1330,21 +1330,21 @@ func (h *mheap) scavengeLocked(nbytes uintptr) {
released := uintptr(0) released := uintptr(0)
for t := h.free.end(treapIterScav, 0); released < nbytes && t.valid(); { for t := h.free.end(treapIterScav, 0); released < nbytes && t.valid(); {
s := t.span() s := t.span()
r := s.scavenge() start, end := s.physPageBounds()
if r == 0 { if start >= end {
// This span doesn't cover at least one physical page, so skip it. // This span doesn't cover at least one physical page, so skip it.
t = t.prev() t = t.prev()
continue continue
} }
n := t.prev() n := t.prev()
h.free.erase(t) h.free.erase(t)
released += s.scavenge()
// Now that s is scavenged, we must eagerly coalesce it // Now that s is scavenged, we must eagerly coalesce it
// with its neighbors to prevent having two spans with // with its neighbors to prevent having two spans with
// the same scavenged state adjacent to each other. // the same scavenged state adjacent to each other.
h.coalesce(s) h.coalesce(s)
t = n t = n
h.free.insert(s) h.free.insert(s)
released += r
} }
// If we over-scavenged, turn that extra amount into credit. // If we over-scavenged, turn that extra amount into credit.
if released > nbytes { if released > nbytes {
...@@ -1363,13 +1363,13 @@ func (h *mheap) scavengeAllLocked(now, limit uint64) uintptr { ...@@ -1363,13 +1363,13 @@ func (h *mheap) scavengeAllLocked(now, limit uint64) uintptr {
s := t.span() s := t.span()
n := t.next() n := t.next()
if (now - uint64(s.unusedsince)) > limit { if (now - uint64(s.unusedsince)) > limit {
r := s.scavenge() start, end := s.physPageBounds()
if r != 0 { if start < end {
h.free.erase(t) h.free.erase(t)
// See (*mheap).scavenge. released += s.scavenge()
// See (*mheap).scavengeLocked.
h.coalesce(s) h.coalesce(s)
h.free.insert(s) h.free.insert(s)
released += r
} }
} }
t = n t = n
......
...@@ -36,6 +36,25 @@ func maskMatchName(mask, match runtime.TreapIterType) string { ...@@ -36,6 +36,25 @@ func maskMatchName(mask, match runtime.TreapIterType) string {
return fmt.Sprintf("%0*b-%0*b", runtime.TreapIterBits, uint8(mask), runtime.TreapIterBits, uint8(match)) return fmt.Sprintf("%0*b-%0*b", runtime.TreapIterBits, uint8(mask), runtime.TreapIterBits, uint8(match))
} }
func TestTreapFilter(t *testing.T) {
var iterTypes = [...]struct {
mask, match runtime.TreapIterType
filter runtime.TreapIterFilter // expected filter
}{
{0, 0, 0x3},
{runtime.TreapIterScav, 0, 0x1},
{runtime.TreapIterScav, runtime.TreapIterScav, 0x2},
{0, runtime.TreapIterScav, 0x0},
}
for _, it := range iterTypes {
t.Run(maskMatchName(it.mask, it.match), func(t *testing.T) {
if f := runtime.TreapFilter(it.mask, it.match); f != it.filter {
t.Fatalf("got %#x, want %#x", f, it.filter)
}
})
}
}
// This test ensures that the treap implementation in the runtime // This test ensures that the treap implementation in the runtime
// maintains all stated invariants after different sequences of // maintains all stated invariants after different sequences of
// insert, removeSpan, find, and erase. Invariants specific to the // insert, removeSpan, find, and erase. Invariants specific to the
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment