Commit a62b5723 authored by Michael Anthony Knyszek's avatar Michael Anthony Knyszek Committed by Michael Knyszek

runtime: scavenge huge spans first

This change adds two new treap iteration types: one for large
unscavenged spans (contain at least one huge page) and one for small
unscavenged spans. This allows us to scavenge the huge spans first by
first iterating over the large ones, then the small ones.

Also, since we now depend on physHugePageSize being a power of two,
ensure that that's the case when it's retrieved from the OS.

For #30333.

Change-Id: I51662740205ad5e4905404a0856f5f2b2d2a5680
Reviewed-on: https://go-review.googlesource.com/c/go/+/174399
Run-TryBot: Michael Knyszek <mknyszek@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: default avatarAustin Clements <austin@google.com>
parent fa8470a8
......@@ -550,6 +550,7 @@ type TreapIterType treapIterType
const (
TreapIterScav TreapIterType = TreapIterType(treapIterScav)
TreapIterHuge = TreapIterType(treapIterHuge)
TreapIterBits = treapIterBits
)
......
......@@ -273,7 +273,8 @@ type treapIterType uint8
const (
treapIterScav treapIterType = 1 << iota // scavenged spans
treapIterBits = iota
treapIterHuge // spans containing at least one huge page
treapIterBits = iota
)
// treapIterFilter is a bitwise filter of different spans by binary
......@@ -318,6 +319,9 @@ func (s *mspan) treapFilter() treapIterFilter {
if s.scavenged {
have |= treapIterScav
}
if s.hugePages() > 0 {
have |= treapIterHuge
}
return treapIterFilter(uint32(1) << (0x1f & have))
}
......
......@@ -521,6 +521,25 @@ func (h *mheap) coalesce(s *mspan) {
}
}
// hugePages returns the number of aligned physical huge pages in the memory
// regioned owned by this mspan.
func (s *mspan) hugePages() uintptr {
if physHugePageSize == 0 || s.npages < physHugePageSize/pageSize {
return 0
}
start := s.base()
end := start + s.npages*pageSize
if physHugePageSize > pageSize {
// Round start and end in.
start = (start + physHugePageSize - 1) &^ (physHugePageSize - 1)
end &^= physHugePageSize - 1
}
if start < end {
return (end - start) / physHugePageSize
}
return 0
}
func (s *mspan) scavenge() uintptr {
// start and end must be rounded in, otherwise madvise
// will round them *out* and release more memory
......@@ -1324,27 +1343,30 @@ func (h *mheap) scavengeLocked(nbytes uintptr) {
h.scavengeCredit -= nbytes
return
}
// Iterate over the unscavenged spans in the treap backwards (from highest
// address to lowest address) scavenging spans until we've reached our
// quota of nbytes.
released := uintptr(0)
for t := h.free.end(treapIterScav, 0); released < nbytes && t.valid(); {
s := t.span()
start, end := s.physPageBounds()
if start >= end {
// This span doesn't cover at least one physical page, so skip it.
t = t.prev()
continue
// Iterate over spans with huge pages first, then spans without.
const mask = treapIterScav | treapIterHuge
for _, match := range []treapIterType{treapIterHuge, 0} {
// Iterate over the treap backwards (from highest address to lowest address)
// scavenging spans until we've reached our quota of nbytes.
for t := h.free.end(mask, match); released < nbytes && t.valid(); {
s := t.span()
start, end := s.physPageBounds()
if start >= end {
// This span doesn't cover at least one physical page, so skip it.
t = t.prev()
continue
}
n := t.prev()
h.free.erase(t)
released += s.scavenge()
// Now that s is scavenged, we must eagerly coalesce it
// with its neighbors to prevent having two spans with
// the same scavenged state adjacent to each other.
h.coalesce(s)
t = n
h.free.insert(s)
}
n := t.prev()
h.free.erase(t)
released += s.scavenge()
// Now that s is scavenged, we must eagerly coalesce it
// with its neighbors to prevent having two spans with
// the same scavenged state adjacent to each other.
h.coalesce(s)
t = n
h.free.insert(s)
}
// If we over-scavenged, turn that extra amount into credit.
if released > nbytes {
......
......@@ -270,8 +270,8 @@ func getHugePageSize() uintptr {
return 0
}
n := read(fd, noescape(unsafe.Pointer(&numbuf[0])), int32(len(numbuf)))
closefd(fd)
if n <= 0 {
closefd(fd)
return 0
}
l := n - 1 // remove trailing newline
......@@ -279,7 +279,10 @@ func getHugePageSize() uintptr {
if !ok || v < 0 {
v = 0
}
closefd(fd)
if v&(v-1) != 0 {
// v is not a power of 2
return 0
}
return uintptr(v)
}
......
......@@ -41,9 +41,11 @@ func TestTreapFilter(t *testing.T) {
mask, match runtime.TreapIterType
filter runtime.TreapIterFilter // expected filter
}{
{0, 0, 0x3},
{runtime.TreapIterScav, 0, 0x1},
{runtime.TreapIterScav, runtime.TreapIterScav, 0x2},
{0, 0, 0xf},
{runtime.TreapIterScav, 0, 0x5},
{runtime.TreapIterScav, runtime.TreapIterScav, 0xa},
{runtime.TreapIterScav | runtime.TreapIterHuge, runtime.TreapIterHuge, 0x4},
{runtime.TreapIterScav | runtime.TreapIterHuge, 0, 0x1},
{0, runtime.TreapIterScav, 0x0},
}
for _, it := range iterTypes {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment