runtime: add scavenging code for new page allocator

This change adds a scavenger for the new page allocator along with
tests. The scavenger walks over the heap backwards once per GC, looking
for memory to scavenge. It walks across the heap without any lock held,
searching optimistically. If it finds what appears to be a scavenging
candidate it acquires the heap lock and attempts to verify it. Upon
verification it then scavenges.

Notably, unlike the old scavenger, it doesn't show any preference for
huge pages and instead follows a more strict last-page-first policy.

Updates #35112.

Change-Id: I0621ef73c999a471843eab2d1307ae5679dd18d6
Reviewed-on: https://go-review.googlesource.com/c/go/+/195697Reviewed-by: Keith Randall <khr@golang.org>
Reviewed-by: Austin Clements <austin@google.com>

src/runtime/export_test.go

@@ -38,7 +38,7 @@ var Nanotime = nanotime
 var NetpollBreak = netpollBreak
 var Usleep = usleep
 
-var PageSize = pageSize
+var PhysPageSize = physPageSize
 var PhysHugePageSize = physHugePageSize
 
 var NetpollGenericInit = netpollGenericInit
@@ -733,6 +733,7 @@ func RunGetgThreadSwitchTest() {
 }
 
 const (
+	PageSize         = pageSize
 	PallocChunkPages = pallocChunkPages
 )
 
@@ -825,6 +826,26 @@ func StringifyPallocBits(b *PallocBits, r BitRange) string {
 	return str
 }
 
+// Expose pallocData for testing.
+type PallocData pallocData
+
+func (d *PallocData) FindScavengeCandidate(searchIdx uint, min, max uintptr) (uint, uint) {
+	return (*pallocData)(d).findScavengeCandidate(searchIdx, min, max)
+}
+func (d *PallocData) AllocRange(i, n uint) { (*pallocData)(d).allocRange(i, n) }
+func (d *PallocData) ScavengedSetRange(i, n uint) {
+	(*pallocData)(d).scavenged.setRange(i, n)
+}
+func (d *PallocData) PallocBits() *PallocBits {
+	return (*PallocBits)(&(*pallocData)(d).pallocBits)
+}
+func (d *PallocData) Scavenged() *PallocBits {
+	return (*PallocBits)(&(*pallocData)(d).scavenged)
+}
+
+// Expose fillAligned for testing.
+func FillAligned(x uint64, m uint) uint64 { return fillAligned(x, m) }
+
 // Expose chunk index type.
 type ChunkIdx chunkIdx
 
@@ -837,8 +858,14 @@ func (p *PageAlloc) Free(base, npages uintptr)    { (*pageAlloc)(p).free(base, n
 func (p *PageAlloc) Bounds() (ChunkIdx, ChunkIdx) {
 	return ChunkIdx((*pageAlloc)(p).start), ChunkIdx((*pageAlloc)(p).end)
 }
-func (p *PageAlloc) PallocBits(i ChunkIdx) *PallocBits {
-	return (*PallocBits)(&((*pageAlloc)(p).chunks[i]))
+func (p *PageAlloc) PallocData(i ChunkIdx) *PallocData {
+	return (*PallocData)(&((*pageAlloc)(p).chunks[i]))
+}
+func (p *PageAlloc) Scavenge(nbytes uintptr) (r uintptr) {
+	systemstack(func() {
+		r = (*pageAlloc)(p).scavenge(nbytes)
+	})
+	return
 }
 
 // BitRange represents a range over a bitmap.
@@ -847,14 +874,25 @@ type BitRange struct {
 }
 
 // NewPageAlloc creates a new page allocator for testing and
-// initializes it with the chunks map. Each key represents a chunk
-// index and each value is a series of bit ranges to set within that
-// chunk.
-func NewPageAlloc(chunks map[ChunkIdx][]BitRange) *PageAlloc {
+// initializes it with the scav and chunks maps. Each key in these maps
+// represents a chunk index and each value is a series of bit ranges to
+// set within each bitmap's chunk.
+//
+// The initialization of the pageAlloc preserves the invariant that if a
+// scavenged bit is set the alloc bit is necessarily unset, so some
+// of the bits described by scav may be cleared in the final bitmap if
+// ranges in chunks overlap with them.
+//
+// scav is optional, and if nil, the scavenged bitmap will be cleared
+// (as opposed to all 1s, which it usually is). Furthermore, every
+// chunk index in scav must appear in chunks; ones that do not are
+// ignored.
+func NewPageAlloc(chunks, scav map[ChunkIdx][]BitRange) *PageAlloc {
 	p := new(pageAlloc)
 
 	// We've got an entry, so initialize the pageAlloc.
 	p.init(new(mutex), nil)
+	p.test = true
 
 	for i, init := range chunks {
 		addr := chunkBase(chunkIdx(i))
@@ -864,6 +902,25 @@ func NewPageAlloc(chunks map[ChunkIdx][]BitRange) *PageAlloc {
 
 		// Initialize the bitmap and update pageAlloc metadata.
 		chunk := &p.chunks[chunkIndex(addr)]
+
+		// Clear all the scavenged bits which grow set.
+		chunk.scavenged.clearRange(0, pallocChunkPages)
+
+		// Apply scavenge state if applicable.
+		if scav != nil {
+			if scvg, ok := scav[i]; ok {
+				for _, s := range scvg {
+					// Ignore the case of s.N == 0. setRange doesn't handle
+					// it and it's a no-op anyway.
+					if s.N != 0 {
+						chunk.scavenged.setRange(s.I, s.N)
+					}
+				}
+			}
+		}
+		p.resetScavengeAddr()
+
+		// Apply alloc state.
 		for _, s := range init {
 			// Ignore the case of s.N == 0. allocRange doesn't handle
 			// it and it's a no-op anyway.

src/runtime/malloc.go

@@ -308,7 +308,7 @@ const (
 	//
 	// On other platforms, the user address space is contiguous
 	// and starts at 0, so no offset is necessary.
-	arenaBaseOffset uintptr = sys.GoarchAmd64 * (1 << 47)
+	arenaBaseOffset = sys.GoarchAmd64 * (1 << 47)
 
 	// Max number of threads to run garbage collection.
 	// 2, 3, and 4 are all plausible maximums depending

src/runtime/mpagealloc.go

@@ -80,6 +80,11 @@ const (
 	// value in the shifted address space, but searchAddr is stored as a regular
 	// memory address. See arenaBaseOffset for details.
 	maxSearchAddr = ^uintptr(0) - arenaBaseOffset
+
+	// Minimum scavAddr value, which indicates that the scavenger is done.
+	//
+	// minScavAddr + arenaBaseOffset == 0
+	minScavAddr = (^arenaBaseOffset + 1) & uintptrMask
 )
 
 // Global chunk index.
@@ -171,7 +176,7 @@ type pageAlloc struct {
 	// TODO(mknyszek): Consider changing the definition of the bitmap
 	// such that 1 means free and 0 means in-use so that summaries and
 	// the bitmaps align better on zero-values.
-	chunks []pallocBits
+	chunks []pallocData
 
 	// The address to start an allocation search with.
 	//
@@ -185,6 +190,9 @@ type pageAlloc struct {
 	// space on architectures with segmented address spaces.
 	searchAddr uintptr
 
+	// The address to start a scavenge candidate search with.
+	scavAddr uintptr
+
 	// start and end represent the chunk indices
 	// which pageAlloc knows about. It assumes
 	// chunks in the range [start, end) are
@@ -198,6 +206,9 @@ type pageAlloc struct {
 	// sysStat is the runtime memstat to update when new system
 	// memory is committed by the pageAlloc for allocation metadata.
 	sysStat *uint64
+
+	// Whether or not this struct is being used in tests.
+	test bool
 }
 
 func (s *pageAlloc) init(mheapLock *mutex, sysStat *uint64) {
@@ -217,6 +228,9 @@ func (s *pageAlloc) init(mheapLock *mutex, sysStat *uint64) {
 	// Start with the searchAddr in a state indicating there's no free memory.
 	s.searchAddr = maxSearchAddr
 
+	// Start with the scavAddr in a state indicating there's nothing more to do.
+	s.scavAddr = minScavAddr
+
 	// Reserve space for the bitmap and put this reservation
 	// into the chunks slice.
 	const maxChunks = (1 << heapAddrBits) / pallocChunkBytes
@@ -225,7 +239,7 @@ func (s *pageAlloc) init(mheapLock *mutex, sysStat *uint64) {
 		throw("failed to reserve page bitmap memory")
 	}
 	sl := notInHeapSlice{(*notInHeap)(r), 0, maxChunks}
-	s.chunks = *(*[]pallocBits)(unsafe.Pointer(&sl))
+	s.chunks = *(*[]pallocData)(unsafe.Pointer(&sl))
 
 	// Set the mheapLock.
 	s.mheapLock = mheapLock
@@ -350,6 +364,13 @@ func (s *pageAlloc) grow(base, size uintptr) {
 		s.searchAddr = base
 	}
 
+	// Newly-grown memory is always considered scavenged.
+	//
+	// Set all the bits in the scavenged bitmaps high.
+	for c := chunkIndex(base); c < chunkIndex(limit); c++ {
+		s.chunks[c].scavenged.setRange(0, pallocChunkPages)
+	}
+
 	// Update summaries accordingly. The grow acts like a free, so
 	// we need to ensure this newly-free memory is visible in the
 	// summaries.

src/runtime/mpagealloc_test.go

@@ -23,8 +23,12 @@ func checkPageAlloc(t *testing.T, want, got *PageAlloc) {
 
 	for i := gotStart; i < gotEnd; i++ {
 		// Check the bitmaps.
-		if !checkPallocBits(t, got.PallocBits(i), want.PallocBits(i)) {
-			t.Logf("in chunk %d", i)
+		gb, wb := got.PallocData(i), want.PallocData(i)
+		if !checkPallocBits(t, gb.PallocBits(), wb.PallocBits()) {
+			t.Logf("in chunk %d (mallocBits)", i)
+		}
+		if !checkPallocBits(t, gb.Scavenged(), wb.Scavenged()) {
+			t.Logf("in chunk %d (scavenged)", i)
 		}
 	}
 	// TODO(mknyszek): Verify summaries too?
@@ -310,7 +314,7 @@ func TestPageAllocAlloc(t *testing.T) {
 	for name, v := range tests {
 		v := v
 		t.Run(name, func(t *testing.T) {
-			b := NewPageAlloc(v.before)
+			b := NewPageAlloc(v.before, nil)
 			defer FreePageAlloc(b)
 
 			for iter, i := range v.hits {
@@ -318,7 +322,7 @@ func TestPageAllocAlloc(t *testing.T) {
 					t.Fatalf("bad alloc #%d: want 0x%x, got 0x%x", iter+1, i.base, a)
 				}
 			}
-			want := NewPageAlloc(v.after)
+			want := NewPageAlloc(v.after, nil)
 			defer FreePageAlloc(want)
 
 			checkPageAlloc(t, want, b)
@@ -335,7 +339,7 @@ func TestPageAllocExhaust(t *testing.T) {
 			for i := ChunkIdx(0); i < 4; i++ {
 				bDesc[BaseChunkIdx+i] = []BitRange{}
 			}
-			b := NewPageAlloc(bDesc)
+			b := NewPageAlloc(bDesc, nil)
 			defer FreePageAlloc(b)
 
 			// Allocate into b with npages until we've exhausted the heap.
@@ -366,7 +370,7 @@ func TestPageAllocExhaust(t *testing.T) {
 					wantDesc[BaseChunkIdx+i] = []BitRange{}
 				}
 			}
-			want := NewPageAlloc(wantDesc)
+			want := NewPageAlloc(wantDesc, nil)
 			defer FreePageAlloc(want)
 
 			// Check to make sure the heap b matches what we want.
@@ -590,14 +594,15 @@ func TestPageAllocFree(t *testing.T) {
 	for name, v := range tests {
 		v := v
 		t.Run(name, func(t *testing.T) {
-			b := NewPageAlloc(v.before)
+			b := NewPageAlloc(v.before, nil)
 			defer FreePageAlloc(b)
+
 			for _, addr := range v.frees {
 				b.Free(addr, v.npages)
 			}
-
-			want := NewPageAlloc(v.after)
+			want := NewPageAlloc(v.after, nil)
 			defer FreePageAlloc(want)
+
 			checkPageAlloc(t, want, b)
 		})
 	}
@@ -641,7 +646,7 @@ func TestPageAllocAllocAndFree(t *testing.T) {
 	for name, v := range tests {
 		v := v
 		t.Run(name, func(t *testing.T) {
-			b := NewPageAlloc(v.init)
+			b := NewPageAlloc(v.init, nil)
 			defer FreePageAlloc(b)
 
 			for iter, i := range v.hits {

src/runtime/mpallocbits.go

@@ -131,7 +131,7 @@ var consec8tab = [256]uint{
 	4, 3, 2, 2, 2, 1, 1, 1, 3, 2, 1, 1, 2, 1, 1, 0,
 }
 
-// summarize returns a packed summary of the bitmap in mallocBits.
+// summarize returns a packed summary of the bitmap in pallocBits.
 func (b *pallocBits) summarize() pallocSum {
 	// TODO(mknyszek): There may be something more clever to be done
 	// here to make the summarize operation more efficient. For example,
@@ -332,3 +332,28 @@ func findBitRange64(c uint64, n uint) uint {
 	}
 	return i
 }
+
+// pallocData encapsulates pallocBits and a bitmap for
+// whether or not a given page is scavenged in a single
+// structure. It's effectively a pallocBits with
+// additional functionality.
+type pallocData struct {
+	pallocBits
+	scavenged pageBits
+}
+
+// allocRange sets bits [i, i+n) in the bitmap to 1 and
+// updates the scavenged bits appropriately.
+func (m *pallocData) allocRange(i, n uint) {
+	// Clear the scavenged bits when we alloc the range.
+	m.pallocBits.allocRange(i, n)
+	m.scavenged.clearRange(i, n)
+}
+
+// allocAll sets every bit in the bitmap to 1 and updates
+// the scavenged bits appropriately.
+func (m *pallocData) allocAll() {
+	// Clear the scavenged bits when we alloc the range.
+	m.pallocBits.allocAll()
+	m.scavenged.clearAll()
+}