runtime: fix memory allocator on Plan 9

Previously, the memory allocator on Plan 9 did
not free memory properly. It was only able to
free the last allocated block.

This change implements a variant of the
Kernighan & Ritchie memory allocator with
coalescing and splitting.

The most notable differences are:

- no header is prefixing the allocated blocks, since
  the size is always specified when calling sysFree,
- the free list is nil-terminated instead of circular.

Fixes #9736.
Fixes #9803.
Fixes #9952.

Change-Id: I00d533714e4144a0012f69820d31cbb0253031a3
Reviewed-on: https://go-review.googlesource.com/5524Reviewed-by: Dmitry Vyukov <dvyukov@google.com>

src/runtime/mem_plan9.go

@@ -6,9 +6,111 @@ package runtime
 
 import "unsafe"
 
+const memDebug = false
+
 var bloc uintptr
 var memlock mutex
 
+type memHdr struct {
+	next *memHdr
+	size uintptr
+}
+
+var memFreelist *memHdr // sorted in ascending order
+
+func memAlloc(n uintptr) unsafe.Pointer {
+	n = memRound(n)
+	var prevp *memHdr
+	for p := memFreelist; p != nil; p = p.next {
+		if p.size >= n {
+			if p.size == n {
+				if prevp != nil {
+					prevp.next = p.next
+				} else {
+					memFreelist = p.next
+				}
+			} else {
+				p.size -= n
+				p = (*memHdr)(add(unsafe.Pointer(p), p.size))
+			}
+			memclr(unsafe.Pointer(p), unsafe.Sizeof(memHdr{}))
+			return unsafe.Pointer(p)
+		}
+		prevp = p
+	}
+	return sbrk(n)
+}
+
+func memFree(ap unsafe.Pointer, n uintptr) {
+	n = memRound(n)
+	memclr(ap, n)
+	bp := (*memHdr)(ap)
+	bp.size = n
+	bpn := uintptr(ap)
+	if memFreelist == nil {
+		bp.next = nil
+		memFreelist = bp
+		return
+	}
+	p := memFreelist
+	if bpn < uintptr(unsafe.Pointer(p)) {
+		memFreelist = bp
+		if bpn+bp.size == uintptr(unsafe.Pointer(p)) {
+			bp.size += p.size
+			bp.next = p.next
+			memclr(unsafe.Pointer(p), unsafe.Sizeof(memHdr{}))
+		} else {
+			bp.next = p
+		}
+		return
+	}
+	for ; p.next != nil; p = p.next {
+		if bpn > uintptr(unsafe.Pointer(p)) && bpn < uintptr(unsafe.Pointer(p.next)) {
+			break
+		}
+	}
+	if bpn+bp.size == uintptr(unsafe.Pointer(p.next)) {
+		bp.size += p.next.size
+		bp.next = p.next.next
+		memclr(unsafe.Pointer(p.next), unsafe.Sizeof(memHdr{}))
+	} else {
+		bp.next = p.next
+	}
+	if uintptr(unsafe.Pointer(p))+p.size == bpn {
+		p.size += bp.size
+		p.next = bp.next
+		memclr(unsafe.Pointer(bp), unsafe.Sizeof(memHdr{}))
+	} else {
+		p.next = bp
+	}
+}
+
+func memCheck() {
+	if memDebug == false {
+		return
+	}
+	for p := memFreelist; p != nil && p.next != nil; p = p.next {
+		if uintptr(unsafe.Pointer(p)) == uintptr(unsafe.Pointer(p.next)) {
+			print("runtime: ", unsafe.Pointer(p), " == ", unsafe.Pointer(p.next), "\n")
+			throw("mem: infinite loop")
+		}
+		if uintptr(unsafe.Pointer(p)) > uintptr(unsafe.Pointer(p.next)) {
+			print("runtime: ", unsafe.Pointer(p), " > ", unsafe.Pointer(p.next), "\n")
+			throw("mem: unordered list")
+		}
+		if uintptr(unsafe.Pointer(p))+p.size > uintptr(unsafe.Pointer(p.next)) {
+			print("runtime: ", unsafe.Pointer(p), "+", p.size, " > ", unsafe.Pointer(p.next), "\n")
+			throw("mem: overlapping blocks")
+		}
+		for b := add(unsafe.Pointer(p), unsafe.Sizeof(memHdr{})); uintptr(b) < uintptr(unsafe.Pointer(p))+p.size; b = add(b, 1) {
+			if *(*byte)(b) != 0 {
+				print("runtime: value at addr ", b, " with offset ", uintptr(b)-uintptr(unsafe.Pointer(p)), " in block ", p, " of size ", p.size, " is not zero\n")
+				throw("mem: uninitialised memory")
+			}
+		}
+	}
+}
+
 func memRound(p uintptr) uintptr {
 	return (p + _PAGESIZE - 1) &^ (_PAGESIZE - 1)
 }
@@ -18,21 +120,21 @@ func initBloc() {
 }
 
 func sbrk(n uintptr) unsafe.Pointer {
-	lock(&memlock)
 	// Plan 9 sbrk from /sys/src/libc/9sys/sbrk.c
 	bl := bloc
 	n = memRound(n)
 	if brk_(unsafe.Pointer(bl+n)) < 0 {
-		unlock(&memlock)
 		return nil
 	}
 	bloc += n
-	unlock(&memlock)
 	return unsafe.Pointer(bl)
 }
 
 func sysAlloc(n uintptr, stat *uint64) unsafe.Pointer {
-	p := sbrk(n)
+	lock(&memlock)
+	p := memAlloc(n)
+	memCheck()
+	unlock(&memlock)
 	if p != nil {
 		xadd64(stat, int64(n))
 	}
@@ -42,14 +144,8 @@ func sysAlloc(n uintptr, stat *uint64) unsafe.Pointer {
 func sysFree(v unsafe.Pointer, n uintptr, stat *uint64) {
 	xadd64(stat, -int64(n))
 	lock(&memlock)
-	// from tiny/mem.c
-	// Push pointer back if this is a free
-	// of the most recent sysAlloc.
-	n = memRound(n)
-	if bloc == uintptr(v)+n {
-		bloc -= n
-		memclr(unsafe.Pointer(bloc), n)
-	}
+	memFree(v, n)
+	memCheck()
 	unlock(&memlock)
 }
 
@@ -70,5 +166,9 @@ func sysFault(v unsafe.Pointer, n uintptr) {
 
 func sysReserve(v unsafe.Pointer, n uintptr, reserved *bool) unsafe.Pointer {
 	*reserved = true
-	return sbrk(n)
+	lock(&memlock)
+	p := memAlloc(n)
+	memCheck()
+	unlock(&memlock)
+	return p
 }