runtime: inform arena placement using sbrk(0)

On 32-bit architectures (or if we fail to map a 64-bit-style arena),
we try to map the heap arena just above the end of the process image.
While we can accept any address, using lower addresses is preferable
because lower addresses cause us to map less of the heap bitmap.

However, if a program is linked against C code that has global
constructors, those constructors may call brk/sbrk to allocate memory
(e.g., many C malloc implementations do this for small allocations).
The brk also starts just above the process image, so this may adjust
the brk past the beginning of where we want to put the heap arena. In
this case, the kernel will pick a different address for the arena and
it will usually be very high (at least, as these things go in a 32-bit
address space).

Fix this by consulting the current value of the brk and using this in
addition to the end of the process image to compute the initial arena
placement.

This is implemented only on Linux currently, since we have no evidence
that it's an issue on any other OSes.

Fixes #19831.

Change-Id: Id64b45d08d8c91e4f50d92d0339146250b04f2f8
Reviewed-on: https://go-review.googlesource.com/39810
Run-TryBot: Austin Clements <austin@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Ian Lance Taylor <iant@golang.org>

src/runtime/malloc.go

@@ -312,6 +312,15 @@ func mallocinit() {
 		// When that gets used up, we'll start asking the kernel
 		// for any memory anywhere.
 
+		// We want to start the arena low, but if we're linked
+		// against C code, it's possible global constructors
+		// have called malloc and adjusted the process' brk.
+		// Query the brk so we can avoid trying to map the
+		// arena over it (which will cause the kernel to put
+		// the arena somewhere else, likely at a high
+		// address).
+		procBrk := sbrk0()
+
 		// If we fail to allocate, try again with a smaller arena.
 		// This is necessary on Android L where we share a process
 		// with ART, which reserves virtual memory aggressively.
@@ -336,6 +345,12 @@ func mallocinit() {
 			// to a MB boundary.
 			p = round(firstmoduledata.end+(1<<18), 1<<20)
 			pSize = bitmapSize + spansSize + arenaSize + _PageSize
+			if p <= procBrk && procBrk < p+pSize {
+				// Move the start above the brk,
+				// leaving some room for future brk
+				// expansion.
+				p = round(procBrk+(1<<20), 1<<20)
+			}
 			p = uintptr(sysReserve(unsafe.Pointer(p), pSize, &reserved))
 			if p != 0 {
 				break

src/runtime/stubs_linux.go

+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build linux
+
+package runtime
+
+func sbrk0() uintptr

src/runtime/stubs_nonlinux.go

+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !linux
+
+package runtime
+
+// sbrk0 returns the current process brk, or 0 if not implemented.
+func sbrk0() uintptr {
+	return 0
+}

src/runtime/sys_linux_386.s

@@ -596,3 +596,12 @@ TEXT runtime·socket(SB),NOSPLIT,$0-16
 	INVOKE_SYSCALL
 	MOVL	AX, ret+12(FP)
 	RET
+
+// func sbrk0() uintptr
+TEXT runtime·sbrk0(SB),NOSPLIT,$0-4
+	// Implemented as brk(NULL).
+	MOVL	$45, AX  // syscall - brk
+	MOVL	$0, BX  // NULL
+	INVOKE_SYSCALL
+	MOVL	AX, ret+0(FP)
+	RET

src/runtime/sys_linux_amd64.s

@@ -598,3 +598,12 @@ TEXT runtime·socket(SB),NOSPLIT,$0-20
 	SYSCALL
 	MOVL	AX, ret+16(FP)
 	RET
+
+// func sbrk0() uintptr
+TEXT runtime·sbrk0(SB),NOSPLIT,$0-8
+	// Implemented as brk(NULL).
+	MOVQ	$0, DI
+	MOVL	$12, AX  // syscall entry
+	SYSCALL
+	MOVQ	AX, ret+0(FP)
+	RET

src/runtime/sys_linux_arm.s

@@ -48,6 +48,7 @@
 #define SYS_access (SYS_BASE + 33)
 #define SYS_connect (SYS_BASE + 283)
 #define SYS_socket (SYS_BASE + 281)
+#define SYS_brk (SYS_BASE + 45)
 
 #define ARM_BASE (SYS_BASE + 0x0f0000)
 
@@ -504,3 +505,12 @@ TEXT runtime·socket(SB),NOSPLIT,$0
 	SWI	$0
 	MOVW	R0, ret+12(FP)
 	RET
+
+// func sbrk0() uintptr
+TEXT runtime·sbrk0(SB),NOSPLIT,$0-4
+	// Implemented as brk(NULL).
+	MOVW	$0, R0
+	MOVW	$SYS_brk, R7
+	SWI	$0
+	MOVW	R0, ret+0(FP)
+	RET

src/runtime/sys_linux_arm64.s

@@ -46,6 +46,7 @@
 #define SYS_faccessat		48
 #define SYS_socket		198
 #define SYS_connect		203
+#define SYS_brk			214
 
 TEXT runtime·exit(SB),NOSPLIT,$-8-4
 	MOVW	code+0(FP), R0
@@ -483,3 +484,12 @@ TEXT runtime·socket(SB),NOSPLIT,$0-20
 	SVC
 	MOVW	R0, ret+16(FP)
 	RET
+
+// func sbrk0() uintptr
+TEXT runtime·sbrk0(SB),NOSPLIT,$0-8
+	// Implemented as brk(NULL).
+	MOVD	$0, R0
+	MOVD	$SYS_brk, R8
+	SVC
+	MOVD	R0, ret+0(FP)
+	RET

src/runtime/sys_linux_mips64x.s

@@ -45,6 +45,7 @@
 #define SYS_epoll_wait		5209
 #define SYS_clock_gettime	5222
 #define SYS_epoll_create1	5285
+#define SYS_brk			5012
 
 TEXT runtime·exit(SB),NOSPLIT,$-8-4
 	MOVW	code+0(FP), R4
@@ -426,3 +427,12 @@ TEXT runtime·closeonexec(SB),NOSPLIT,$-8
 	MOVV	$SYS_fcntl, R2
 	SYSCALL
 	RET
+
+// func sbrk0() uintptr
+TEXT runtime·sbrk0(SB),NOSPLIT,$-8-8
+	// Implemented as brk(NULL).
+	MOVV	$0, R4
+	MOVV	$SYS_brk, R2
+	SYSCALL
+	MOVV	R2, ret+0(FP)
+	RET

src/runtime/sys_linux_mipsx.s

@@ -45,6 +45,7 @@
 #define SYS_epoll_wait		    4250
 #define SYS_clock_gettime	    4263
 #define SYS_epoll_create1	    4326
+#define SYS_brk			    4045
 
 TEXT runtime·exit(SB),NOSPLIT,$0-4
 	MOVW	code+0(FP), R4
@@ -465,3 +466,12 @@ TEXT runtime·closeonexec(SB),NOSPLIT,$0-4
 	MOVW	$SYS_fcntl, R2
 	SYSCALL
 	RET
+
+// func sbrk0() uintptr
+TEXT runtime·sbrk0(SB),NOSPLIT,$0-4
+	// Implemented as brk(NULL).
+	MOVW	$0, R4
+	MOVW	$SYS_brk, R2
+	SYSCALL
+	MOVW	R2, ret+0(FP)
+	RET

src/runtime/sys_linux_ppc64x.s

@@ -21,6 +21,7 @@
 #define SYS_close		  6
 #define SYS_getpid		 20
 #define SYS_kill		 37
+#define SYS_brk			 45
 #define SYS_fcntl		 55
 #define SYS_gettimeofday	 78
 #define SYS_select		 82	// always return -ENOSYS
@@ -422,3 +423,11 @@ TEXT runtime·closeonexec(SB),NOSPLIT|NOFRAME,$0
 	MOVD    $1, R5  // FD_CLOEXEC
 	SYSCALL	$SYS_fcntl
 	RET
+
+// func sbrk0() uintptr
+TEXT runtime·sbrk0(SB),NOSPLIT|NOFRAME,$0
+	// Implemented as brk(NULL).
+	MOVD	$0, R3
+	SYSCALL	$SYS_brk
+	MOVD	R3, ret+0(FP)
+	RET

src/runtime/sys_linux_s390x.s

@@ -16,6 +16,7 @@
 #define SYS_close                 6
 #define SYS_getpid               20
 #define SYS_kill                 37
+#define SYS_brk			 45
 #define SYS_fcntl                55
 #define SYS_gettimeofday         78
 #define SYS_mmap                 90
@@ -434,3 +435,12 @@ TEXT runtime·closeonexec(SB),NOSPLIT|NOFRAME,$0
 	MOVW	$SYS_fcntl, R1
 	SYSCALL
 	RET
+
+// func sbrk0() uintptr
+TEXT runtime·sbrk0(SB),NOSPLIT|NOFRAME,$0-8
+	// Implemented as brk(NULL).
+	MOVD	$0, R2
+	MOVW	$SYS_brk, R1
+	SYSCALL
+	MOVD	R2, ret+0(FP)
+	RET