runtime: protect growslice against newcap*et.size overflow

The check of uintptr(newcap) > maxSliceCap(et.size) in addition
to capmem > _MaxMem is needed to prevent a reproducible overflow
on 32bit architectures.

On 64bit platforms this problem is less likely to occur as allocation
of a sufficiently large array or slice to be append is likely to
already exhaust available memory before the call to append can be made.

Example program that without the fix in this CL does segfault on 386:

type T [1<<27 + 1]int64

var d T
var s []T

func main() {
        s = append(s, d, d, d, d)
        print(len(s), "\n")
}

Fixes #21586

Change-Id: Ib4185435826ef43df71ba0f789e19f5bf9a347e6
Reviewed-on: https://go-review.googlesource.com/55133
Run-TryBot: Martin Möhrmann <moehrmann@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Ian Lance Taylor <iant@golang.org>

src/runtime/slice.go

@@ -125,6 +125,7 @@ func growslice(et *_type, old slice, cap int) slice {
 		}
 	}
 
+	var overflow bool
 	var lenmem, newlenmem, capmem uintptr
 	const ptrSize = unsafe.Sizeof((*byte)(nil))
 	switch et.size {
@@ -132,20 +133,37 @@ func growslice(et *_type, old slice, cap int) slice {
 		lenmem = uintptr(old.len)
 		newlenmem = uintptr(cap)
 		capmem = roundupsize(uintptr(newcap))
+		overflow = uintptr(newcap) > _MaxMem
 		newcap = int(capmem)
 	case ptrSize:
 		lenmem = uintptr(old.len) * ptrSize
 		newlenmem = uintptr(cap) * ptrSize
 		capmem = roundupsize(uintptr(newcap) * ptrSize)
+		overflow = uintptr(newcap) > _MaxMem/ptrSize
 		newcap = int(capmem / ptrSize)
 	default:
 		lenmem = uintptr(old.len) * et.size
 		newlenmem = uintptr(cap) * et.size
 		capmem = roundupsize(uintptr(newcap) * et.size)
+		overflow = uintptr(newcap) > maxSliceCap(et.size)
 		newcap = int(capmem / et.size)
 	}
 
-	if cap < old.cap || capmem > _MaxMem {
+	// The check of overflow (uintptr(newcap) > maxSliceCap(et.size))
+	// in addition to capmem > _MaxMem is needed to prevent an overflow
+	// which can be used to trigger a segfault on 32bit architectures
+	// with this example program:
+	//
+	// type T [1<<27 + 1]int64
+	//
+	// var d T
+	// var s []T
+	//
+	// func main() {
+	//   s = append(s, d, d, d, d)
+	//   print(len(s), "\n")
+	// }
+	if cap < old.cap || overflow || capmem > _MaxMem {
 		panic(errorString("growslice: cap out of range"))
 	}