runtime: map reserved memory as NORESERVE on solaris

This changes makes it so that sysReserve, which creates a PROT_NONE
mapping, maps that memory as NORESERVE. Before this change, relatively
large PROT_NONE mappings could cause fork to fail with ENOMEM, reported
as "not enough space". Presumably this refers to swap space, since
adding this flag causes the failures to go away.

This helps unblock page allocator work, since it allows us to make large
PROT_NONE mappings on solaris safely.

Updates #35112.

Change-Id: Ic3cba310c626e93d5db0f27269e2569bb7bc393e
Reviewed-on: https://go-review.googlesource.com/c/go/+/205759
Run-TryBot: Michael Knyszek <mknyszek@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Cherry Zhang <cherryyz@google.com>

src/runtime/mem_bsd.go

@@ -44,8 +44,18 @@ func sysFault(v unsafe.Pointer, n uintptr) {
 	mmap(v, n, _PROT_NONE, _MAP_ANON|_MAP_PRIVATE|_MAP_FIXED, -1, 0)
 }
 
+// Indicates not to reserve swap space for the mapping.
+const _sunosMAP_NORESERVE = 0x40
+
 func sysReserve(v unsafe.Pointer, n uintptr) unsafe.Pointer {
-	p, err := mmap(v, n, _PROT_NONE, _MAP_ANON|_MAP_PRIVATE, -1, 0)
+	flags := int32(_MAP_ANON | _MAP_PRIVATE)
+	if GOOS == "solaris" || GOOS == "illumos" {
+		// Be explicit that we don't want to reserve swap space
+		// for PROT_NONE anonymous mappings. This avoids an issue
+		// wherein large mappings can cause fork to fail.
+		flags |= _sunosMAP_NORESERVE
+	}
+	p, err := mmap(v, n, _PROT_NONE, flags, -1, 0)
 	if err != 0 {
 		return nil
 	}