runtime: fix mmap error return on linux.

Fixes #1511 again.

R=golang-dev, iant
CC=golang-dev
https://golang.org/cl/4527070

src/pkg/runtime/linux/mem.c

@@ -39,13 +39,19 @@ runtime·SysFree(void *v, uintptr n)
 void*
 runtime·SysReserve(void *v, uintptr n)
 {
+	void *p;
+
 	// On 64-bit, people with ulimit -v set complain if we reserve too
 	// much address space.  Instead, assume that the reservation is okay
 	// and check the assumption in SysMap.
 	if(sizeof(void*) == 8)
 		return v;
 	
-	return runtime·mmap(v, n, PROT_NONE, MAP_ANON|MAP_PRIVATE, -1, 0);
+	p = runtime·mmap(v, n, PROT_NONE, MAP_ANON|MAP_PRIVATE, -1, 0);
+	if(p < (void*)4096) {
+		return nil;
+	}
+	return p;
 }
 
 enum
@@ -63,6 +69,8 @@ runtime·SysMap(void *v, uintptr n)
 	// On 64-bit, we don't actually have v reserved, so tread carefully.
 	if(sizeof(void*) == 8) {
 		p = runtime·mmap(v, n, PROT_READ|PROT_WRITE|PROT_EXEC, MAP_ANON|MAP_PRIVATE, -1, 0);
+		if(p == (void*)ENOMEM)
+			runtime·throw("runtime: out of memory");
 		if(p != v) {
 			runtime·printf("runtime: address space conflict: map(%p) = %p\n", v, p);
 			runtime·throw("runtime: address space conflict");
@@ -71,7 +79,7 @@ runtime·SysMap(void *v, uintptr n)
 	}
 
 	p = runtime·mmap(v, n, PROT_READ|PROT_WRITE|PROT_EXEC, MAP_ANON|MAP_FIXED|MAP_PRIVATE, -1, 0);
-	if(p == (void*)-ENOMEM)
+	if(p == (void*)ENOMEM)
 		runtime·throw("runtime: out of memory");
 	if(p != v)
 		runtime·throw("runtime: cannot map pages in arena address space");