runtime: grow stack by copying

On stack overflow, if all frames on the stack are
copyable, we copy the frames to a new stack twice
as large as the old one.  During GC, if a G is using
less than 1/4 of its stack, copy the stack to a stack
half its size.

TODO
- Do something about C frames.  When a C frame is in the
  stack segment, it isn't copyable.  We allocate a new segment
  in this case.
  - For idempotent C code, we can abort it, copy the stack,
    then retry.  I'm working on a separate CL for this.
  - For other C code, we can raise the stackguard
    to the lowest Go frame so the next call that Go frame
    makes triggers a copy, which will then succeed.
- Pick a starting stack size?

The plan is that eventually we reach a point where the
stack contains only copyable frames.

LGTM=rsc
R=dvyukov, rsc
CC=golang-codereviews
https://golang.org/cl/54650044

src/pkg/runtime/malloc.h

@@ -175,6 +175,9 @@ struct MLink
 // location if that one is unavailable.
 //
 // SysMap maps previously reserved address space for use.
+//
+// SysFault marks a (already SysAlloc'd) region to fault
+// if accessed.  Used only for debugging the runtime.
 
 void*	runtime·SysAlloc(uintptr nbytes, uint64 *stat);
 void	runtime·SysFree(void *v, uintptr nbytes, uint64 *stat);
@@ -182,6 +185,7 @@ void	runtime·SysUnused(void *v, uintptr nbytes);
 void	runtime·SysUsed(void *v, uintptr nbytes);
 void	runtime·SysMap(void *v, uintptr nbytes, uint64 *stat);
 void*	runtime·SysReserve(void *v, uintptr nbytes);
+void	runtime·SysFault(void *v, uintptr nbytes);
 
 // FixAlloc is a simple free-list allocator for fixed size objects.
 // Malloc uses a FixAlloc wrapped around SysAlloc to manages its
@@ -572,6 +576,31 @@ enum
 	DebugTypeAtBlockEnd = 0,
 };
 
+// Information from the compiler about the layout of stack frames.
+typedef struct BitVector BitVector;
+struct BitVector
+{
+	int32 n; // # of bits
+	uint32 data[];
+};
+typedef struct StackMap StackMap;
+struct StackMap
+{
+	int32 n;
+	uint32 data[];
+};
+enum {
+	// Pointer map
+	BitsPerPointer = 2,
+	BitsNoPointer = 0,
+	BitsPointer = 1,
+	BitsIface = 2,
+	BitsEface = 3,
+};
+// Returns pointer map data for the given stackmap index
+// (the index is encoded in PCDATA_StackMapIndex).
+BitVector*	runtime·stackmapdata(StackMap *stackmap, int32 n);
+
 // defined in mgc0.go
 void	runtime·gc_m_ptr(Eface*);
 void	runtime·gc_itab_ptr(Eface*);

src/pkg/runtime/mem_darwin.c

@@ -41,6 +41,12 @@ runtime·SysFree(void *v, uintptr n, uint64 *stat)
 	runtime·munmap(v, n);
 }
 
+void
+runtime·SysFault(void *v, uintptr n)
+{
+	runtime·mmap(v, n, PROT_NONE, 0, -1, 0);
+}
+
 void*
 runtime·SysReserve(void *v, uintptr n)
 {

src/pkg/runtime/mem_dragonfly.c

@@ -45,6 +45,12 @@ runtime·SysFree(void *v, uintptr n, uint64 *stat)
 	runtime·munmap(v, n);
 }
 
+void
+runtime·SysFault(void *v, uintptr n)
+{
+	runtime·mmap(v, n, PROT_NONE, 0, -1, 0);
+}
+
 void*
 runtime·SysReserve(void *v, uintptr n)
 {

src/pkg/runtime/mem_freebsd.c

@@ -45,6 +45,12 @@ runtime·SysFree(void *v, uintptr n, uint64 *stat)
 	runtime·munmap(v, n);
 }
 
+void
+runtime·SysFault(void *v, uintptr n)
+{
+	runtime·mmap(v, n, PROT_NONE, 0, -1, 0);
+}
+
 void*
 runtime·SysReserve(void *v, uintptr n)
 {

src/pkg/runtime/mem_linux.c

@@ -92,6 +92,12 @@ runtime·SysFree(void *v, uintptr n, uint64 *stat)
 	runtime·munmap(v, n);
 }
 
+void
+runtime·SysFault(void *v, uintptr n)
+{
+	runtime·mmap(v, n, PROT_NONE, 0, -1, 0);
+}
+
 void*
 runtime·SysReserve(void *v, uintptr n)
 {

src/pkg/runtime/mem_netbsd.c

@@ -45,6 +45,12 @@ runtime·SysFree(void *v, uintptr n, uint64 *stat)
 	runtime·munmap(v, n);
 }
 
+void
+runtime·SysFault(void *v, uintptr n)
+{
+	runtime·mmap(v, n, PROT_NONE, 0, -1, 0);
+}
+
 void*
 runtime·SysReserve(void *v, uintptr n)
 {

src/pkg/runtime/mem_openbsd.c

@@ -45,6 +45,12 @@ runtime·SysFree(void *v, uintptr n, uint64 *stat)
 	runtime·munmap(v, n);
 }
 
+void
+runtime·SysFault(void *v, uintptr n)
+{
+	runtime·mmap(v, n, PROT_NONE, 0, -1, 0);
+}
+
 void*
 runtime·SysReserve(void *v, uintptr n)
 {

src/pkg/runtime/mem_plan9.c

@@ -67,6 +67,12 @@ runtime·SysMap(void *v, uintptr nbytes, uint64 *stat)
 	USED(v, nbytes, stat);
 }
 
+void
+runtime·SysFault(void *v, uintptr n)
+{
+	USED(v, nbytes);
+}
+
 void*
 runtime·SysReserve(void *v, uintptr nbytes)
 {

src/pkg/runtime/mem_solaris.c

@@ -46,6 +46,12 @@ runtime·SysFree(void *v, uintptr n, uint64 *stat)
 	runtime·munmap(v, n);
 }
 
+void
+runtime·SysFault(void *v, uintptr n)
+{
+	runtime·mmap(v, n, PROT_NONE, 0, -1, 0);
+}
+
 void*
 runtime·SysReserve(void *v, uintptr n)
 {

src/pkg/runtime/mem_windows.c

@@ -15,12 +15,15 @@ enum {
 	MEM_RELEASE = 0x8000,
 	
 	PAGE_READWRITE = 0x0004,
+	PAGE_NOACCESS = 0x0001,
 };
 
 #pragma dynimport runtime·VirtualAlloc VirtualAlloc "kernel32.dll"
 #pragma dynimport runtime·VirtualFree VirtualFree "kernel32.dll"
+#pragma dynimport runtime·VirtualProtect VirtualProtect "kernel32.dll"
 extern void *runtime·VirtualAlloc;
 extern void *runtime·VirtualFree;
+extern void *runtime·VirtualProtect;
 
 void*
 runtime·SysAlloc(uintptr n, uint64 *stat)
@@ -60,6 +63,16 @@ runtime·SysFree(void *v, uintptr n, uint64 *stat)
 		runtime·throw("runtime: failed to release pages");
 }
 
+void
+runtime·SysFault(void *v, uintptr n)
+{
+	uintptr r, old;
+
+	r = (uintptr)runtime·stdcall(runtime·VirtualProtect, 4, v, n, (uintptr)PAGE_NOACCESS, &old);
+	if(r == 0)
+		runtime·throw("runtime: failed to protect pages");
+}
+
 void*
 runtime·SysReserve(void *v, uintptr n)
 {

src/pkg/runtime/mgc0.c

@@ -84,13 +84,6 @@ enum {
 	LOOP = 2,
 	PC_BITS = PRECISE | LOOP,
 
-	// Pointer map
-	BitsPerPointer = 2,
-	BitsNoPointer = 0,
-	BitsPointer = 1,
-	BitsIface = 2,
-	BitsEface = 3,
-
 	RootData	= 0,
 	RootBss		= 1,
 	RootFinalizers	= 2,
@@ -265,7 +258,7 @@ static Workbuf* handoff(Workbuf*);
 static void	gchelperstart(void);
 static void	addfinroots(void *wbufp, void *v);
 static void	flushallmcaches(void);
-static void	scanframe(Stkframe *frame, void *wbufp);
+static bool	scanframe(Stkframe *frame, void *wbufp);
 static void	addstackroots(G *gp, Workbuf **wbufp);
 
 static FuncVal runfinqv = {runfinq};
@@ -1445,22 +1438,8 @@ handoff(Workbuf *b)
 
 extern byte pclntab[]; // base for f->ptrsoff
 
-typedef struct BitVector BitVector;
-struct BitVector
-{
-	int32 n;
-	uint32 data[];
-};
-
-typedef struct StackMap StackMap;
-struct StackMap
-{
-	int32 n;
-	uint32 data[];
-};
-
-static BitVector*
-stackmapdata(StackMap *stackmap, int32 n)
+BitVector*
+runtime·stackmapdata(StackMap *stackmap, int32 n)
 {
 	BitVector *bv;
 	uint32 *ptr;
@@ -1531,7 +1510,7 @@ scanbitvector(byte *scanp, BitVector *bv, bool afterprologue, void *wbufp)
 }
 
 // Scan a stack frame: local variables and function arguments/results.
-static void
+static bool
 scanframe(Stkframe *frame, void *wbufp)
 {
 	Func *f;
@@ -1576,7 +1555,7 @@ scanframe(Stkframe *frame, void *wbufp)
 					pcdata, stackmap->n, runtime·funcname(f), targetpc);
 				runtime·throw("scanframe: bad symbol table");
 			}
-			bv = stackmapdata(stackmap, pcdata);
+			bv = runtime·stackmapdata(stackmap, pcdata);
 			size = (bv->n * PtrSize) / BitsPerPointer;
 			scanbitvector(frame->varp - size, bv, afterprologue, wbufp);
 		}
@@ -1586,10 +1565,11 @@ scanframe(Stkframe *frame, void *wbufp)
 	// Use pointer information if known.
 	stackmap = runtime·funcdata(f, FUNCDATA_ArgsPointerMaps);
 	if(stackmap != nil) {
-		bv = stackmapdata(stackmap, pcdata);
+		bv = runtime·stackmapdata(stackmap, pcdata);
 		scanbitvector(frame->argp, bv, true, wbufp);
 	} else
 		enqueue1(wbufp, (Obj){frame->argp, frame->arglen, 0});
+	return true;
 }
 
 static void
@@ -1620,6 +1600,10 @@ addstackroots(G *gp, Workbuf **wbufp)
 		runtime·throw("can't scan our own stack");
 	if((mp = gp->m) != nil && mp->helpgc)
 		runtime·throw("can't scan gchelper stack");
+
+	// Shrink stack if not much of it is being used.
+	runtime·shrinkstack(gp);
+
 	if(gp->syscallstack != (uintptr)nil) {
 		// Scanning another goroutine that is about to enter or might
 		// have just exited a system call. It may be executing code such

src/pkg/runtime/proc.c

@@ -173,6 +173,11 @@ runtime·schedinit(void)
 	runtime·allp = runtime·malloc((MaxGomaxprocs+1)*sizeof(runtime·allp[0]));
 	procresize(procs);
 
+	runtime·copystack = runtime·precisestack;
+	p = runtime·getenv("GOCOPYSTACK");
+	if(p != nil && !runtime·strcmp(p, (byte*)"0"))
+		runtime·copystack = false;
+
 	mstats.enablegc = 1;
 
 	if(raceenabled)
@@ -187,6 +192,15 @@ static FuncVal scavenger = {runtime·MHeap_Scavenger};
 static FuncVal initDone = { runtime·unlockOSThread };
 
 // The main goroutine.
+// Note: C frames in general are not copyable during stack growth, for two reasons:
+//   1) We don't know where in a frame to find pointers to other stack locations.
+//   2) There's no guarantee that globals or heap values do not point into the frame.
+//
+// The C frame for runtime.main is copyable, because:
+//   1) There are no pointers to other stack locations in the frame
+//      (d.fn points at a global, d.link is nil, d.argp is -1).
+//   2) The only pointer into this frame is from the defer chain,
+//      which is explicitly handled during stack copying.
 void
 runtime·main(void)
 {
@@ -1870,8 +1884,20 @@ allgadd(G *gp)
 static void
 gfput(P *p, G *gp)
 {
+	uintptr stksize;
+
 	if(gp->stackguard - StackGuard != gp->stack0)
 		runtime·throw("invalid stack in gfput");
+	stksize = gp->stackbase + sizeof(Stktop) - gp->stack0;
+	if(stksize != FixedStack) {
+		// non-standard stack size - free it.
+		runtime·stackfree((void*)gp->stack0, stksize);
+		gp->stacksize = 0;
+		gp->stack0 = 0;
+		gp->stackguard = 0;
+		gp->stackguard0 = 0;
+		gp->stackbase = 0;
+	}
 	gp->schedlink = p->gfree;
 	p->gfree = gp;
 	p->gfreecnt++;
@@ -1894,6 +1920,7 @@ static G*
 gfget(P *p)
 {
 	G *gp;
+	byte *stk;
 
 retry:
 	gp = p->gfree;
@@ -1912,6 +1939,24 @@ retry:
 	if(gp) {
 		p->gfree = gp->schedlink;
 		p->gfreecnt--;
+
+		if(gp->stack0 == 0) {
+			// Stack was deallocated in gfput.  Allocate a new one.
+			if(g == m->g0) {
+				stk = runtime·stackalloc(FixedStack);
+			} else {
+				g->param = (void*)FixedStack;
+				runtime·mcall(mstackalloc);
+				stk = g->param;
+				g->param = nil;
+			}
+			gp->stacksize = FixedStack;
+			gp->stack0 = (uintptr)stk;
+			gp->stackbase = (uintptr)stk + FixedStack - sizeof(Stktop);
+			gp->stackguard = (uintptr)stk + StackGuard;
+			gp->stackguard0 = gp->stackguard;
+			runtime·memclr((byte*)gp->stackbase, sizeof(Stktop));
+		}
 	}
 	return gp;
 }

src/pkg/runtime/runtime.h

@@ -589,6 +589,7 @@ struct DebugVars
 };
 
 extern bool runtime·precisestack;
+extern bool runtime·copystack;
 
 /*
  * defined macros
@@ -732,7 +733,7 @@ struct Stkframe
 	uintptr	arglen;	// number of bytes at argp
 };
 
-int32	runtime·gentraceback(uintptr, uintptr, uintptr, G*, int32, uintptr*, int32, void(*)(Stkframe*, void*), void*, bool);
+int32	runtime·gentraceback(uintptr, uintptr, uintptr, G*, int32, uintptr*, int32, bool(*)(Stkframe*, void*), void*, bool);
 void	runtime·traceback(uintptr pc, uintptr sp, uintptr lr, G* gp);
 void	runtime·tracebackothers(G*);
 bool	runtime·haszeroargs(uintptr pc);
@@ -860,6 +861,7 @@ int8*	runtime·funcname(Func*);
 int32	runtime·pcdatavalue(Func*, int32, uintptr);
 void*	runtime·stackalloc(uint32);
 void	runtime·stackfree(void*, uintptr);
+void	runtime·shrinkstack(G*);
 MCache*	runtime·allocmcache(void);
 void	runtime·freemcache(MCache*);
 void	runtime·mallocinit(void);

src/pkg/runtime/stack.h

@@ -76,7 +76,7 @@ enum {
 	// The minimum stack segment size to allocate.
 	// If the amount needed for the splitting frame + StackExtra
 	// is less than this number, the stack will have this size instead.
-	StackMin = 8192,
+	StackMin = 4096,
 	FixedStack = StackMin + StackSystem,
 
 	// Functions that need frames bigger than this use an extra

src/pkg/runtime/traceback_arm.c

@@ -10,7 +10,7 @@
 void runtime·sigpanic(void);
 
 int32
-runtime·gentraceback(uintptr pc0, uintptr sp0, uintptr lr0, G *gp, int32 skip, uintptr *pcbuf, int32 max, void (*callback)(Stkframe*, void*), void *v, bool printall)
+runtime·gentraceback(uintptr pc0, uintptr sp0, uintptr lr0, G *gp, int32 skip, uintptr *pcbuf, int32 max, bool (*callback)(Stkframe*, void*), void *v, bool printall)
 {
 	int32 i, n, nprint, line;
 	uintptr x, tracepc;
@@ -140,8 +140,10 @@ runtime·gentraceback(uintptr pc0, uintptr sp0, uintptr lr0, G *gp, int32 skip,
 
 		if(pcbuf != nil)
 			pcbuf[n] = frame.pc;
-		if(callback != nil)
-			callback(&frame, v);
+		if(callback != nil) {
+			if(!callback(&frame, v))
+				return n;
+		}
 		if(printing) {
 			if(printall || runtime·showframe(f, gp)) {
 				// Print during crash.

src/pkg/runtime/traceback_x86.c

@@ -19,7 +19,7 @@ void runtime·sigpanic(void);
 // collector (callback != nil).  A little clunky to merge these, but avoids
 // duplicating the code and all its subtlety.
 int32
-runtime·gentraceback(uintptr pc0, uintptr sp0, uintptr lr0, G *gp, int32 skip, uintptr *pcbuf, int32 max, void (*callback)(Stkframe*, void*), void *v, bool printall)
+runtime·gentraceback(uintptr pc0, uintptr sp0, uintptr lr0, G *gp, int32 skip, uintptr *pcbuf, int32 max, bool (*callback)(Stkframe*, void*), void *v, bool printall)
 {
 	int32 i, n, nprint, line;
 	uintptr tracepc;
@@ -151,8 +151,10 @@ runtime·gentraceback(uintptr pc0, uintptr sp0, uintptr lr0, G *gp, int32 skip,
 
 		if(pcbuf != nil)
 			pcbuf[n] = frame.pc;
-		if(callback != nil)
-			callback(&frame, v);
+		if(callback != nil) {
+			if(!callback(&frame, v))
+				return n;
+		}
 		if(printing) {
 			if(printall || runtime·showframe(f, gp)) {
 				// Print during crash.