proc.c 39.1 KB
Newer Older
Ken Thompson's avatar
Ken Thompson committed
1 2 3 4 5
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

#include "runtime.h"
Russ Cox's avatar
Russ Cox committed
6 7
#include "arch_GOARCH.h"
#include "defs_GOOS_GOARCH.h"
Russ Cox's avatar
Russ Cox committed
8
#include "malloc.h"
Russ Cox's avatar
Russ Cox committed
9
#include "os_GOOS.h"
10
#include "stack.h"
Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
11
#include "race.h"
12
#include "type.h"
Ken Thompson's avatar
Ken Thompson committed
13

14 15
bool	runtime·iscgo;

16
static void schedule(G*);
17

18 19
typedef struct Sched Sched;

20 21
M	runtime·m0;
G	runtime·g0;	// idle goroutine for m0
22

Ken Thompson's avatar
Ken Thompson committed
23 24
static	int32	debug	= 0;

25
int32	runtime·gcwaiting;
26

27 28 29 30 31 32 33
G*	runtime·allg;
G*	runtime·lastg;
M*	runtime·allm;

int8*	runtime·goos;
int32	runtime·ncpu;

Russ Cox's avatar
Russ Cox committed
34 35 36
// Go scheduler
//
// The go scheduler's job is to match ready-to-run goroutines (`g's)
37 38 39 40
// with waiting-for-work schedulers (`m's).  If there are ready g's
// and no waiting m's, ready() will start a new m running in a new
// OS thread, so that all ready g's can run simultaneously, up to a limit.
// For now, m's never go away.
Russ Cox's avatar
Russ Cox committed
41
//
42 43 44 45 46 47
// By default, Go keeps only one kernel thread (m) running user code
// at a single time; other threads may be blocked in the operating system.
// Setting the environment variable $GOMAXPROCS or calling
// runtime.GOMAXPROCS() will change the number of user threads
// allowed to execute simultaneously.  $GOMAXPROCS is thus an
// approximation of the maximum number of cores to use.
Russ Cox's avatar
Russ Cox committed
48 49
//
// Even a program that can run without deadlock in a single process
50 51
// might use more m's if given the chance.  For example, the prime
// sieve will use as many m's as there are primes (up to runtime·sched.mmax),
Russ Cox's avatar
Russ Cox committed
52
// allowing different stages of the pipeline to execute in parallel.
53
// We could revisit this choice, only kicking off new m's for blocking
Russ Cox's avatar
Russ Cox committed
54 55 56 57 58 59 60
// system calls, but that would limit the amount of parallel computation
// that go would try to do.
//
// In general, one could imagine all sorts of refinements to the
// scheduler, but the goal now is just to get something working on
// Linux and OS X.

61 62
struct Sched {
	Lock;
Russ Cox's avatar
Russ Cox committed
63

64
	G *gfree;	// available g's (status == Gdead)
65
	int64 goidgen;
Russ Cox's avatar
Russ Cox committed
66

67
	G *ghead;	// g's waiting to run
Russ Cox's avatar
Russ Cox committed
68
	G *gtail;
69 70 71
	int32 gwait;	// number of g's waiting to run
	int32 gcount;	// number of g's that are alive
	int32 grunning;	// number of g's running on cpu or in syscall
Russ Cox's avatar
Russ Cox committed
72

73 74 75
	M *mhead;	// m's waiting for work
	int32 mwait;	// number of m's waiting for work
	int32 mcount;	// number of m's that have been created
Russ Cox's avatar
Russ Cox committed
76

77 78
	volatile uint32 atomic;	// atomic scheduling word (see below)

Russ Cox's avatar
Russ Cox committed
79
	int32 profilehz;	// cpu profiling rate
80

81 82
	bool init;  // running initialization
	bool lockmain;  // init called runtime.LockOSThread
83

84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127
	Note	stopped;	// one g can set waitstop and wait here for m's to stop
};

// The atomic word in sched is an atomic uint32 that
// holds these fields.
//
//	[15 bits] mcpu		number of m's executing on cpu
//	[15 bits] mcpumax	max number of m's allowed on cpu
//	[1 bit] waitstop	some g is waiting on stopped
//	[1 bit] gwaiting	gwait != 0
//
// These fields are the information needed by entersyscall
// and exitsyscall to decide whether to coordinate with the
// scheduler.  Packing them into a single machine word lets
// them use a fast path with a single atomic read/write and
// no lock/unlock.  This greatly reduces contention in
// syscall- or cgo-heavy multithreaded programs.
//
// Except for entersyscall and exitsyscall, the manipulations
// to these fields only happen while holding the schedlock,
// so the routines holding schedlock only need to worry about
// what entersyscall and exitsyscall do, not the other routines
// (which also use the schedlock).
//
// In particular, entersyscall and exitsyscall only read mcpumax,
// waitstop, and gwaiting.  They never write them.  Thus, writes to those
// fields can be done (holding schedlock) without fear of write conflicts.
// There may still be logic conflicts: for example, the set of waitstop must
// be conditioned on mcpu >= mcpumax or else the wait may be a
// spurious sleep.  The Promela model in proc.p verifies these accesses.
enum {
	mcpuWidth = 15,
	mcpuMask = (1<<mcpuWidth) - 1,
	mcpuShift = 0,
	mcpumaxShift = mcpuShift + mcpuWidth,
	waitstopShift = mcpumaxShift + mcpuWidth,
	gwaitingShift = waitstopShift+1,

	// The max value of GOMAXPROCS is constrained
	// by the max value we can store in the bit fields
	// of the atomic word.  Reserve a few high values
	// so that we can detect accidental decrement
	// beyond zero.
	maxgomaxprocs = mcpuMask - 10,
128 129
};

130 131 132 133 134
#define atomic_mcpu(v)		(((v)>>mcpuShift)&mcpuMask)
#define atomic_mcpumax(v)	(((v)>>mcpumaxShift)&mcpuMask)
#define atomic_waitstop(v)	(((v)>>waitstopShift)&1)
#define atomic_gwaiting(v)	(((v)>>gwaitingShift)&1)

135
Sched runtime·sched;
Wei Guangjing's avatar
Wei Guangjing committed
136
int32 runtime·gomaxprocs;
137
bool runtime·singleproc;
138

139 140
static bool canaddmcpu(void);

141
// An m that is waiting for notewakeup(&m->havenextg).  This may
142 143 144 145 146 147
// only be accessed while the scheduler lock is held.  This is used to
// minimize the number of times we call notewakeup while the scheduler
// lock is held, since the m will normally move quickly to lock the
// scheduler itself, producing lock contention.
static M* mwakeup;

Russ Cox's avatar
Russ Cox committed
148 149 150 151
// Scheduling helpers.  Sched must be locked.
static void gput(G*);	// put/get on ghead/gtail
static G* gget(void);
static void mput(M*);	// put/get on mhead
152
static M* mget(G*);
Russ Cox's avatar
Russ Cox committed
153 154
static void gfput(G*);	// put/get on gfree
static G* gfget(void);
155
static void matchmg(void);	// match m's to g's
Russ Cox's avatar
Russ Cox committed
156
static void readylocked(G*);	// ready, but sched is locked
157
static void mnextg(M*, G*);
158
static void mcommoninit(M*);
Russ Cox's avatar
Russ Cox committed
159

160 161 162 163 164 165 166 167 168 169 170 171 172 173 174
void
setmcpumax(uint32 n)
{
	uint32 v, w;

	for(;;) {
		v = runtime·sched.atomic;
		w = v;
		w &= ~(mcpuMask<<mcpumaxShift);
		w |= n<<mcpumaxShift;
		if(runtime·cas(&runtime·sched.atomic, v, w))
			break;
	}
}

175 176 177
// Keep trace of scavenger's goroutine for deadlock detection.
static G *scvg;

Russ Cox's avatar
Russ Cox committed
178 179 180 181 182
// The bootstrap sequence is:
//
//	call osinit
//	call schedinit
//	make & queue new G
183
//	call runtime·mstart
Russ Cox's avatar
Russ Cox committed
184
//
185
// The new G calls runtime·main.
Russ Cox's avatar
Russ Cox committed
186
void
187
runtime·schedinit(void)
Russ Cox's avatar
Russ Cox committed
188 189 190
{
	int32 n;
	byte *p;
Russ Cox's avatar
Russ Cox committed
191

192
	m->nomemprof++;
193
	runtime·mallocinit();
194 195
	mcommoninit(m);

196
	runtime·goargs();
197
	runtime·goenvs();
Russ Cox's avatar
Russ Cox committed
198

199
	// For debugging:
Russ Cox's avatar
Russ Cox committed
200 201
	// Allocate internal symbol table representation now,
	// so that we don't need to call malloc when we crash.
Russ Cox's avatar
Russ Cox committed
202
	// runtime·findfunc(0);
203

Russ Cox's avatar
Russ Cox committed
204
	runtime·gomaxprocs = 1;
205
	p = runtime·getenv("GOMAXPROCS");
206 207 208
	if(p != nil && (n = runtime·atoi(p)) != 0) {
		if(n > maxgomaxprocs)
			n = maxgomaxprocs;
Russ Cox's avatar
Russ Cox committed
209
		runtime·gomaxprocs = n;
210
	}
211 212 213
	// wait for the main goroutine to start before taking
	// GOMAXPROCS into account.
	setmcpumax(1);
214
	runtime·singleproc = runtime·gomaxprocs == 1;
215

216 217 218 219
	canaddmcpu();	// mcpu++ to account for bootstrap m
	m->helpgc = 1;	// flag to tell schedule() to mcpu--
	runtime·sched.grunning++;

220
	mstats.enablegc = 1;
221
	m->nomemprof--;
Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
222 223 224

	if(raceenabled)
		runtime·raceinit();
Russ Cox's avatar
Russ Cox committed
225 226
}

227 228 229 230 231 232 233 234 235 236 237 238 239 240
extern void main·init(void);
extern void main·main(void);

// The main goroutine.
void
runtime·main(void)
{
	// Lock the main goroutine onto this, the main OS thread,
	// during initialization.  Most programs won't care, but a few
	// do require certain calls to be made by the main thread.
	// Those can arrange for main.main to run in the main thread
	// by calling runtime.LockOSThread during initialization
	// to preserve the lock.
	runtime·LockOSThread();
241 242
	// From now on, newgoroutines may use non-main threads.
	setmcpumax(runtime·gomaxprocs);
243
	runtime·sched.init = true;
Russ Cox's avatar
Russ Cox committed
244
	scvg = runtime·newproc1((byte*)runtime·MHeap_Scavenger, nil, 0, 0, runtime·main);
245
	scvg->issystem = true;
246 247 248 249 250
	main·init();
	runtime·sched.init = false;
	if(!runtime·sched.lockmain)
		runtime·UnlockOSThread();

251 252 253 254 255
	// The deadlock detection has false negatives.
	// Let scvg start up, to eliminate the false negative
	// for the trivial program func main() { select{} }.
	runtime·gosched();

256
	main·main();
Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
257 258
	if(raceenabled)
		runtime·racefini();
259 260 261 262 263
	runtime·exit(0);
	for(;;)
		*(int32*)runtime·main = 0;
}

264 265 266 267 268 269 270 271 272 273 274
// Lock the scheduler.
static void
schedlock(void)
{
	runtime·lock(&runtime·sched);
}

// Unlock the scheduler.
static void
schedunlock(void)
{
275
	M *mp;
276

277
	mp = mwakeup;
278 279
	mwakeup = nil;
	runtime·unlock(&runtime·sched);
280 281
	if(mp != nil)
		runtime·notewakeup(&mp->havenextg);
282 283
}

Ken Thompson's avatar
Ken Thompson committed
284
void
285
runtime·goexit(void)
Ken Thompson's avatar
Ken Thompson committed
286
{
Russ Cox's avatar
Russ Cox committed
287
	g->status = Gmoribund;
288
	runtime·gosched();
Ken Thompson's avatar
Ken Thompson committed
289 290
}

Russ Cox's avatar
Russ Cox committed
291
void
Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
292
runtime·goroutineheader(G *gp)
Russ Cox's avatar
Russ Cox committed
293 294 295
{
	int8 *status;

Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
296
	switch(gp->status) {
Russ Cox's avatar
Russ Cox committed
297 298 299 300 301 302 303 304 305 306 307 308 309
	case Gidle:
		status = "idle";
		break;
	case Grunnable:
		status = "runnable";
		break;
	case Grunning:
		status = "running";
		break;
	case Gsyscall:
		status = "syscall";
		break;
	case Gwaiting:
Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
310 311
		if(gp->waitreason)
			status = gp->waitreason;
Russ Cox's avatar
Russ Cox committed
312 313 314 315 316 317 318 319 320 321
		else
			status = "waiting";
		break;
	case Gmoribund:
		status = "moribund";
		break;
	default:
		status = "???";
		break;
	}
322
	runtime·printf("goroutine %D [%s]:\n", gp->goid, status);
Russ Cox's avatar
Russ Cox committed
323 324
}

325
void
326
runtime·tracebackothers(G *me)
327
{
Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
328
	G *gp;
329
	int32 traceback;
330

331
	traceback = runtime·gotraceback();
Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
332 333
	for(gp = runtime·allg; gp != nil; gp = gp->alllink) {
		if(gp == me || gp->status == Gdead)
334
			continue;
335 336
		if(gp->issystem && traceback < 2)
			continue;
Russ Cox's avatar
Russ Cox committed
337
		runtime·printf("\n");
Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
338 339
		runtime·goroutineheader(gp);
		runtime·traceback(gp->sched.pc, (byte*)gp->sched.sp, 0, gp);
340 341 342
	}
}

Russ Cox's avatar
Russ Cox committed
343 344 345 346 347 348 349 350 351 352 353 354
// Mark this g as m's idle goroutine.
// This functionality might be used in environments where programs
// are limited to a single thread, to simulate a select-driven
// network server.  It is not exposed via the standard runtime API.
void
runtime·idlegoroutine(void)
{
	if(g->idlem != nil)
		runtime·throw("g is already an idle goroutine");
	g->idlem = m;
}

355
static void
Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
356
mcommoninit(M *mp)
357
{
Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
358 359
	mp->id = runtime·sched.mcount++;
	mp->fastrand = 0x49f6428aUL + mp->id + runtime·cputicks();
360

Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
361 362
	if(mp->mcache == nil)
		mp->mcache = runtime·allocmcache();
363

Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
364
	runtime·callers(1, mp->createstack, nelem(mp->createstack));
365

366 367
	// Add to runtime·allm so garbage collector doesn't free m
	// when it is just in a register or thread-local storage.
Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
368
	mp->alllink = runtime·allm;
369
	// runtime·NumCgoCall() iterates over allm w/o schedlock,
370
	// so we need to publish it safely.
Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
371
	runtime·atomicstorep(&runtime·allm, mp);
372 373
}

374 375 376 377 378 379 380 381 382 383 384 385 386 387 388
// Try to increment mcpu.  Report whether succeeded.
static bool
canaddmcpu(void)
{
	uint32 v;

	for(;;) {
		v = runtime·sched.atomic;
		if(atomic_mcpu(v) >= atomic_mcpumax(v))
			return 0;
		if(runtime·cas(&runtime·sched.atomic, v, v+(1<<mcpuShift)))
			return 1;
	}
}

Russ Cox's avatar
Russ Cox committed
389
// Put on `g' queue.  Sched must be locked.
390
static void
Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
391
gput(G *gp)
Ken Thompson's avatar
Ken Thompson committed
392
{
Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
393
	M *mp;
394 395

	// If g is wired, hand it off directly.
Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
396 397
	if((mp = gp->lockedm) != nil && canaddmcpu()) {
		mnextg(mp, gp);
398 399
		return;
	}
400

Russ Cox's avatar
Russ Cox committed
401
	// If g is the idle goroutine for an m, hand it off.
Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
402 403
	if(gp->idlem != nil) {
		if(gp->idlem->idleg != nil) {
404
			runtime·printf("m%d idle out of sync: g%D g%D\n",
Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
405 406
				gp->idlem->id,
				gp->idlem->idleg->goid, gp->goid);
Russ Cox's avatar
Russ Cox committed
407 408
			runtime·throw("runtime: double idle");
		}
Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
409
		gp->idlem->idleg = gp;
Russ Cox's avatar
Russ Cox committed
410 411
		return;
	}
412

Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
413
	gp->schedlink = nil;
414
	if(runtime·sched.ghead == nil)
Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
415
		runtime·sched.ghead = gp;
416
	else
Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
417 418
		runtime·sched.gtail->schedlink = gp;
	runtime·sched.gtail = gp;
419 420 421 422 423 424 425 426 427 428 429 430

	// increment gwait.
	// if it transitions to nonzero, set atomic gwaiting bit.
	if(runtime·sched.gwait++ == 0)
		runtime·xadd(&runtime·sched.atomic, 1<<gwaitingShift);
}

// Report whether gget would return something.
static bool
haveg(void)
{
	return runtime·sched.ghead != nil || m->idleg != nil;
Russ Cox's avatar
Russ Cox committed
431 432 433 434 435 436
}

// Get from `g' queue.  Sched must be locked.
static G*
gget(void)
{
Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
437
	G *gp;
Russ Cox's avatar
Russ Cox committed
438

Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
439 440 441
	gp = runtime·sched.ghead;
	if(gp) {
		runtime·sched.ghead = gp->schedlink;
442 443
		if(runtime·sched.ghead == nil)
			runtime·sched.gtail = nil;
444 445 446 447
		// decrement gwait.
		// if it transitions to zero, clear atomic gwaiting bit.
		if(--runtime·sched.gwait == 0)
			runtime·xadd(&runtime·sched.atomic, -1<<gwaitingShift);
Russ Cox's avatar
Russ Cox committed
448
	} else if(m->idleg != nil) {
Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
449
		gp = m->idleg;
Russ Cox's avatar
Russ Cox committed
450
		m->idleg = nil;
Russ Cox's avatar
Russ Cox committed
451
	}
Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
452
	return gp;
Russ Cox's avatar
Russ Cox committed
453 454 455 456
}

// Put on `m' list.  Sched must be locked.
static void
Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
457
mput(M *mp)
Russ Cox's avatar
Russ Cox committed
458
{
Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
459 460
	mp->schedlink = runtime·sched.mhead;
	runtime·sched.mhead = mp;
461
	runtime·sched.mwait++;
Russ Cox's avatar
Russ Cox committed
462 463
}

464
// Get an `m' to run `g'.  Sched must be locked.
Russ Cox's avatar
Russ Cox committed
465
static M*
Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
466
mget(G *gp)
Russ Cox's avatar
Russ Cox committed
467
{
Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
468
	M *mp;
Russ Cox's avatar
Russ Cox committed
469

470
	// if g has its own m, use it.
Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
471 472
	if(gp && (mp = gp->lockedm) != nil)
		return mp;
473 474

	// otherwise use general m pool.
Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
475 476
	if((mp = runtime·sched.mhead) != nil) {
		runtime·sched.mhead = mp->schedlink;
477
		runtime·sched.mwait--;
Russ Cox's avatar
Russ Cox committed
478
	}
Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
479
	return mp;
Russ Cox's avatar
Russ Cox committed
480 481 482
}

// Mark g ready to run.
483
void
Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
484
runtime·ready(G *gp)
485
{
486
	schedlock();
Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
487
	readylocked(gp);
488
	schedunlock();
Russ Cox's avatar
Russ Cox committed
489 490
}

491 492 493
// Mark g ready to run.  Sched is already locked.
// G might be running already and about to stop.
// The sched lock protects g->status from changing underfoot.
Russ Cox's avatar
Russ Cox committed
494
static void
Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
495
readylocked(G *gp)
Russ Cox's avatar
Russ Cox committed
496
{
Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
497
	if(gp->m) {
498 499
		// Running on another machine.
		// Ready it when it stops.
Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
500
		gp->readyonstop = 1;
501 502
		return;
	}
Russ Cox's avatar
Russ Cox committed
503 504

	// Mark runnable.
Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
505
	if(gp->status == Grunnable || gp->status == Grunning) {
506
		runtime·printf("goroutine %D has status %d\n", gp->goid, gp->status);
507
		runtime·throw("bad g->status in ready");
508
	}
Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
509
	gp->status = Grunnable;
Russ Cox's avatar
Russ Cox committed
510

Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
511
	gput(gp);
512
	matchmg();
Russ Cox's avatar
Russ Cox committed
513
}
514

515 516 517 518 519
static void
nop(void)
{
}

520 521 522 523
// Same as readylocked but a different symbol so that
// debuggers can set a breakpoint here and catch all
// new goroutines.
static void
Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
524
newprocreadylocked(G *gp)
525
{
526
	nop();	// avoid inlining in 6l
Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
527
	readylocked(gp);
528 529
}

530
// Pass g to m for running.
531
// Caller has already incremented mcpu.
532
static void
Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
533
mnextg(M *mp, G *gp)
534
{
535
	runtime·sched.grunning++;
Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
536 537 538
	mp->nextg = gp;
	if(mp->waitnextg) {
		mp->waitnextg = 0;
539 540
		if(mwakeup != nil)
			runtime·notewakeup(&mwakeup->havenextg);
Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
541
		mwakeup = mp;
542 543 544
	}
}

Russ Cox's avatar
Russ Cox committed
545 546
// Get the next goroutine that m should run.
// Sched must be locked on entry, is unlocked on exit.
547
// Makes sure that at most $GOMAXPROCS g's are
548
// running on cpus (not in system calls) at any given time.
Russ Cox's avatar
Russ Cox committed
549 550
static G*
nextgandunlock(void)
551 552
{
	G *gp;
553
	uint32 v;
554

555
top:
556 557
	if(atomic_mcpu(runtime·sched.atomic) >= maxgomaxprocs)
		runtime·throw("negative mcpu");
558

559 560
	// If there is a g waiting as m->nextg, the mcpu++
	// happened before it was passed to mnextg.
561 562 563
	if(m->nextg != nil) {
		gp = m->nextg;
		m->nextg = nil;
564
		schedunlock();
565 566 567
		return gp;
	}

568 569 570 571
	if(m->lockedg != nil) {
		// We can only run one g, and it's not available.
		// Make sure some other cpu is running to handle
		// the ordinary run queue.
Hector Chu's avatar
Hector Chu committed
572
		if(runtime·sched.gwait != 0) {
573
			matchmg();
Hector Chu's avatar
Hector Chu committed
574 575 576 577 578 579 580 581
			// m->lockedg might have been on the queue.
			if(m->nextg != nil) {
				gp = m->nextg;
				m->nextg = nil;
				schedunlock();
				return gp;
			}
		}
582 583
	} else {
		// Look for work on global queue.
584 585 586 587 588
		while(haveg() && canaddmcpu()) {
			gp = gget();
			if(gp == nil)
				runtime·throw("gget inconsistency");

589 590 591 592
			if(gp->lockedm) {
				mnextg(gp->lockedm, gp);
				continue;
			}
593
			runtime·sched.grunning++;
594
			schedunlock();
595
			return gp;
596
		}
597 598 599 600 601 602 603 604 605 606 607 608 609 610

		// The while loop ended either because the g queue is empty
		// or because we have maxed out our m procs running go
		// code (mcpu >= mcpumax).  We need to check that
		// concurrent actions by entersyscall/exitsyscall cannot
		// invalidate the decision to end the loop.
		//
		// We hold the sched lock, so no one else is manipulating the
		// g queue or changing mcpumax.  Entersyscall can decrement
		// mcpu, but if does so when there is something on the g queue,
		// the gwait bit will be set, so entersyscall will take the slow path
		// and use the sched lock.  So it cannot invalidate our decision.
		//
		// Wait on global m queue.
611
		mput(m);
Ken Thompson's avatar
Ken Thompson committed
612
	}
613

Russ Cox's avatar
Russ Cox committed
614
	// Look for deadlock situation.
615 616 617 618 619 620 621 622 623 624 625 626 627 628
	// There is a race with the scavenger that causes false negatives:
	// if the scavenger is just starting, then we have
	//	scvg != nil && grunning == 0 && gwait == 0
	// and we do not detect a deadlock.  It is possible that we should
	// add that case to the if statement here, but it is too close to Go 1
	// to make such a subtle change.  Instead, we work around the
	// false negative in trivial programs by calling runtime.gosched
	// from the main goroutine just before main.main.
	// See runtime·main above.
	//
	// On a related note, it is also possible that the scvg == nil case is
	// wrong and should include gwait, but that does not happen in
	// standard Go programs, which all start the scavenger.
	//
Mikio Hara's avatar
Mikio Hara committed
629 630 631
	if((scvg == nil && runtime·sched.grunning == 0) ||
	   (scvg != nil && runtime·sched.grunning == 1 && runtime·sched.gwait == 0 &&
	    (scvg->status == Grunning || scvg->status == Gsyscall))) {
632
		m->throwing = -1;  // do not dump full stacks
Mikio Hara's avatar
Mikio Hara committed
633 634
		runtime·throw("all goroutines are asleep - deadlock!");
	}
635

Russ Cox's avatar
Russ Cox committed
636
	m->nextg = nil;
637
	m->waitnextg = 1;
638
	runtime·noteclear(&m->havenextg);
639 640 641 642 643

	// Stoptheworld is waiting for all but its cpu to go to stop.
	// Entersyscall might have decremented mcpu too, but if so
	// it will see the waitstop and take the slow path.
	// Exitsyscall never increments mcpu beyond mcpumax.
644
	v = runtime·atomicload(&runtime·sched.atomic);
645 646 647
	if(atomic_waitstop(v) && atomic_mcpu(v) <= atomic_mcpumax(v)) {
		// set waitstop = 0 (known to be 1)
		runtime·xadd(&runtime·sched.atomic, -1<<waitstopShift);
648
		runtime·notewakeup(&runtime·sched.stopped);
Russ Cox's avatar
Russ Cox committed
649
	}
650
	schedunlock();
Russ Cox's avatar
Russ Cox committed
651

652
	runtime·notesleep(&m->havenextg);
653 654 655 656 657 658
	if(m->helpgc) {
		runtime·gchelper();
		m->helpgc = 0;
		runtime·lock(&runtime·sched);
		goto top;
	}
Russ Cox's avatar
Russ Cox committed
659
	if((gp = m->nextg) == nil)
660
		runtime·throw("bad m->nextg in nextgoroutine");
Russ Cox's avatar
Russ Cox committed
661
	m->nextg = nil;
662
	return gp;
Ken Thompson's avatar
Ken Thompson committed
663 664
}

665
int32
666
runtime·gcprocs(void)
667
{
668 669 670
	int32 n;
	
	// Figure out how many CPUs to use during GC.
671
	// Limited by gomaxprocs, number of actual CPUs, and MaxGcproc.
672 673 674 675 676 677 678 679 680
	n = runtime·gomaxprocs;
	if(n > runtime·ncpu)
		n = runtime·ncpu;
	if(n > MaxGcproc)
		n = MaxGcproc;
	if(n > runtime·sched.mwait+1) // one M is currently running
		n = runtime·sched.mwait+1;
	return n;
}
681

682 683 684 685 686
void
runtime·helpgc(int32 nproc)
{
	M *mp;
	int32 n;
687 688

	runtime·lock(&runtime·sched);
689 690 691 692
	for(n = 1; n < nproc; n++) { // one M is currently running
		mp = mget(nil);
		if(mp == nil)
			runtime·throw("runtime·gcprocs inconsistency");
693 694 695
		mp->helpgc = 1;
		mp->waitnextg = 0;
		runtime·notewakeup(&mp->havenextg);
696 697 698 699
	}
	runtime·unlock(&runtime·sched);
}

700
void
701
runtime·stoptheworld(void)
702
{
703 704
	uint32 v;

705
	schedlock();
706
	runtime·gcwaiting = 1;
707 708 709 710 711 712 713 714 715

	setmcpumax(1);

	// while mcpu > 1
	for(;;) {
		v = runtime·sched.atomic;
		if(atomic_mcpu(v) <= 1)
			break;

Russ Cox's avatar
Russ Cox committed
716 717
		// It would be unsafe for multiple threads to be using
		// the stopped note at once, but there is only
718
		// ever one thread doing garbage collection.
719
		runtime·noteclear(&runtime·sched.stopped);
720 721 722 723 724 725 726 727
		if(atomic_waitstop(v))
			runtime·throw("invalid waitstop");

		// atomic { waitstop = 1 }, predicated on mcpu <= 1 check above
		// still being true.
		if(!runtime·cas(&runtime·sched.atomic, v, v+(1<<waitstopShift)))
			continue;

728
		schedunlock();
729
		runtime·notesleep(&runtime·sched.stopped);
730
		schedlock();
731
	}
732
	runtime·singleproc = runtime·gomaxprocs == 1;
733
	schedunlock();
734 735 736
}

void
737
runtime·starttheworld(void)
738
{
Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
739
	M *mp;
740 741 742 743 744 745 746 747
	int32 max;
	
	// Figure out how many CPUs GC could possibly use.
	max = runtime·gomaxprocs;
	if(max > runtime·ncpu)
		max = runtime·ncpu;
	if(max > MaxGcproc)
		max = MaxGcproc;
748

749
	schedlock();
750
	runtime·gcwaiting = 0;
751
	setmcpumax(runtime·gomaxprocs);
752
	matchmg();
753 754 755 756 757 758 759 760
	if(runtime·gcprocs() < max && canaddmcpu()) {
		// If GC could have used another helper proc, start one now,
		// in the hope that it will be available next time.
		// It would have been even better to start it before the collection,
		// but doing so requires allocating memory, so it's tricky to
		// coordinate.  This lazy approach works out in practice:
		// we don't mind if the first couple gc rounds don't have quite
		// the maximum number of procs.
761 762 763 764 765
		// canaddmcpu above did mcpu++
		// (necessary, because m will be doing various
		// initialization work so is definitely running),
		// but m is not running a specific goroutine,
		// so set the helpgc flag as a signal to m's
766
		// first schedule(nil) to mcpu-- and grunning--.
Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
767 768
		mp = runtime·newm();
		mp->helpgc = 1;
769
		runtime·sched.grunning++;
770
	}
771
	schedunlock();
772 773
}

Russ Cox's avatar
Russ Cox committed
774 775
// Called to start an M.
void
776
runtime·mstart(void)
Russ Cox's avatar
Russ Cox committed
777
{
778 779 780 781 782
	// It is used by windows-386 only. Unfortunately, seh needs
	// to be located on os stack, and mstart runs on os stack
	// for both m0 and m.
	SEH seh;

783
	if(g != m->g0)
784
		runtime·throw("bad runtime·mstart");
785 786 787 788 789 790

	// Record top of stack for use by mcall.
	// Once we call schedule we're never coming back,
	// so other calls can reuse this stack space.
	runtime·gosave(&m->g0->sched);
	m->g0->sched.pc = (void*)-1;  // make sure it is never used
791
	m->seh = &seh;
792
	runtime·asminit();
793
	runtime·minit();
794 795 796 797 798 799

	// Install signal handlers; after minit so that minit can
	// prepare the thread to be able to handle the signals.
	if(m == &runtime·m0)
		runtime·initsig();

800
	schedule(nil);
801 802 803 804

	// TODO(brainman): This point is never reached, because scheduler
	// does not release os threads at the moment. But once this path
	// is enabled, we must remove our seh here.
Russ Cox's avatar
Russ Cox committed
805 806
}

Russ Cox's avatar
Russ Cox committed
807 808 809 810 811 812 813 814 815 816 817 818 819
// When running with cgo, we call libcgo_thread_start
// to start threads for us so that we can play nicely with
// foreign code.
void (*libcgo_thread_start)(void*);

typedef struct CgoThreadStart CgoThreadStart;
struct CgoThreadStart
{
	M *m;
	G *g;
	void (*fn)(void);
};

820
// Kick off new m's as needed (up to mcpumax).
821 822 823 824
// Sched is locked.
static void
matchmg(void)
{
825 826
	G *gp;
	M *mp;
827

828
	if(m->mallocing || m->gcing)
829
		return;
830 831

	while(haveg() && canaddmcpu()) {
832 833
		gp = gget();
		if(gp == nil)
834
			runtime·throw("gget inconsistency");
835

836 837
		// Find the m that will run gp.
		if((mp = mget(gp)) == nil)
838
			mp = runtime·newm();
839 840 841 842
		mnextg(mp, gp);
	}
}

843
// Create a new m.  It will start off with a call to runtime·mstart.
844 845
M*
runtime·newm(void)
846
{
Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
847
	M *mp;
848
	static Type *mtype;  // The Go type M
849

850 851 852 853 854 855 856
	if(mtype == nil) {
		Eface e;
		runtime·gc_m_ptr(&e);
		mtype = ((PtrType*)e.type)->elem;
	}

	mp = runtime·cnew(mtype);
Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
857
	mcommoninit(mp);
858 859 860 861 862 863 864

	if(runtime·iscgo) {
		CgoThreadStart ts;

		if(libcgo_thread_start == nil)
			runtime·throw("libcgo_thread_start missing");
		// pthread_create will make us a stack.
Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
865 866 867
		mp->g0 = runtime·malg(-1);
		ts.m = mp;
		ts.g = mp->g0;
868 869 870 871 872
		ts.fn = runtime·mstart;
		runtime·asmcgocall(libcgo_thread_start, &ts);
	} else {
		if(Windows)
			// windows will layout sched stack on os stack
Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
873
			mp->g0 = runtime·malg(-1);
874
		else
Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
875 876
			mp->g0 = runtime·malg(8192);
		runtime·newosproc(mp, mp->g0, (byte*)mp->g0->stackbase, runtime·mstart);
877
	}
878

Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
879
	return mp;
880 881
}

882 883 884 885
// One round of scheduler: find a goroutine and run it.
// The argument is the goroutine that was running before
// schedule was called, or nil if this is the first call.
// Never returns.
Russ Cox's avatar
Russ Cox committed
886
static void
887
schedule(G *gp)
Ken Thompson's avatar
Ken Thompson committed
888
{
Russ Cox's avatar
Russ Cox committed
889
	int32 hz;
890
	uint32 v;
Russ Cox's avatar
Russ Cox committed
891

892
	schedlock();
893
	if(gp != nil) {
Russ Cox's avatar
Russ Cox committed
894
		// Just finished running gp.
895
		gp->m = nil;
896
		runtime·sched.grunning--;
897

898 899 900 901 902
		// atomic { mcpu-- }
		v = runtime·xadd(&runtime·sched.atomic, -1<<mcpuShift);
		if(atomic_mcpu(v) > maxgomaxprocs)
			runtime·throw("negative mcpu in scheduler");

Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
903
		switch(gp->status) {
Russ Cox's avatar
Russ Cox committed
904
		case Grunnable:
905
		case Gdead:
Russ Cox's avatar
Russ Cox committed
906
			// Shouldn't have been running!
907
			runtime·throw("bad gp->status in sched");
908
		case Grunning:
Russ Cox's avatar
Russ Cox committed
909 910
			gp->status = Grunnable;
			gput(gp);
911
			break;
Russ Cox's avatar
Russ Cox committed
912
		case Gmoribund:
Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
913 914
			if(raceenabled)
				runtime·racegoend(gp->goid);
Russ Cox's avatar
Russ Cox committed
915
			gp->status = Gdead;
916 917 918 919
			if(gp->lockedm) {
				gp->lockedm = nil;
				m->lockedg = nil;
			}
Russ Cox's avatar
Russ Cox committed
920
			gp->idlem = nil;
921
			runtime·unwindstack(gp, nil);
Russ Cox's avatar
Russ Cox committed
922
			gfput(gp);
923 924
			if(--runtime·sched.gcount == 0)
				runtime·exit(0);
925 926
			break;
		}
Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
927
		if(gp->readyonstop) {
928 929 930
			gp->readyonstop = 0;
			readylocked(gp);
		}
931
	} else if(m->helpgc) {
932
		// Bootstrap m or new m started by starttheworld.
933 934 935 936
		// atomic { mcpu-- }
		v = runtime·xadd(&runtime·sched.atomic, -1<<mcpuShift);
		if(atomic_mcpu(v) > maxgomaxprocs)
			runtime·throw("negative mcpu in scheduler");
937 938
		// Compensate for increment in starttheworld().
		runtime·sched.grunning--;
939
		m->helpgc = 0;
940 941 942 943
	} else if(m->nextg != nil) {
		// New m started by matchmg.
	} else {
		runtime·throw("invalid m state in scheduler");
944 945
	}

946
	// Find (or wait for) g to run.  Unlocks runtime·sched.
Russ Cox's avatar
Russ Cox committed
947
	gp = nextgandunlock();
948
	gp->readyonstop = 0;
949
	gp->status = Grunning;
Ken Thompson's avatar
Ken Thompson committed
950
	m->curg = gp;
951
	gp->m = m;
952

Russ Cox's avatar
Russ Cox committed
953 954 955 956 957
	// Check whether the profiler needs to be turned on or off.
	hz = runtime·sched.profilehz;
	if(m->profilehz != hz)
		runtime·resetcpuprofiler(hz);

958 959
	if(gp->sched.pc == (byte*)runtime·goexit) {	// kickoff
		runtime·gogocall(&gp->sched, (void(*)(void))gp->entry);
Ken Thompson's avatar
Ken Thompson committed
960
	}
961
	runtime·gogo(&gp->sched, 0);
Ken Thompson's avatar
Ken Thompson committed
962 963
}

Russ Cox's avatar
Russ Cox committed
964 965 966 967
// Enter scheduler.  If g->status is Grunning,
// re-queues g and runs everyone else who is waiting
// before running g again.  If g->status is Gmoribund,
// kills off g.
968 969 970
// Cannot split stack because it is called from exitsyscall.
// See comment below.
#pragma textflag 7
971
void
972
runtime·gosched(void)
973
{
974
	if(m->locks != 0)
975
		runtime·throw("gosched holding locks");
976
	if(g == m->g0)
977
		runtime·throw("gosched of g0");
978
	runtime·mcall(schedule);
979 980
}

981 982 983 984 985 986 987 988 989 990 991 992
// Puts the current goroutine into a waiting state and unlocks the lock.
// The goroutine can be made runnable again by calling runtime·ready(gp).
void
runtime·park(void (*unlockf)(Lock*), Lock *lock, int8 *reason)
{
	g->status = Gwaiting;
	g->waitreason = reason;
	if(unlockf)
		unlockf(lock);
	runtime·gosched();
}

993 994
// The goroutine g is about to enter a system call.
// Record that it's not using the cpu anymore.
995 996
// This is called only from the go syscall library and cgocall,
// not from the low-level system calls used by the runtime.
997
//
998
// Entersyscall cannot split the stack: the runtime·gosave must
999 1000
// make g->sched refer to the caller's stack segment, because
// entersyscall is going to return immediately after.
1001 1002
// It's okay to call matchmg and notewakeup even after
// decrementing mcpu, because we haven't released the
1003
// sched lock yet, so the garbage collector cannot be running.
1004
#pragma textflag 7
1005
void
1006
runtime·entersyscall(void)
Russ Cox's avatar
Russ Cox committed
1007
{
1008
	uint32 v;
1009

1010 1011 1012
	if(m->profilehz > 0)
		runtime·setprof(false);

1013 1014 1015 1016 1017
	// Leave SP around for gc and traceback.
	runtime·gosave(&g->sched);
	g->gcsp = g->sched.sp;
	g->gcstack = g->stackbase;
	g->gcguard = g->stackguard;
1018
	g->status = Gsyscall;
1019
	if(g->gcsp < g->gcguard-StackGuard || g->gcstack < g->gcsp) {
1020 1021
		// runtime·printf("entersyscall inconsistent %p [%p,%p]\n",
		//	g->gcsp, g->gcguard-StackGuard, g->gcstack);
1022 1023
		runtime·throw("entersyscall");
	}
1024 1025 1026 1027 1028 1029 1030

	// Fast path.
	// The slow path inside the schedlock/schedunlock will get
	// through without stopping if it does:
	//	mcpu--
	//	gwait not true
	//	waitstop && mcpu <= mcpumax not true
1031
	// If we can do the same with a single atomic add,
1032
	// then we can skip the locks.
1033 1034 1035
	v = runtime·xadd(&runtime·sched.atomic, -1<<mcpuShift);
	if(!atomic_gwaiting(v) && (!atomic_waitstop(v) || atomic_mcpu(v) > atomic_mcpumax(v)))
		return;
1036 1037

	schedlock();
1038
	v = runtime·atomicload(&runtime·sched.atomic);
1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051
	if(atomic_gwaiting(v)) {
		matchmg();
		v = runtime·atomicload(&runtime·sched.atomic);
	}
	if(atomic_waitstop(v) && atomic_mcpu(v) <= atomic_mcpumax(v)) {
		runtime·xadd(&runtime·sched.atomic, -1<<waitstopShift);
		runtime·notewakeup(&runtime·sched.stopped);
	}

	// Re-save sched in case one of the calls
	// (notewakeup, matchmg) triggered something using it.
	runtime·gosave(&g->sched);

1052
	schedunlock();
1053 1054 1055 1056 1057 1058 1059
}

// The goroutine g exited its system call.
// Arrange for it to run on a cpu again.
// This is called only from the go syscall library, not
// from the low-level system calls used by the runtime.
void
1060
runtime·exitsyscall(void)
1061
{
1062
	uint32 v;
1063 1064

	// Fast path.
1065
	// If we can do the mcpu++ bookkeeping and
1066 1067 1068
	// find that we still have mcpu <= mcpumax, then we can
	// start executing Go code immediately, without having to
	// schedlock/schedunlock.
1069 1070 1071 1072 1073 1074
	v = runtime·xadd(&runtime·sched.atomic, (1<<mcpuShift));
	if(m->profilehz == runtime·sched.profilehz && atomic_mcpu(v) <= atomic_mcpumax(v)) {
		// There's a cpu for us, so we can run.
		g->status = Grunning;
		// Garbage collector isn't running (since we are),
		// so okay to clear gcstack.
1075
		g->gcstack = (uintptr)nil;
1076 1077 1078

		if(m->profilehz > 0)
			runtime·setprof(true);
1079
		return;
Russ Cox's avatar
Russ Cox committed
1080
	}
1081

1082 1083 1084 1085 1086
	// Tell scheduler to put g back on the run queue:
	// mostly equivalent to g->status = Grunning,
	// but keeps the garbage collector from thinking
	// that g is running right now, which it's not.
	g->readyonstop = 1;
Russ Cox's avatar
Russ Cox committed
1087

1088
	// All the cpus are taken.
1089
	// The scheduler will ready g and put this m to sleep.
1090
	// When the scheduler takes g away from m,
1091 1092
	// it will undo the runtime·sched.mcpu++ above.
	runtime·gosched();
1093

1094 1095 1096 1097 1098 1099
	// Gosched returned, so we're allowed to run now.
	// Delete the gcstack information that we left for
	// the garbage collector during the system call.
	// Must wait until now because until gosched returns
	// we don't know for sure that the garbage collector
	// is not running.
1100
	g->gcstack = (uintptr)nil;
Russ Cox's avatar
Russ Cox committed
1101 1102
}

1103 1104 1105
// Called from runtime·lessstack when returning from a function which
// allocated a new stack segment.  The function's return value is in
// m->cret.
Ken Thompson's avatar
Ken Thompson committed
1106
void
1107
runtime·oldstack(void)
Ken Thompson's avatar
Ken Thompson committed
1108
{
1109 1110
	Stktop *top;
	Gobuf label;
1111
	uint32 argsize;
Russ Cox's avatar
Russ Cox committed
1112
	uintptr cret;
1113 1114
	byte *sp, *old;
	uintptr *src, *dst, *dstend;
1115
	G *gp;
1116
	int64 goid;
Ken Thompson's avatar
Ken Thompson committed
1117

1118
//printf("oldstack m->cret=%p\n", m->cret);
Ken Thompson's avatar
Ken Thompson committed
1119

1120 1121
	gp = m->curg;
	top = (Stktop*)gp->stackbase;
1122
	old = (byte*)gp->stackguard - StackGuard;
Ken Thompson's avatar
Ken Thompson committed
1123
	sp = (byte*)top;
1124
	argsize = top->argsize;
1125 1126
	if(argsize > 0) {
		sp -= argsize;
1127 1128 1129 1130 1131
		dst = (uintptr*)top->argp;
		dstend = dst + argsize/sizeof(*dst);
		src = (uintptr*)sp;
		while(dst < dstend)
			*dst++ = *src++;
Ken Thompson's avatar
Ken Thompson committed
1132
	}
1133
	goid = top->gobuf.g->goid;	// fault if g is bad, before gogo
1134
	USED(goid);
Ken Thompson's avatar
Ken Thompson committed
1135

1136 1137 1138 1139 1140
	label = top->gobuf;
	gp->stackbase = (uintptr)top->stackbase;
	gp->stackguard = (uintptr)top->stackguard;
	if(top->free != 0)
		runtime·stackfree(old, top->free);
Ken Thompson's avatar
Ken Thompson committed
1141

Russ Cox's avatar
Russ Cox committed
1142 1143
	cret = m->cret;
	m->cret = 0;  // drop reference
1144
	runtime·gogo(&label, cret);
1145 1146
}

1147 1148 1149 1150 1151
// Called from reflect·call or from runtime·morestack when a new
// stack segment is needed.  Allocate a new stack big enough for
// m->moreframesize bytes, copy m->moreargsize bytes to the new frame,
// and then act as though runtime·lessstack called the function at
// m->morepc.
Ken Thompson's avatar
Ken Thompson committed
1152
void
1153
runtime·newstack(void)
Ken Thompson's avatar
Ken Thompson committed
1154
{
1155
	int32 framesize, minalloc, argsize;
Ken Thompson's avatar
Ken Thompson committed
1156 1157
	Stktop *top;
	byte *stk, *sp;
1158
	uintptr *src, *dst, *dstend;
1159
	G *gp;
1160
	Gobuf label;
1161 1162
	bool reflectcall;
	uintptr free;
Ken Thompson's avatar
Ken Thompson committed
1163

1164 1165
	framesize = m->moreframesize;
	argsize = m->moreargsize;
1166
	gp = m->curg;
Russ Cox's avatar
Russ Cox committed
1167

1168 1169
	if(m->morebuf.sp < gp->stackguard - StackGuard) {
		runtime·printf("runtime: split stack overflow: %p < %p\n", m->morebuf.sp, gp->stackguard - StackGuard);
1170 1171
		runtime·throw("runtime: split stack overflow");
	}
1172 1173 1174 1175
	if(argsize % sizeof(uintptr) != 0) {
		runtime·printf("runtime: stack split with misaligned argsize %d\n", argsize);
		runtime·throw("runtime: stack split argsize");
	}
Russ Cox's avatar
Russ Cox committed
1176

1177
	minalloc = 0;
1178
	reflectcall = framesize==1;
1179
	if(reflectcall) {
1180
		framesize = 0;
1181 1182 1183 1184 1185 1186 1187
		// moreframesize_minalloc is only set in runtime·gc(),
		// that calls newstack via reflect·call().
		minalloc = m->moreframesize_minalloc;
		m->moreframesize_minalloc = 0;
		if(framesize < minalloc)
			framesize = minalloc;
	}
1188

1189
	if(reflectcall && minalloc == 0 && m->morebuf.sp - sizeof(Stktop) - argsize - 32 > gp->stackguard) {
1190
		// special case: called from reflect.call (framesize==1)
1191 1192 1193 1194 1195
		// to call code with an arbitrary argument size,
		// and we have enough space on the current stack.
		// the new Stktop* is necessary to unwind, but
		// we don't need to create a new segment.
		top = (Stktop*)(m->morebuf.sp - sizeof(*top));
1196
		stk = (byte*)gp->stackguard - StackGuard;
1197
		free = 0;
1198 1199
	} else {
		// allocate new segment.
1200
		framesize += argsize;
1201
		framesize += StackExtra;	// room for more functions, Stktop.
1202 1203 1204
		if(framesize < StackMin)
			framesize = StackMin;
		framesize += StackSystem;
1205 1206
		stk = runtime·stackalloc(framesize);
		top = (Stktop*)(stk+framesize-sizeof(*top));
1207
		free = framesize;
1208
	}
1209

1210 1211
	if(0) {
		runtime·printf("newstack framesize=%d argsize=%d morepc=%p moreargp=%p gobuf=%p, %p top=%p old=%p\n",
1212
			framesize, argsize, m->morepc, m->moreargp, m->morebuf.pc, m->morebuf.sp, top, gp->stackbase);
1213
	}
Ken Thompson's avatar
Ken Thompson committed
1214

1215 1216
	top->stackbase = (byte*)gp->stackbase;
	top->stackguard = (byte*)gp->stackguard;
1217
	top->gobuf = m->morebuf;
1218 1219
	top->argp = m->moreargp;
	top->argsize = argsize;
1220
	top->free = free;
Russ Cox's avatar
Russ Cox committed
1221 1222
	m->moreargp = nil;
	m->morebuf.pc = nil;
1223
	m->morebuf.sp = (uintptr)nil;
1224

Russ Cox's avatar
Russ Cox committed
1225
	// copy flag from panic
1226 1227
	top->panic = gp->ispanic;
	gp->ispanic = false;
Ken Thompson's avatar
Ken Thompson committed
1228

1229 1230
	gp->stackbase = (uintptr)top;
	gp->stackguard = (uintptr)stk + StackGuard;
Ken Thompson's avatar
Ken Thompson committed
1231 1232

	sp = (byte*)top;
1233 1234
	if(argsize > 0) {
		sp -= argsize;
1235 1236 1237 1238 1239
		dst = (uintptr*)sp;
		dstend = dst + argsize/sizeof(*dst);
		src = (uintptr*)top->argp;
		while(dst < dstend)
			*dst++ = *src++;
1240 1241 1242 1243 1244
	}
	if(thechar == '5') {
		// caller would have saved its LR below args.
		sp -= sizeof(void*);
		*(void**)sp = nil;
Ken Thompson's avatar
Ken Thompson committed
1245 1246
	}

1247 1248
	// Continue as if lessstack had just called m->morepc
	// (the PC that decided to grow the stack).
1249
	label.sp = (uintptr)sp;
1250
	label.pc = (byte*)runtime·lessstack;
1251
	label.g = m->curg;
1252
	runtime·gogocall(&label, m->morepc);
Ken Thompson's avatar
Ken Thompson committed
1253 1254 1255 1256

	*(int32*)345 = 123;	// never return
}

1257 1258 1259 1260
// Hook used by runtime·malg to call runtime·stackalloc on the
// scheduler stack.  This exists because runtime·stackalloc insists
// on being called on the scheduler stack, to avoid trying to grow
// the stack while allocating a new stack segment.
1261 1262 1263 1264 1265 1266 1267
static void
mstackalloc(G *gp)
{
	gp->param = runtime·stackalloc((uintptr)gp->param);
	runtime·gogo(&gp->sched, 0);
}

1268
// Allocate a new g, with a stack big enough for stacksize bytes.
1269
G*
1270
runtime·malg(int32 stacksize)
1271
{
1272
	G *newg;
1273
	byte *stk;
1274

1275 1276 1277 1278
	if(StackTop < sizeof(Stktop)) {
		runtime·printf("runtime: SizeofStktop=%d, should be >=%d\n", (int32)StackTop, (int32)sizeof(Stktop));
		runtime·throw("runtime: bad stack.h");
	}
1279

1280
	newg = runtime·malloc(sizeof(G));
1281
	if(stacksize >= 0) {
1282 1283 1284 1285 1286 1287
		if(g == m->g0) {
			// running on scheduler stack already.
			stk = runtime·stackalloc(StackSystem + stacksize);
		} else {
			// have to call stackalloc on scheduler stack.
			g->param = (void*)(StackSystem + stacksize);
1288
			runtime·mcall(mstackalloc);
1289 1290 1291
			stk = g->param;
			g->param = nil;
		}
1292 1293 1294 1295
		newg->stack0 = (uintptr)stk;
		newg->stackguard = (uintptr)stk + StackGuard;
		newg->stackbase = (uintptr)stk + StackSystem + stacksize - sizeof(Stktop);
		runtime·memclr((byte*)newg->stackbase, sizeof(Stktop));
1296
	}
1297
	return newg;
1298 1299
}

1300 1301 1302 1303 1304 1305 1306
// Create a new g running fn with siz bytes of arguments.
// Put it on the queue of g's waiting to run.
// The compiler turns a go statement into a call to this.
// Cannot split the stack because it assumes that the arguments
// are available sequentially after &fn; they would not be
// copied if a stack split occurred.  It's OK for this to call
// functions that split the stack.
1307 1308
#pragma textflag 7
void
1309
runtime·newproc(int32 siz, byte* fn, ...)
1310
{
1311
	byte *argp;
1312

1313 1314 1315 1316
	if(thechar == '5')
		argp = (byte*)(&fn+2);  // skip caller's saved LR
	else
		argp = (byte*)(&fn+1);
1317
	runtime·newproc1(fn, argp, siz, 0, runtime·getcallerpc(&siz));
1318 1319
}

1320 1321 1322 1323
// Create a new g running fn with narg bytes of arguments starting
// at argp and returning nret bytes of results.  callerpc is the
// address of the go statement that created this.  The new g is put
// on the queue of g's waiting to run.
1324
G*
1325
runtime·newproc1(byte *fn, byte *argp, int32 narg, int32 nret, void *callerpc)
1326
{
1327
	byte *sp;
1328
	G *newg;
1329
	int32 siz;
1330
	int64 goid;
1331

1332 1333
//printf("newproc1 %p %p narg=%d nret=%d\n", fn, argp, narg, nret);
	siz = narg + nret;
1334
	siz = (siz+7) & ~7;
1335

1336 1337 1338 1339 1340 1341
	// We could instead create a secondary stack frame
	// and make it look like goexit was on the original but
	// the call to the actual goroutine function was split.
	// Not worth it: this is almost always an error.
	if(siz > StackMin - 1024)
		runtime·throw("runtime.newproc: function arguments too large for new goroutine");
1342

1343
	goid = runtime·xadd64((uint64*)&runtime·sched.goidgen, 1);
Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
1344 1345 1346
	if(raceenabled)
		runtime·racegostart(goid, callerpc);

1347
	schedlock();
1348

Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
1349
	if((newg = gfget()) != nil) {
1350
		if(newg->stackguard - StackGuard != newg->stack0)
1351
			runtime·throw("invalid stack in newg");
1352
	} else {
1353
		newg = runtime·malg(StackMin);
Russ Cox's avatar
Russ Cox committed
1354 1355 1356 1357 1358
		if(runtime·lastg == nil)
			runtime·allg = newg;
		else
			runtime·lastg->alllink = newg;
		runtime·lastg = newg;
1359
	}
Russ Cox's avatar
Russ Cox committed
1360 1361
	newg->status = Gwaiting;
	newg->waitreason = "new goroutine";
1362

1363
	sp = (byte*)newg->stackbase;
1364
	sp -= siz;
1365
	runtime·memmove(sp, argp, narg);
1366 1367 1368 1369 1370
	if(thechar == '5') {
		// caller's LR
		sp -= sizeof(void*);
		*(void**)sp = nil;
	}
1371

1372
	newg->sched.sp = (uintptr)sp;
1373
	newg->sched.pc = (byte*)runtime·goexit;
1374 1375
	newg->sched.g = newg;
	newg->entry = fn;
1376
	newg->gopc = (uintptr)callerpc;
1377

1378
	runtime·sched.gcount++;
Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
1379
	newg->goid = goid;
1380

1381
	newprocreadylocked(newg);
1382
	schedunlock();
1383

1384
	return newg;
1385 1386 1387
//printf(" goid=%d\n", newg->goid);
}

1388 1389
// Put on gfree list.  Sched must be locked.
static void
Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
1390
gfput(G *gp)
1391
{
Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
1392
	if(gp->stackguard - StackGuard != gp->stack0)
1393
		runtime·throw("invalid stack in gfput");
Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
1394 1395
	gp->schedlink = runtime·sched.gfree;
	runtime·sched.gfree = gp;
1396 1397 1398 1399 1400 1401
}

// Get from gfree list.  Sched must be locked.
static G*
gfget(void)
{
Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
1402
	G *gp;
1403

Dmitriy Vyukov's avatar
Dmitriy Vyukov committed
1404 1405 1406 1407
	gp = runtime·sched.gfree;
	if(gp)
		runtime·sched.gfree = gp->schedlink;
	return gp;
1408
}
1409

1410
void
1411
runtime·Breakpoint(void)
1412
{
1413
	runtime·breakpoint();
1414 1415 1416
}

void
1417
runtime·Gosched(void)
1418
{
1419
	runtime·gosched();
1420 1421
}

1422
// Implementation of runtime.GOMAXPROCS.
1423
// delete when scheduler is stronger
1424
int32
1425
runtime·gomaxprocsfunc(int32 n)
1426
{
1427
	int32 ret;
1428
	uint32 v;
1429

1430
	schedlock();
Russ Cox's avatar
Russ Cox committed
1431
	ret = runtime·gomaxprocs;
1432
	if(n <= 0)
1433
		n = ret;
1434 1435
	if(n > maxgomaxprocs)
		n = maxgomaxprocs;
Russ Cox's avatar
Russ Cox committed
1436
	runtime·gomaxprocs = n;
1437 1438
	if(runtime·gomaxprocs > 1)
		runtime·singleproc = false;
1439 1440 1441
 	if(runtime·gcwaiting != 0) {
 		if(atomic_mcpumax(runtime·sched.atomic) != 1)
 			runtime·throw("invalid mcpumax during gc");
1442 1443 1444
		schedunlock();
		return ret;
	}
1445 1446 1447 1448 1449 1450 1451

	setmcpumax(n);

	// If there are now fewer allowed procs
	// than procs running, stop.
	v = runtime·atomicload(&runtime·sched.atomic);
	if(atomic_mcpu(v) > n) {
1452
		schedunlock();
1453
		runtime·gosched();
1454
		return ret;
1455 1456 1457
	}
	// handle more procs
	matchmg();
1458
	schedunlock();
1459
	return ret;
1460 1461
}

1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472
void
runtime·LockOSThread(void)
{
	if(m == &runtime·m0 && runtime·sched.init) {
		runtime·sched.lockmain = true;
		return;
	}
	m->lockedg = g;
	g->lockedm = m;
}

1473
void
1474
runtime·UnlockOSThread(void)
1475
{
1476 1477 1478 1479
	if(m == &runtime·m0 && runtime·sched.init) {
		runtime·sched.lockmain = false;
		return;
	}
1480 1481 1482 1483
	m->lockedg = nil;
	g->lockedm = nil;
}

1484 1485 1486 1487 1488 1489
bool
runtime·lockedOSThread(void)
{
	return g->lockedm != nil && m->lockedg != nil;
}

1490 1491 1492 1493 1494 1495 1496 1497
// for testing of callbacks
void
runtime·golockedOSThread(bool ret)
{
	ret = runtime·lockedOSThread();
	FLUSH(&ret);
}

1498 1499
// for testing of wire, unwire
void
1500
runtime·mid(uint32 ret)
1501 1502 1503 1504
{
	ret = m->id;
	FLUSH(&ret);
}
Keith Rarick's avatar
Keith Rarick committed
1505 1506

void
Russ Cox's avatar
Russ Cox committed
1507
runtime·NumGoroutine(intgo ret)
Keith Rarick's avatar
Keith Rarick committed
1508 1509 1510 1511
{
	ret = runtime·sched.gcount;
	FLUSH(&ret);
}
1512

1513 1514 1515 1516 1517 1518
int32
runtime·gcount(void)
{
	return runtime·sched.gcount;
}

1519 1520 1521 1522 1523
int32
runtime·mcount(void)
{
	return runtime·sched.mcount;
}
1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535

void
runtime·badmcall(void)  // called from assembly
{
	runtime·throw("runtime: mcall called on m->g0 stack");
}

void
runtime·badmcall2(void)  // called from assembly
{
	runtime·throw("runtime: mcall function returned");
}
Russ Cox's avatar
Russ Cox committed
1536 1537 1538 1539 1540 1541 1542 1543

static struct {
	Lock;
	void (*fn)(uintptr*, int32);
	int32 hz;
	uintptr pcbuf[100];
} prof;

1544
// Called if we receive a SIGPROF signal.
Russ Cox's avatar
Russ Cox committed
1545 1546 1547 1548
void
runtime·sigprof(uint8 *pc, uint8 *sp, uint8 *lr, G *gp)
{
	int32 n;
1549

Russ Cox's avatar
Russ Cox committed
1550 1551
	if(prof.fn == nil || prof.hz == 0)
		return;
1552

Russ Cox's avatar
Russ Cox committed
1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563
	runtime·lock(&prof);
	if(prof.fn == nil) {
		runtime·unlock(&prof);
		return;
	}
	n = runtime·gentraceback(pc, sp, lr, gp, 0, prof.pcbuf, nelem(prof.pcbuf));
	if(n > 0)
		prof.fn(prof.pcbuf, n);
	runtime·unlock(&prof);
}

1564
// Arrange to call fn with a traceback hz times a second.
Russ Cox's avatar
Russ Cox committed
1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587
void
runtime·setcpuprofilerate(void (*fn)(uintptr*, int32), int32 hz)
{
	// Force sane arguments.
	if(hz < 0)
		hz = 0;
	if(hz == 0)
		fn = nil;
	if(fn == nil)
		hz = 0;

	// Stop profiler on this cpu so that it is safe to lock prof.
	// if a profiling signal came in while we had prof locked,
	// it would deadlock.
	runtime·resetcpuprofiler(0);

	runtime·lock(&prof);
	prof.fn = fn;
	prof.hz = hz;
	runtime·unlock(&prof);
	runtime·lock(&runtime·sched);
	runtime·sched.profilehz = hz;
	runtime·unlock(&runtime·sched);
1588

Russ Cox's avatar
Russ Cox committed
1589 1590 1591
	if(hz != 0)
		runtime·resetcpuprofiler(hz);
}