* comment, clean up scheduler

* rewrite lock implementation to be correct
  (tip: never assume that an algorithm you found
  in a linux man page is correct.)
* delete unneeded void* arg from clone fn
* replace Rendez with Note
* comment mal better
* use 6c -w, fix warnings
* mark all assembly functions 7

R=r
DELTA=828  (338 added, 221 deleted, 269 changed)
OCL=13884
CL=13886

src/runtime/Makefile

@@ -49,10 +49,10 @@ clean:
 	rm -f *.$(O) *.a runtime.acid
 
 %.$O:	%.c
-	$(CC) $<
+	$(CC) -w $<
 
 sys_file.$O:	sys_file.c sys_types.h $(OS_H)
-	$(CC) -D$(GOARCH)_$(GOOS) $<
+	$(CC) -w -D$(GOARCH)_$(GOOS) $<
 
 %.$O:	%.s
 	$(AS) $<

src/runtime/amd64_linux.h

@@ -48,6 +48,6 @@ struct stat {
 
 // Linux-specific system calls
 int64	futex(uint32*, int32, uint32, struct timespec*, uint32*, uint32);
-int64	clone(int32, void*, M*, G*, void(*)(void*), void*);
+int64	clone(int32, void*, M*, G*, void(*)(void));
 int64	select(int32, void*, void*, void*, void*);
 

src/runtime/chan.c

@@ -487,7 +487,7 @@ sys·selectgo(Select *sel)
 	SudoG *sg;
 	G *gp;
 
-	byte *ae, *as;
+	byte *as;
 
 	if(xxx) {
 		prints("selectgo: sel=");
@@ -630,6 +630,8 @@ sys·selectgo(Select *sel)
 asynr:
 asyns:
 	throw("asyn");
+	return;	// compiler doesn't know throw doesn't return
+
 gotr:
 	// recv path to wakeup the sender (sg)
 	if(xxx) {

src/runtime/map.c

@@ -199,7 +199,6 @@ out:
 void
 sys·mapassign1(Hmap *m, ...)
 {
-	Link **ll;
 	byte *ak, *av;
 
 	ak = (byte*)&m + m->ko;

src/runtime/print.c

@@ -8,7 +8,6 @@
 void
 dump(byte *p, int32 n)
 {
-	uint32 v;
 	int32 i;
 
 	for(i=0; i<n; i++) {

src/runtime/rt0_amd64.s

@@ -41,7 +41,7 @@ TEXT	_rt0_amd64(SB),7,$-8
 	PUSHQ	$main·main(SB)		// entry
 	PUSHQ	$16			// arg size
 	CALL	sys·newproc(SB)
-	CALL	gom0init(SB)
+	CALL	m0init(SB)
 	POPQ	AX
 	POPQ	AX
 

src/runtime/rt1_amd64_darwin.c

@@ -191,7 +191,7 @@ sys·sleep(int64 ms)
 void
 lock(Lock *l)
 {
-	if(xadd(&l->key, 1) == 1)
+	if(cas(&l->key, 0, 1))
 		return;
 	unimplemented("lock wait");
 }
@@ -199,43 +199,33 @@ lock(Lock *l)
 void
 unlock(Lock *l)
 {
-	if(xadd(&l->key, -1) == 0)
+	if(cas(&l->key, 1, 0))
 		return;
 	unimplemented("unlock wakeup");
 }
 
 void
-rsleep(Rendez *r)
+noteclear(Note *n)
 {
-	unimplemented("rsleep");
-
-	// dumb implementation:
-	r->sleeping = 1;
-	unlock(r->l);
-	while(r->sleeping)
-		;
-	lock(r->l);
+	n->lock.key = 0;
+	lock(&n->lock);
 }
 
 void
-rwakeup(Rendez *r)
+notesleep(Note *n)
 {
-	unimplemented("rwakeup");
-
-	// dumb implementation:
-	r->sleeping = 0;
+	lock(&n->lock);
+	unlock(&n->lock);
 }
 
 void
-rwakeupandunlock(Rendez *r)
+notewakeup(Note *n)
 {
-	// dumb implementation:
-	rwakeup(r);
-	unlock(r->l);
+	unlock(&n->lock);
 }
 
 void
-newosproc(M *mm, G *gg, void *stk, void (*fn)(void*), void *arg)
+newosproc(M *mm, G *gg, void *stk, void (*fn)(void))
 {
 	unimplemented("newosproc");
 }

src/runtime/rt1_amd64_linux.c

@@ -138,8 +138,6 @@ typedef struct sigaction {
 void
 sighandler(int32 sig, siginfo* info, void** context)
 {
-	int32 i;
-
 	if(sig < 0 || sig >= NSIG){
 		prints("Signal ");
 		sys·printint(sig);
@@ -179,16 +177,14 @@ initsig(void)
 		}
 }
 
-// Linux futex.  The simple cases really are simple:
+// Linux futex.
 //
-//	futex(addr, FUTEX_WAIT, val, duration, _, _)
-//		Inside the kernel, atomically check that *addr == val
-//		and go to sleep for at most duration.
+//	futexsleep(uint32 *addr, uint32 val)
+//	futexwakeup(uint32 *addr)
 //
-//	futex(addr, FUTEX_WAKE, val, _, _, _)
-//		Wake up at least val procs sleeping on addr.
-//
-// (Of course, they have added more complicated things since then.)
+// Futexsleep atomically checks if *addr == val and if so, sleeps on addr.
+// Futexwakeup wakes up one thread sleeping on addr.
+// Futexsleep is allowed to wake up spuriously.
 
 enum
 {	
@@ -199,9 +195,9 @@ enum
 	EAGAIN = 11,
 };
 
-// TODO(rsc) I tried using 1<<40 here but it woke up (-ETIMEDOUT).
+// TODO(rsc) I tried using 1<<40 here but futex woke up (-ETIMEDOUT).
 // I wonder if the timespec that gets to the kernel
-// actually has two 32-bit numbers in it, so that
+// actually has two 32-bit numbers in it, so tha
 // a 64-bit 1<<40 ends up being 0 seconds,
 // 1<<8 nanoseconds.
 static struct timespec longtime =
@@ -210,69 +206,106 @@ static struct timespec longtime =
 	0
 };
 
+// Atomically,
+//	if(*addr == val) sleep
+// Might be woken up spuriously; that's allowed.
 static void
-efutex(uint32 *addr, int32 op, int32 val, struct timespec *ts)
+futexsleep(uint32 *addr, uint32 val)
 {
 	int64 ret;
 	
-again:
-	ret = futex(addr, op, val, ts, nil, 0);
-
-	// These happen when you use a debugger, among other times.
-	if(ret == -EAGAIN || ret == -EINTR){
-		// If we were sleeping, it's okay to wake up early.
-		if(op == FUTEX_WAIT)
+	ret = futex(addr, FUTEX_WAIT, val, &longtime, nil, 0);
+	if(ret >= 0 || ret == -EAGAIN || ret == -EINTR)
 		return;
 
-		// If we were waking someone up, we don't know
-		// whether that succeeded, so wake someone else up too.
-		if(op == FUTEX_WAKE){
-prints("futexwake ");
-sys·printint(ret);
-prints("\n");
-			goto again;
-		}
-	}
-
-	if(ret < 0){
-		prints("futex error addr=");
+	prints("futexsleep addr=");
 	sys·printpointer(addr);
-		prints(" op=");
-		sys·printint(op);
 	prints(" val=");
 	sys·printint(val);
-		prints(" ts=");
-		sys·printpointer(ts);
 	prints(" returned ");
-		sys·printint(-ret);
+	sys·printint(ret);
 	prints("\n");
-		*(int32*)101 = 202;
-	}
+	*(int32*)0x1005 = 0x1005;
 }
 
+// If any procs are sleeping on addr, wake up at least one.
+static void
+futexwakeup(uint32 *addr)
+{
+	int64 ret;
+	
+	ret = futex(addr, FUTEX_WAKE, 1, nil, nil, 0);
+
+	if(ret >= 0)
+		return;
+
+	// I don't know that futex wakeup can return
+	// EAGAIN or EINTR, but if it does, it would be 
+	// safe to loop and call futex again.
+
+	prints("futexwakeup addr=");
+	sys·printpointer(addr);
+	prints(" returned ");
+	sys·printint(ret);
+	prints("\n");
+	*(int32*)0x1006 = 0x1006;
+}
+
+
 // Lock and unlock.
-// A zeroed Lock is unlocked (no need to initialize each lock).
-// The l->key is either 0 (unlocked), 1 (locked), or >=2 (contended).
+//
+// The lock state is a single 32-bit word that holds
+// a 31-bit count of threads waiting for the lock
+// and a single bit (the low bit) saying whether the lock is held.
+// The uncontended case runs entirely in user space.
+// When contention is detected, we defer to the kernel (futex).
+//
+// A reminder: compare-and-swap cas(addr, old, new) does
+//	if(*addr == old) { *addr = new; return 1; }
+// 	else return 0;
+// but atomically.
 
 void
 lock(Lock *l)
 {
 	uint32 v;
 
-	if(l->key != 0) *(int32*)0x1001 = 0x1001;
-	l->key = 1;
+again:
+	v = l->key;
+	if((v&1) == 0){
+		if(cas(&l->key, v, v|1)){
+			// Lock wasn't held; we grabbed it.
 			return;
+		}
+		goto again;
+	}
 	
-	for(;;){
-		// Try for lock.  If we incremented it from 0 to 1, we win.
-		if((v=xadd(&l->key, 1)) == 1)
-			return;
+	// Lock was held; try to add ourselves to the waiter count.
+	if(!cas(&l->key, v, v+2))
+		goto again;
 	
-		// We lose.  It was already >=1 and is now >=2.
-		// Use futex to atomically check that the value is still
-		// what we think it is and go to sleep.
-		efutex(&l->key, FUTEX_WAIT, v, &longtime);
+	// We're accounted for, now sleep in the kernel.
+	//
+	// We avoid the obvious lock/unlock race because
+	// the kernel won't put us to sleep if l->key has
+	// changed underfoot and is no longer v+2.
+	//
+	// We only really care that (v&1) == 1 (the lock is held),
+	// and in fact there is a futex variant that could
+	// accomodate that check, but let's not get carried away.)
+	futexsleep(&l->key, v+2);
+	
+	// We're awake: remove ourselves from the count.
+	for(;;){
+		v = l->key;
+		if(v < 2)
+			throw("bad lock key");
+		if(cas(&l->key, v, v-2))
+			break;
 	}
+	
+	// Try for the lock again.
+	goto again;
 }
 
 void
@@ -280,68 +313,54 @@ unlock(Lock *l)
 {
 	uint32 v;
 
-	if(l->key != 1) *(int32*)0x1002 = 0x1002;
-	l->key = 0;
-	return;
-
-	// Unlock the lock.  If we decremented from 1 to 0, wasn't contended.
-	if((v=xadd(&l->key, -1)) == 0)
-		return;
+	// Atomically get value and clear lock bit.
+again:
+	v = l->key;
+	if((v&1) == 0)
+		throw("unlock of unlocked lock");
+	if(!cas(&l->key, v, v&~1))
+		goto again;
 
-	// The lock was contended.  Mark it as unlocked and wake a waiter.
-	l->key = 0;
-	efutex(&l->key, FUTEX_WAKE, 1, nil);
+	// If there were waiters, wake one.
+	if(v & ~1)
+		futexwakeup(&l->key);
 }
 
-// Sleep and wakeup (see description in runtime.h)
+
+// One-time notifications.
+//
+// Since the lock/unlock implementation already
+// takes care of sleeping in the kernel, we just reuse it.
+// (But it's a weird use, so it gets its own interface.)
+//
+// We use a lock to represent the event:
+// unlocked == event has happened.
+// Thus the lock starts out locked, and to wait for the
+// event you try to lock the lock.  To signal the event,
+// you unlock the lock.
 
 void
-rsleep(Rendez *r)
+noteclear(Note *n)
 {
-	// Record that we're about to go to sleep and drop the lock.
-	r->sleeping = 1;
-	unlock(r->l);
-	
-	// Go to sleep if r->sleeping is still 1.
-	efutex(&r->sleeping, FUTEX_WAIT, 1, &longtime);
-
-	// Reacquire the lock.
-	lock(r->l);
+	n->lock.key = 0;	// memset(n, 0, sizeof *n)
+	lock(&n->lock);
 }
 
 void
-rwakeup(Rendez *r)
+notewakeup(Note *n)
 {
-	if(!r->sleeping)
-		return;
-
-	// Clear the sleeping flag in case sleeper
-	// is between unlock and futex.
-	r->sleeping = 0;
-	
-	// Wake up if actually made it to sleep.
-	efutex(&r->sleeping, FUTEX_WAKE, 1, nil);
+	unlock(&n->lock);
 }
 
-// Like rwakeup(r), unlock(r->l), but drops the lock before
-// waking the other proc.  This reduces bouncing back and forth
-// in the scheduler: the first thing the other proc wants to do
-// is acquire r->l, so it helps to unlock it before we wake him.
 void
-rwakeupandunlock(Rendez *r)
+notesleep(Note *n)
 {
-	int32 wassleeping;
-	
-	if(!r->sleeping){
-		unlock(r->l);
-		return;
-	}
-
-	r->sleeping = 0;
-	unlock(r->l);
-	efutex(&r->sleeping, FUTEX_WAKE, 1, nil);
+	lock(&n->lock);
+	unlock(&n->lock);	// Let other sleepers find out too.
 }
 
+
+// Clone, the Linux rfork.
 enum
 {
 	CLONE_VM = 0x100,
@@ -365,7 +384,7 @@ enum
 };
 
 void
-newosproc(M *mm, G *gg, void *stk, void (*fn)(void*), void *arg)
+newosproc(M *m, G *g, void *stk, void (*fn)(void))
 {
 	int64 ret;
 	int32 flags;
@@ -382,20 +401,18 @@ newosproc(M *mm, G *gg, void *stk, void (*fn)(void*), void *arg)
 	if(0){
 		prints("newosproc stk=");
 		sys·printpointer(stk);
-		prints(" mm=");
-		sys·printpointer(mm);
-		prints(" gg=");
-		sys·printpointer(gg);
+		prints(" m=");
+		sys·printpointer(m);
+		prints(" g=");
+		sys·printpointer(g);
 		prints(" fn=");
 		sys·printpointer(fn);
-		prints(" arg=");
-		sys·printpointer(arg);
 		prints(" clone=");
 		sys·printpointer(clone);
 		prints("\n");
 	}
 
-	ret = clone(flags, stk, mm, gg, fn, arg);
+	ret = clone(flags, stk, m, g, fn);
 	if(ret < 0)
 		*(int32*)123 = 123;
 }

src/runtime/runtime.c

@@ -71,6 +71,7 @@ rnd(uint32 n, uint32 m)
 	return n;
 }
 
+// Convenient wrapper around mmap.
 static void*
 brk(uint32 n)
 {
@@ -81,12 +82,15 @@ brk(uint32 n)
 	return v;
 }
 
-
+// Allocate n bytes of memory.  Note that this gets used
+// to allocate new stack segments, so at each call to a function
+// you have to ask yourself "would it be okay to call mal recursively
+// right here?"  The answer is yes unless we're in the middle of
+// editing the malloc state in m->mem.
 void*
 mal(uint32 n)
 {
 	byte* v;
-	Mem *mem;
 
 	// round to keep everything 64-bit aligned
 	n = rnd(n, 8);
@@ -94,17 +98,19 @@ mal(uint32 n)
 	// be careful.  calling any function might invoke
 	// mal to allocate more stack.
 	if(n > NHUNK) {
-		// this call is okay - calling mal recursively
-		// won't change anything we depend on.
 		v = brk(n);
 	} else {
 		// allocate a new hunk if this one is too small
 		if(n > m->mem.nhunk) {
-			// better not to call brk here - it might grow the stack,
-			// causing a call to mal and the allocation of a 
-			// new hunk behind our backs.  then we'd toss away
-			// almost all of that new hunk and replace it.
-			// that'd just be a memory leak - the code would still run.
+			// here we're in the middle of editing m->mem
+			// (we're about to overwrite m->mem.hunk),
+			// so we can't call brk - it might call mal to grow the
+			// stack, and the recursive call would allocate a new
+			// hunk, and then once brk returned we'd immediately
+			// overwrite that hunk with our own.
+			// (the net result would be a memory leak, not a crash.)
+			// so we have to call sys·mmap directly - it is written
+			// in assembly and tagged not to grow the stack.
 			m->mem.hunk =
 				sys·mmap(nil, NHUNK, PROT_READ|PROT_WRITE,
 					MAP_ANON|MAP_PRIVATE, 0, 0);
@@ -136,7 +142,7 @@ hashmap(Sigi *si, Sigs *ss)
 	byte *sname, *iname;
 	Map *m;
 
-	h = ((uint32)si + (uint32)ss) % nelem(hash);
+	h = ((uint32)(uint64)si + (uint32)(uint64)ss) % nelem(hash);
 	for(m=hash[h]; m!=nil; m=m->link) {
 		if(m->si == si && m->ss == ss) {
 			if(m->bad) {
@@ -301,9 +307,9 @@ enum
 	NANSIGN		= 1<<31,
 };
 
-static	uint64	uvnan		= 0x7FF0000000000001;
-static	uint64	uvinf		= 0x7FF0000000000000;
-static	uint64	uvneginf	= 0xFFF0000000000000;
+static	uint64	uvnan		= 0x7FF0000000000001ULL;
+static	uint64	uvinf		= 0x7FF0000000000000ULL;
+static	uint64	uvneginf	= 0xFFF0000000000000ULL;
 
 static int32
 isInf(float64 d, int32 sign)
@@ -338,7 +344,7 @@ isNaN(float64 d)
 	uint64 x;
 
 	x = *(uint64*)&d;
-	return ((uint32)x>>32)==0x7FF00000 && !isInf(d, 0);
+	return (uint32)(x>>32)==0x7FF00000 && !isInf(d, 0);
 }
 
 static float64
@@ -424,7 +430,7 @@ modf(float64 d, float64 *ip)
 	return d - dd;
 }
 
-// func frexp(float64) (float64, int32); // break fp into exp,fract
+// func frexp(float64) (float64, int32); // break fp into exp,frac
 void
 sys·frexp(float64 din, float64 dou, int32 iou)
 {
@@ -432,7 +438,7 @@ sys·frexp(float64 din, float64 dou, int32 iou)
 	FLUSH(&dou);
 }
 
-//func	ldexp(int32, float64) float64;	// make fp from exp,fract
+//func	ldexp(int32, float64) float64;	// make fp from exp,frac
 void
 sys·ldexp(float64 din, int32 ein, float64 dou)
 {
@@ -441,7 +447,7 @@ sys·ldexp(float64 din, int32 ein, float64 dou)
 }
 
 //func	modf(float64) (float64, float64);	// break fp into double+double
-float64
+void
 sys·modf(float64 din, float64 integer, float64 fraction)
 {
 	fraction = modf(din, &integer);
@@ -593,6 +599,7 @@ out:
 	FLUSH(&s);
 }
 
+void
 check(void)
 {
 	int8 a;
@@ -638,18 +645,6 @@ check(void)
 	initsig();
 }
 
-uint32
-xadd(uint32 *val, uint32 delta)
-{
-	uint32 v;
-	
-	for(;;){
-		v = *val;
-		if(cas(val, v, v+delta))
-			return v+delta;
-	}
-}
-
 /*
  * map and chan helpers for
  * dealing with unknown types
@@ -657,6 +652,7 @@ xadd(uint32 *val, uint32 delta)
 static uint64
 memhash(uint32 s, void *a)
 {
+	USED(s, a);
 	prints("memhash\n");
 	return 0x12345;
 }
@@ -718,6 +714,7 @@ memcopy(uint32 s, void *a, void *b)
 static uint64
 stringhash(uint32 s, string *a)
 {
+	USED(s, a);
 	prints("stringhash\n");
 	return 0x12345;
 }
@@ -725,18 +722,21 @@ stringhash(uint32 s, string *a)
 static uint32
 stringequal(uint32 s, string *a, string *b)
 {
+	USED(s);
 	return cmpstring(*a, *b) == 0;
 }
 
 static void
 stringprint(uint32 s, string *a)
 {
+	USED(s);
 	sys·printstring(*a);
 }
 
 static void
 stringcopy(uint32 s, string *a, string *b)
 {
+	USED(s);
 	if(b == nil) {
 		*a = nil;
 		return;
@@ -747,6 +747,7 @@ stringcopy(uint32 s, string *a, string *b)
 static uint64
 pointerhash(uint32 s, void **a)
 {
+	USED(s, a);
 	prints("pointerhash\n");
 	return 0x12345;
 }
@@ -754,6 +755,7 @@ pointerhash(uint32 s, void **a)
 static uint32
 pointerequal(uint32 s, void **a, void **b)
 {
+	USED(s, a, b);
 	prints("pointerequal\n");
 	return 0;
 }
@@ -761,12 +763,14 @@ pointerequal(uint32 s, void **a, void **b)
 static void
 pointerprint(uint32 s, void **a)
 {
+	USED(s, a);
 	prints("pointerprint\n");
 }
 
 static void
 pointercopy(uint32 s, void **a, void **b)
 {
+	USED(s);
 	if(b == nil) {
 		*a = nil;
 		return;
@@ -777,8 +781,8 @@ pointercopy(uint32 s, void **a, void **b)
 Alg
 algarray[3] =
 {
-	{	&memhash,	&memequal,	&memprint,	&memcopy	},  // 0
-	{	&stringhash,	&stringequal,	&stringprint,	&stringcopy	},  // 1
-//	{	&pointerhash,	&pointerequal,	&pointerprint,	&pointercopy	},  // 2
-	{	&memhash,	&memequal,	&memprint,	&memcopy	},  // 2 - treat pointers as ints
+	{	memhash,	memequal,	memprint,	memcopy	},  // 0
+	{	stringhash,	stringequal,	stringprint,	stringcopy	},  // 1
+//	{	pointerhash,	pointerequal,	pointerprint,	pointercopy	},  // 2
+	{	memhash,	memequal,	memprint,	memcopy	},  // 2 - treat pointers as ints
 };

src/runtime/runtime.h

@@ -43,7 +43,7 @@ typedef	struct	M		M;
 typedef	struct	Stktop		Stktop;
 typedef	struct	Alg		Alg;
 typedef	struct	Lock		Lock;
-typedef	struct	Rendez	Rendez;
+typedef	struct	Note	Note;
 typedef	struct	Mem		Mem;
 
 /*
@@ -62,6 +62,7 @@ enum
 	Grunnable,
 	Grunning,
 	Gwaiting,
+	Gmoribund,
 	Gdead,
 };
 enum
@@ -77,10 +78,9 @@ struct	Lock
 {
 	uint32	key;
 };
-struct	Rendez
+struct	Note
 {
-	Lock*	l;
-	uint32	sleeping;	// someone is sleeping (Linux)
+	Lock	lock;
 };
 struct String
 {
@@ -124,8 +124,8 @@ struct	G
 	int16	status;
 	int32	goid;
 	int32	selgen;		// valid sudog pointer
-	G*	runlink;
-	Lock	runlock;
+	G*	schedlink;
+	Note	stopped;
 	M*	m;	// for debuggers
 };
 struct	Mem
@@ -147,9 +147,10 @@ struct	M
 	byte*	moresp;
 	int32	siz1;
 	int32	siz2;
-	Rendez	waitr;
-	M*	waitlink;
-	int32	pid;	// for debuggers
+	Note	havenextg;
+	G*	nextg;
+	M*	schedlink;
+	int32	procid;	// for debuggers
 	Mem	mem;
 };
 struct	Stktop
@@ -224,36 +225,34 @@ int32	write(int32, void*, int32);
 void	close(int32);
 int32	fstat(int32, void*);
 bool	cas(uint32*, uint32, uint32);
-uint32	xadd(uint32*, uint32);
 void	exit1(int32);
 void	ready(G*);
 byte*	getenv(int8*);
 int32	atoi(byte*);
-void	newosproc(M *mm, G *gg, void *stk, void (*fn)(void*), void *arg);
+void	newosproc(M *m, G *g, void *stk, void (*fn)(void));
 int32	getprocid(void);
 
 /*
  * mutual exclusion locks.  in the uncontended case,
  * as fast as spin locks (just a few user-level instructions),
  * but on the contention path they sleep in the kernel.
+ * a zeroed Lock is unlocked (no need to initialize each lock).
  */
 void	lock(Lock*);
 void	unlock(Lock*);
-void	lockinit(Lock*);
 
 /*
- * sleep and wakeup.
- * a Rendez is somewhere to sleep.  it is protected by the lock r->l.
- * the caller must acquire r->l, check the condition, and if the 
- * condition is false, call rsleep.  rsleep will atomically drop the lock
- * and go to sleep.  a subsequent rwakeup (caller must hold r->l)
- * will wake up the guy who is rsleeping.  the lock keeps rsleep and
- * rwakeup from missing each other.
- * n.b. only one proc can rsleep on a given rendez at a time.
+ * sleep and wakeup on one-time events.
+ * before any calls to notesleep or notewakeup, 
+ * must call noteclear to initialize the Note.
+ * then, any number of threads can call notesleep
+ * and exactly one thread can call notewakeup (once).
+ * once notewakeup has been called, all the notesleeps
+ * will return.  future notesleeps will return immediately.
  */
-void	rsleep(Rendez*);
-void	rwakeup(Rendez*);
-void	rwakeupandunlock(Rendez*);
+void	noteclear(Note*);
+void	notesleep(Note*);
+void	notewakeup(Note*);
 
 /*
  * low level go -called

src/runtime/string.c

@@ -45,8 +45,6 @@ out:
 static void
 prbounds(int8* s, int32 a, int32 b, int32 c)
 {
-	int32 i;
-
 	prints(s);
 	prints(" ");
 	sys·printint(a);
@@ -115,7 +113,6 @@ strcmp(byte *s1, byte *s2)
 void
 sys·slicestring(string si, int32 lindex, int32 hindex, string so)
 {
-	string s, str;
 	int32 l;
 
 	if(si == nil)
@@ -154,8 +151,6 @@ sys·indexstring(string s, int32 i, byte b)
 void
 sys·intstring(int64 v, string s)
 {
-	int32 l;
-
 	s = mal(sizeof(s->len)+8);
 	s->len = runetochar(s->str, v);
 	FLUSH(&s);

src/runtime/sys_amd64_darwin.s

@@ -7,21 +7,24 @@
 //
 
 // TODO(rsc): Either sys·exit or exit1 is wrong!
-TEXT	sys·exit(SB),1,$-8
+// It looks like sys·exit is correct (exits the entire program)
+// and exit1 should be mimicking the OS X library routine
+// __bsdthread_terminate.
+TEXT	sys·exit(SB),7,$-8
 	MOVL	8(SP), DI		// arg 1 exit status
 	MOVL	$(0x2000000+1), AX	// syscall entry
 	SYSCALL
 	CALL	notok(SB)
 	RET
 
-TEXT	exit1(SB),1,$-8
+TEXT	exit1(SB),7,$-8
 	MOVL	8(SP), DI		// arg 1 exit status
 	MOVL	$(0x2000000+1), AX	// syscall entry
 	SYSCALL
 	CALL	notok(SB)
 	RET
 
-TEXT	sys·write(SB),1,$-8
+TEXT	sys·write(SB),7,$-8
 	MOVL	8(SP), DI		// arg 1 fid
 	MOVQ	16(SP), SI		// arg 2 buf
 	MOVL	24(SP), DX		// arg 3 count
@@ -31,7 +34,7 @@ TEXT	sys·write(SB),1,$-8
 	CALL	notok(SB)
 	RET
 
-TEXT	open(SB),1,$-8
+TEXT	open(SB),7,$-8
 	MOVQ	8(SP), DI
 	MOVL	16(SP), SI
 	MOVL	20(SP), DX
@@ -40,20 +43,20 @@ TEXT	open(SB),1,$-8
 	SYSCALL
 	RET
 
-TEXT	close(SB),1,$-8
+TEXT	close(SB),7,$-8
 	MOVL	8(SP), DI
 	MOVL	$(0x2000000+6), AX	// syscall entry
 	SYSCALL
 	RET
 
-TEXT	fstat(SB),1,$-8
+TEXT	fstat(SB),7,$-8
 	MOVL	8(SP), DI
 	MOVQ	16(SP), SI
 	MOVL	$(0x2000000+339), AX	// syscall entry; really fstat64
 	SYSCALL
 	RET
 
-TEXT	read(SB),1,$-8
+TEXT	read(SB),7,$-8
 	MOVL	8(SP), DI
 	MOVQ	16(SP), SI
 	MOVL	24(SP), DX
@@ -61,7 +64,7 @@ TEXT	read(SB),1,$-8
 	SYSCALL
 	RET
 
-TEXT	write(SB),1,$-8
+TEXT	write(SB),7,$-8
 	MOVL	8(SP), DI
 	MOVQ	16(SP), SI
 	MOVL	24(SP), DX
@@ -69,7 +72,7 @@ TEXT	write(SB),1,$-8
 	SYSCALL
 	RET
 
-TEXT	sys·sigaction(SB),1,$-8
+TEXT	sys·sigaction(SB),7,$-8
 	MOVL	8(SP), DI		// arg 1 sig
 	MOVQ	16(SP), SI		// arg 2 act
 	MOVQ	24(SP), DX		// arg 3 oact
@@ -81,7 +84,7 @@ TEXT	sys·sigaction(SB),1,$-8
 	CALL	notok(SB)
 	RET
 
-TEXT sigtramp(SB),1,$24
+TEXT sigtramp(SB),7,$24
 	MOVL	DX,0(SP)
 	MOVQ	CX,8(SP)
 	MOVQ	R8,16(SP)
@@ -101,7 +104,7 @@ TEXT	sys·mmap(SB),7,$-8
 	CALL	notok(SB)
 	RET
 
-TEXT	notok(SB),1,$-8
+TEXT	notok(SB),7,$-8
 	MOVL	$0xf1, BP
 	MOVQ	BP, (BP)
 	RET
@@ -117,12 +120,12 @@ TEXT	sys·memclr(SB),7,$-8
 	STOSQ
 	RET
 
-TEXT	sys·getcallerpc+0(SB),1,$0
+TEXT	sys·getcallerpc+0(SB),7,$0
 	MOVQ	x+0(FP),AX		// addr of first arg
 	MOVQ	-8(AX),AX		// get calling pc
 	RET
 
-TEXT	sys·setcallerpc+0(SB),1,$0
+TEXT	sys·setcallerpc+0(SB),7,$0
 	MOVQ	x+0(FP),AX		// addr of first arg
 	MOVQ	x+8(FP), BX
 	MOVQ	BX, -8(AX)		// set calling pc

src/runtime/sys_amd64_linux.s

@@ -6,19 +6,19 @@
 // System calls and other sys.stuff for AMD64, Linux
 //
 
-TEXT	sys·exit(SB),1,$0-8
+TEXT	sys·exit(SB),7,$0-8
 	MOVL	8(SP), DI
-	MOVL	$231, AX	// force all os threads to exit
+	MOVL	$231, AX	// exitgroup - force all os threads to exi
 	SYSCALL
 	RET
 
-TEXT exit1(SB),1,$0-8
+TEXT exit1(SB),7,$0-8
 	MOVL	8(SP), DI
-	MOVL	$60, AX	// exit the current os thread
+	MOVL	$60, AX	// exit - exit the current os thread
 	SYSCALL
 	RET
 
-TEXT	open(SB),1,$0-16
+TEXT	open(SB),7,$0-16
 	MOVQ	8(SP), DI
 	MOVL	16(SP), SI
 	MOVL	20(SP), DX
@@ -26,20 +26,20 @@ TEXT	open(SB),1,$0-16
 	SYSCALL
 	RET
 
-TEXT	close(SB),1,$0-8
+TEXT	close(SB),7,$0-8
 	MOVL	8(SP), DI
 	MOVL	$3, AX			// syscall entry
 	SYSCALL
 	RET
 
-TEXT	fstat(SB),1,$0-16
+TEXT	fstat(SB),7,$0-16
 	MOVL	8(SP), DI
 	MOVQ	16(SP), SI
 	MOVL	$5, AX			// syscall entry
 	SYSCALL
 	RET
 
-TEXT	read(SB),1,$0-24
+TEXT	read(SB),7,$0-24
 	MOVL	8(SP), DI
 	MOVQ	16(SP), SI
 	MOVL	24(SP), DX
@@ -47,7 +47,7 @@ TEXT	read(SB),1,$0-24
 	SYSCALL
 	RET
 
-TEXT	write(SB),1,$0-24
+TEXT	write(SB),7,$0-24
 	MOVL	8(SP), DI
 	MOVQ	16(SP), SI
 	MOVL	24(SP), DX
@@ -55,7 +55,7 @@ TEXT	write(SB),1,$0-24
 	SYSCALL
 	RET
 
-TEXT	sys·write(SB),1,$0-24
+TEXT	sys·write(SB),7,$0-24
 	MOVL	8(SP), DI
 	MOVQ	16(SP), SI
 	MOVL	24(SP), DX
@@ -63,7 +63,7 @@ TEXT	sys·write(SB),1,$0-24
 	SYSCALL
 	RET
 
-TEXT	sys·rt_sigaction(SB),1,$0-32
+TEXT	sys·rt_sigaction(SB),7,$0-32
 	MOVL	8(SP), DI
 	MOVQ	16(SP), SI
 	MOVQ	24(SP), DX
@@ -72,7 +72,7 @@ TEXT	sys·rt_sigaction(SB),1,$0-32
 	SYSCALL
 	RET
 
-TEXT	sigtramp(SB),1,$24-16
+TEXT	sigtramp(SB),7,$24-16
 	MOVQ	DI,0(SP)
 	MOVQ	SI,8(SP)
 	MOVQ	DX,16(SP)
@@ -118,12 +118,12 @@ TEXT	sys·memclr(SB),7,$0-16
 	STOSQ
 	RET
 
-TEXT	sys·getcallerpc+0(SB),1,$0
+TEXT	sys·getcallerpc+0(SB),7,$0
 	MOVQ	x+0(FP),AX		// addr of first arg
 	MOVQ	-8(AX),AX		// get calling pc
 	RET
 
-TEXT	sys·setcallerpc+0(SB),1,$0
+TEXT	sys·setcallerpc+0(SB),7,$0
 	MOVQ	x+0(FP),AX		// addr of first arg
 	MOVQ	x+8(FP), BX
 	MOVQ	BX, -8(AX)		// set calling pc
@@ -131,7 +131,7 @@ TEXT	sys·setcallerpc+0(SB),1,$0
 
 // int64 futex(int32 *uaddr, int32 op, int32 val,
 //	struct timespec *timeout, int32 *uaddr2, int32 val2);
-TEXT futex(SB),1,$0
+TEXT futex(SB),7,$0
 	MOVQ	8(SP), DI
 	MOVL	16(SP), SI
 	MOVL	20(SP), DX
@@ -142,17 +142,16 @@ TEXT futex(SB),1,$0
 	SYSCALL
 	RET
 
-// int64 clone(int32 flags, void *stack, M *m, G *g, void (*fn)(void*), void *arg);
+// int64 clone(int32 flags, void *stack, M *m, G *g, void (*fn)(void));
 TEXT clone(SB),7,$0
-	MOVL	8(SP), DI
-	MOVQ	16(SP), SI
+	MOVL	flags+8(SP), DI
+	MOVQ	stack+16(SP), SI
 	
-	// Copy m, g, fn, arg off parent stack for use by child.
+	// Copy m, g, fn off parent stack for use by child.
 	// Careful: Linux system call clobbers CX and R11.
-	MOVQ	24(SP), R8
-	MOVQ	32(SP), R9
-	MOVQ	40(SP), R12
-	MOVQ	48(SP), R13
+	MOVQ	m+24(SP), R8
+	MOVQ	g+32(SP), R9
+	MOVQ	fn+40(SP), R12
 
 	MOVL	$56, AX
 	SYSCALL
@@ -162,21 +161,20 @@ TEXT clone(SB),7,$0
 	JEQ	2(PC)
 	RET
 	
-	// In child, call fn(arg) on new stack
+	// In child, call fn on new stack
 	MOVQ	SI, SP
 	MOVQ	R8, R14	// m
 	MOVQ	R9, R15	// g
-	PUSHQ	R13
 	CALL	R12
 	
-	// It shouldn't return.  If it does, exit
+	// It shouldn't return.  If it does, exi
 	MOVL	$111, DI
 	MOVL	$60, AX
 	SYSCALL
 	JMP	-3(PC)	// keep exiting
 
 // int64 select(int32, void*, void*, void*, void*)
-TEXT select(SB),1,$0
+TEXT select(SB),7,$0
 	MOVL	8(SP), DI
 	MOVQ	16(SP), SI
 	MOVQ	24(SP), DX
@@ -194,7 +192,7 @@ TEXT select(SB),1,$0
 // the calling thread, not the fake one.
 //
 // int32 getprocid(void)
-TEXT getprocid(SB),1,$0
+TEXT getprocid(SB),7,$0
 	MOVL	$186, AX
 	SYSCALL
 	RET