Commit e03bce15 authored by Peter Collingbourne's avatar Peter Collingbourne Committed by Russ Cox

cmd/cc, runtime: eliminate use of the unnamed substructure C extension

Eliminating use of this extension makes it easier to port the Go runtime
to other compilers. This CL also disables the extension in cc to prevent
accidental use.

LGTM=rsc, khr
R=rsc, aram, khr, dvyukov
CC=axwalk, golang-codereviews
https://golang.org/cl/106790044
parent 6cee4d3e
...@@ -1481,12 +1481,9 @@ edecl(int c, Type *t, Sym *s) ...@@ -1481,12 +1481,9 @@ edecl(int c, Type *t, Sym *s)
{ {
Type *t1; Type *t1;
if(s == S) { if(s == S)
if(!typesu[t->etype]) diag(Z, "unnamed structure elements not supported");
diag(Z, "unnamed structure element must be struct/union"); else
if(c != CXXX)
diag(Z, "unnamed structure element cannot have class");
} else
if(c != CXXX) if(c != CXXX)
diag(Z, "structure element cannot have class: %s", s->name); diag(Z, "structure element cannot have class: %s", s->name);
t1 = t; t1 = t;
......
...@@ -11,7 +11,7 @@ ...@@ -11,7 +11,7 @@
typedef struct Callbacks Callbacks; typedef struct Callbacks Callbacks;
struct Callbacks { struct Callbacks {
Lock; Lock lock;
WinCallbackContext* ctxt[cb_max]; WinCallbackContext* ctxt[cb_max];
int32 n; int32 n;
}; };
...@@ -44,13 +44,13 @@ runtime·compilecallback(Eface fn, bool cleanstack) ...@@ -44,13 +44,13 @@ runtime·compilecallback(Eface fn, bool cleanstack)
argsize += sizeof(uintptr); argsize += sizeof(uintptr);
} }
runtime·lock(&cbs); runtime·lock(&cbs.lock);
if(runtime·cbctxts == nil) if(runtime·cbctxts == nil)
runtime·cbctxts = &(cbs.ctxt[0]); runtime·cbctxts = &(cbs.ctxt[0]);
n = cbs.n; n = cbs.n;
for(i=0; i<n; i++) { for(i=0; i<n; i++) {
if(cbs.ctxt[i]->gobody == fn.data && cbs.ctxt[i]->cleanstack == cleanstack) { if(cbs.ctxt[i]->gobody == fn.data && cbs.ctxt[i]->cleanstack == cleanstack) {
runtime·unlock(&cbs); runtime·unlock(&cbs.lock);
// runtime·callbackasm is just a series of CALL instructions // runtime·callbackasm is just a series of CALL instructions
// (each is 5 bytes long), and we want callback to arrive at // (each is 5 bytes long), and we want callback to arrive at
// correspondent call instruction instead of start of // correspondent call instruction instead of start of
...@@ -70,7 +70,7 @@ runtime·compilecallback(Eface fn, bool cleanstack) ...@@ -70,7 +70,7 @@ runtime·compilecallback(Eface fn, bool cleanstack)
c->restorestack = 0; c->restorestack = 0;
cbs.ctxt[n] = c; cbs.ctxt[n] = c;
cbs.n++; cbs.n++;
runtime·unlock(&cbs); runtime·unlock(&cbs.lock);
// as before // as before
return (byte*)runtime·callbackasm + n * 5; return (byte*)runtime·callbackasm + n * 5;
......
...@@ -136,7 +136,7 @@ chansend(ChanType *t, Hchan *c, byte *ep, bool block, void *pc) ...@@ -136,7 +136,7 @@ chansend(ChanType *t, Hchan *c, byte *ep, bool block, void *pc)
mysg.releasetime = -1; mysg.releasetime = -1;
} }
runtime·lock(c); runtime·lock(&c->lock);
if(raceenabled) if(raceenabled)
runtime·racereadpc(c, pc, chansend); runtime·racereadpc(c, pc, chansend);
if(c->closed) if(c->closed)
...@@ -149,7 +149,7 @@ chansend(ChanType *t, Hchan *c, byte *ep, bool block, void *pc) ...@@ -149,7 +149,7 @@ chansend(ChanType *t, Hchan *c, byte *ep, bool block, void *pc)
if(sg != nil) { if(sg != nil) {
if(raceenabled) if(raceenabled)
racesync(c, sg); racesync(c, sg);
runtime·unlock(c); runtime·unlock(&c->lock);
gp = sg->g; gp = sg->g;
gp->param = sg; gp->param = sg;
...@@ -162,7 +162,7 @@ chansend(ChanType *t, Hchan *c, byte *ep, bool block, void *pc) ...@@ -162,7 +162,7 @@ chansend(ChanType *t, Hchan *c, byte *ep, bool block, void *pc)
} }
if(!block) { if(!block) {
runtime·unlock(c); runtime·unlock(&c->lock);
return false; return false;
} }
...@@ -171,10 +171,10 @@ chansend(ChanType *t, Hchan *c, byte *ep, bool block, void *pc) ...@@ -171,10 +171,10 @@ chansend(ChanType *t, Hchan *c, byte *ep, bool block, void *pc)
mysg.selectdone = nil; mysg.selectdone = nil;
g->param = nil; g->param = nil;
enqueue(&c->sendq, &mysg); enqueue(&c->sendq, &mysg);
runtime·parkunlock(c, "chan send"); runtime·parkunlock(&c->lock, "chan send");
if(g->param == nil) { if(g->param == nil) {
runtime·lock(c); runtime·lock(&c->lock);
if(!c->closed) if(!c->closed)
runtime·throw("chansend: spurious wakeup"); runtime·throw("chansend: spurious wakeup");
goto closed; goto closed;
...@@ -191,16 +191,16 @@ asynch: ...@@ -191,16 +191,16 @@ asynch:
if(c->qcount >= c->dataqsiz) { if(c->qcount >= c->dataqsiz) {
if(!block) { if(!block) {
runtime·unlock(c); runtime·unlock(&c->lock);
return false; return false;
} }
mysg.g = g; mysg.g = g;
mysg.elem = nil; mysg.elem = nil;
mysg.selectdone = nil; mysg.selectdone = nil;
enqueue(&c->sendq, &mysg); enqueue(&c->sendq, &mysg);
runtime·parkunlock(c, "chan send"); runtime·parkunlock(&c->lock, "chan send");
runtime·lock(c); runtime·lock(&c->lock);
goto asynch; goto asynch;
} }
...@@ -217,18 +217,18 @@ asynch: ...@@ -217,18 +217,18 @@ asynch:
sg = dequeue(&c->recvq); sg = dequeue(&c->recvq);
if(sg != nil) { if(sg != nil) {
gp = sg->g; gp = sg->g;
runtime·unlock(c); runtime·unlock(&c->lock);
if(sg->releasetime) if(sg->releasetime)
sg->releasetime = runtime·cputicks(); sg->releasetime = runtime·cputicks();
runtime·ready(gp); runtime·ready(gp);
} else } else
runtime·unlock(c); runtime·unlock(&c->lock);
if(mysg.releasetime > 0) if(mysg.releasetime > 0)
runtime·blockevent(mysg.releasetime - t0, 2); runtime·blockevent(mysg.releasetime - t0, 2);
return true; return true;
closed: closed:
runtime·unlock(c); runtime·unlock(&c->lock);
runtime·panicstring("send on closed channel"); runtime·panicstring("send on closed channel");
return false; // not reached return false; // not reached
} }
...@@ -262,7 +262,7 @@ chanrecv(ChanType *t, Hchan* c, byte *ep, bool block, bool *received) ...@@ -262,7 +262,7 @@ chanrecv(ChanType *t, Hchan* c, byte *ep, bool block, bool *received)
mysg.releasetime = -1; mysg.releasetime = -1;
} }
runtime·lock(c); runtime·lock(&c->lock);
if(c->dataqsiz > 0) if(c->dataqsiz > 0)
goto asynch; goto asynch;
...@@ -273,7 +273,7 @@ chanrecv(ChanType *t, Hchan* c, byte *ep, bool block, bool *received) ...@@ -273,7 +273,7 @@ chanrecv(ChanType *t, Hchan* c, byte *ep, bool block, bool *received)
if(sg != nil) { if(sg != nil) {
if(raceenabled) if(raceenabled)
racesync(c, sg); racesync(c, sg);
runtime·unlock(c); runtime·unlock(&c->lock);
if(ep != nil) if(ep != nil)
c->elemtype->alg->copy(c->elemsize, ep, sg->elem); c->elemtype->alg->copy(c->elemsize, ep, sg->elem);
...@@ -289,7 +289,7 @@ chanrecv(ChanType *t, Hchan* c, byte *ep, bool block, bool *received) ...@@ -289,7 +289,7 @@ chanrecv(ChanType *t, Hchan* c, byte *ep, bool block, bool *received)
} }
if(!block) { if(!block) {
runtime·unlock(c); runtime·unlock(&c->lock);
return false; return false;
} }
...@@ -298,10 +298,10 @@ chanrecv(ChanType *t, Hchan* c, byte *ep, bool block, bool *received) ...@@ -298,10 +298,10 @@ chanrecv(ChanType *t, Hchan* c, byte *ep, bool block, bool *received)
mysg.selectdone = nil; mysg.selectdone = nil;
g->param = nil; g->param = nil;
enqueue(&c->recvq, &mysg); enqueue(&c->recvq, &mysg);
runtime·parkunlock(c, "chan receive"); runtime·parkunlock(&c->lock, "chan receive");
if(g->param == nil) { if(g->param == nil) {
runtime·lock(c); runtime·lock(&c->lock);
if(!c->closed) if(!c->closed)
runtime·throw("chanrecv: spurious wakeup"); runtime·throw("chanrecv: spurious wakeup");
goto closed; goto closed;
...@@ -319,7 +319,7 @@ asynch: ...@@ -319,7 +319,7 @@ asynch:
goto closed; goto closed;
if(!block) { if(!block) {
runtime·unlock(c); runtime·unlock(&c->lock);
if(received != nil) if(received != nil)
*received = false; *received = false;
return false; return false;
...@@ -328,9 +328,9 @@ asynch: ...@@ -328,9 +328,9 @@ asynch:
mysg.elem = nil; mysg.elem = nil;
mysg.selectdone = nil; mysg.selectdone = nil;
enqueue(&c->recvq, &mysg); enqueue(&c->recvq, &mysg);
runtime·parkunlock(c, "chan receive"); runtime·parkunlock(&c->lock, "chan receive");
runtime·lock(c); runtime·lock(&c->lock);
goto asynch; goto asynch;
} }
...@@ -349,12 +349,12 @@ asynch: ...@@ -349,12 +349,12 @@ asynch:
sg = dequeue(&c->sendq); sg = dequeue(&c->sendq);
if(sg != nil) { if(sg != nil) {
gp = sg->g; gp = sg->g;
runtime·unlock(c); runtime·unlock(&c->lock);
if(sg->releasetime) if(sg->releasetime)
sg->releasetime = runtime·cputicks(); sg->releasetime = runtime·cputicks();
runtime·ready(gp); runtime·ready(gp);
} else } else
runtime·unlock(c); runtime·unlock(&c->lock);
if(received != nil) if(received != nil)
*received = true; *received = true;
...@@ -369,7 +369,7 @@ closed: ...@@ -369,7 +369,7 @@ closed:
*received = false; *received = false;
if(raceenabled) if(raceenabled)
runtime·raceacquire(c); runtime·raceacquire(c);
runtime·unlock(c); runtime·unlock(&c->lock);
if(mysg.releasetime > 0) if(mysg.releasetime > 0)
runtime·blockevent(mysg.releasetime - t0, 2); runtime·blockevent(mysg.releasetime - t0, 2);
return true; return true;
...@@ -617,7 +617,7 @@ sellock(Select *sel) ...@@ -617,7 +617,7 @@ sellock(Select *sel)
c0 = sel->lockorder[i]; c0 = sel->lockorder[i];
if(c0 && c0 != c) { if(c0 && c0 != c) {
c = sel->lockorder[i]; c = sel->lockorder[i];
runtime·lock(c); runtime·lock(&c->lock);
} }
} }
} }
...@@ -645,7 +645,7 @@ selunlock(Select *sel) ...@@ -645,7 +645,7 @@ selunlock(Select *sel)
c = sel->lockorder[i]; c = sel->lockorder[i];
if(i>0 && sel->lockorder[i-1] == c) if(i>0 && sel->lockorder[i-1] == c)
continue; // will unlock it on the next iteration continue; // will unlock it on the next iteration
runtime·unlock(c); runtime·unlock(&c->lock);
} }
} }
...@@ -1067,9 +1067,9 @@ closechan(Hchan *c, void *pc) ...@@ -1067,9 +1067,9 @@ closechan(Hchan *c, void *pc)
if(c == nil) if(c == nil)
runtime·panicstring("close of nil channel"); runtime·panicstring("close of nil channel");
runtime·lock(c); runtime·lock(&c->lock);
if(c->closed) { if(c->closed) {
runtime·unlock(c); runtime·unlock(&c->lock);
runtime·panicstring("close of closed channel"); runtime·panicstring("close of closed channel");
} }
...@@ -1104,7 +1104,7 @@ closechan(Hchan *c, void *pc) ...@@ -1104,7 +1104,7 @@ closechan(Hchan *c, void *pc)
runtime·ready(gp); runtime·ready(gp);
} }
runtime·unlock(c); runtime·unlock(&c->lock);
} }
func reflect·chanlen(c *Hchan) (len int) { func reflect·chanlen(c *Hchan) (len int) {
......
...@@ -38,7 +38,7 @@ struct Hchan ...@@ -38,7 +38,7 @@ struct Hchan
uintgo recvx; // receive index uintgo recvx; // receive index
WaitQ recvq; // list of recv waiters WaitQ recvq; // list of recv waiters
WaitQ sendq; // list of send waiters WaitQ sendq; // list of send waiters
Lock; Lock lock;
}; };
// Buffer follows Hchan immediately in memory. // Buffer follows Hchan immediately in memory.
......
...@@ -21,9 +21,6 @@ struct Ureg ...@@ -21,9 +21,6 @@ struct Ureg
uint32 pc; /* pc */ uint32 pc; /* pc */
uint32 cs; /* old context */ uint32 cs; /* old context */
uint32 flags; /* old flags */ uint32 flags; /* old flags */
union {
uint32 usp;
uint32 sp; uint32 sp;
};
uint32 ss; /* old stack segment */ uint32 ss; /* old stack segment */
}; };
...@@ -515,7 +515,7 @@ dumproots(void) ...@@ -515,7 +515,7 @@ dumproots(void)
if(sp->kind != KindSpecialFinalizer) if(sp->kind != KindSpecialFinalizer)
continue; continue;
spf = (SpecialFinalizer*)sp; spf = (SpecialFinalizer*)sp;
p = (byte*)((s->start << PageShift) + spf->offset); p = (byte*)((s->start << PageShift) + spf->special.offset);
dumpfinalizer(p, spf->fn, spf->fint, spf->ot); dumpfinalizer(p, spf->fn, spf->fint, spf->ot);
} }
} }
...@@ -695,7 +695,7 @@ dumpmemprof(void) ...@@ -695,7 +695,7 @@ dumpmemprof(void)
if(sp->kind != KindSpecialProfile) if(sp->kind != KindSpecialProfile)
continue; continue;
spp = (SpecialProfile*)sp; spp = (SpecialProfile*)sp;
p = (byte*)((s->start << PageShift) + spp->offset); p = (byte*)((s->start << PageShift) + spp->special.offset);
dumpint(TagAllocSample); dumpint(TagAllocSample);
dumpint((uintptr)p); dumpint((uintptr)p);
dumpint((uintptr)spp->b); dumpint((uintptr)spp->b);
......
...@@ -42,7 +42,7 @@ itab(InterfaceType *inter, Type *type, int32 canfail) ...@@ -42,7 +42,7 @@ itab(InterfaceType *inter, Type *type, int32 canfail)
} }
// compiler has provided some good hash codes for us. // compiler has provided some good hash codes for us.
h = inter->hash; h = inter->typ.hash;
h += 17 * type->hash; h += 17 * type->hash;
// TODO(rsc): h += 23 * x->mhash ? // TODO(rsc): h += 23 * x->mhash ?
h %= nelem(hash); h %= nelem(hash);
...@@ -98,7 +98,7 @@ search: ...@@ -98,7 +98,7 @@ search:
throw: throw:
// didn't find method // didn't find method
runtime·newTypeAssertionError( runtime·newTypeAssertionError(
nil, type->string, inter->string, nil, type->string, inter->typ.string,
iname, &err); iname, &err);
if(locked) if(locked)
runtime·unlock(&ifacelock); runtime·unlock(&ifacelock);
...@@ -231,7 +231,7 @@ assertI2Tret(Type *t, Iface i, byte *ret) ...@@ -231,7 +231,7 @@ assertI2Tret(Type *t, Iface i, byte *ret)
} }
if(tab->type != t) { if(tab->type != t) {
runtime·newTypeAssertionError( runtime·newTypeAssertionError(
tab->inter->string, tab->type->string, t->string, tab->inter->typ.string, tab->type->string, t->string,
nil, &err); nil, &err);
runtime·panic(err); runtime·panic(err);
} }
...@@ -332,7 +332,7 @@ func assertI2E(inter *InterfaceType, i Iface) (ret Eface) { ...@@ -332,7 +332,7 @@ func assertI2E(inter *InterfaceType, i Iface) (ret Eface) {
if(tab == nil) { if(tab == nil) {
// explicit conversions require non-nil interface value. // explicit conversions require non-nil interface value.
runtime·newTypeAssertionError( runtime·newTypeAssertionError(
nil, nil, inter->string, nil, nil, inter->typ.string,
nil, &err); nil, &err);
runtime·panic(err); runtime·panic(err);
} }
...@@ -377,7 +377,7 @@ runtime·ifaceI2I(InterfaceType *inter, Iface i, Iface *ret) ...@@ -377,7 +377,7 @@ runtime·ifaceI2I(InterfaceType *inter, Iface i, Iface *ret)
if(tab == nil) { if(tab == nil) {
// explicit conversions require non-nil interface value. // explicit conversions require non-nil interface value.
runtime·newTypeAssertionError( runtime·newTypeAssertionError(
nil, nil, inter->string, nil, nil, inter->typ.string,
nil, &err); nil, &err);
runtime·panic(err); runtime·panic(err);
} }
...@@ -414,7 +414,7 @@ runtime·ifaceE2I(InterfaceType *inter, Eface e, Iface *ret) ...@@ -414,7 +414,7 @@ runtime·ifaceE2I(InterfaceType *inter, Eface e, Iface *ret)
if(t == nil) { if(t == nil) {
// explicit conversions require non-nil interface value. // explicit conversions require non-nil interface value.
runtime·newTypeAssertionError( runtime·newTypeAssertionError(
nil, nil, inter->string, nil, nil, inter->typ.string,
nil, &err); nil, &err);
runtime·panic(err); runtime·panic(err);
} }
...@@ -462,7 +462,7 @@ func assertE2E(inter *InterfaceType, e Eface) (ret Eface) { ...@@ -462,7 +462,7 @@ func assertE2E(inter *InterfaceType, e Eface) (ret Eface) {
if(t == nil) { if(t == nil) {
// explicit conversions require non-nil interface value. // explicit conversions require non-nil interface value.
runtime·newTypeAssertionError( runtime·newTypeAssertionError(
nil, nil, inter->string, nil, nil, inter->typ.string,
nil, &err); nil, &err);
runtime·panic(err); runtime·panic(err);
} }
......
...@@ -45,9 +45,9 @@ runtime·mlookup(void *v, byte **base, uintptr *size, MSpan **sp) ...@@ -45,9 +45,9 @@ runtime·mlookup(void *v, byte **base, uintptr *size, MSpan **sp)
g->m->mcache->local_nlookup++; g->m->mcache->local_nlookup++;
if (sizeof(void*) == 4 && g->m->mcache->local_nlookup >= (1<<30)) { if (sizeof(void*) == 4 && g->m->mcache->local_nlookup >= (1<<30)) {
// purge cache stats to prevent overflow // purge cache stats to prevent overflow
runtime·lock(&runtime·mheap); runtime·lock(&runtime·mheap.lock);
runtime·purgecachedstats(g->m->mcache); runtime·purgecachedstats(g->m->mcache);
runtime·unlock(&runtime·mheap); runtime·unlock(&runtime·mheap.lock);
} }
s = runtime·MHeap_LookupMaybe(&runtime·mheap, v); s = runtime·MHeap_LookupMaybe(&runtime·mheap, v);
...@@ -341,7 +341,7 @@ runtime·MHeap_SysAlloc(MHeap *h, uintptr n) ...@@ -341,7 +341,7 @@ runtime·MHeap_SysAlloc(MHeap *h, uintptr n)
static struct static struct
{ {
Lock; Lock lock;
byte* pos; byte* pos;
byte* end; byte* end;
} persistent; } persistent;
...@@ -370,19 +370,19 @@ runtime·persistentalloc(uintptr size, uintptr align, uint64 *stat) ...@@ -370,19 +370,19 @@ runtime·persistentalloc(uintptr size, uintptr align, uint64 *stat)
align = 8; align = 8;
if(size >= PersistentAllocMaxBlock) if(size >= PersistentAllocMaxBlock)
return runtime·SysAlloc(size, stat); return runtime·SysAlloc(size, stat);
runtime·lock(&persistent); runtime·lock(&persistent.lock);
persistent.pos = (byte*)ROUND((uintptr)persistent.pos, align); persistent.pos = (byte*)ROUND((uintptr)persistent.pos, align);
if(persistent.pos + size > persistent.end) { if(persistent.pos + size > persistent.end) {
persistent.pos = runtime·SysAlloc(PersistentAllocChunk, &mstats.other_sys); persistent.pos = runtime·SysAlloc(PersistentAllocChunk, &mstats.other_sys);
if(persistent.pos == nil) { if(persistent.pos == nil) {
runtime·unlock(&persistent); runtime·unlock(&persistent.lock);
runtime·throw("runtime: cannot allocate memory"); runtime·throw("runtime: cannot allocate memory");
} }
persistent.end = persistent.pos + PersistentAllocChunk; persistent.end = persistent.pos + PersistentAllocChunk;
} }
p = persistent.pos; p = persistent.pos;
persistent.pos += size; persistent.pos += size;
runtime·unlock(&persistent); runtime·unlock(&persistent.lock);
if(stat != &mstats.other_sys) { if(stat != &mstats.other_sys) {
// reaccount the allocation against provided stat // reaccount the allocation against provided stat
runtime·xadd64(stat, size); runtime·xadd64(stat, size);
......
...@@ -370,7 +370,7 @@ struct Special ...@@ -370,7 +370,7 @@ struct Special
typedef struct SpecialFinalizer SpecialFinalizer; typedef struct SpecialFinalizer SpecialFinalizer;
struct SpecialFinalizer struct SpecialFinalizer
{ {
Special; Special special;
FuncVal* fn; FuncVal* fn;
uintptr nret; uintptr nret;
Type* fint; Type* fint;
...@@ -382,7 +382,7 @@ typedef struct Bucket Bucket; // from mprof.h ...@@ -382,7 +382,7 @@ typedef struct Bucket Bucket; // from mprof.h
typedef struct SpecialProfile SpecialProfile; typedef struct SpecialProfile SpecialProfile;
struct SpecialProfile struct SpecialProfile
{ {
Special; Special special;
Bucket* b; Bucket* b;
}; };
...@@ -438,7 +438,7 @@ void runtime·MSpanList_Remove(MSpan *span); // from whatever list it is in ...@@ -438,7 +438,7 @@ void runtime·MSpanList_Remove(MSpan *span); // from whatever list it is in
// Central list of free objects of a given size. // Central list of free objects of a given size.
struct MCentral struct MCentral
{ {
Lock; Lock lock;
int32 sizeclass; int32 sizeclass;
MSpan nonempty; // list of spans with a free object MSpan nonempty; // list of spans with a free object
MSpan empty; // list of spans with no free objects (or cached in an MCache) MSpan empty; // list of spans with no free objects (or cached in an MCache)
...@@ -454,7 +454,7 @@ bool runtime·MCentral_FreeSpan(MCentral *c, MSpan *s, int32 n, MLink *start, ML ...@@ -454,7 +454,7 @@ bool runtime·MCentral_FreeSpan(MCentral *c, MSpan *s, int32 n, MLink *start, ML
// but all the other global data is here too. // but all the other global data is here too.
struct MHeap struct MHeap
{ {
Lock; Lock lock;
MSpan free[MaxMHeapList]; // free lists of given length MSpan free[MaxMHeapList]; // free lists of given length
MSpan freelarge; // free lists length >= MaxMHeapList MSpan freelarge; // free lists length >= MaxMHeapList
MSpan busy[MaxMHeapList]; // busy lists of large objects of given length MSpan busy[MaxMHeapList]; // busy lists of large objects of given length
...@@ -483,7 +483,7 @@ struct MHeap ...@@ -483,7 +483,7 @@ struct MHeap
// spaced CacheLineSize bytes apart, so that each MCentral.Lock // spaced CacheLineSize bytes apart, so that each MCentral.Lock
// gets its own cache line. // gets its own cache line.
struct { struct {
MCentral; MCentral mcentral;
byte pad[CacheLineSize]; byte pad[CacheLineSize];
} central[NumSizeClasses]; } central[NumSizeClasses];
......
...@@ -22,9 +22,9 @@ runtime·allocmcache(void) ...@@ -22,9 +22,9 @@ runtime·allocmcache(void)
MCache *c; MCache *c;
int32 i; int32 i;
runtime·lock(&runtime·mheap); runtime·lock(&runtime·mheap.lock);
c = runtime·FixAlloc_Alloc(&runtime·mheap.cachealloc); c = runtime·FixAlloc_Alloc(&runtime·mheap.cachealloc);
runtime·unlock(&runtime·mheap); runtime·unlock(&runtime·mheap.lock);
runtime·memclr((byte*)c, sizeof(*c)); runtime·memclr((byte*)c, sizeof(*c));
for(i = 0; i < NumSizeClasses; i++) for(i = 0; i < NumSizeClasses; i++)
c->alloc[i] = &emptymspan; c->alloc[i] = &emptymspan;
...@@ -45,10 +45,10 @@ freemcache(MCache *c) ...@@ -45,10 +45,10 @@ freemcache(MCache *c)
runtime·MCache_ReleaseAll(c); runtime·MCache_ReleaseAll(c);
runtime·stackcache_clear(c); runtime·stackcache_clear(c);
runtime·gcworkbuffree(c->gcworkbuf); runtime·gcworkbuffree(c->gcworkbuf);
runtime·lock(&runtime·mheap); runtime·lock(&runtime·mheap.lock);
runtime·purgecachedstats(c); runtime·purgecachedstats(c);
runtime·FixAlloc_Free(&runtime·mheap.cachealloc, c); runtime·FixAlloc_Free(&runtime·mheap.cachealloc, c);
runtime·unlock(&runtime·mheap); runtime·unlock(&runtime·mheap.lock);
} }
static void static void
...@@ -85,7 +85,7 @@ runtime·MCache_Refill(MCache *c, int32 sizeclass) ...@@ -85,7 +85,7 @@ runtime·MCache_Refill(MCache *c, int32 sizeclass)
s->incache = false; s->incache = false;
// Get a new cached span from the central lists. // Get a new cached span from the central lists.
s = runtime·MCentral_CacheSpan(&runtime·mheap.central[sizeclass]); s = runtime·MCentral_CacheSpan(&runtime·mheap.central[sizeclass].mcentral);
if(s == nil) if(s == nil)
runtime·throw("out of memory"); runtime·throw("out of memory");
if(s->freelist == nil) { if(s->freelist == nil) {
...@@ -106,7 +106,7 @@ runtime·MCache_ReleaseAll(MCache *c) ...@@ -106,7 +106,7 @@ runtime·MCache_ReleaseAll(MCache *c)
for(i=0; i<NumSizeClasses; i++) { for(i=0; i<NumSizeClasses; i++) {
s = c->alloc[i]; s = c->alloc[i];
if(s != &emptymspan) { if(s != &emptymspan) {
runtime·MCentral_UncacheSpan(&runtime·mheap.central[i], s); runtime·MCentral_UncacheSpan(&runtime·mheap.central[i].mcentral, s);
c->alloc[i] = &emptymspan; c->alloc[i] = &emptymspan;
} }
} }
......
...@@ -37,14 +37,14 @@ runtime·MCentral_CacheSpan(MCentral *c) ...@@ -37,14 +37,14 @@ runtime·MCentral_CacheSpan(MCentral *c)
int32 cap, n; int32 cap, n;
uint32 sg; uint32 sg;
runtime·lock(c); runtime·lock(&c->lock);
sg = runtime·mheap.sweepgen; sg = runtime·mheap.sweepgen;
retry: retry:
for(s = c->nonempty.next; s != &c->nonempty; s = s->next) { for(s = c->nonempty.next; s != &c->nonempty; s = s->next) {
if(s->sweepgen == sg-2 && runtime·cas(&s->sweepgen, sg-2, sg-1)) { if(s->sweepgen == sg-2 && runtime·cas(&s->sweepgen, sg-2, sg-1)) {
runtime·unlock(c); runtime·unlock(&c->lock);
runtime·MSpan_Sweep(s); runtime·MSpan_Sweep(s);
runtime·lock(c); runtime·lock(&c->lock);
// the span could have been moved to heap, retry // the span could have been moved to heap, retry
goto retry; goto retry;
} }
...@@ -63,9 +63,9 @@ retry: ...@@ -63,9 +63,9 @@ retry:
runtime·MSpanList_Remove(s); runtime·MSpanList_Remove(s);
// swept spans are at the end of the list // swept spans are at the end of the list
runtime·MSpanList_InsertBack(&c->empty, s); runtime·MSpanList_InsertBack(&c->empty, s);
runtime·unlock(c); runtime·unlock(&c->lock);
runtime·MSpan_Sweep(s); runtime·MSpan_Sweep(s);
runtime·lock(c); runtime·lock(&c->lock);
// the span could be moved to nonempty or heap, retry // the span could be moved to nonempty or heap, retry
goto retry; goto retry;
} }
...@@ -80,7 +80,7 @@ retry: ...@@ -80,7 +80,7 @@ retry:
// Replenish central list if empty. // Replenish central list if empty.
if(!MCentral_Grow(c)) { if(!MCentral_Grow(c)) {
runtime·unlock(c); runtime·unlock(&c->lock);
return nil; return nil;
} }
goto retry; goto retry;
...@@ -95,7 +95,7 @@ havespan: ...@@ -95,7 +95,7 @@ havespan:
runtime·MSpanList_Remove(s); runtime·MSpanList_Remove(s);
runtime·MSpanList_InsertBack(&c->empty, s); runtime·MSpanList_InsertBack(&c->empty, s);
s->incache = true; s->incache = true;
runtime·unlock(c); runtime·unlock(&c->lock);
return s; return s;
} }
...@@ -105,7 +105,7 @@ runtime·MCentral_UncacheSpan(MCentral *c, MSpan *s) ...@@ -105,7 +105,7 @@ runtime·MCentral_UncacheSpan(MCentral *c, MSpan *s)
{ {
int32 cap, n; int32 cap, n;
runtime·lock(c); runtime·lock(&c->lock);
s->incache = false; s->incache = false;
...@@ -118,7 +118,7 @@ runtime·MCentral_UncacheSpan(MCentral *c, MSpan *s) ...@@ -118,7 +118,7 @@ runtime·MCentral_UncacheSpan(MCentral *c, MSpan *s)
runtime·MSpanList_Remove(s); runtime·MSpanList_Remove(s);
runtime·MSpanList_Insert(&c->nonempty, s); runtime·MSpanList_Insert(&c->nonempty, s);
} }
runtime·unlock(c); runtime·unlock(&c->lock);
} }
// Free n objects from a span s back into the central free list c. // Free n objects from a span s back into the central free list c.
...@@ -130,7 +130,7 @@ runtime·MCentral_FreeSpan(MCentral *c, MSpan *s, int32 n, MLink *start, MLink * ...@@ -130,7 +130,7 @@ runtime·MCentral_FreeSpan(MCentral *c, MSpan *s, int32 n, MLink *start, MLink *
{ {
if(s->incache) if(s->incache)
runtime·throw("freespan into cached span"); runtime·throw("freespan into cached span");
runtime·lock(c); runtime·lock(&c->lock);
// Move to nonempty if necessary. // Move to nonempty if necessary.
if(s->freelist == nil) { if(s->freelist == nil) {
...@@ -150,7 +150,7 @@ runtime·MCentral_FreeSpan(MCentral *c, MSpan *s, int32 n, MLink *start, MLink * ...@@ -150,7 +150,7 @@ runtime·MCentral_FreeSpan(MCentral *c, MSpan *s, int32 n, MLink *start, MLink *
runtime·atomicstore(&s->sweepgen, runtime·mheap.sweepgen); runtime·atomicstore(&s->sweepgen, runtime·mheap.sweepgen);
if(s->ref != 0) { if(s->ref != 0) {
runtime·unlock(c); runtime·unlock(&c->lock);
return false; return false;
} }
...@@ -158,7 +158,7 @@ runtime·MCentral_FreeSpan(MCentral *c, MSpan *s, int32 n, MLink *start, MLink * ...@@ -158,7 +158,7 @@ runtime·MCentral_FreeSpan(MCentral *c, MSpan *s, int32 n, MLink *start, MLink *
runtime·MSpanList_Remove(s); runtime·MSpanList_Remove(s);
s->needzero = 1; s->needzero = 1;
s->freelist = nil; s->freelist = nil;
runtime·unlock(c); runtime·unlock(&c->lock);
runtime·unmarkspan((byte*)(s->start<<PageShift), s->npages<<PageShift); runtime·unmarkspan((byte*)(s->start<<PageShift), s->npages<<PageShift);
runtime·MHeap_Free(&runtime·mheap, s, 0); runtime·MHeap_Free(&runtime·mheap, s, 0);
return true; return true;
...@@ -174,14 +174,14 @@ MCentral_Grow(MCentral *c) ...@@ -174,14 +174,14 @@ MCentral_Grow(MCentral *c)
byte *p; byte *p;
MSpan *s; MSpan *s;
runtime·unlock(c); runtime·unlock(&c->lock);
npages = runtime·class_to_allocnpages[c->sizeclass]; npages = runtime·class_to_allocnpages[c->sizeclass];
size = runtime·class_to_size[c->sizeclass]; size = runtime·class_to_size[c->sizeclass];
n = (npages << PageShift) / size; n = (npages << PageShift) / size;
s = runtime·MHeap_Alloc(&runtime·mheap, npages, c->sizeclass, 0, 1); s = runtime·MHeap_Alloc(&runtime·mheap, npages, c->sizeclass, 0, 1);
if(s == nil) { if(s == nil) {
// TODO(rsc): Log out of memory // TODO(rsc): Log out of memory
runtime·lock(c); runtime·lock(&c->lock);
return false; return false;
} }
...@@ -198,7 +198,7 @@ MCentral_Grow(MCentral *c) ...@@ -198,7 +198,7 @@ MCentral_Grow(MCentral *c)
*tailp = nil; *tailp = nil;
runtime·markspan((byte*)(s->start<<PageShift), size, n, size*n < (s->npages<<PageShift)); runtime·markspan((byte*)(s->start<<PageShift), size, n, size*n < (s->npages<<PageShift));
runtime·lock(c); runtime·lock(&c->lock);
runtime·MSpanList_Insert(&c->nonempty, s); runtime·MSpanList_Insert(&c->nonempty, s);
return true; return true;
} }
...@@ -539,7 +539,7 @@ markroot(ParFor *desc, uint32 i) ...@@ -539,7 +539,7 @@ markroot(ParFor *desc, uint32 i)
// retain everything it points to. // retain everything it points to.
spf = (SpecialFinalizer*)sp; spf = (SpecialFinalizer*)sp;
// A finalizer can be set for an inner byte of an object, find object beginning. // A finalizer can be set for an inner byte of an object, find object beginning.
p = (void*)((s->start << PageShift) + spf->offset/s->elemsize*s->elemsize); p = (void*)((s->start << PageShift) + spf->special.offset/s->elemsize*s->elemsize);
scanblock(p, s->elemsize, nil); scanblock(p, s->elemsize, nil);
scanblock((void*)&spf->fn, PtrSize, ScanConservatively); scanblock((void*)&spf->fn, PtrSize, ScanConservatively);
} }
...@@ -1043,7 +1043,7 @@ runtime·MSpan_Sweep(MSpan *s) ...@@ -1043,7 +1043,7 @@ runtime·MSpan_Sweep(MSpan *s)
c->local_nsmallfree[cl] += nfree; c->local_nsmallfree[cl] += nfree;
c->local_cachealloc -= nfree * size; c->local_cachealloc -= nfree * size;
runtime·xadd64(&mstats.next_gc, -(uint64)(nfree * size * (runtime·gcpercent + 100)/100)); runtime·xadd64(&mstats.next_gc, -(uint64)(nfree * size * (runtime·gcpercent + 100)/100));
res = runtime·MCentral_FreeSpan(&runtime·mheap.central[cl], s, nfree, head.next, end); res = runtime·MCentral_FreeSpan(&runtime·mheap.central[cl].mcentral, s, nfree, head.next, end);
// MCentral_FreeSpan updates sweepgen // MCentral_FreeSpan updates sweepgen
} }
return res; return res;
...@@ -1308,10 +1308,10 @@ runtime·gc(int32 force) ...@@ -1308,10 +1308,10 @@ runtime·gc(int32 force)
return; return;
if(runtime·gcpercent == GcpercentUnknown) { // first time through if(runtime·gcpercent == GcpercentUnknown) { // first time through
runtime·lock(&runtime·mheap); runtime·lock(&runtime·mheap.lock);
if(runtime·gcpercent == GcpercentUnknown) if(runtime·gcpercent == GcpercentUnknown)
runtime·gcpercent = runtime·readgogc(); runtime·gcpercent = runtime·readgogc();
runtime·unlock(&runtime·mheap); runtime·unlock(&runtime·mheap.lock);
} }
if(runtime·gcpercent < 0) if(runtime·gcpercent < 0)
return; return;
...@@ -1560,7 +1560,7 @@ runtime∕debug·readGCStats(Slice *pauses) ...@@ -1560,7 +1560,7 @@ runtime∕debug·readGCStats(Slice *pauses)
// Pass back: pauses, last gc (absolute time), number of gc, total pause ns. // Pass back: pauses, last gc (absolute time), number of gc, total pause ns.
p = (uint64*)pauses->array; p = (uint64*)pauses->array;
runtime·lock(&runtime·mheap); runtime·lock(&runtime·mheap.lock);
n = mstats.numgc; n = mstats.numgc;
if(n > nelem(mstats.pause_ns)) if(n > nelem(mstats.pause_ns))
n = nelem(mstats.pause_ns); n = nelem(mstats.pause_ns);
...@@ -1575,7 +1575,7 @@ runtime∕debug·readGCStats(Slice *pauses) ...@@ -1575,7 +1575,7 @@ runtime∕debug·readGCStats(Slice *pauses)
p[n] = mstats.last_gc; p[n] = mstats.last_gc;
p[n+1] = mstats.numgc; p[n+1] = mstats.numgc;
p[n+2] = mstats.pause_total_ns; p[n+2] = mstats.pause_total_ns;
runtime·unlock(&runtime·mheap); runtime·unlock(&runtime·mheap.lock);
pauses->len = n+3; pauses->len = n+3;
} }
...@@ -1583,14 +1583,14 @@ int32 ...@@ -1583,14 +1583,14 @@ int32
runtime·setgcpercent(int32 in) { runtime·setgcpercent(int32 in) {
int32 out; int32 out;
runtime·lock(&runtime·mheap); runtime·lock(&runtime·mheap.lock);
if(runtime·gcpercent == GcpercentUnknown) if(runtime·gcpercent == GcpercentUnknown)
runtime·gcpercent = runtime·readgogc(); runtime·gcpercent = runtime·readgogc();
out = runtime·gcpercent; out = runtime·gcpercent;
if(in < 0) if(in < 0)
in = -1; in = -1;
runtime·gcpercent = in; runtime·gcpercent = in;
runtime·unlock(&runtime·mheap); runtime·unlock(&runtime·mheap.lock);
return out; return out;
} }
...@@ -1670,11 +1670,11 @@ runfinq(void) ...@@ -1670,11 +1670,11 @@ runfinq(void)
} else if(((InterfaceType*)f->fint)->mhdr.len == 0) { } else if(((InterfaceType*)f->fint)->mhdr.len == 0) {
// convert to empty interface // convert to empty interface
ef = (Eface*)frame; ef = (Eface*)frame;
ef->type = f->ot; ef->type = &f->ot->typ;
ef->data = f->arg; ef->data = f->arg;
} else { } else {
// convert to interface with methods, via empty interface. // convert to interface with methods, via empty interface.
ef1.type = f->ot; ef1.type = &f->ot->typ;
ef1.data = f->arg; ef1.data = f->arg;
if(!runtime·ifaceE2I2((InterfaceType*)f->fint, ef1, (Iface*)frame)) if(!runtime·ifaceE2I2((InterfaceType*)f->fint, ef1, (Iface*)frame))
runtime·throw("invalid type conversion in runfinq"); runtime·throw("invalid type conversion in runfinq");
......
...@@ -70,7 +70,7 @@ runtime·MHeap_Init(MHeap *h) ...@@ -70,7 +70,7 @@ runtime·MHeap_Init(MHeap *h)
runtime·MSpanList_Init(&h->freelarge); runtime·MSpanList_Init(&h->freelarge);
runtime·MSpanList_Init(&h->busylarge); runtime·MSpanList_Init(&h->busylarge);
for(i=0; i<nelem(h->central); i++) for(i=0; i<nelem(h->central); i++)
runtime·MCentral_Init(&h->central[i], i); runtime·MCentral_Init(&h->central[i].mcentral, i);
} }
void void
...@@ -106,9 +106,9 @@ retry: ...@@ -106,9 +106,9 @@ retry:
runtime·MSpanList_Remove(s); runtime·MSpanList_Remove(s);
// swept spans are at the end of the list // swept spans are at the end of the list
runtime·MSpanList_InsertBack(list, s); runtime·MSpanList_InsertBack(list, s);
runtime·unlock(h); runtime·unlock(&h->lock);
n += runtime·MSpan_Sweep(s); n += runtime·MSpan_Sweep(s);
runtime·lock(h); runtime·lock(&h->lock);
if(n >= npages) if(n >= npages)
return n; return n;
// the span could have been moved elsewhere // the span could have been moved elsewhere
...@@ -153,7 +153,7 @@ MHeap_Reclaim(MHeap *h, uintptr npage) ...@@ -153,7 +153,7 @@ MHeap_Reclaim(MHeap *h, uintptr npage)
} }
// Now sweep everything that is not yet swept. // Now sweep everything that is not yet swept.
runtime·unlock(h); runtime·unlock(&h->lock);
for(;;) { for(;;) {
n = runtime·sweepone(); n = runtime·sweepone();
if(n == -1) // all spans are swept if(n == -1) // all spans are swept
...@@ -162,7 +162,7 @@ MHeap_Reclaim(MHeap *h, uintptr npage) ...@@ -162,7 +162,7 @@ MHeap_Reclaim(MHeap *h, uintptr npage)
if(reclaimed >= npage) if(reclaimed >= npage)
break; break;
} }
runtime·lock(h); runtime·lock(&h->lock);
} }
// Allocate a new span of npage pages from the heap for GC'd memory // Allocate a new span of npage pages from the heap for GC'd memory
...@@ -174,7 +174,7 @@ mheap_alloc(MHeap *h, uintptr npage, int32 sizeclass, bool large) ...@@ -174,7 +174,7 @@ mheap_alloc(MHeap *h, uintptr npage, int32 sizeclass, bool large)
if(g != g->m->g0) if(g != g->m->g0)
runtime·throw("mheap_alloc not on M stack"); runtime·throw("mheap_alloc not on M stack");
runtime·lock(h); runtime·lock(&h->lock);
// To prevent excessive heap growth, before allocating n pages // To prevent excessive heap growth, before allocating n pages
// we need to sweep and reclaim at least n pages. // we need to sweep and reclaim at least n pages.
...@@ -207,7 +207,7 @@ mheap_alloc(MHeap *h, uintptr npage, int32 sizeclass, bool large) ...@@ -207,7 +207,7 @@ mheap_alloc(MHeap *h, uintptr npage, int32 sizeclass, bool large)
runtime·MSpanList_InsertBack(&h->busylarge, s); runtime·MSpanList_InsertBack(&h->busylarge, s);
} }
} }
runtime·unlock(h); runtime·unlock(&h->lock);
return s; return s;
} }
...@@ -259,7 +259,7 @@ runtime·MHeap_AllocStack(MHeap *h, uintptr npage) ...@@ -259,7 +259,7 @@ runtime·MHeap_AllocStack(MHeap *h, uintptr npage)
if(g != g->m->g0) if(g != g->m->g0)
runtime·throw("mheap_allocstack not on M stack"); runtime·throw("mheap_allocstack not on M stack");
runtime·lock(h); runtime·lock(&h->lock);
s = MHeap_AllocSpanLocked(h, npage); s = MHeap_AllocSpanLocked(h, npage);
if(s != nil) { if(s != nil) {
s->state = MSpanStack; s->state = MSpanStack;
...@@ -267,7 +267,7 @@ runtime·MHeap_AllocStack(MHeap *h, uintptr npage) ...@@ -267,7 +267,7 @@ runtime·MHeap_AllocStack(MHeap *h, uintptr npage)
s->ref = 0; s->ref = 0;
mstats.stacks_inuse += s->npages<<PageShift; mstats.stacks_inuse += s->npages<<PageShift;
} }
runtime·unlock(h); runtime·unlock(&h->lock);
return s; return s;
} }
...@@ -460,7 +460,7 @@ mheap_free(MHeap *h, MSpan *s, int32 acct) ...@@ -460,7 +460,7 @@ mheap_free(MHeap *h, MSpan *s, int32 acct)
{ {
if(g != g->m->g0) if(g != g->m->g0)
runtime·throw("mheap_free not on M stack"); runtime·throw("mheap_free not on M stack");
runtime·lock(h); runtime·lock(&h->lock);
mstats.heap_alloc += g->m->mcache->local_cachealloc; mstats.heap_alloc += g->m->mcache->local_cachealloc;
g->m->mcache->local_cachealloc = 0; g->m->mcache->local_cachealloc = 0;
if(acct) { if(acct) {
...@@ -468,7 +468,7 @@ mheap_free(MHeap *h, MSpan *s, int32 acct) ...@@ -468,7 +468,7 @@ mheap_free(MHeap *h, MSpan *s, int32 acct)
mstats.heap_objects--; mstats.heap_objects--;
} }
MHeap_FreeSpanLocked(h, s); MHeap_FreeSpanLocked(h, s);
runtime·unlock(h); runtime·unlock(&h->lock);
} }
static void static void
...@@ -504,10 +504,10 @@ runtime·MHeap_FreeStack(MHeap *h, MSpan *s) ...@@ -504,10 +504,10 @@ runtime·MHeap_FreeStack(MHeap *h, MSpan *s)
if(g != g->m->g0) if(g != g->m->g0)
runtime·throw("mheap_freestack not on M stack"); runtime·throw("mheap_freestack not on M stack");
s->needzero = 1; s->needzero = 1;
runtime·lock(h); runtime·lock(&h->lock);
mstats.stacks_inuse -= s->npages<<PageShift; mstats.stacks_inuse -= s->npages<<PageShift;
MHeap_FreeSpanLocked(h, s); MHeap_FreeSpanLocked(h, s);
runtime·unlock(h); runtime·unlock(&h->lock);
} }
static void static void
...@@ -626,9 +626,9 @@ scavenge(int32 k, uint64 now, uint64 limit) ...@@ -626,9 +626,9 @@ scavenge(int32 k, uint64 now, uint64 limit)
static void static void
scavenge_m(G *gp) scavenge_m(G *gp)
{ {
runtime·lock(&runtime·mheap); runtime·lock(&runtime·mheap.lock);
scavenge(g->m->scalararg[0], g->m->scalararg[1], g->m->scalararg[2]); scavenge(g->m->scalararg[0], g->m->scalararg[1], g->m->scalararg[2]);
runtime·unlock(&runtime·mheap); runtime·unlock(&runtime·mheap.lock);
runtime·gogo(&gp->sched); runtime·gogo(&gp->sched);
} }
...@@ -865,12 +865,12 @@ runtime·addfinalizer(void *p, FuncVal *f, uintptr nret, Type *fint, PtrType *ot ...@@ -865,12 +865,12 @@ runtime·addfinalizer(void *p, FuncVal *f, uintptr nret, Type *fint, PtrType *ot
runtime·lock(&runtime·mheap.speciallock); runtime·lock(&runtime·mheap.speciallock);
s = runtime·FixAlloc_Alloc(&runtime·mheap.specialfinalizeralloc); s = runtime·FixAlloc_Alloc(&runtime·mheap.specialfinalizeralloc);
runtime·unlock(&runtime·mheap.speciallock); runtime·unlock(&runtime·mheap.speciallock);
s->kind = KindSpecialFinalizer; s->special.kind = KindSpecialFinalizer;
s->fn = f; s->fn = f;
s->nret = nret; s->nret = nret;
s->fint = fint; s->fint = fint;
s->ot = ot; s->ot = ot;
if(addspecial(p, s)) if(addspecial(p, &s->special))
return true; return true;
// There was an old finalizer // There was an old finalizer
...@@ -903,9 +903,9 @@ runtime·setprofilebucket(void *p, Bucket *b) ...@@ -903,9 +903,9 @@ runtime·setprofilebucket(void *p, Bucket *b)
runtime·lock(&runtime·mheap.speciallock); runtime·lock(&runtime·mheap.speciallock);
s = runtime·FixAlloc_Alloc(&runtime·mheap.specialprofilealloc); s = runtime·FixAlloc_Alloc(&runtime·mheap.specialprofilealloc);
runtime·unlock(&runtime·mheap.speciallock); runtime·unlock(&runtime·mheap.speciallock);
s->kind = KindSpecialProfile; s->special.kind = KindSpecialProfile;
s->b = b; s->b = b;
if(!addspecial(p, s)) if(!addspecial(p, &s->special))
runtime·throw("setprofilebucket: profile already set"); runtime·throw("setprofilebucket: profile already set");
} }
......
...@@ -92,20 +92,20 @@ MProf_GC(void) ...@@ -92,20 +92,20 @@ MProf_GC(void)
Bucket *b; Bucket *b;
for(b=mbuckets; b; b=b->allnext) { for(b=mbuckets; b; b=b->allnext) {
b->allocs += b->prev_allocs; b->data.mp.allocs += b->data.mp.prev_allocs;
b->frees += b->prev_frees; b->data.mp.frees += b->data.mp.prev_frees;
b->alloc_bytes += b->prev_alloc_bytes; b->data.mp.alloc_bytes += b->data.mp.prev_alloc_bytes;
b->free_bytes += b->prev_free_bytes; b->data.mp.free_bytes += b->data.mp.prev_free_bytes;
b->prev_allocs = b->recent_allocs; b->data.mp.prev_allocs = b->data.mp.recent_allocs;
b->prev_frees = b->recent_frees; b->data.mp.prev_frees = b->data.mp.recent_frees;
b->prev_alloc_bytes = b->recent_alloc_bytes; b->data.mp.prev_alloc_bytes = b->data.mp.recent_alloc_bytes;
b->prev_free_bytes = b->recent_free_bytes; b->data.mp.prev_free_bytes = b->data.mp.recent_free_bytes;
b->recent_allocs = 0; b->data.mp.recent_allocs = 0;
b->recent_frees = 0; b->data.mp.recent_frees = 0;
b->recent_alloc_bytes = 0; b->data.mp.recent_alloc_bytes = 0;
b->recent_free_bytes = 0; b->data.mp.recent_free_bytes = 0;
} }
} }
...@@ -129,8 +129,8 @@ runtime·MProf_Malloc(void *p, uintptr size) ...@@ -129,8 +129,8 @@ runtime·MProf_Malloc(void *p, uintptr size)
nstk = runtime·callers(1, stk, nelem(stk)); nstk = runtime·callers(1, stk, nelem(stk));
runtime·lock(&proflock); runtime·lock(&proflock);
b = stkbucket(MProf, size, stk, nstk, true); b = stkbucket(MProf, size, stk, nstk, true);
b->recent_allocs++; b->data.mp.recent_allocs++;
b->recent_alloc_bytes += size; b->data.mp.recent_alloc_bytes += size;
runtime·unlock(&proflock); runtime·unlock(&proflock);
// Setprofilebucket locks a bunch of other mutexes, so we call it outside of proflock. // Setprofilebucket locks a bunch of other mutexes, so we call it outside of proflock.
...@@ -160,8 +160,8 @@ runtime·mprofMalloc_m(void) ...@@ -160,8 +160,8 @@ runtime·mprofMalloc_m(void)
nstk = runtime·gcallers(g->m->curg, 1, stk, nelem(stk)); nstk = runtime·gcallers(g->m->curg, 1, stk, nelem(stk));
runtime·lock(&proflock); runtime·lock(&proflock);
b = stkbucket(MProf, size, stk, nstk, true); b = stkbucket(MProf, size, stk, nstk, true);
b->recent_allocs++; b->data.mp.recent_allocs++;
b->recent_alloc_bytes += size; b->data.mp.recent_alloc_bytes += size;
runtime·unlock(&proflock); runtime·unlock(&proflock);
// Setprofilebucket locks a bunch of other mutexes, so we call it outside of proflock. // Setprofilebucket locks a bunch of other mutexes, so we call it outside of proflock.
...@@ -177,11 +177,11 @@ runtime·MProf_Free(Bucket *b, uintptr size, bool freed) ...@@ -177,11 +177,11 @@ runtime·MProf_Free(Bucket *b, uintptr size, bool freed)
{ {
runtime·lock(&proflock); runtime·lock(&proflock);
if(freed) { if(freed) {
b->recent_frees++; b->data.mp.recent_frees++;
b->recent_free_bytes += size; b->data.mp.recent_free_bytes += size;
} else { } else {
b->prev_frees++; b->data.mp.prev_frees++;
b->prev_free_bytes += size; b->data.mp.prev_free_bytes += size;
} }
runtime·unlock(&proflock); runtime·unlock(&proflock);
} }
...@@ -221,8 +221,8 @@ runtime·blockevent(int64 cycles, int32 skip) ...@@ -221,8 +221,8 @@ runtime·blockevent(int64 cycles, int32 skip)
nstk = runtime·callers(skip, stk, nelem(stk)); nstk = runtime·callers(skip, stk, nelem(stk));
runtime·lock(&proflock); runtime·lock(&proflock);
b = stkbucket(BProf, 0, stk, nstk, true); b = stkbucket(BProf, 0, stk, nstk, true);
b->count++; b->data.bp.count++;
b->cycles += cycles; b->data.bp.cycles += cycles;
runtime·unlock(&proflock); runtime·unlock(&proflock);
} }
...@@ -242,10 +242,10 @@ record(Record *r, Bucket *b) ...@@ -242,10 +242,10 @@ record(Record *r, Bucket *b)
{ {
int32 i; int32 i;
r->alloc_bytes = b->alloc_bytes; r->alloc_bytes = b->data.mp.alloc_bytes;
r->free_bytes = b->free_bytes; r->free_bytes = b->data.mp.free_bytes;
r->alloc_objects = b->allocs; r->alloc_objects = b->data.mp.allocs;
r->free_objects = b->frees; r->free_objects = b->data.mp.frees;
for(i=0; i<b->nstk && i<nelem(r->stk); i++) for(i=0; i<b->nstk && i<nelem(r->stk); i++)
r->stk[i] = b->stk[i]; r->stk[i] = b->stk[i];
for(; i<nelem(r->stk); i++) for(; i<nelem(r->stk); i++)
...@@ -261,9 +261,9 @@ func MemProfile(p Slice, include_inuse_zero bool) (n int, ok bool) { ...@@ -261,9 +261,9 @@ func MemProfile(p Slice, include_inuse_zero bool) (n int, ok bool) {
n = 0; n = 0;
clear = true; clear = true;
for(b=mbuckets; b; b=b->allnext) { for(b=mbuckets; b; b=b->allnext) {
if(include_inuse_zero || b->alloc_bytes != b->free_bytes) if(include_inuse_zero || b->data.mp.alloc_bytes != b->data.mp.free_bytes)
n++; n++;
if(b->allocs != 0 || b->frees != 0) if(b->data.mp.allocs != 0 || b->data.mp.frees != 0)
clear = false; clear = false;
} }
if(clear) { if(clear) {
...@@ -275,7 +275,7 @@ func MemProfile(p Slice, include_inuse_zero bool) (n int, ok bool) { ...@@ -275,7 +275,7 @@ func MemProfile(p Slice, include_inuse_zero bool) (n int, ok bool) {
MProf_GC(); MProf_GC();
n = 0; n = 0;
for(b=mbuckets; b; b=b->allnext) for(b=mbuckets; b; b=b->allnext)
if(include_inuse_zero || b->alloc_bytes != b->free_bytes) if(include_inuse_zero || b->data.mp.alloc_bytes != b->data.mp.free_bytes)
n++; n++;
} }
ok = false; ok = false;
...@@ -283,7 +283,7 @@ func MemProfile(p Slice, include_inuse_zero bool) (n int, ok bool) { ...@@ -283,7 +283,7 @@ func MemProfile(p Slice, include_inuse_zero bool) (n int, ok bool) {
ok = true; ok = true;
r = (Record*)p.array; r = (Record*)p.array;
for(b=mbuckets; b; b=b->allnext) for(b=mbuckets; b; b=b->allnext)
if(include_inuse_zero || b->alloc_bytes != b->free_bytes) if(include_inuse_zero || b->data.mp.alloc_bytes != b->data.mp.free_bytes)
record(r++, b); record(r++, b);
} }
runtime·unlock(&proflock); runtime·unlock(&proflock);
...@@ -296,7 +296,7 @@ runtime·iterate_memprof(void (*callback)(Bucket*, uintptr, uintptr*, uintptr, u ...@@ -296,7 +296,7 @@ runtime·iterate_memprof(void (*callback)(Bucket*, uintptr, uintptr*, uintptr, u
runtime·lock(&proflock); runtime·lock(&proflock);
for(b=mbuckets; b; b=b->allnext) { for(b=mbuckets; b; b=b->allnext) {
callback(b, b->nstk, b->stk, b->size, b->allocs, b->frees); callback(b, b->nstk, b->stk, b->size, b->data.mp.allocs, b->data.mp.frees);
} }
runtime·unlock(&proflock); runtime·unlock(&proflock);
} }
...@@ -323,8 +323,8 @@ func BlockProfile(p Slice) (n int, ok bool) { ...@@ -323,8 +323,8 @@ func BlockProfile(p Slice) (n int, ok bool) {
ok = true; ok = true;
r = (BRecord*)p.array; r = (BRecord*)p.array;
for(b=bbuckets; b; b=b->allnext, r++) { for(b=bbuckets; b; b=b->allnext, r++) {
r->count = b->count; r->count = b->data.bp.count;
r->cycles = b->cycles; r->cycles = b->data.bp.cycles;
for(i=0; i<b->nstk && i<nelem(r->stk); i++) for(i=0; i<b->nstk && i<nelem(r->stk); i++)
r->stk[i] = b->stk[i]; r->stk[i] = b->stk[i];
for(; i<nelem(r->stk); i++) for(; i<nelem(r->stk); i++)
......
...@@ -42,13 +42,13 @@ struct Bucket ...@@ -42,13 +42,13 @@ struct Bucket
uintptr recent_alloc_bytes; uintptr recent_alloc_bytes;
uintptr recent_free_bytes; uintptr recent_free_bytes;
}; } mp;
struct // typ == BProf struct // typ == BProf
{ {
int64 count; int64 count;
int64 cycles; int64 cycles;
}; } bp;
}; } data;
uintptr hash; // hash of size + stk uintptr hash; // hash of size + stk
uintptr size; uintptr size;
uintptr nstk; uintptr nstk;
......
...@@ -48,7 +48,7 @@ struct PollDesc ...@@ -48,7 +48,7 @@ struct PollDesc
// pollReset, pollWait, pollWaitCanceled and runtime·netpollready (IO rediness notification) // pollReset, pollWait, pollWaitCanceled and runtime·netpollready (IO rediness notification)
// proceed w/o taking the lock. So closing, rg, rd, wg and wd are manipulated // proceed w/o taking the lock. So closing, rg, rd, wg and wd are manipulated
// in a lock-free way by all operations. // in a lock-free way by all operations.
Lock; // protectes the following fields Lock lock; // protectes the following fields
uintptr fd; uintptr fd;
bool closing; bool closing;
uintptr seq; // protects from stale timers and ready notifications uintptr seq; // protects from stale timers and ready notifications
...@@ -63,7 +63,7 @@ struct PollDesc ...@@ -63,7 +63,7 @@ struct PollDesc
static struct static struct
{ {
Lock; Lock lock;
PollDesc* first; PollDesc* first;
// PollDesc objects must be type-stable, // PollDesc objects must be type-stable,
// because we can get ready notification from epoll/kqueue // because we can get ready notification from epoll/kqueue
...@@ -95,7 +95,7 @@ func runtime_pollServerInit() { ...@@ -95,7 +95,7 @@ func runtime_pollServerInit() {
func runtime_pollOpen(fd uintptr) (pd *PollDesc, errno int) { func runtime_pollOpen(fd uintptr) (pd *PollDesc, errno int) {
pd = allocPollDesc(); pd = allocPollDesc();
runtime·lock(pd); runtime·lock(&pd->lock);
if(pd->wg != nil && pd->wg != READY) if(pd->wg != nil && pd->wg != READY)
runtime·throw("runtime_pollOpen: blocked write on free descriptor"); runtime·throw("runtime_pollOpen: blocked write on free descriptor");
if(pd->rg != nil && pd->rg != READY) if(pd->rg != nil && pd->rg != READY)
...@@ -107,7 +107,7 @@ func runtime_pollOpen(fd uintptr) (pd *PollDesc, errno int) { ...@@ -107,7 +107,7 @@ func runtime_pollOpen(fd uintptr) (pd *PollDesc, errno int) {
pd->rd = 0; pd->rd = 0;
pd->wg = nil; pd->wg = nil;
pd->wd = 0; pd->wd = 0;
runtime·unlock(pd); runtime·unlock(&pd->lock);
errno = runtime·netpollopen(fd, pd); errno = runtime·netpollopen(fd, pd);
} }
...@@ -120,10 +120,10 @@ func runtime_pollClose(pd *PollDesc) { ...@@ -120,10 +120,10 @@ func runtime_pollClose(pd *PollDesc) {
if(pd->rg != nil && pd->rg != READY) if(pd->rg != nil && pd->rg != READY)
runtime·throw("runtime_pollClose: blocked read on closing descriptor"); runtime·throw("runtime_pollClose: blocked read on closing descriptor");
runtime·netpollclose(pd->fd); runtime·netpollclose(pd->fd);
runtime·lock(&pollcache); runtime·lock(&pollcache.lock);
pd->link = pollcache.first; pd->link = pollcache.first;
pollcache.first = pd; pollcache.first = pd;
runtime·unlock(&pollcache); runtime·unlock(&pollcache.lock);
} }
func runtime_pollReset(pd *PollDesc, mode int) (err int) { func runtime_pollReset(pd *PollDesc, mode int) (err int) {
...@@ -164,9 +164,9 @@ func runtime_pollWaitCanceled(pd *PollDesc, mode int) { ...@@ -164,9 +164,9 @@ func runtime_pollWaitCanceled(pd *PollDesc, mode int) {
func runtime_pollSetDeadline(pd *PollDesc, d int64, mode int) { func runtime_pollSetDeadline(pd *PollDesc, d int64, mode int) {
G *rg, *wg; G *rg, *wg;
runtime·lock(pd); runtime·lock(&pd->lock);
if(pd->closing) { if(pd->closing) {
runtime·unlock(pd); runtime·unlock(&pd->lock);
return; return;
} }
pd->seq++; // invalidate current timers pd->seq++; // invalidate current timers
...@@ -218,7 +218,7 @@ func runtime_pollSetDeadline(pd *PollDesc, d int64, mode int) { ...@@ -218,7 +218,7 @@ func runtime_pollSetDeadline(pd *PollDesc, d int64, mode int) {
rg = netpollunblock(pd, 'r', false); rg = netpollunblock(pd, 'r', false);
if(pd->wd < 0) if(pd->wd < 0)
wg = netpollunblock(pd, 'w', false); wg = netpollunblock(pd, 'w', false);
runtime·unlock(pd); runtime·unlock(&pd->lock);
if(rg) if(rg)
runtime·ready(rg); runtime·ready(rg);
if(wg) if(wg)
...@@ -228,7 +228,7 @@ func runtime_pollSetDeadline(pd *PollDesc, d int64, mode int) { ...@@ -228,7 +228,7 @@ func runtime_pollSetDeadline(pd *PollDesc, d int64, mode int) {
func runtime_pollUnblock(pd *PollDesc) { func runtime_pollUnblock(pd *PollDesc) {
G *rg, *wg; G *rg, *wg;
runtime·lock(pd); runtime·lock(&pd->lock);
if(pd->closing) if(pd->closing)
runtime·throw("runtime_pollUnblock: already closing"); runtime·throw("runtime_pollUnblock: already closing");
pd->closing = true; pd->closing = true;
...@@ -244,7 +244,7 @@ func runtime_pollUnblock(pd *PollDesc) { ...@@ -244,7 +244,7 @@ func runtime_pollUnblock(pd *PollDesc) {
runtime·deltimer(&pd->wt); runtime·deltimer(&pd->wt);
pd->wt.fv = nil; pd->wt.fv = nil;
} }
runtime·unlock(pd); runtime·unlock(&pd->lock);
if(rg) if(rg)
runtime·ready(rg); runtime·ready(rg);
if(wg) if(wg)
...@@ -272,13 +272,13 @@ runtime·netpollclosing(PollDesc *pd) ...@@ -272,13 +272,13 @@ runtime·netpollclosing(PollDesc *pd)
void void
runtime·netpolllock(PollDesc *pd) runtime·netpolllock(PollDesc *pd)
{ {
runtime·lock(pd); runtime·lock(&pd->lock);
} }
void void
runtime·netpollunlock(PollDesc *pd) runtime·netpollunlock(PollDesc *pd)
{ {
runtime·unlock(pd); runtime·unlock(&pd->lock);
} }
// make pd ready, newly runnable goroutines (if any) are enqueued info gpp list // make pd ready, newly runnable goroutines (if any) are enqueued info gpp list
...@@ -396,10 +396,10 @@ deadlineimpl(int64 now, Eface arg, bool read, bool write) ...@@ -396,10 +396,10 @@ deadlineimpl(int64 now, Eface arg, bool read, bool write)
// If it's stale, ignore the timer event. // If it's stale, ignore the timer event.
seq = (uintptr)arg.type; seq = (uintptr)arg.type;
rg = wg = nil; rg = wg = nil;
runtime·lock(pd); runtime·lock(&pd->lock);
if(seq != pd->seq) { if(seq != pd->seq) {
// The descriptor was reused or timers were reset. // The descriptor was reused or timers were reset.
runtime·unlock(pd); runtime·unlock(&pd->lock);
return; return;
} }
if(read) { if(read) {
...@@ -416,7 +416,7 @@ deadlineimpl(int64 now, Eface arg, bool read, bool write) ...@@ -416,7 +416,7 @@ deadlineimpl(int64 now, Eface arg, bool read, bool write)
runtime·atomicstorep(&pd->wt.fv, nil); // full memory barrier between store to wd and load of wg in netpollunblock runtime·atomicstorep(&pd->wt.fv, nil); // full memory barrier between store to wd and load of wg in netpollunblock
wg = netpollunblock(pd, 'w', false); wg = netpollunblock(pd, 'w', false);
} }
runtime·unlock(pd); runtime·unlock(&pd->lock);
if(rg) if(rg)
runtime·ready(rg); runtime·ready(rg);
if(wg) if(wg)
...@@ -447,7 +447,7 @@ allocPollDesc(void) ...@@ -447,7 +447,7 @@ allocPollDesc(void)
PollDesc *pd; PollDesc *pd;
uint32 i, n; uint32 i, n;
runtime·lock(&pollcache); runtime·lock(&pollcache.lock);
if(pollcache.first == nil) { if(pollcache.first == nil) {
n = PollBlockSize/sizeof(*pd); n = PollBlockSize/sizeof(*pd);
if(n == 0) if(n == 0)
...@@ -462,6 +462,6 @@ allocPollDesc(void) ...@@ -462,6 +462,6 @@ allocPollDesc(void)
} }
pd = pollcache.first; pd = pollcache.first;
pollcache.first = pd->link; pollcache.first = pd->link;
runtime·unlock(&pollcache); runtime·unlock(&pollcache.lock);
return pd; return pd;
} }
...@@ -26,7 +26,7 @@ ...@@ -26,7 +26,7 @@
typedef struct Sched Sched; typedef struct Sched Sched;
struct Sched { struct Sched {
Lock; Lock lock;
uint64 goidgen; uint64 goidgen;
...@@ -371,7 +371,7 @@ mcommoninit(M *mp) ...@@ -371,7 +371,7 @@ mcommoninit(M *mp)
mp->fastrand = 0x49f6428aUL + mp->id + runtime·cputicks(); mp->fastrand = 0x49f6428aUL + mp->id + runtime·cputicks();
runtime·lock(&runtime·sched); runtime·lock(&runtime·sched.lock);
mp->id = runtime·sched.mcount++; mp->id = runtime·sched.mcount++;
checkmcount(); checkmcount();
runtime·mpreinit(mp); runtime·mpreinit(mp);
...@@ -382,7 +382,7 @@ mcommoninit(M *mp) ...@@ -382,7 +382,7 @@ mcommoninit(M *mp)
// runtime·NumCgoCall() iterates over allm w/o schedlock, // runtime·NumCgoCall() iterates over allm w/o schedlock,
// so we need to publish it safely. // so we need to publish it safely.
runtime·atomicstorep(&runtime·allm, mp); runtime·atomicstorep(&runtime·allm, mp);
runtime·unlock(&runtime·sched); runtime·unlock(&runtime·sched.lock);
} }
// Mark gp ready to run. // Mark gp ready to run.
...@@ -411,7 +411,7 @@ runtime·gcprocs(void) ...@@ -411,7 +411,7 @@ runtime·gcprocs(void)
// Figure out how many CPUs to use during GC. // Figure out how many CPUs to use during GC.
// Limited by gomaxprocs, number of actual CPUs, and MaxGcproc. // Limited by gomaxprocs, number of actual CPUs, and MaxGcproc.
runtime·lock(&runtime·sched); runtime·lock(&runtime·sched.lock);
n = runtime·gomaxprocs; n = runtime·gomaxprocs;
if(n > runtime·ncpu) if(n > runtime·ncpu)
n = runtime·ncpu; n = runtime·ncpu;
...@@ -419,7 +419,7 @@ runtime·gcprocs(void) ...@@ -419,7 +419,7 @@ runtime·gcprocs(void)
n = MaxGcproc; n = MaxGcproc;
if(n > runtime·sched.nmidle+1) // one M is currently running if(n > runtime·sched.nmidle+1) // one M is currently running
n = runtime·sched.nmidle+1; n = runtime·sched.nmidle+1;
runtime·unlock(&runtime·sched); runtime·unlock(&runtime·sched.lock);
return n; return n;
} }
...@@ -428,14 +428,14 @@ needaddgcproc(void) ...@@ -428,14 +428,14 @@ needaddgcproc(void)
{ {
int32 n; int32 n;
runtime·lock(&runtime·sched); runtime·lock(&runtime·sched.lock);
n = runtime·gomaxprocs; n = runtime·gomaxprocs;
if(n > runtime·ncpu) if(n > runtime·ncpu)
n = runtime·ncpu; n = runtime·ncpu;
if(n > MaxGcproc) if(n > MaxGcproc)
n = MaxGcproc; n = MaxGcproc;
n -= runtime·sched.nmidle+1; // one M is currently running n -= runtime·sched.nmidle+1; // one M is currently running
runtime·unlock(&runtime·sched); runtime·unlock(&runtime·sched.lock);
return n > 0; return n > 0;
} }
...@@ -445,7 +445,7 @@ runtime·helpgc(int32 nproc) ...@@ -445,7 +445,7 @@ runtime·helpgc(int32 nproc)
M *mp; M *mp;
int32 n, pos; int32 n, pos;
runtime·lock(&runtime·sched); runtime·lock(&runtime·sched.lock);
pos = 0; pos = 0;
for(n = 1; n < nproc; n++) { // one M is currently running for(n = 1; n < nproc; n++) { // one M is currently running
if(runtime·allp[pos]->mcache == g->m->mcache) if(runtime·allp[pos]->mcache == g->m->mcache)
...@@ -458,7 +458,7 @@ runtime·helpgc(int32 nproc) ...@@ -458,7 +458,7 @@ runtime·helpgc(int32 nproc)
pos++; pos++;
runtime·notewakeup(&mp->park); runtime·notewakeup(&mp->park);
} }
runtime·unlock(&runtime·sched); runtime·unlock(&runtime·sched.lock);
} }
// Similar to stoptheworld but best-effort and can be called several times. // Similar to stoptheworld but best-effort and can be called several times.
...@@ -497,7 +497,7 @@ runtime·stoptheworld(void) ...@@ -497,7 +497,7 @@ runtime·stoptheworld(void)
P *p; P *p;
bool wait; bool wait;
runtime·lock(&runtime·sched); runtime·lock(&runtime·sched.lock);
runtime·sched.stopwait = runtime·gomaxprocs; runtime·sched.stopwait = runtime·gomaxprocs;
runtime·atomicstore((uint32*)&runtime·sched.gcwaiting, 1); runtime·atomicstore((uint32*)&runtime·sched.gcwaiting, 1);
preemptall(); preemptall();
...@@ -517,7 +517,7 @@ runtime·stoptheworld(void) ...@@ -517,7 +517,7 @@ runtime·stoptheworld(void)
runtime·sched.stopwait--; runtime·sched.stopwait--;
} }
wait = runtime·sched.stopwait > 0; wait = runtime·sched.stopwait > 0;
runtime·unlock(&runtime·sched); runtime·unlock(&runtime·sched.lock);
// wait for remaining P's to stop voluntarily // wait for remaining P's to stop voluntarily
if(wait) { if(wait) {
...@@ -557,7 +557,7 @@ runtime·starttheworld(void) ...@@ -557,7 +557,7 @@ runtime·starttheworld(void)
gp = runtime·netpoll(false); // non-blocking gp = runtime·netpoll(false); // non-blocking
injectglist(gp); injectglist(gp);
add = needaddgcproc(); add = needaddgcproc();
runtime·lock(&runtime·sched); runtime·lock(&runtime·sched.lock);
if(newprocs) { if(newprocs) {
procresize(newprocs); procresize(newprocs);
newprocs = 0; newprocs = 0;
...@@ -581,7 +581,7 @@ runtime·starttheworld(void) ...@@ -581,7 +581,7 @@ runtime·starttheworld(void)
runtime·sched.sysmonwait = false; runtime·sched.sysmonwait = false;
runtime·notewakeup(&runtime·sched.sysmonnote); runtime·notewakeup(&runtime·sched.sysmonnote);
} }
runtime·unlock(&runtime·sched); runtime·unlock(&runtime·sched.lock);
while(p1) { while(p1) {
p = p1; p = p1;
...@@ -964,9 +964,9 @@ stopm(void) ...@@ -964,9 +964,9 @@ stopm(void)
} }
retry: retry:
runtime·lock(&runtime·sched); runtime·lock(&runtime·sched.lock);
mput(g->m); mput(g->m);
runtime·unlock(&runtime·sched); runtime·unlock(&runtime·sched.lock);
runtime·notesleep(&g->m->park); runtime·notesleep(&g->m->park);
runtime·noteclear(&g->m->park); runtime·noteclear(&g->m->park);
if(g->m->helpgc) { if(g->m->helpgc) {
...@@ -993,18 +993,18 @@ startm(P *p, bool spinning) ...@@ -993,18 +993,18 @@ startm(P *p, bool spinning)
M *mp; M *mp;
void (*fn)(void); void (*fn)(void);
runtime·lock(&runtime·sched); runtime·lock(&runtime·sched.lock);
if(p == nil) { if(p == nil) {
p = pidleget(); p = pidleget();
if(p == nil) { if(p == nil) {
runtime·unlock(&runtime·sched); runtime·unlock(&runtime·sched.lock);
if(spinning) if(spinning)
runtime·xadd(&runtime·sched.nmspinning, -1); runtime·xadd(&runtime·sched.nmspinning, -1);
return; return;
} }
} }
mp = mget(); mp = mget();
runtime·unlock(&runtime·sched); runtime·unlock(&runtime·sched.lock);
if(mp == nil) { if(mp == nil) {
fn = nil; fn = nil;
if(spinning) if(spinning)
...@@ -1037,28 +1037,28 @@ handoffp(P *p) ...@@ -1037,28 +1037,28 @@ handoffp(P *p)
startm(p, true); startm(p, true);
return; return;
} }
runtime·lock(&runtime·sched); runtime·lock(&runtime·sched.lock);
if(runtime·sched.gcwaiting) { if(runtime·sched.gcwaiting) {
p->status = Pgcstop; p->status = Pgcstop;
if(--runtime·sched.stopwait == 0) if(--runtime·sched.stopwait == 0)
runtime·notewakeup(&runtime·sched.stopnote); runtime·notewakeup(&runtime·sched.stopnote);
runtime·unlock(&runtime·sched); runtime·unlock(&runtime·sched.lock);
return; return;
} }
if(runtime·sched.runqsize) { if(runtime·sched.runqsize) {
runtime·unlock(&runtime·sched); runtime·unlock(&runtime·sched.lock);
startm(p, false); startm(p, false);
return; return;
} }
// If this is the last running P and nobody is polling network, // If this is the last running P and nobody is polling network,
// need to wakeup another M to poll network. // need to wakeup another M to poll network.
if(runtime·sched.npidle == runtime·gomaxprocs-1 && runtime·atomicload64(&runtime·sched.lastpoll) != 0) { if(runtime·sched.npidle == runtime·gomaxprocs-1 && runtime·atomicload64(&runtime·sched.lastpoll) != 0) {
runtime·unlock(&runtime·sched); runtime·unlock(&runtime·sched.lock);
startm(p, false); startm(p, false);
return; return;
} }
pidleput(p); pidleput(p);
runtime·unlock(&runtime·sched); runtime·unlock(&runtime·sched.lock);
} }
// Tries to add one more P to execute G's. // Tries to add one more P to execute G's.
...@@ -1130,11 +1130,11 @@ gcstopm(void) ...@@ -1130,11 +1130,11 @@ gcstopm(void)
runtime·xadd(&runtime·sched.nmspinning, -1); runtime·xadd(&runtime·sched.nmspinning, -1);
} }
p = releasep(); p = releasep();
runtime·lock(&runtime·sched); runtime·lock(&runtime·sched.lock);
p->status = Pgcstop; p->status = Pgcstop;
if(--runtime·sched.stopwait == 0) if(--runtime·sched.stopwait == 0)
runtime·notewakeup(&runtime·sched.stopnote); runtime·notewakeup(&runtime·sched.stopnote);
runtime·unlock(&runtime·sched); runtime·unlock(&runtime·sched.lock);
stopm(); stopm();
} }
...@@ -1187,9 +1187,9 @@ top: ...@@ -1187,9 +1187,9 @@ top:
return gp; return gp;
// global runq // global runq
if(runtime·sched.runqsize) { if(runtime·sched.runqsize) {
runtime·lock(&runtime·sched); runtime·lock(&runtime·sched.lock);
gp = globrunqget(g->m->p, 0); gp = globrunqget(g->m->p, 0);
runtime·unlock(&runtime·sched); runtime·unlock(&runtime·sched.lock);
if(gp) if(gp)
return gp; return gp;
} }
...@@ -1223,19 +1223,19 @@ top: ...@@ -1223,19 +1223,19 @@ top:
} }
stop: stop:
// return P and block // return P and block
runtime·lock(&runtime·sched); runtime·lock(&runtime·sched.lock);
if(runtime·sched.gcwaiting) { if(runtime·sched.gcwaiting) {
runtime·unlock(&runtime·sched); runtime·unlock(&runtime·sched.lock);
goto top; goto top;
} }
if(runtime·sched.runqsize) { if(runtime·sched.runqsize) {
gp = globrunqget(g->m->p, 0); gp = globrunqget(g->m->p, 0);
runtime·unlock(&runtime·sched); runtime·unlock(&runtime·sched.lock);
return gp; return gp;
} }
p = releasep(); p = releasep();
pidleput(p); pidleput(p);
runtime·unlock(&runtime·sched); runtime·unlock(&runtime·sched.lock);
if(g->m->spinning) { if(g->m->spinning) {
g->m->spinning = false; g->m->spinning = false;
runtime·xadd(&runtime·sched.nmspinning, -1); runtime·xadd(&runtime·sched.nmspinning, -1);
...@@ -1244,9 +1244,9 @@ stop: ...@@ -1244,9 +1244,9 @@ stop:
for(i = 0; i < runtime·gomaxprocs; i++) { for(i = 0; i < runtime·gomaxprocs; i++) {
p = runtime·allp[i]; p = runtime·allp[i];
if(p && p->runqhead != p->runqtail) { if(p && p->runqhead != p->runqtail) {
runtime·lock(&runtime·sched); runtime·lock(&runtime·sched.lock);
p = pidleget(); p = pidleget();
runtime·unlock(&runtime·sched); runtime·unlock(&runtime·sched.lock);
if(p) { if(p) {
acquirep(p); acquirep(p);
goto top; goto top;
...@@ -1263,9 +1263,9 @@ stop: ...@@ -1263,9 +1263,9 @@ stop:
gp = runtime·netpoll(true); // block until new work is available gp = runtime·netpoll(true); // block until new work is available
runtime·atomicstore64(&runtime·sched.lastpoll, runtime·nanotime()); runtime·atomicstore64(&runtime·sched.lastpoll, runtime·nanotime());
if(gp) { if(gp) {
runtime·lock(&runtime·sched); runtime·lock(&runtime·sched.lock);
p = pidleget(); p = pidleget();
runtime·unlock(&runtime·sched); runtime·unlock(&runtime·sched.lock);
if(p) { if(p) {
acquirep(p); acquirep(p);
injectglist(gp->schedlink); injectglist(gp->schedlink);
...@@ -1308,14 +1308,14 @@ injectglist(G *glist) ...@@ -1308,14 +1308,14 @@ injectglist(G *glist)
if(glist == nil) if(glist == nil)
return; return;
runtime·lock(&runtime·sched); runtime·lock(&runtime·sched.lock);
for(n = 0; glist; n++) { for(n = 0; glist; n++) {
gp = glist; gp = glist;
glist = gp->schedlink; glist = gp->schedlink;
gp->status = Grunnable; gp->status = Grunnable;
globrunqput(gp); globrunqput(gp);
} }
runtime·unlock(&runtime·sched); runtime·unlock(&runtime·sched.lock);
for(; n && runtime·sched.npidle; n--) for(; n && runtime·sched.npidle; n--)
startm(nil, false); startm(nil, false);
...@@ -1351,9 +1351,9 @@ top: ...@@ -1351,9 +1351,9 @@ top:
// This is a fancy way to say tick%61==0, // This is a fancy way to say tick%61==0,
// it uses 2 MUL instructions instead of a single DIV and so is faster on modern processors. // it uses 2 MUL instructions instead of a single DIV and so is faster on modern processors.
if(tick - (((uint64)tick*0x4325c53fu)>>36)*61 == 0 && runtime·sched.runqsize > 0) { if(tick - (((uint64)tick*0x4325c53fu)>>36)*61 == 0 && runtime·sched.runqsize > 0) {
runtime·lock(&runtime·sched); runtime·lock(&runtime·sched.lock);
gp = globrunqget(g->m->p, 1); gp = globrunqget(g->m->p, 1);
runtime·unlock(&runtime·sched); runtime·unlock(&runtime·sched.lock);
if(gp) if(gp)
resetspinning(); resetspinning();
} }
...@@ -1459,9 +1459,9 @@ runtime·gosched0(G *gp) ...@@ -1459,9 +1459,9 @@ runtime·gosched0(G *gp)
{ {
gp->status = Grunnable; gp->status = Grunnable;
dropg(); dropg();
runtime·lock(&runtime·sched); runtime·lock(&runtime·sched.lock);
globrunqput(gp); globrunqput(gp);
runtime·unlock(&runtime·sched); runtime·unlock(&runtime·sched.lock);
schedule(); schedule();
} }
...@@ -1551,12 +1551,12 @@ void ...@@ -1551,12 +1551,12 @@ void
} }
if(runtime·atomicload(&runtime·sched.sysmonwait)) { // TODO: fast atomic if(runtime·atomicload(&runtime·sched.sysmonwait)) { // TODO: fast atomic
runtime·lock(&runtime·sched); runtime·lock(&runtime·sched.lock);
if(runtime·atomicload(&runtime·sched.sysmonwait)) { if(runtime·atomicload(&runtime·sched.sysmonwait)) {
runtime·atomicstore(&runtime·sched.sysmonwait, 0); runtime·atomicstore(&runtime·sched.sysmonwait, 0);
runtime·notewakeup(&runtime·sched.sysmonnote); runtime·notewakeup(&runtime·sched.sysmonnote);
} }
runtime·unlock(&runtime·sched); runtime·unlock(&runtime·sched.lock);
save(runtime·getcallerpc(&dummy), runtime·getcallersp(&dummy)); save(runtime·getcallerpc(&dummy), runtime·getcallersp(&dummy));
} }
...@@ -1564,12 +1564,12 @@ void ...@@ -1564,12 +1564,12 @@ void
g->m->p->m = nil; g->m->p->m = nil;
runtime·atomicstore(&g->m->p->status, Psyscall); runtime·atomicstore(&g->m->p->status, Psyscall);
if(runtime·sched.gcwaiting) { if(runtime·sched.gcwaiting) {
runtime·lock(&runtime·sched); runtime·lock(&runtime·sched.lock);
if (runtime·sched.stopwait > 0 && runtime·cas(&g->m->p->status, Psyscall, Pgcstop)) { if (runtime·sched.stopwait > 0 && runtime·cas(&g->m->p->status, Psyscall, Pgcstop)) {
if(--runtime·sched.stopwait == 0) if(--runtime·sched.stopwait == 0)
runtime·notewakeup(&runtime·sched.stopnote); runtime·notewakeup(&runtime·sched.stopnote);
} }
runtime·unlock(&runtime·sched); runtime·unlock(&runtime·sched.lock);
save(runtime·getcallerpc(&dummy), runtime·getcallersp(&dummy)); save(runtime·getcallerpc(&dummy), runtime·getcallersp(&dummy));
} }
...@@ -1685,13 +1685,13 @@ exitsyscallfast(void) ...@@ -1685,13 +1685,13 @@ exitsyscallfast(void)
// Try to get any other idle P. // Try to get any other idle P.
g->m->p = nil; g->m->p = nil;
if(runtime·sched.pidle) { if(runtime·sched.pidle) {
runtime·lock(&runtime·sched); runtime·lock(&runtime·sched.lock);
p = pidleget(); p = pidleget();
if(p && runtime·atomicload(&runtime·sched.sysmonwait)) { if(p && runtime·atomicload(&runtime·sched.sysmonwait)) {
runtime·atomicstore(&runtime·sched.sysmonwait, 0); runtime·atomicstore(&runtime·sched.sysmonwait, 0);
runtime·notewakeup(&runtime·sched.sysmonnote); runtime·notewakeup(&runtime·sched.sysmonnote);
} }
runtime·unlock(&runtime·sched); runtime·unlock(&runtime·sched.lock);
if(p) { if(p) {
acquirep(p); acquirep(p);
return true; return true;
...@@ -1709,7 +1709,7 @@ exitsyscall0(G *gp) ...@@ -1709,7 +1709,7 @@ exitsyscall0(G *gp)
gp->status = Grunnable; gp->status = Grunnable;
dropg(); dropg();
runtime·lock(&runtime·sched); runtime·lock(&runtime·sched.lock);
p = pidleget(); p = pidleget();
if(p == nil) if(p == nil)
globrunqput(gp); globrunqput(gp);
...@@ -1717,7 +1717,7 @@ exitsyscall0(G *gp) ...@@ -1717,7 +1717,7 @@ exitsyscall0(G *gp)
runtime·atomicstore(&runtime·sched.sysmonwait, 0); runtime·atomicstore(&runtime·sched.sysmonwait, 0);
runtime·notewakeup(&runtime·sched.sysmonnote); runtime·notewakeup(&runtime·sched.sysmonnote);
} }
runtime·unlock(&runtime·sched); runtime·unlock(&runtime·sched.lock);
if(p) { if(p) {
acquirep(p); acquirep(p);
execute(gp); // Never returns. execute(gp); // Never returns.
...@@ -2069,13 +2069,13 @@ runtime·gomaxprocsfunc(int32 n) ...@@ -2069,13 +2069,13 @@ runtime·gomaxprocsfunc(int32 n)
if(n > MaxGomaxprocs) if(n > MaxGomaxprocs)
n = MaxGomaxprocs; n = MaxGomaxprocs;
runtime·lock(&runtime·sched); runtime·lock(&runtime·sched.lock);
ret = runtime·gomaxprocs; ret = runtime·gomaxprocs;
if(n <= 0 || n == ret) { if(n <= 0 || n == ret) {
runtime·unlock(&runtime·sched); runtime·unlock(&runtime·sched.lock);
return ret; return ret;
} }
runtime·unlock(&runtime·sched); runtime·unlock(&runtime·sched.lock);
runtime·semacquire(&runtime·worldsema, false); runtime·semacquire(&runtime·worldsema, false);
g->m->gcing = 1; g->m->gcing = 1;
...@@ -2192,7 +2192,7 @@ runtime·badreflectcall(void) // called from assembly ...@@ -2192,7 +2192,7 @@ runtime·badreflectcall(void) // called from assembly
} }
static struct { static struct {
Lock; Lock lock;
void (*fn)(uintptr*, int32); void (*fn)(uintptr*, int32);
int32 hz; int32 hz;
uintptr pcbuf[100]; uintptr pcbuf[100];
...@@ -2300,9 +2300,9 @@ runtime·sigprof(uint8 *pc, uint8 *sp, uint8 *lr, G *gp, M *mp) ...@@ -2300,9 +2300,9 @@ runtime·sigprof(uint8 *pc, uint8 *sp, uint8 *lr, G *gp, M *mp)
((uint8*)runtime·gogo <= pc && pc < (uint8*)runtime·gogo + RuntimeGogoBytes)) ((uint8*)runtime·gogo <= pc && pc < (uint8*)runtime·gogo + RuntimeGogoBytes))
traceback = false; traceback = false;
runtime·lock(&prof); runtime·lock(&prof.lock);
if(prof.fn == nil) { if(prof.fn == nil) {
runtime·unlock(&prof); runtime·unlock(&prof.lock);
mp->mallocing--; mp->mallocing--;
return; return;
} }
...@@ -2341,7 +2341,7 @@ runtime·sigprof(uint8 *pc, uint8 *sp, uint8 *lr, G *gp, M *mp) ...@@ -2341,7 +2341,7 @@ runtime·sigprof(uint8 *pc, uint8 *sp, uint8 *lr, G *gp, M *mp)
} }
} }
prof.fn(prof.pcbuf, n); prof.fn(prof.pcbuf, n);
runtime·unlock(&prof); runtime·unlock(&prof.lock);
mp->mallocing--; mp->mallocing--;
} }
...@@ -2366,13 +2366,13 @@ runtime·setcpuprofilerate(void (*fn)(uintptr*, int32), int32 hz) ...@@ -2366,13 +2366,13 @@ runtime·setcpuprofilerate(void (*fn)(uintptr*, int32), int32 hz)
// it would deadlock. // it would deadlock.
runtime·resetcpuprofiler(0); runtime·resetcpuprofiler(0);
runtime·lock(&prof); runtime·lock(&prof.lock);
prof.fn = fn; prof.fn = fn;
prof.hz = hz; prof.hz = hz;
runtime·unlock(&prof); runtime·unlock(&prof.lock);
runtime·lock(&runtime·sched); runtime·lock(&runtime·sched.lock);
runtime·sched.profilehz = hz; runtime·sched.profilehz = hz;
runtime·unlock(&runtime·sched); runtime·unlock(&runtime·sched.lock);
if(hz != 0) if(hz != 0)
runtime·resetcpuprofiler(hz); runtime·resetcpuprofiler(hz);
...@@ -2510,11 +2510,11 @@ releasep(void) ...@@ -2510,11 +2510,11 @@ releasep(void)
static void static void
incidlelocked(int32 v) incidlelocked(int32 v)
{ {
runtime·lock(&runtime·sched); runtime·lock(&runtime·sched.lock);
runtime·sched.nmidlelocked += v; runtime·sched.nmidlelocked += v;
if(v > 0) if(v > 0)
checkdead(); checkdead();
runtime·unlock(&runtime·sched); runtime·unlock(&runtime·sched.lock);
} }
// Check for deadlock situation. // Check for deadlock situation.
...@@ -2583,16 +2583,16 @@ sysmon(void) ...@@ -2583,16 +2583,16 @@ sysmon(void)
runtime·usleep(delay); runtime·usleep(delay);
if(runtime·debug.schedtrace <= 0 && if(runtime·debug.schedtrace <= 0 &&
(runtime·sched.gcwaiting || runtime·atomicload(&runtime·sched.npidle) == runtime·gomaxprocs)) { // TODO: fast atomic (runtime·sched.gcwaiting || runtime·atomicload(&runtime·sched.npidle) == runtime·gomaxprocs)) { // TODO: fast atomic
runtime·lock(&runtime·sched); runtime·lock(&runtime·sched.lock);
if(runtime·atomicload(&runtime·sched.gcwaiting) || runtime·atomicload(&runtime·sched.npidle) == runtime·gomaxprocs) { if(runtime·atomicload(&runtime·sched.gcwaiting) || runtime·atomicload(&runtime·sched.npidle) == runtime·gomaxprocs) {
runtime·atomicstore(&runtime·sched.sysmonwait, 1); runtime·atomicstore(&runtime·sched.sysmonwait, 1);
runtime·unlock(&runtime·sched); runtime·unlock(&runtime·sched.lock);
runtime·notesleep(&runtime·sched.sysmonnote); runtime·notesleep(&runtime·sched.sysmonnote);
runtime·noteclear(&runtime·sched.sysmonnote); runtime·noteclear(&runtime·sched.sysmonnote);
idle = 0; idle = 0;
delay = 20; delay = 20;
} else } else
runtime·unlock(&runtime·sched); runtime·unlock(&runtime·sched.lock);
} }
// poll network if not polled for more than 10ms // poll network if not polled for more than 10ms
lastpoll = runtime·atomicload64(&runtime·sched.lastpoll); lastpoll = runtime·atomicload64(&runtime·sched.lastpoll);
...@@ -2757,7 +2757,7 @@ runtime·schedtrace(bool detailed) ...@@ -2757,7 +2757,7 @@ runtime·schedtrace(bool detailed)
if(starttime == 0) if(starttime == 0)
starttime = now; starttime = now;
runtime·lock(&runtime·sched); runtime·lock(&runtime·sched.lock);
runtime·printf("SCHED %Dms: gomaxprocs=%d idleprocs=%d threads=%d spinningthreads=%d idlethreads=%d runqueue=%d", runtime·printf("SCHED %Dms: gomaxprocs=%d idleprocs=%d threads=%d spinningthreads=%d idlethreads=%d runqueue=%d",
(now-starttime)/1000000, runtime·gomaxprocs, runtime·sched.npidle, runtime·sched.mcount, (now-starttime)/1000000, runtime·gomaxprocs, runtime·sched.npidle, runtime·sched.mcount,
runtime·sched.nmspinning, runtime·sched.nmidle, runtime·sched.runqsize); runtime·sched.nmspinning, runtime·sched.nmidle, runtime·sched.runqsize);
...@@ -2793,7 +2793,7 @@ runtime·schedtrace(bool detailed) ...@@ -2793,7 +2793,7 @@ runtime·schedtrace(bool detailed)
} }
} }
if(!detailed) { if(!detailed) {
runtime·unlock(&runtime·sched); runtime·unlock(&runtime·sched.lock);
return; return;
} }
for(mp = runtime·allm; mp; mp = mp->alllink) { for(mp = runtime·allm; mp; mp = mp->alllink) {
...@@ -2825,7 +2825,7 @@ runtime·schedtrace(bool detailed) ...@@ -2825,7 +2825,7 @@ runtime·schedtrace(bool detailed)
lockedm ? lockedm->id : -1); lockedm ? lockedm->id : -1);
} }
runtime·unlock(&allglock); runtime·unlock(&allglock);
runtime·unlock(&runtime·sched); runtime·unlock(&runtime·sched.lock);
} }
// Put mp on midle list. // Put mp on midle list.
...@@ -2981,9 +2981,9 @@ runqputslow(P *p, G *gp, uint32 h, uint32 t) ...@@ -2981,9 +2981,9 @@ runqputslow(P *p, G *gp, uint32 h, uint32 t)
for(i=0; i<n; i++) for(i=0; i<n; i++)
batch[i]->schedlink = batch[i+1]; batch[i]->schedlink = batch[i+1];
// Now put the batch on global queue. // Now put the batch on global queue.
runtime·lock(&runtime·sched); runtime·lock(&runtime·sched.lock);
globrunqputbatch(batch[0], batch[n], n+1); globrunqputbatch(batch[0], batch[n], n+1);
runtime·unlock(&runtime·sched); runtime·unlock(&runtime·sched.lock);
return true; return true;
} }
...@@ -3146,11 +3146,11 @@ runtime·setmaxthreads(int32 in) ...@@ -3146,11 +3146,11 @@ runtime·setmaxthreads(int32 in)
{ {
int32 out; int32 out;
runtime·lock(&runtime·sched); runtime·lock(&runtime·sched.lock);
out = runtime·sched.maxmcount; out = runtime·sched.maxmcount;
runtime·sched.maxmcount = in; runtime·sched.maxmcount = in;
checkmcount(); checkmcount();
runtime·unlock(&runtime·sched); runtime·unlock(&runtime·sched.lock);
return out; return out;
} }
......
...@@ -375,7 +375,7 @@ struct M ...@@ -375,7 +375,7 @@ struct M
struct P struct P
{ {
Lock; Lock lock;
int32 id; int32 id;
uint32 status; // one of Pidle/Prunning/... uint32 status; // one of Pidle/Prunning/...
...@@ -506,7 +506,7 @@ enum { ...@@ -506,7 +506,7 @@ enum {
struct Timers struct Timers
{ {
Lock; Lock lock;
G *timerproc; G *timerproc;
bool sleeping; bool sleeping;
bool rescheduling; bool rescheduling;
......
...@@ -36,7 +36,7 @@ struct SemaWaiter ...@@ -36,7 +36,7 @@ struct SemaWaiter
typedef struct SemaRoot SemaRoot; typedef struct SemaRoot SemaRoot;
struct SemaRoot struct SemaRoot
{ {
Lock; Lock lock;
SemaWaiter* head; SemaWaiter* head;
SemaWaiter* tail; SemaWaiter* tail;
// Number of waiters. Read w/o the lock. // Number of waiters. Read w/o the lock.
...@@ -48,7 +48,7 @@ struct SemaRoot ...@@ -48,7 +48,7 @@ struct SemaRoot
struct semtable struct semtable
{ {
SemaRoot; SemaRoot root;
uint8 pad[CacheLineSize-sizeof(SemaRoot)]; uint8 pad[CacheLineSize-sizeof(SemaRoot)];
}; };
#pragma dataflag NOPTR /* mark semtable as 'no pointers', hiding from garbage collector */ #pragma dataflag NOPTR /* mark semtable as 'no pointers', hiding from garbage collector */
...@@ -57,7 +57,7 @@ static struct semtable semtable[SEMTABLESZ]; ...@@ -57,7 +57,7 @@ static struct semtable semtable[SEMTABLESZ];
static SemaRoot* static SemaRoot*
semroot(uint32 *addr) semroot(uint32 *addr)
{ {
return &semtable[((uintptr)addr >> 3) % SEMTABLESZ]; return &semtable[((uintptr)addr >> 3) % SEMTABLESZ].root;
} }
static void static void
...@@ -125,19 +125,19 @@ runtime·semacquire(uint32 volatile *addr, bool profile) ...@@ -125,19 +125,19 @@ runtime·semacquire(uint32 volatile *addr, bool profile)
s.releasetime = -1; s.releasetime = -1;
} }
for(;;) { for(;;) {
runtime·lock(root); runtime·lock(&root->lock);
// Add ourselves to nwait to disable "easy case" in semrelease. // Add ourselves to nwait to disable "easy case" in semrelease.
runtime·xadd(&root->nwait, 1); runtime·xadd(&root->nwait, 1);
// Check cansemacquire to avoid missed wakeup. // Check cansemacquire to avoid missed wakeup.
if(cansemacquire(addr)) { if(cansemacquire(addr)) {
runtime·xadd(&root->nwait, -1); runtime·xadd(&root->nwait, -1);
runtime·unlock(root); runtime·unlock(&root->lock);
return; return;
} }
// Any semrelease after the cansemacquire knows we're waiting // Any semrelease after the cansemacquire knows we're waiting
// (we set nwait above), so go to sleep. // (we set nwait above), so go to sleep.
semqueue(root, addr, &s); semqueue(root, addr, &s);
runtime·parkunlock(root, "semacquire"); runtime·parkunlock(&root->lock, "semacquire");
if(cansemacquire(addr)) { if(cansemacquire(addr)) {
if(t0) if(t0)
runtime·blockevent(s.releasetime - t0, 3); runtime·blockevent(s.releasetime - t0, 3);
...@@ -162,11 +162,11 @@ runtime·semrelease(uint32 volatile *addr) ...@@ -162,11 +162,11 @@ runtime·semrelease(uint32 volatile *addr)
return; return;
// Harder case: search for a waiter and wake it. // Harder case: search for a waiter and wake it.
runtime·lock(root); runtime·lock(&root->lock);
if(runtime·atomicload(&root->nwait) == 0) { if(runtime·atomicload(&root->nwait) == 0) {
// The count is already consumed by another goroutine, // The count is already consumed by another goroutine,
// so no need to wake up another goroutine. // so no need to wake up another goroutine.
runtime·unlock(root); runtime·unlock(&root->lock);
return; return;
} }
for(s = root->head; s; s = s->next) { for(s = root->head; s; s = s->next) {
...@@ -176,7 +176,7 @@ runtime·semrelease(uint32 volatile *addr) ...@@ -176,7 +176,7 @@ runtime·semrelease(uint32 volatile *addr)
break; break;
} }
} }
runtime·unlock(root); runtime·unlock(&root->lock);
if(s) { if(s) {
if(s->releasetime) if(s->releasetime)
s->releasetime = runtime·cputicks(); s->releasetime = runtime·cputicks();
...@@ -206,7 +206,7 @@ func runtime_Semrelease(addr *uint32) { ...@@ -206,7 +206,7 @@ func runtime_Semrelease(addr *uint32) {
typedef struct SyncSema SyncSema; typedef struct SyncSema SyncSema;
struct SyncSema struct SyncSema
{ {
Lock; Lock lock;
SemaWaiter* head; SemaWaiter* head;
SemaWaiter* tail; SemaWaiter* tail;
}; };
...@@ -233,7 +233,7 @@ func runtime_Syncsemacquire(s *SyncSema) { ...@@ -233,7 +233,7 @@ func runtime_Syncsemacquire(s *SyncSema) {
w.releasetime = -1; w.releasetime = -1;
} }
runtime·lock(s); runtime·lock(&s->lock);
if(s->head && s->head->nrelease > 0) { if(s->head && s->head->nrelease > 0) {
// have pending release, consume it // have pending release, consume it
wake = nil; wake = nil;
...@@ -244,7 +244,7 @@ func runtime_Syncsemacquire(s *SyncSema) { ...@@ -244,7 +244,7 @@ func runtime_Syncsemacquire(s *SyncSema) {
if(s->head == nil) if(s->head == nil)
s->tail = nil; s->tail = nil;
} }
runtime·unlock(s); runtime·unlock(&s->lock);
if(wake) if(wake)
runtime·ready(wake->g); runtime·ready(wake->g);
} else { } else {
...@@ -254,7 +254,7 @@ func runtime_Syncsemacquire(s *SyncSema) { ...@@ -254,7 +254,7 @@ func runtime_Syncsemacquire(s *SyncSema) {
else else
s->tail->next = &w; s->tail->next = &w;
s->tail = &w; s->tail = &w;
runtime·parkunlock(s, "semacquire"); runtime·parkunlock(&s->lock, "semacquire");
if(t0) if(t0)
runtime·blockevent(w.releasetime - t0, 2); runtime·blockevent(w.releasetime - t0, 2);
} }
...@@ -269,7 +269,7 @@ func runtime_Syncsemrelease(s *SyncSema, n uint32) { ...@@ -269,7 +269,7 @@ func runtime_Syncsemrelease(s *SyncSema, n uint32) {
w.next = nil; w.next = nil;
w.releasetime = 0; w.releasetime = 0;
runtime·lock(s); runtime·lock(&s->lock);
while(w.nrelease > 0 && s->head && s->head->nrelease < 0) { while(w.nrelease > 0 && s->head && s->head->nrelease < 0) {
// have pending acquire, satisfy it // have pending acquire, satisfy it
wake = s->head; wake = s->head;
...@@ -288,7 +288,7 @@ func runtime_Syncsemrelease(s *SyncSema, n uint32) { ...@@ -288,7 +288,7 @@ func runtime_Syncsemrelease(s *SyncSema, n uint32) {
else else
s->tail->next = &w; s->tail->next = &w;
s->tail = &w; s->tail = &w;
runtime·parkunlock(s, "semarelease"); runtime·parkunlock(&s->lock, "semarelease");
} else } else
runtime·unlock(s); runtime·unlock(&s->lock);
} }
...@@ -33,7 +33,7 @@ package runtime ...@@ -33,7 +33,7 @@ package runtime
#pragma textflag NOPTR #pragma textflag NOPTR
static struct { static struct {
Note; Note note;
uint32 mask[(NSIG+31)/32]; uint32 mask[(NSIG+31)/32];
uint32 wanted[(NSIG+31)/32]; uint32 wanted[(NSIG+31)/32];
uint32 recv[(NSIG+31)/32]; uint32 recv[(NSIG+31)/32];
...@@ -72,7 +72,7 @@ runtime·sigsend(int32 s) ...@@ -72,7 +72,7 @@ runtime·sigsend(int32 s)
new = HASSIGNAL; new = HASSIGNAL;
if(runtime·cas(&sig.state, old, new)) { if(runtime·cas(&sig.state, old, new)) {
if (old == HASWAITER) if (old == HASWAITER)
runtime·notewakeup(&sig); runtime·notewakeup(&sig.note);
break; break;
} }
} }
...@@ -108,8 +108,8 @@ func signal_recv() (m uint32) { ...@@ -108,8 +108,8 @@ func signal_recv() (m uint32) {
new = HASWAITER; new = HASWAITER;
if(runtime·cas(&sig.state, old, new)) { if(runtime·cas(&sig.state, old, new)) {
if (new == HASWAITER) { if (new == HASWAITER) {
runtime·notetsleepg(&sig, -1); runtime·notetsleepg(&sig.note, -1);
runtime·noteclear(&sig); runtime·noteclear(&sig.note);
} }
break; break;
} }
...@@ -139,7 +139,7 @@ func signal_enable(s uint32) { ...@@ -139,7 +139,7 @@ func signal_enable(s uint32) {
// to use for initialization. It does not pass // to use for initialization. It does not pass
// signal information in m. // signal information in m.
sig.inuse = true; // enable reception of signals; cannot disable sig.inuse = true; // enable reception of signals; cannot disable
runtime·noteclear(&sig); runtime·noteclear(&sig.note);
return; return;
} }
......
...@@ -92,9 +92,9 @@ runtime·tsleep(int64 ns, int8 *reason) ...@@ -92,9 +92,9 @@ runtime·tsleep(int64 ns, int8 *reason)
t.period = 0; t.period = 0;
t.fv = &readyv; t.fv = &readyv;
t.arg.data = g; t.arg.data = g;
runtime·lock(&timers); runtime·lock(&timers.lock);
addtimer(&t); addtimer(&t);
runtime·parkunlock(&timers, reason); runtime·parkunlock(&timers.lock, reason);
} }
static FuncVal timerprocv = {timerproc}; static FuncVal timerprocv = {timerproc};
...@@ -102,9 +102,9 @@ static FuncVal timerprocv = {timerproc}; ...@@ -102,9 +102,9 @@ static FuncVal timerprocv = {timerproc};
void void
runtime·addtimer(Timer *t) runtime·addtimer(Timer *t)
{ {
runtime·lock(&timers); runtime·lock(&timers.lock);
addtimer(t); addtimer(t);
runtime·unlock(&timers); runtime·unlock(&timers.lock);
} }
// Add a timer to the heap and start or kick the timer proc // Add a timer to the heap and start or kick the timer proc
...@@ -165,14 +165,14 @@ runtime·deltimer(Timer *t) ...@@ -165,14 +165,14 @@ runtime·deltimer(Timer *t)
i = t->i; i = t->i;
USED(i); USED(i);
runtime·lock(&timers); runtime·lock(&timers.lock);
// t may not be registered anymore and may have // t may not be registered anymore and may have
// a bogus i (typically 0, if generated by Go). // a bogus i (typically 0, if generated by Go).
// Verify it before proceeding. // Verify it before proceeding.
i = t->i; i = t->i;
if(i < 0 || i >= timers.len || timers.t[i] != t) { if(i < 0 || i >= timers.len || timers.t[i] != t) {
runtime·unlock(&timers); runtime·unlock(&timers.lock);
return false; return false;
} }
...@@ -188,7 +188,7 @@ runtime·deltimer(Timer *t) ...@@ -188,7 +188,7 @@ runtime·deltimer(Timer *t)
} }
if(debug) if(debug)
dumptimers("deltimer"); dumptimers("deltimer");
runtime·unlock(&timers); runtime·unlock(&timers.lock);
return true; return true;
} }
...@@ -205,7 +205,7 @@ timerproc(void) ...@@ -205,7 +205,7 @@ timerproc(void)
Eface arg; Eface arg;
for(;;) { for(;;) {
runtime·lock(&timers); runtime·lock(&timers.lock);
timers.sleeping = false; timers.sleeping = false;
now = runtime·nanotime(); now = runtime·nanotime();
for(;;) { for(;;) {
...@@ -230,7 +230,7 @@ timerproc(void) ...@@ -230,7 +230,7 @@ timerproc(void)
} }
f = (void*)t->fv->fn; f = (void*)t->fv->fn;
arg = t->arg; arg = t->arg;
runtime·unlock(&timers); runtime·unlock(&timers.lock);
if(raceenabled) if(raceenabled)
runtime·raceacquire(t); runtime·raceacquire(t);
f(now, arg); f(now, arg);
...@@ -242,20 +242,20 @@ timerproc(void) ...@@ -242,20 +242,20 @@ timerproc(void)
arg.data = nil; arg.data = nil;
USED(&arg); USED(&arg);
runtime·lock(&timers); runtime·lock(&timers.lock);
} }
if(delta < 0) { if(delta < 0) {
// No timers left - put goroutine to sleep. // No timers left - put goroutine to sleep.
timers.rescheduling = true; timers.rescheduling = true;
g->isbackground = true; g->isbackground = true;
runtime·parkunlock(&timers, "timer goroutine (idle)"); runtime·parkunlock(&timers.lock, "timer goroutine (idle)");
g->isbackground = false; g->isbackground = false;
continue; continue;
} }
// At least one timer pending. Sleep until then. // At least one timer pending. Sleep until then.
timers.sleeping = true; timers.sleeping = true;
runtime·noteclear(&timers.waitnote); runtime·noteclear(&timers.waitnote);
runtime·unlock(&timers); runtime·unlock(&timers.lock);
runtime·notetsleepg(&timers.waitnote, delta); runtime·notetsleepg(&timers.waitnote, delta);
} }
} }
......
...@@ -66,14 +66,14 @@ struct IMethod ...@@ -66,14 +66,14 @@ struct IMethod
struct InterfaceType struct InterfaceType
{ {
Type; Type typ;
Slice mhdr; Slice mhdr;
IMethod m[]; IMethod m[];
}; };
struct MapType struct MapType
{ {
Type; Type typ;
Type *key; Type *key;
Type *elem; Type *elem;
Type *bucket; // internal type representing a hash bucket Type *bucket; // internal type representing a hash bucket
...@@ -87,20 +87,20 @@ struct MapType ...@@ -87,20 +87,20 @@ struct MapType
struct ChanType struct ChanType
{ {
Type; Type typ;
Type *elem; Type *elem;
uintptr dir; uintptr dir;
}; };
struct SliceType struct SliceType
{ {
Type; Type typ;
Type *elem; Type *elem;
}; };
struct FuncType struct FuncType
{ {
Type; Type typ;
bool dotdotdot; bool dotdotdot;
Slice in; Slice in;
Slice out; Slice out;
...@@ -108,6 +108,6 @@ struct FuncType ...@@ -108,6 +108,6 @@ struct FuncType
struct PtrType struct PtrType
{ {
Type; Type typ;
Type *elem; Type *elem;
}; };
...@@ -42,25 +42,15 @@ typedef signed char schar; ...@@ -42,25 +42,15 @@ typedef signed char schar;
#define SIGN(n) (1UL<<(n-1)) #define SIGN(n) (1UL<<(n-1))
typedef struct Vlong Vlong; typedef union Vlong Vlong;
struct Vlong union Vlong
{ {
union
{
long long v; long long v;
struct struct
{ {
ulong lo; ulong lo;
ulong hi; ulong hi;
}; } v2;
struct
{
ushort lols;
ushort loms;
ushort hils;
ushort hims;
};
};
}; };
void runtime·abort(void); void runtime·abort(void);
...@@ -68,15 +58,15 @@ void runtime·abort(void); ...@@ -68,15 +58,15 @@ void runtime·abort(void);
void void
_d2v(Vlong *y, double d) _d2v(Vlong *y, double d)
{ {
union { double d; struct Vlong; } x; union { double d; Vlong vl; } x;
ulong xhi, xlo, ylo, yhi; ulong xhi, xlo, ylo, yhi;
int sh; int sh;
x.d = d; x.d = d;
xhi = (x.hi & 0xfffff) | 0x100000; xhi = (x.vl.v2.hi & 0xfffff) | 0x100000;
xlo = x.lo; xlo = x.vl.v2.lo;
sh = 1075 - ((x.hi >> 20) & 0x7ff); sh = 1075 - ((x.vl.v2.hi >> 20) & 0x7ff);
ylo = 0; ylo = 0;
yhi = 0; yhi = 0;
...@@ -109,7 +99,7 @@ _d2v(Vlong *y, double d) ...@@ -109,7 +99,7 @@ _d2v(Vlong *y, double d)
yhi = d; /* causes something awful */ yhi = d; /* causes something awful */
} }
} }
if(x.hi & SIGN(32)) { if(x.vl.v2.hi & SIGN(32)) {
if(ylo != 0) { if(ylo != 0) {
ylo = -ylo; ylo = -ylo;
yhi = ~yhi; yhi = ~yhi;
...@@ -117,8 +107,8 @@ _d2v(Vlong *y, double d) ...@@ -117,8 +107,8 @@ _d2v(Vlong *y, double d)
yhi = -yhi; yhi = -yhi;
} }
y->hi = yhi; y->v2.hi = yhi;
y->lo = ylo; y->v2.lo = ylo;
} }
void void
...@@ -131,15 +121,15 @@ _f2v(Vlong *y, float f) ...@@ -131,15 +121,15 @@ _f2v(Vlong *y, float f)
double double
_v2d(Vlong x) _v2d(Vlong x)
{ {
if(x.hi & SIGN(32)) { if(x.v2.hi & SIGN(32)) {
if(x.lo) { if(x.v2.lo) {
x.lo = -x.lo; x.v2.lo = -x.v2.lo;
x.hi = ~x.hi; x.v2.hi = ~x.v2.hi;
} else } else
x.hi = -x.hi; x.v2.hi = -x.v2.hi;
return -((long)x.hi*4294967296. + x.lo); return -((long)x.v2.hi*4294967296. + x.v2.lo);
} }
return (long)x.hi*4294967296. + x.lo; return (long)x.v2.hi*4294967296. + x.v2.lo;
} }
float float
...@@ -157,10 +147,10 @@ slowdodiv(Vlong num, Vlong den, Vlong *q, Vlong *r) ...@@ -157,10 +147,10 @@ slowdodiv(Vlong num, Vlong den, Vlong *q, Vlong *r)
ulong numlo, numhi, denhi, denlo, quohi, quolo, t; ulong numlo, numhi, denhi, denlo, quohi, quolo, t;
int i; int i;
numhi = num.hi; numhi = num.v2.hi;
numlo = num.lo; numlo = num.v2.lo;
denhi = den.hi; denhi = den.v2.hi;
denlo = den.lo; denlo = den.v2.lo;
/* /*
* get a divide by zero * get a divide by zero
...@@ -204,12 +194,12 @@ slowdodiv(Vlong num, Vlong den, Vlong *q, Vlong *r) ...@@ -204,12 +194,12 @@ slowdodiv(Vlong num, Vlong den, Vlong *q, Vlong *r)
} }
if(q) { if(q) {
q->lo = quolo; q->v2.lo = quolo;
q->hi = quohi; q->v2.hi = quohi;
} }
if(r) { if(r) {
r->lo = numlo; r->v2.lo = numlo;
r->hi = numhi; r->v2.hi = numhi;
} }
} }
...@@ -219,46 +209,46 @@ dodiv(Vlong num, Vlong den, Vlong *qp, Vlong *rp) ...@@ -219,46 +209,46 @@ dodiv(Vlong num, Vlong den, Vlong *qp, Vlong *rp)
ulong n; ulong n;
Vlong x, q, r; Vlong x, q, r;
if(den.hi > num.hi || (den.hi == num.hi && den.lo > num.lo)){ if(den.v2.hi > num.v2.hi || (den.v2.hi == num.v2.hi && den.v2.lo > num.v2.lo)){
if(qp) { if(qp) {
qp->hi = 0; qp->v2.hi = 0;
qp->lo = 0; qp->v2.lo = 0;
} }
if(rp) { if(rp) {
rp->hi = num.hi; rp->v2.hi = num.v2.hi;
rp->lo = num.lo; rp->v2.lo = num.v2.lo;
} }
return; return;
} }
if(den.hi != 0){ if(den.v2.hi != 0){
q.hi = 0; q.v2.hi = 0;
n = num.hi/den.hi; n = num.v2.hi/den.v2.hi;
if(_mul64by32(&x, den, n) || x.hi > num.hi || (x.hi == num.hi && x.lo > num.lo)) if(_mul64by32(&x, den, n) || x.v2.hi > num.v2.hi || (x.v2.hi == num.v2.hi && x.v2.lo > num.v2.lo))
slowdodiv(num, den, &q, &r); slowdodiv(num, den, &q, &r);
else { else {
q.lo = n; q.v2.lo = n;
r.v = num.v - x.v; r.v = num.v - x.v;
} }
} else { } else {
if(num.hi >= den.lo){ if(num.v2.hi >= den.v2.lo){
if(den.lo == 0) if(den.v2.lo == 0)
runtime·panicdivide(); runtime·panicdivide();
q.hi = n = num.hi/den.lo; q.v2.hi = n = num.v2.hi/den.v2.lo;
num.hi -= den.lo*n; num.v2.hi -= den.v2.lo*n;
} else { } else {
q.hi = 0; q.v2.hi = 0;
} }
q.lo = _div64by32(num, den.lo, &r.lo); q.v2.lo = _div64by32(num, den.v2.lo, &r.v2.lo);
r.hi = 0; r.v2.hi = 0;
} }
if(qp) { if(qp) {
qp->lo = q.lo; qp->v2.lo = q.v2.lo;
qp->hi = q.hi; qp->v2.hi = q.v2.hi;
} }
if(rp) { if(rp) {
rp->lo = r.lo; rp->v2.lo = r.v2.lo;
rp->hi = r.hi; rp->v2.hi = r.v2.hi;
} }
} }
...@@ -266,11 +256,11 @@ void ...@@ -266,11 +256,11 @@ void
_divvu(Vlong *q, Vlong n, Vlong d) _divvu(Vlong *q, Vlong n, Vlong d)
{ {
if(n.hi == 0 && d.hi == 0) { if(n.v2.hi == 0 && d.v2.hi == 0) {
if(d.lo == 0) if(d.v2.lo == 0)
runtime·panicdivide(); runtime·panicdivide();
q->hi = 0; q->v2.hi = 0;
q->lo = n.lo / d.lo; q->v2.lo = n.v2.lo / d.v2.lo;
return; return;
} }
dodiv(n, d, q, 0); dodiv(n, d, q, 0);
...@@ -286,11 +276,11 @@ void ...@@ -286,11 +276,11 @@ void
_modvu(Vlong *r, Vlong n, Vlong d) _modvu(Vlong *r, Vlong n, Vlong d)
{ {
if(n.hi == 0 && d.hi == 0) { if(n.v2.hi == 0 && d.v2.hi == 0) {
if(d.lo == 0) if(d.v2.lo == 0)
runtime·panicdivide(); runtime·panicdivide();
r->hi = 0; r->v2.hi = 0;
r->lo = n.lo % d.lo; r->v2.lo = n.v2.lo % d.v2.lo;
return; return;
} }
dodiv(n, d, 0, r); dodiv(n, d, 0, r);
...@@ -306,12 +296,12 @@ static void ...@@ -306,12 +296,12 @@ static void
vneg(Vlong *v) vneg(Vlong *v)
{ {
if(v->lo == 0) { if(v->v2.lo == 0) {
v->hi = -v->hi; v->v2.hi = -v->v2.hi;
return; return;
} }
v->lo = -v->lo; v->v2.lo = -v->v2.lo;
v->hi = ~v->hi; v->v2.hi = ~v->v2.hi;
} }
void void
...@@ -319,24 +309,24 @@ _divv(Vlong *q, Vlong n, Vlong d) ...@@ -319,24 +309,24 @@ _divv(Vlong *q, Vlong n, Vlong d)
{ {
long nneg, dneg; long nneg, dneg;
if(n.hi == (((long)n.lo)>>31) && d.hi == (((long)d.lo)>>31)) { if(n.v2.hi == (((long)n.v2.lo)>>31) && d.v2.hi == (((long)d.v2.lo)>>31)) {
if((long)n.lo == -0x80000000 && (long)d.lo == -1) { if((long)n.v2.lo == -0x80000000 && (long)d.v2.lo == -1) {
// special case: 32-bit -0x80000000 / -1 causes divide error, // special case: 32-bit -0x80000000 / -1 causes divide error,
// but it's okay in this 64-bit context. // but it's okay in this 64-bit context.
q->lo = 0x80000000; q->v2.lo = 0x80000000;
q->hi = 0; q->v2.hi = 0;
return; return;
} }
if(d.lo == 0) if(d.v2.lo == 0)
runtime·panicdivide(); runtime·panicdivide();
q->lo = (long)n.lo / (long)d.lo; q->v2.lo = (long)n.v2.lo / (long)d.v2.lo;
q->hi = ((long)q->lo) >> 31; q->v2.hi = ((long)q->v2.lo) >> 31;
return; return;
} }
nneg = n.hi >> 31; nneg = n.v2.hi >> 31;
if(nneg) if(nneg)
vneg(&n); vneg(&n);
dneg = d.hi >> 31; dneg = d.v2.hi >> 31;
if(dneg) if(dneg)
vneg(&d); vneg(&d);
dodiv(n, d, q, 0); dodiv(n, d, q, 0);
...@@ -355,24 +345,24 @@ _modv(Vlong *r, Vlong n, Vlong d) ...@@ -355,24 +345,24 @@ _modv(Vlong *r, Vlong n, Vlong d)
{ {
long nneg, dneg; long nneg, dneg;
if(n.hi == (((long)n.lo)>>31) && d.hi == (((long)d.lo)>>31)) { if(n.v2.hi == (((long)n.v2.lo)>>31) && d.v2.hi == (((long)d.v2.lo)>>31)) {
if((long)n.lo == -0x80000000 && (long)d.lo == -1) { if((long)n.v2.lo == -0x80000000 && (long)d.v2.lo == -1) {
// special case: 32-bit -0x80000000 % -1 causes divide error, // special case: 32-bit -0x80000000 % -1 causes divide error,
// but it's okay in this 64-bit context. // but it's okay in this 64-bit context.
r->lo = 0; r->v2.lo = 0;
r->hi = 0; r->v2.hi = 0;
return; return;
} }
if(d.lo == 0) if(d.v2.lo == 0)
runtime·panicdivide(); runtime·panicdivide();
r->lo = (long)n.lo % (long)d.lo; r->v2.lo = (long)n.v2.lo % (long)d.v2.lo;
r->hi = ((long)r->lo) >> 31; r->v2.hi = ((long)r->v2.lo) >> 31;
return; return;
} }
nneg = n.hi >> 31; nneg = n.v2.hi >> 31;
if(nneg) if(nneg)
vneg(&n); vneg(&n);
dneg = d.hi >> 31; dneg = d.v2.hi >> 31;
if(dneg) if(dneg)
vneg(&d); vneg(&d);
dodiv(n, d, 0, r); dodiv(n, d, 0, r);
...@@ -391,24 +381,24 @@ _rshav(Vlong *r, Vlong a, int b) ...@@ -391,24 +381,24 @@ _rshav(Vlong *r, Vlong a, int b)
{ {
long t; long t;
t = a.hi; t = a.v2.hi;
if(b >= 32) { if(b >= 32) {
r->hi = t>>31; r->v2.hi = t>>31;
if(b >= 64) { if(b >= 64) {
/* this is illegal re C standard */ /* this is illegal re C standard */
r->lo = t>>31; r->v2.lo = t>>31;
return; return;
} }
r->lo = t >> (b-32); r->v2.lo = t >> (b-32);
return; return;
} }
if(b <= 0) { if(b <= 0) {
r->hi = t; r->v2.hi = t;
r->lo = a.lo; r->v2.lo = a.v2.lo;
return; return;
} }
r->hi = t >> b; r->v2.hi = t >> b;
r->lo = (t << (32-b)) | (a.lo >> b); r->v2.lo = (t << (32-b)) | (a.v2.lo >> b);
} }
void void
...@@ -416,24 +406,24 @@ _rshlv(Vlong *r, Vlong a, int b) ...@@ -416,24 +406,24 @@ _rshlv(Vlong *r, Vlong a, int b)
{ {
ulong t; ulong t;
t = a.hi; t = a.v2.hi;
if(b >= 32) { if(b >= 32) {
r->hi = 0; r->v2.hi = 0;
if(b >= 64) { if(b >= 64) {
/* this is illegal re C standard */ /* this is illegal re C standard */
r->lo = 0; r->v2.lo = 0;
return; return;
} }
r->lo = t >> (b-32); r->v2.lo = t >> (b-32);
return; return;
} }
if(b <= 0) { if(b <= 0) {
r->hi = t; r->v2.hi = t;
r->lo = a.lo; r->v2.lo = a.v2.lo;
return; return;
} }
r->hi = t >> b; r->v2.hi = t >> b;
r->lo = (t << (32-b)) | (a.lo >> b); r->v2.lo = (t << (32-b)) | (a.v2.lo >> b);
} }
#pragma textflag NOSPLIT #pragma textflag NOSPLIT
...@@ -442,89 +432,89 @@ _lshv(Vlong *r, Vlong a, int b) ...@@ -442,89 +432,89 @@ _lshv(Vlong *r, Vlong a, int b)
{ {
ulong t; ulong t;
t = a.lo; t = a.v2.lo;
if(b >= 32) { if(b >= 32) {
r->lo = 0; r->v2.lo = 0;
if(b >= 64) { if(b >= 64) {
/* this is illegal re C standard */ /* this is illegal re C standard */
r->hi = 0; r->v2.hi = 0;
return; return;
} }
r->hi = t << (b-32); r->v2.hi = t << (b-32);
return; return;
} }
if(b <= 0) { if(b <= 0) {
r->lo = t; r->v2.lo = t;
r->hi = a.hi; r->v2.hi = a.v2.hi;
return; return;
} }
r->lo = t << b; r->v2.lo = t << b;
r->hi = (t >> (32-b)) | (a.hi << b); r->v2.hi = (t >> (32-b)) | (a.v2.hi << b);
} }
void void
_andv(Vlong *r, Vlong a, Vlong b) _andv(Vlong *r, Vlong a, Vlong b)
{ {
r->hi = a.hi & b.hi; r->v2.hi = a.v2.hi & b.v2.hi;
r->lo = a.lo & b.lo; r->v2.lo = a.v2.lo & b.v2.lo;
} }
void void
_orv(Vlong *r, Vlong a, Vlong b) _orv(Vlong *r, Vlong a, Vlong b)
{ {
r->hi = a.hi | b.hi; r->v2.hi = a.v2.hi | b.v2.hi;
r->lo = a.lo | b.lo; r->v2.lo = a.v2.lo | b.v2.lo;
} }
void void
_xorv(Vlong *r, Vlong a, Vlong b) _xorv(Vlong *r, Vlong a, Vlong b)
{ {
r->hi = a.hi ^ b.hi; r->v2.hi = a.v2.hi ^ b.v2.hi;
r->lo = a.lo ^ b.lo; r->v2.lo = a.v2.lo ^ b.v2.lo;
} }
void void
_vpp(Vlong *l, Vlong *r) _vpp(Vlong *l, Vlong *r)
{ {
l->hi = r->hi; l->v2.hi = r->v2.hi;
l->lo = r->lo; l->v2.lo = r->v2.lo;
r->lo++; r->v2.lo++;
if(r->lo == 0) if(r->v2.lo == 0)
r->hi++; r->v2.hi++;
} }
void void
_vmm(Vlong *l, Vlong *r) _vmm(Vlong *l, Vlong *r)
{ {
l->hi = r->hi; l->v2.hi = r->v2.hi;
l->lo = r->lo; l->v2.lo = r->v2.lo;
if(r->lo == 0) if(r->v2.lo == 0)
r->hi--; r->v2.hi--;
r->lo--; r->v2.lo--;
} }
void void
_ppv(Vlong *l, Vlong *r) _ppv(Vlong *l, Vlong *r)
{ {
r->lo++; r->v2.lo++;
if(r->lo == 0) if(r->v2.lo == 0)
r->hi++; r->v2.hi++;
l->hi = r->hi; l->v2.hi = r->v2.hi;
l->lo = r->lo; l->v2.lo = r->v2.lo;
} }
void void
_mmv(Vlong *l, Vlong *r) _mmv(Vlong *l, Vlong *r)
{ {
if(r->lo == 0) if(r->v2.lo == 0)
r->hi--; r->v2.hi--;
r->lo--; r->v2.lo--;
l->hi = r->hi; l->v2.hi = r->v2.hi;
l->lo = r->lo; l->v2.lo = r->v2.lo;
} }
void void
...@@ -532,67 +522,67 @@ _vasop(Vlong *ret, void *lv, void fn(Vlong*, Vlong, Vlong), int type, Vlong rv) ...@@ -532,67 +522,67 @@ _vasop(Vlong *ret, void *lv, void fn(Vlong*, Vlong, Vlong), int type, Vlong rv)
{ {
Vlong t, u; Vlong t, u;
u.lo = 0; u.v2.lo = 0;
u.hi = 0; u.v2.hi = 0;
switch(type) { switch(type) {
default: default:
runtime·abort(); runtime·abort();
break; break;
case 1: /* schar */ case 1: /* schar */
t.lo = *(schar*)lv; t.v2.lo = *(schar*)lv;
t.hi = t.lo >> 31; t.v2.hi = t.v2.lo >> 31;
fn(&u, t, rv); fn(&u, t, rv);
*(schar*)lv = u.lo; *(schar*)lv = u.v2.lo;
break; break;
case 2: /* uchar */ case 2: /* uchar */
t.lo = *(uchar*)lv; t.v2.lo = *(uchar*)lv;
t.hi = 0; t.v2.hi = 0;
fn(&u, t, rv); fn(&u, t, rv);
*(uchar*)lv = u.lo; *(uchar*)lv = u.v2.lo;
break; break;
case 3: /* short */ case 3: /* short */
t.lo = *(short*)lv; t.v2.lo = *(short*)lv;
t.hi = t.lo >> 31; t.v2.hi = t.v2.lo >> 31;
fn(&u, t, rv); fn(&u, t, rv);
*(short*)lv = u.lo; *(short*)lv = u.v2.lo;
break; break;
case 4: /* ushort */ case 4: /* ushort */
t.lo = *(ushort*)lv; t.v2.lo = *(ushort*)lv;
t.hi = 0; t.v2.hi = 0;
fn(&u, t, rv); fn(&u, t, rv);
*(ushort*)lv = u.lo; *(ushort*)lv = u.v2.lo;
break; break;
case 9: /* int */ case 9: /* int */
t.lo = *(int*)lv; t.v2.lo = *(int*)lv;
t.hi = t.lo >> 31; t.v2.hi = t.v2.lo >> 31;
fn(&u, t, rv); fn(&u, t, rv);
*(int*)lv = u.lo; *(int*)lv = u.v2.lo;
break; break;
case 10: /* uint */ case 10: /* uint */
t.lo = *(uint*)lv; t.v2.lo = *(uint*)lv;
t.hi = 0; t.v2.hi = 0;
fn(&u, t, rv); fn(&u, t, rv);
*(uint*)lv = u.lo; *(uint*)lv = u.v2.lo;
break; break;
case 5: /* long */ case 5: /* long */
t.lo = *(long*)lv; t.v2.lo = *(long*)lv;
t.hi = t.lo >> 31; t.v2.hi = t.v2.lo >> 31;
fn(&u, t, rv); fn(&u, t, rv);
*(long*)lv = u.lo; *(long*)lv = u.v2.lo;
break; break;
case 6: /* ulong */ case 6: /* ulong */
t.lo = *(ulong*)lv; t.v2.lo = *(ulong*)lv;
t.hi = 0; t.v2.hi = 0;
fn(&u, t, rv); fn(&u, t, rv);
*(ulong*)lv = u.lo; *(ulong*)lv = u.v2.lo;
break; break;
case 7: /* vlong */ case 7: /* vlong */
...@@ -610,8 +600,8 @@ _p2v(Vlong *ret, void *p) ...@@ -610,8 +600,8 @@ _p2v(Vlong *ret, void *p)
long t; long t;
t = (ulong)p; t = (ulong)p;
ret->lo = t; ret->v2.lo = t;
ret->hi = 0; ret->v2.hi = 0;
} }
void void
...@@ -620,8 +610,8 @@ _sl2v(Vlong *ret, long sl) ...@@ -620,8 +610,8 @@ _sl2v(Vlong *ret, long sl)
long t; long t;
t = sl; t = sl;
ret->lo = t; ret->v2.lo = t;
ret->hi = t >> 31; ret->v2.hi = t >> 31;
} }
void void
...@@ -630,8 +620,8 @@ _ul2v(Vlong *ret, ulong ul) ...@@ -630,8 +620,8 @@ _ul2v(Vlong *ret, ulong ul)
long t; long t;
t = ul; t = ul;
ret->lo = t; ret->v2.lo = t;
ret->hi = 0; ret->v2.hi = 0;
} }
void void
...@@ -640,8 +630,8 @@ _si2v(Vlong *ret, int si) ...@@ -640,8 +630,8 @@ _si2v(Vlong *ret, int si)
long t; long t;
t = si; t = si;
ret->lo = t; ret->v2.lo = t;
ret->hi = t >> 31; ret->v2.hi = t >> 31;
} }
void void
...@@ -650,8 +640,8 @@ _ui2v(Vlong *ret, uint ui) ...@@ -650,8 +640,8 @@ _ui2v(Vlong *ret, uint ui)
long t; long t;
t = ui; t = ui;
ret->lo = t; ret->v2.lo = t;
ret->hi = 0; ret->v2.hi = 0;
} }
void void
...@@ -660,8 +650,8 @@ _sh2v(Vlong *ret, long sh) ...@@ -660,8 +650,8 @@ _sh2v(Vlong *ret, long sh)
long t; long t;
t = (sh << 16) >> 16; t = (sh << 16) >> 16;
ret->lo = t; ret->v2.lo = t;
ret->hi = t >> 31; ret->v2.hi = t >> 31;
} }
void void
...@@ -670,8 +660,8 @@ _uh2v(Vlong *ret, ulong ul) ...@@ -670,8 +660,8 @@ _uh2v(Vlong *ret, ulong ul)
long t; long t;
t = ul & 0xffff; t = ul & 0xffff;
ret->lo = t; ret->v2.lo = t;
ret->hi = 0; ret->v2.hi = 0;
} }
void void
...@@ -680,8 +670,8 @@ _sc2v(Vlong *ret, long uc) ...@@ -680,8 +670,8 @@ _sc2v(Vlong *ret, long uc)
long t; long t;
t = (uc << 24) >> 24; t = (uc << 24) >> 24;
ret->lo = t; ret->v2.lo = t;
ret->hi = t >> 31; ret->v2.hi = t >> 31;
} }
void void
...@@ -690,8 +680,8 @@ _uc2v(Vlong *ret, ulong ul) ...@@ -690,8 +680,8 @@ _uc2v(Vlong *ret, ulong ul)
long t; long t;
t = ul & 0xff; t = ul & 0xff;
ret->lo = t; ret->v2.lo = t;
ret->hi = 0; ret->v2.hi = 0;
} }
long long
...@@ -699,7 +689,7 @@ _v2sc(Vlong rv) ...@@ -699,7 +689,7 @@ _v2sc(Vlong rv)
{ {
long t; long t;
t = rv.lo & 0xff; t = rv.v2.lo & 0xff;
return (t << 24) >> 24; return (t << 24) >> 24;
} }
...@@ -707,7 +697,7 @@ long ...@@ -707,7 +697,7 @@ long
_v2uc(Vlong rv) _v2uc(Vlong rv)
{ {
return rv.lo & 0xff; return rv.v2.lo & 0xff;
} }
long long
...@@ -715,7 +705,7 @@ _v2sh(Vlong rv) ...@@ -715,7 +705,7 @@ _v2sh(Vlong rv)
{ {
long t; long t;
t = rv.lo & 0xffff; t = rv.v2.lo & 0xffff;
return (t << 16) >> 16; return (t << 16) >> 16;
} }
...@@ -723,107 +713,107 @@ long ...@@ -723,107 +713,107 @@ long
_v2uh(Vlong rv) _v2uh(Vlong rv)
{ {
return rv.lo & 0xffff; return rv.v2.lo & 0xffff;
} }
long long
_v2sl(Vlong rv) _v2sl(Vlong rv)
{ {
return rv.lo; return rv.v2.lo;
} }
long long
_v2ul(Vlong rv) _v2ul(Vlong rv)
{ {
return rv.lo; return rv.v2.lo;
} }
long long
_v2si(Vlong rv) _v2si(Vlong rv)
{ {
return rv.lo; return rv.v2.lo;
} }
long long
_v2ui(Vlong rv) _v2ui(Vlong rv)
{ {
return rv.lo; return rv.v2.lo;
} }
int int
_testv(Vlong rv) _testv(Vlong rv)
{ {
return rv.lo || rv.hi; return rv.v2.lo || rv.v2.hi;
} }
int int
_eqv(Vlong lv, Vlong rv) _eqv(Vlong lv, Vlong rv)
{ {
return lv.lo == rv.lo && lv.hi == rv.hi; return lv.v2.lo == rv.v2.lo && lv.v2.hi == rv.v2.hi;
} }
int int
_nev(Vlong lv, Vlong rv) _nev(Vlong lv, Vlong rv)
{ {
return lv.lo != rv.lo || lv.hi != rv.hi; return lv.v2.lo != rv.v2.lo || lv.v2.hi != rv.v2.hi;
} }
int int
_ltv(Vlong lv, Vlong rv) _ltv(Vlong lv, Vlong rv)
{ {
return (long)lv.hi < (long)rv.hi || return (long)lv.v2.hi < (long)rv.v2.hi ||
(lv.hi == rv.hi && lv.lo < rv.lo); (lv.v2.hi == rv.v2.hi && lv.v2.lo < rv.v2.lo);
} }
int int
_lev(Vlong lv, Vlong rv) _lev(Vlong lv, Vlong rv)
{ {
return (long)lv.hi < (long)rv.hi || return (long)lv.v2.hi < (long)rv.v2.hi ||
(lv.hi == rv.hi && lv.lo <= rv.lo); (lv.v2.hi == rv.v2.hi && lv.v2.lo <= rv.v2.lo);
} }
int int
_gtv(Vlong lv, Vlong rv) _gtv(Vlong lv, Vlong rv)
{ {
return (long)lv.hi > (long)rv.hi || return (long)lv.v2.hi > (long)rv.v2.hi ||
(lv.hi == rv.hi && lv.lo > rv.lo); (lv.v2.hi == rv.v2.hi && lv.v2.lo > rv.v2.lo);
} }
int int
_gev(Vlong lv, Vlong rv) _gev(Vlong lv, Vlong rv)
{ {
return (long)lv.hi > (long)rv.hi || return (long)lv.v2.hi > (long)rv.v2.hi ||
(lv.hi == rv.hi && lv.lo >= rv.lo); (lv.v2.hi == rv.v2.hi && lv.v2.lo >= rv.v2.lo);
} }
int int
_lov(Vlong lv, Vlong rv) _lov(Vlong lv, Vlong rv)
{ {
return lv.hi < rv.hi || return lv.v2.hi < rv.v2.hi ||
(lv.hi == rv.hi && lv.lo < rv.lo); (lv.v2.hi == rv.v2.hi && lv.v2.lo < rv.v2.lo);
} }
int int
_lsv(Vlong lv, Vlong rv) _lsv(Vlong lv, Vlong rv)
{ {
return lv.hi < rv.hi || return lv.v2.hi < rv.v2.hi ||
(lv.hi == rv.hi && lv.lo <= rv.lo); (lv.v2.hi == rv.v2.hi && lv.v2.lo <= rv.v2.lo);
} }
int int
_hiv(Vlong lv, Vlong rv) _hiv(Vlong lv, Vlong rv)
{ {
return lv.hi > rv.hi || return lv.v2.hi > rv.v2.hi ||
(lv.hi == rv.hi && lv.lo > rv.lo); (lv.v2.hi == rv.v2.hi && lv.v2.lo > rv.v2.lo);
} }
int int
_hsv(Vlong lv, Vlong rv) _hsv(Vlong lv, Vlong rv)
{ {
return lv.hi > rv.hi || return lv.v2.hi > rv.v2.hi ||
(lv.hi == rv.hi && lv.lo >= rv.lo); (lv.v2.hi == rv.v2.hi && lv.v2.lo >= rv.v2.lo);
} }
...@@ -40,21 +40,8 @@ typedef signed char schar; ...@@ -40,21 +40,8 @@ typedef signed char schar;
typedef struct Vlong Vlong; typedef struct Vlong Vlong;
struct Vlong struct Vlong
{ {
union
{
struct
{
ulong lo; ulong lo;
ulong hi; ulong hi;
};
struct
{
ushort lols;
ushort loms;
ushort hils;
ushort hims;
};
};
}; };
void runtime·abort(void); void runtime·abort(void);
...@@ -82,15 +69,15 @@ _subv(Vlong *r, Vlong a, Vlong b) ...@@ -82,15 +69,15 @@ _subv(Vlong *r, Vlong a, Vlong b)
void void
_d2v(Vlong *y, double d) _d2v(Vlong *y, double d)
{ {
union { double d; struct Vlong; } x; union { double d; Vlong vl; } x;
ulong xhi, xlo, ylo, yhi; ulong xhi, xlo, ylo, yhi;
int sh; int sh;
x.d = d; x.d = d;
xhi = (x.hi & 0xfffff) | 0x100000; xhi = (x.vl.hi & 0xfffff) | 0x100000;
xlo = x.lo; xlo = x.vl.lo;
sh = 1075 - ((x.hi >> 20) & 0x7ff); sh = 1075 - ((x.vl.hi >> 20) & 0x7ff);
ylo = 0; ylo = 0;
yhi = 0; yhi = 0;
...@@ -123,7 +110,7 @@ _d2v(Vlong *y, double d) ...@@ -123,7 +110,7 @@ _d2v(Vlong *y, double d)
yhi = d; /* causes something awful */ yhi = d; /* causes something awful */
} }
} }
if(x.hi & SIGN(32)) { if(x.vl.hi & SIGN(32)) {
if(ylo != 0) { if(ylo != 0) {
ylo = -ylo; ylo = -ylo;
yhi = ~yhi; yhi = ~yhi;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment