Skip to content

G
go
Commit 013fa63c authored by Jan Ziak's avatar Jan Ziak Committed by Russ Cox
Browse files
Options

runtime: struct Obj in mgc0.c and buffers in scanblock() 

Details:

- This CL is the conceptual skeleton of code found in CL 6114046

- The garbage collector uses struct Obj to specify memory blocks

- scanblock() is putting found memory blocks into an intermediate buffer
  (xbuf) before adding/flushing them to the main work buffer (wbuf)

- The main loop in scanblock() is replaced with a skeleton code that
  in the future will be able to recognize the type of objects and
  thus will improve the garbage collector's precision.
  For now, all objects are simply sequences of pointers so
  the precision of the garbage collector remains unchanged.

- The code plugs .gcdata and .gcbss sections into the garbage collector.
  scanblock() in this CL is unable to make any use of this.

R=rsc, dvyukov, remyoudompheng
CC=dave, golang-dev, minux.ma
https://golang.org/cl/6856121
parent c00371ea
Show whitespace changes
Inline Side-by-side
Showing with 387 additions and 141 deletions
src/pkg/runtime/mgc0.c
View file @ 013fa63c
......@@ -8,16 +8,19 @@
#include "arch_GOARCH.h"
#include "malloc.h"
#include "stack.h"
#include "mgc0.h"
#include "race.h"
enum {
Debug = 0,
DebugMark = 0, // run second pass to check mark
DataBlock = 8*1024,
// Four bits per word (see #defines below).
wordsPerBitmapWord = sizeof(void*)*8/4,
bitShift = sizeof(void*)*8/4,
handoffThreshold = 4,
IntermediateBufferCapacity = 64,
};
// Bits in per-word bitmap.
......@@ -70,12 +73,24 @@ uint32 runtime·worldsema = 1;
static int32 gctrace;
typedef struct Obj Obj;
struct Obj
{
byte *p; // data pointer
uintptr n; // size of data in bytes
uintptr ti; // type info
};
// The size of Workbuf is N*PageSize.
typedef struct Workbuf Workbuf;
struct Workbuf
{
#define SIZE (2*PageSize-sizeof(LFNode)-sizeof(uintptr))
LFNode node; // must be first
uintptr nobj;
byte *obj[512-(sizeof(LFNode)+sizeof(uintptr))/sizeof(byte*)];
Obj obj[SIZE/sizeof(Obj) - 1];
uint8 _padding[SIZE%sizeof(Obj) + sizeof(Obj)];
#undef SIZE
};
typedef struct Finalizer Finalizer;
......@@ -97,9 +112,13 @@ struct FinBlock
};
extern byte data[];
extern byte etext[];
extern byte edata[];
extern byte bss[];
extern byte ebss[];
extern byte gcdata[];
extern byte gcbss[];
static G *fing;
static FinBlock *finq; // list of finalizers that are to be executed
static FinBlock *finc; // cache of free blocks
......@@ -113,13 +132,6 @@ static Workbuf* getfull(Workbuf*);
static void putempty(Workbuf*);
static Workbuf* handoff(Workbuf*);
typedef struct GcRoot GcRoot;
struct GcRoot
{
byte *p;
uintptr n;
};
static struct {
uint64 full; // lock-free list of full blocks
uint64 empty; // lock-free list of empty blocks
......@@ -136,77 +148,122 @@ static struct {
byte *chunk;
uintptr nchunk;
GcRoot *roots;
Obj *roots;
uint32 nroot;
uint32 rootcap;
} work;
// scanblock scans a block of n bytes starting at pointer b for references
// to other objects, scanning any it finds recursively until there are no
// unscanned objects left. Instead of using an explicit recursion, it keeps
// a work list in the Workbuf* structures and loops in the main function
// body. Keeping an explicit work list is easier on the stack allocator and
// more efficient.
enum {
// TODO(atom): to be expanded in a next CL
GC_DEFAULT_PTR = GC_NUM_INSTR,
};
// PtrTarget and BitTarget are structures used by intermediate buffers.
// The intermediate buffers hold GC data before it
// is moved/flushed to the work buffer (Workbuf).
// The size of an intermediate buffer is very small,
// such as 32 or 64 elements.
struct PtrTarget
{
void *p;
uintptr ti;
};
struct BitTarget
{
void *p;
uintptr ti;
uintptr *bitp, shift;
};
struct BufferList
{
struct PtrTarget ptrtarget[IntermediateBufferCapacity];
struct BitTarget bittarget[IntermediateBufferCapacity];
struct BufferList *next;
};
static struct BufferList *bufferList;
static Lock lock;
// flushptrbuf moves data from the PtrTarget buffer to the work buffer.
// The PtrTarget buffer contains blocks irrespective of whether the blocks have been marked or scanned,
// while the work buffer contains blocks which have been marked
// and are prepared to be scanned by the garbage collector.
//
// _wp, _wbuf, _nobj are input/output parameters and are specifying the work buffer.
// bitbuf holds temporary data generated by this function.
//
// A simplified drawing explaining how the todo-list moves from a structure to another:
//
// scanblock
// (find pointers)
// Obj ------> PtrTarget (pointer targets)
// ↑ |
// | | flushptrbuf (1st part,
// | | find block start)
// | ↓
// `--------- BitTarget (pointer targets and the corresponding locations in bitmap)
// flushptrbuf
// (2nd part, mark and enqueue)
static void
scanblock(byte *b, uintptr n)
flushptrbuf(struct PtrTarget *ptrbuf, uintptr n, Obj **_wp, Workbuf **_wbuf, uintptr *_nobj, struct BitTarget *bitbuf)
{
byte *obj, *arena_start, *arena_used, *p;
void **vp;
uintptr size, *bitp, bits, shift, i, j, x, xbits, off, nobj, nproc;
byte *p, *arena_start, *obj;
uintptr size, *bitp, bits, shift, j, x, xbits, off, nobj, ti;
MSpan *s;
PageID k;
void **wp;
Obj *wp;
Workbuf *wbuf;
bool keepworking;
if((intptr)n < 0) {
runtime·printf("scanblock %p %D\n", b, (int64)n);
runtime·throw("scanblock");
}
struct PtrTarget *ptrbuf_end;
struct BitTarget *bitbufpos, *bt;
// Memory arena parameters.
arena_start = runtime·mheap.arena_start;
arena_used = runtime·mheap.arena_used;
nproc = work.nproc;
wbuf = nil; // current work buffer
wp = nil; // storage for next queued pointer (write pointer)
nobj = 0; // number of queued objects
wp = *_wp;
wbuf = *_wbuf;
nobj = *_nobj;
// Scanblock helpers pass b==nil.
// Procs needs to return to make more
// calls to scanblock. But if work.nproc==1 then
// might as well process blocks as soon as we
// have them.
keepworking = b == nil || work.nproc == 1;
ptrbuf_end = ptrbuf + n;
// Align b to a word boundary.
off = (uintptr)b & (PtrSize-1);
if(off != 0) {
b += PtrSize - off;
n -= PtrSize - off;
// If buffer is nearly full, get a new one.
if(wbuf == nil || nobj+n >= nelem(wbuf->obj)) {
if(wbuf != nil)
wbuf->nobj = nobj;
wbuf = getempty(wbuf);
wp = wbuf->obj;
nobj = 0;
if(n >= nelem(wbuf->obj))
runtime·throw("ptrbuf has to be smaller than WorkBuf");
}
for(;;) {
// Each iteration scans the block b of length n, queueing pointers in
// the work buffer.
if(Debug > 1)
runtime·printf("scanblock %p %D\n", b, (int64)n);
// TODO(atom): This block is a branch of an if-then-else statement.
// The single-threaded branch may be added in a next CL.
{
// Multi-threaded version.
vp = (void**)b;
n >>= (2+PtrSize/8); /* n /= PtrSize (4 or 8) */
for(i=0; i<n; i++) {
obj = (byte*)vp[i];
bitbufpos = bitbuf;
// Words outside the arena cannot be pointers.
if((byte*)obj < arena_start || (byte*)obj >= arena_used)
continue;
while(ptrbuf < ptrbuf_end) {
obj = ptrbuf->p;
ti = ptrbuf->ti;
ptrbuf++;
// obj belongs to interval [mheap.arena_start, mheap.arena_used).
if(Debug > 1) {
if(obj < runtime·mheap.arena_start || obj >= runtime·mheap.arena_used)
runtime·throw("object is outside of mheap");
}
// obj may be a pointer to a live object.
// Try to find the beginning of the object.
// Round down to word boundary.
if(((uintptr)obj & ((uintptr)PtrSize-1)) != 0) {
obj = (void*)((uintptr)obj & ~((uintptr)PtrSize-1));
ti = 0;
}
// Find bits for this word.
off = (uintptr*)obj - (uintptr*)arena_start;
......@@ -219,6 +276,8 @@ scanblock(byte *b, uintptr n)
if((bits & (bitAllocated|bitBlockBoundary)) != 0)
goto found;
ti = 0;
// Pointing just past the beginning?
// Scan backward a little to find a block boundary.
for(j=shift; j-->0; ) {
......@@ -245,7 +304,7 @@ scanblock(byte *b, uintptr n)
} else {
if((byte*)obj >= (byte*)s->limit)
continue;
size = runtime·class_to_size[s->sizeclass];
size = s->elemsize;
int32 i = ((byte*)obj - p)/size;
obj = p+i*size;
}
......@@ -258,81 +317,203 @@ scanblock(byte *b, uintptr n)
bits = xbits >> shift;
found:
// If another proc wants a pointer, give it some.
if(work.nwait > 0 && nobj > 4 && work.full == 0) {
wbuf->nobj = nobj;
wbuf = handoff(wbuf);
nobj = wbuf->nobj;
wp = wbuf->obj + nobj;
}
// Now we have bits, bitp, and shift correct for
// obj pointing at the base of the object.
// Only care about allocated and not marked.
if((bits & (bitAllocated|bitMarked)) != bitAllocated)
continue;
if(nproc == 1)
*bitp |= bitMarked<<shift;
else {
for(;;) {
x = *bitp;
if(x & (bitMarked<<shift))
goto continue_obj;
if(runtime·casp((void**)bitp, (void*)x, (void*)(x|(bitMarked<<shift))))
break;
}
*bitbufpos = (struct BitTarget){obj, ti, bitp, shift};
bitbufpos++;
}
runtime·lock(&lock);
for(bt=bitbuf; bt<bitbufpos; bt++){
xbits = *bt->bitp;
bits = xbits >> bt->shift;
if((bits & bitMarked) != 0)
continue;
// Mark the block
*bt->bitp = xbits | (bitMarked << bt->shift);
// If object has no pointers, don't need to scan further.
if((bits & bitNoPointers) != 0)
continue;
obj = bt->p;
// Ask span about size class.
// (Manually inlined copy of MHeap_Lookup.)
x = (uintptr)obj >> PageShift;
if(sizeof(void*) == 8)
x -= (uintptr)arena_start>>PageShift;
s = runtime·mheap.map[x];
PREFETCH(obj);
// If buffer is full, get a new one.
if(wbuf == nil || nobj >= nelem(wbuf->obj)) {
if(wbuf != nil)
*wp = (Obj){obj, s->elemsize, bt->ti};
wp++;
nobj++;
}
runtime·unlock(&lock);
// If another proc wants a pointer, give it some.
if(work.nwait > 0 && nobj > handoffThreshold && work.full == 0) {
wbuf->nobj = nobj;
wbuf = getempty(wbuf);
wp = wbuf->obj;
nobj = 0;
wbuf = handoff(wbuf);
nobj = wbuf->nobj;
wp = wbuf->obj + nobj;
}
*wp++ = obj;
nobj++;
continue_obj:;
}
*_wp = wp;
*_wbuf = wbuf;
*_nobj = nobj;
}
// Program that scans the whole block and treats every block element as a potential pointer
static uintptr defaultProg[2] = {PtrSize, GC_DEFAULT_PTR};
// scanblock scans a block of n bytes starting at pointer b for references
// to other objects, scanning any it finds recursively until there are no
// unscanned objects left. Instead of using an explicit recursion, it keeps
// a work list in the Workbuf* structures and loops in the main function
// body. Keeping an explicit work list is easier on the stack allocator and
// more efficient.
//
// wbuf: current work buffer
// wp: storage for next queued pointer (write pointer)
// nobj: number of queued objects
static void
scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
{
byte *b, *arena_start, *arena_used;
uintptr n, i, end_b;
void *obj;
// TODO(atom): to be expanded in a next CL
struct Frame {uintptr count, b; uintptr *loop_or_ret;};
struct Frame stack_top;
uintptr *pc;
struct BufferList *scanbuffers;
struct PtrTarget *ptrbuf, *ptrbuf_end;
struct BitTarget *bitbuf;
struct PtrTarget *ptrbufpos;
// End of local variable declarations.
if(sizeof(Workbuf) % PageSize != 0)
runtime·throw("scanblock: size of Workbuf is suboptimal");
// Memory arena parameters.
arena_start = runtime·mheap.arena_start;
arena_used = runtime·mheap.arena_used;
// Allocate ptrbuf, bitbuf
{
runtime·lock(&lock);
if(bufferList == nil) {
bufferList = runtime·SysAlloc(sizeof(*bufferList));
bufferList->next = nil;
}
scanbuffers = bufferList;
bufferList = bufferList->next;
ptrbuf = &scanbuffers->ptrtarget[0];
ptrbuf_end = &scanbuffers->ptrtarget[0] + nelem(scanbuffers->ptrtarget);
bitbuf = &scanbuffers->bittarget[0];
runtime·unlock(&lock);
}
ptrbufpos = ptrbuf;
goto next_block;
for(;;) {
// Each iteration scans the block b of length n, queueing pointers in
// the work buffer.
if(Debug > 1) {
runtime·printf("scanblock %p %D\n", b, (int64)n);
}
// TODO(atom): to be replaced in a next CL
pc = defaultProg;
pc++;
stack_top.b = (uintptr)b;
end_b = (uintptr)b + n - PtrSize;
next_instr:
// TODO(atom): to be expanded in a next CL
switch(pc[0]) {
case GC_DEFAULT_PTR:
while(true) {
i = stack_top.b;
if(i > end_b)
goto next_block;
stack_top.b += PtrSize;
obj = *(byte**)i;
if(obj >= arena_start && obj < arena_used) {
*ptrbufpos = (struct PtrTarget){obj, 0};
ptrbufpos++;
if(ptrbufpos == ptrbuf_end)
goto flush_buffers;
}
}
default:
runtime·throw("scanblock: invalid GC instruction");
return;
}
flush_buffers:
flushptrbuf(ptrbuf, ptrbufpos-ptrbuf, &wp, &wbuf, &nobj, bitbuf);
ptrbufpos = ptrbuf;
goto next_instr;
next_block:
// Done scanning [b, b+n). Prepare for the next iteration of
// the loop by setting b and n to the parameters for the next block.
// the loop by setting b, n to the parameters for the next block.
if(nobj == 0) {
flushptrbuf(ptrbuf, ptrbufpos-ptrbuf, &wp, &wbuf, &nobj, bitbuf);
ptrbufpos = ptrbuf;
// Fetch b from the work buffer.
if(nobj == 0) {
if(!keepworking) {
if(wbuf)
putempty(wbuf);
return;
goto endscan;
}
// Emptied our buffer: refill.
wbuf = getfull(wbuf);
if(wbuf == nil)
return;
goto endscan;
nobj = wbuf->nobj;
wp = wbuf->obj + wbuf->nobj;
}
b = *--wp;
nobj--;
}
// Ask span about size class.
// (Manually inlined copy of MHeap_Lookup.)
x = (uintptr)b>>PageShift;
if(sizeof(void*) == 8)
x -= (uintptr)arena_start>>PageShift;
s = runtime·mheap.map[x];
if(s->sizeclass == 0)
n = s->npages<<PageShift;
else
n = runtime·class_to_size[s->sizeclass];
// Fetch b from the work buffer.
--wp;
b = wp->p;
n = wp->n;
nobj--;
}
endscan:
runtime·lock(&lock);
scanbuffers->next = bufferList;
bufferList = scanbuffers;
runtime·unlock(&lock);
}
// debug_scanblock is the debug copy of scanblock.
......@@ -379,13 +560,12 @@ debug_scanblock(byte *b, uintptr n)
continue;
p = (byte*)((uintptr)s->start<<PageShift);
size = s->elemsize;
if(s->sizeclass == 0) {
obj = p;
size = (uintptr)s->npages<<PageShift;
} else {
if((byte*)obj >= (byte*)s->limit)
continue;
size = runtime·class_to_size[s->sizeclass];
int32 i = ((byte*)obj - p)/size;
obj = p+i*size;
}
......@@ -414,11 +594,74 @@ debug_scanblock(byte *b, uintptr n)
}
}
// Append obj to the work buffer.
// _wbuf, _wp, _nobj are input/output parameters and are specifying the work buffer.
static void
enqueue(Obj obj, Workbuf **_wbuf, Obj **_wp, uintptr *_nobj)
{
uintptr nobj, off;
Obj *wp;
Workbuf *wbuf;
if(Debug > 1)
runtime·printf("append obj(%p %D %p)\n", obj.p, (int64)obj.n, obj.ti);
// Align obj.b to a word boundary.
off = (uintptr)obj.p & (PtrSize-1);
if(off != 0) {
obj.p += PtrSize - off;
obj.n -= PtrSize - off;
obj.ti = 0;
}
if(obj.p == nil || obj.n == 0)
return;
// Load work buffer state
wp = *_wp;
wbuf = *_wbuf;
nobj = *_nobj;
// If another proc wants a pointer, give it some.
if(work.nwait > 0 && nobj > handoffThreshold && work.full == 0) {
wbuf->nobj = nobj;
wbuf = handoff(wbuf);
nobj = wbuf->nobj;
wp = wbuf->obj + nobj;
}
// If buffer is full, get a new one.
if(wbuf == nil || nobj >= nelem(wbuf->obj)) {
if(wbuf != nil)
wbuf->nobj = nobj;
wbuf = getempty(wbuf);
wp = wbuf->obj;
nobj = 0;
}
*wp = obj;
wp++;
nobj++;
// Save work buffer state
*_wp = wp;
*_wbuf = wbuf;
*_nobj = nobj;
}
static void
markroot(ParFor *desc, uint32 i)
{
Obj *wp;
Workbuf *wbuf;
uintptr nobj;
USED(&desc);
scanblock(work.roots[i].p, work.roots[i].n);
wp = nil;
wbuf = nil;
nobj = 0;
enqueue(work.roots[i], &wbuf, &wp, &nobj);
scanblock(wbuf, wp, nobj, false);
}
// Get an empty work buffer off the work.empty list,
......@@ -508,25 +751,24 @@ handoff(Workbuf *b)
}
static void
addroot(byte *p, uintptr n)
addroot(Obj obj)
{
uint32 cap;
GcRoot *new;
Obj *new;
if(work.nroot >= work.rootcap) {
cap = PageSize/sizeof(GcRoot);
cap = PageSize/sizeof(Obj);
if(cap < 2*work.rootcap)
cap = 2*work.rootcap;
new = (GcRoot*)runtime·SysAlloc(cap*sizeof(GcRoot));
new = (Obj*)runtime·SysAlloc(cap*sizeof(Obj));
if(work.roots != nil) {
runtime·memmove(new, work.roots, work.rootcap*sizeof(GcRoot));
runtime·SysFree(work.roots, work.rootcap*sizeof(GcRoot));
runtime·memmove(new, work.roots, work.rootcap*sizeof(Obj));
runtime·SysFree(work.roots, work.rootcap*sizeof(Obj));
}
work.roots = new;
work.rootcap = cap;
}
work.roots[work.nroot].p = p;
work.roots[work.nroot].n = n;
work.roots[work.nroot] = obj;
work.nroot++;
}
......@@ -570,7 +812,7 @@ addstackroots(G *gp)
runtime·printf("scanstack inconsistent: g%D#%d sp=%p not in [%p,%p]\n", gp->goid, n, sp, guard-StackGuard, stk);
runtime·throw("scanstack");
}
addroot(sp, (byte*)stk - sp);
addroot((Obj){sp, (byte*)stk - sp, 0});
sp = (byte*)stk->gobuf.sp;
guard = stk->stackguard;
stk = (Stktop*)stk->stackbase;
......@@ -588,7 +830,7 @@ addfinroots(void *v)
runtime·throw("mark - finalizer inconsistency");
// do not mark the finalizer block itself. just mark the things it points at.
addroot(v, size);
addroot((Obj){v, size, 0});
}
static void
......@@ -596,15 +838,15 @@ addroots(void)
{
G *gp;
FinBlock *fb;
byte *p;
MSpan *s, **allspans;
uint32 spanidx;
work.nroot = 0;
// mark data+bss.
for(p=data; p<ebss; p+=DataBlock)
addroot(p, p+DataBlock < ebss ? DataBlock : ebss-p);
// data & bss
// TODO(atom): load balancing
addroot((Obj){data, edata - data, (uintptr)gcdata});
addroot((Obj){bss, ebss - bss, (uintptr)gcbss});
// MSpan.types
allspans = runtime·mheap.allspans;
......@@ -617,12 +859,14 @@ addroots(void)
break;
case MTypes_Words:
case MTypes_Bytes:
addroot((byte*)&s->types.data, sizeof(void*));
// TODO(atom): consider using defaultProg instead of 0
addroot((Obj){(byte*)&s->types.data, sizeof(void*), 0});
break;
}
}
}
// stacks
for(gp=runtime·allg; gp!=nil; gp=gp->alllink) {
switch(gp->status){
default:
......@@ -646,7 +890,7 @@ addroots(void)
runtime·walkfintab(addfinroots);
for(fb=allfin; fb; fb=fb->alllink)
addroot((byte*)fb->fin, fb->cnt*sizeof(fb->fin[0]));
addroot((Obj){(byte*)fb->fin, fb->cnt*sizeof(fb->fin[0]), 0});
}
static bool
......@@ -887,8 +1131,9 @@ runtime·gchelper(void)
{
// parallel mark for over gc roots
runtime·parfordo(work.markfor);
// help other threads scan secondary blocks
scanblock(nil, 0);
scanblock(nil, nil, 0, true);
if(DebugMark) {
// wait while the main thread executes mark(debug_scanblock)
......@@ -1050,26 +1295,27 @@ gc(struct gc_args *args)
obj0 = mstats.nmalloc - mstats.nfree;
}
m->locks++; // disable gc during mallocs in parforalloc
if(work.markfor == nil)
work.markfor = runtime·parforalloc(MaxGcproc);
if(work.sweepfor == nil)
work.sweepfor = runtime·parforalloc(MaxGcproc);
m->locks--;
work.nwait = 0;
work.ndone = 0;
work.debugmarkdone = 0;
work.nproc = runtime·gcprocs();
addroots();
m->locks++; // disable gc during mallocs in parforalloc
if(work.markfor == nil)
work.markfor = runtime·parforalloc(MaxGcproc);
runtime·parforsetup(work.markfor, work.nproc, work.nroot, nil, false, markroot);
if(work.sweepfor == nil)
work.sweepfor = runtime·parforalloc(MaxGcproc);
runtime·parforsetup(work.sweepfor, work.nproc, runtime·mheap.nspan, nil, true, sweepspan);
m->locks--;
if(work.nproc > 1) {
runtime·noteclear(&work.alldone);
runtime·helpgc(work.nproc);
}
runtime·parfordo(work.markfor);
scanblock(nil, 0);
scanblock(nil, nil, 0, true);
if(DebugMark) {
for(i=0; i<work.nroot; i++)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7