Commit 8b8ff164 authored by Russ Cox's avatar Russ Cox

delete playpen copy of malloc

R=r
DELTA=905  (0 added, 905 deleted, 0 changed)
OCL=22663
CL=22690
parent 8fb60768
# Copyright 2009 The Go Authors. All rights reserved.
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file.
CC=6c -w
GC=6g
LD=6l
O=6
TARG=testrandom testrepeat testsizetoclass
default: $(TARG)
%.$O: %.c malloc.h
$(CC) $*.c
%.$O: %.go
$(GC) $*.go
OFILES=\
allocator.$O\
malloc.$O\
mem.$O\
ms.$O\
pagemap.$O\
triv.$O\
testrandom.$O: allocator.$O
testrepeat.$O: allocator.$O
test%: test%.$O $(OFILES)
$(LD) -o $@ $^
clean:
rm -f *.$O $(TARG)
runtime: $(OFILES)
6ar grc $(GOROOT)/lib/lib_$(GOARCH)_$(GOOS).a $^
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package allocator
export func free(*byte)
export func malloc(int) *byte
export func memset(*byte, int, int)
export var footprint int64
export var frozen bool
export func testsizetoclass()
export var allocated int64
export func find(uint64) (obj *byte, size int64, ref *int32, ok bool)
export func gc()
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// General C malloc/free, but intended for Go.
// Same design as tcmalloc:
// see https://www/eng/designdocs/tcmalloc/tcmalloc.html
// TODO:
// * Central free lists.
// * Thread cache stealing.
// * Return memory to the OS.
// * Memory footprint during testrandom is too big.
// * Need to coalesce adjacent free spans.
//
// *** Some way to avoid the ``malloc overflows the stack
// during the stack overflow malloc'' problem.
#include "malloc.h"
Central central;
PageMap spanmap;
// Insert a new span into the map.
static void
insertspan(Span *s)
{
int32 i;
uintptr base;
// TODO: This is likely too slow for large spans.
base = (uintptr)s->base >> PageShift;
for(i=0; i<s->length; i++)
pminsert(&spanmap, base+i, s);
}
// Record that a span has gotten smaller.
static void
shrinkspan(Span *s, int32 newlength)
{
int32 i;
uintptr base;
// TODO: This is unnecessary, because an insertspan is next.
base = (uintptr)s->base >> PageShift;
for(i=newlength; i<s->length; i++)
pminsert(&spanmap, base+i, nil);
s->length = newlength;
}
// Find the span for a given pointer.
static Span*
spanofptr(void *v)
{
return pmlookup(&spanmap, (uintptr)v >> PageShift);
}
static void freespan(Span*);
// Linked list of spans.
// TODO(rsc): Remove - should be able to walk pagemap.
Span *spanfirst;
Span *spanlast;
static void
addtolist(Span *s)
{
if(spanlast) {
s->aprev = spanlast;
s->aprev->anext = s;
} else {
s->aprev = nil;
spanfirst = s;
}
s->anext = nil;
spanlast = s;
}
/*
static void
delfromlist(Span *s)
{
if(s->aprev)
s->aprev->anext = s->anext;
else
spanfirst = s->anext;
if(s->anext)
s->anext->aprev = s->aprev;
else
spanlast = s->aprev;
}
*/
// Allocate a span of at least n pages.
static Span*
allocspan(int32 npage)
{
Span *s, **l, *s1;
int32 allocnpage, i;
// Look in the n-page free lists for big enough n.
for(i=npage; i<nelem(central.free); i++) {
s = central.free[i];
if(s != nil) {
central.free[i] = s->next;
goto havespan;
}
}
// Look in the large list, which has large runs of pages.
for(l=&central.large; (s=*l) != nil; l=&s->next) {
if(s->length >= npage) {
*l = s->next;
s->next = nil;
//if(s->length > npage) printf("Chop span %D for %d\n", s->length, npage);
goto havespan;
}
}
// Otherwise we need more memory.
// TODO: Could try to release this lock while asking for memory.
s = trivalloc(sizeof *s);
allocnpage = npage;
if(allocnpage < (1<<20>>PageShift)) // TODO: Tune
allocnpage = (1<<20>>PageShift);
s->length = allocnpage;
//printf("New span %d for %d\n", allocnpage, npage);
s->base = trivalloc(allocnpage<<PageShift);
insertspan(s);
addtolist(s);
havespan:
// If span is bigger than needed, redistribute the remainder.
if(s->length > npage) {
s1 = trivalloc(sizeof *s);
s1->base = s->base + (npage << PageShift);
s1->length = s->length - npage;
shrinkspan(s, npage);
insertspan(s1);
addtolist(s1);
freespan(s1);
}
s->state = SpanInUse;
return s;
}
// Free a span.
// TODO(rsc): Coalesce adjacent free spans.
static void
freespan(Span *s)
{
Span **l;
Span *ss;
s->state = SpanFree;
if(s->length < nelem(central.free)) {
s->next = central.free[s->length];
central.free[s->length] = s;
} else {
// Keep central.large sorted in
// increasing size for best-fit allocation.
for(l = &central.large; (ss=*l) != nil; l=&ss->next)
if(ss->length >= s->length)
break;
s->next = *l;
*l = s;
}
}
// Small objects are kept on per-size free lists in the M.
// There are SmallFreeClasses (defined in runtime.h) different lists.
int32 classtosize[SmallFreeClasses] = {
/*
seq 8 8 127 | sed 's/$/,/' | fmt
seq 128 16 255 | sed 's/$/,/' | fmt
seq 256 32 511 | sed 's/$/,/' | fmt
seq 512 64 1023 | sed 's/$/,/' | fmt
seq 1024 128 2047 | sed 's/$/,/' | fmt
seq 2048 256 32768 | sed 's/$/,/' | fmt
*/
8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120,
128, 144, 160, 176, 192, 208, 224, 240,
256, 288, 320, 352, 384, 416, 448, 480,
512, 576, 640, 704, 768, 832, 896, 960,
1024, 1152, 1280, 1408, 1536, 1664, 1792, 1920,
2048, 2304, 2560, 2816, 3072, 3328, 3584, 3840, 4096, 4352, 4608,
4864, 5120, 5376, 5632, 5888, 6144, 6400, 6656, 6912, 7168, 7424,
7680, 7936, 8192, 8448, 8704, 8960, 9216, 9472, 9728, 9984, 10240,
10496, 10752, 11008, 11264, 11520, 11776, 12032, 12288, 12544,
12800, 13056, 13312, 13568, 13824, 14080, 14336, 14592, 14848,
15104, 15360, 15616, 15872, 16128, 16384, 16640, 16896, 17152,
17408, 17664, 17920, 18176, 18432, 18688, 18944, 19200, 19456,
19712, 19968, 20224, 20480, 20736, 20992, 21248, 21504, 21760,
22016, 22272, 22528, 22784, 23040, 23296, 23552, 23808, 24064,
24320, 24576, 24832, 25088, 25344, 25600, 25856, 26112, 26368,
26624, 26880, 27136, 27392, 27648, 27904, 28160, 28416, 28672,
28928, 29184, 29440, 29696, 29952, 30208, 30464, 30720, 30976,
31232, 31488, 31744, 32000, 32256, 32512, 32768,
};
enum {
LargeSize = 32768
};
// Trigger compile error if nelem(classtosize) != SmallFreeClasses.
static int32 zzz1[SmallFreeClasses-nelem(classtosize)+1];
static int32 zzz2[nelem(classtosize)-SmallFreeClasses+1];
static int32
sizetoclass(int32 siz)
{
if(siz <= 0)
return 0;
if(siz <= 128)
return (siz-1) >> 3;
if(siz <= 256)
return ((siz-1) >> 4) + 8;
if(siz <= 512)
return ((siz-1) >> 5) + 16;
if(siz <= 1024)
return ((siz-1) >> 6) + 24;
if(siz <= 2048)
return ((siz-1) >> 7) + 32;
if(siz <= 32768)
return ((siz-1) >> 8) + 40;
throw("sizetoclass - invalid size");
return -1;
}
void
allocator·testsizetoclass(void)
{
int32 i, n;
n = 0;
for(i=0; i<nelem(classtosize); i++) {
for(; n <= classtosize[i]; n++) {
if(sizetoclass(n) != i) {
printf("sizetoclass %d = %d want %d\n", n, sizetoclass(n), i);
throw("testsizetoclass");
}
}
}
if (n != 32768+1) {
printf("testsizetoclass stopped at %d\n", n);
throw("testsizetoclass");
}
}
// Grab a bunch of objects of size class cl off the central free list.
// Set *pn to the number of objects returned.
static void*
centralgrab(int32 cl, int32 *pn)
{
byte *p;
Span *s;
int32 chunk, i, n, siz;
// For now there is no central free list.
// Fall back to allocating a new span
// and chopping it up.
chunk = classtosize[cl] * 1024;
if(chunk > 1<<20) {
chunk = 1<<20;
}
chunk = (chunk+PageMask) & ~PageMask;
s = allocspan(chunk>>PageShift);
//printf("New class %d\n", cl);
s->state = SpanInUse;
s->cl = cl;
siz = classtosize[cl];
n = chunk/(siz+sizeof(s->refbase[0]));
p = s->base;
//printf("centralgrab cl=%d siz=%d n=%d\n", cl, siz, n);
for(i=0; i<n; i++) {
if(i < n-1)
*(void**)p = p+siz;
p += siz;
}
s->refbase = (int32*)p;
// TODO(rsc): Remove - only for mark/sweep
for(i=0; i<n; i++)
s->refbase[i] = RefFree;
*pn = n;
return s->base;
}
// Allocate a small object of size class cl.
void*
allocsmall(int32 cl)
{
void **p;
int32 n;
if(cl < 0 || cl >= SmallFreeClasses)
throw("allocsmall - invalid class");
// try m-local cache.
p = m->freelist[cl];
if(p == nil) {
// otherwise grab some blocks from central cache.
lock(&central);
//printf("centralgrab for %d\n", cl);
p = centralgrab(cl, &n);
// TODO: update local counters using n
unlock(&central);
}
//printf("alloc from cl %d %p\n", cl, p);
// advance linked list.
m->freelist[cl] = *p;
// TODO(rsc): If cl > 0, can store ref ptr in *(p+1),
// avoiding call to findobj.
// Or could get rid of RefFree, which is only truly
// necessary for mark/sweep.
int32 *ref;
if(!findobj(p, nil, nil, &ref))
throw("bad findobj");
if(*ref != RefFree)
throw("double alloc");
*ref = 0;
// Blocks on free list are zeroed except for
// the linked list pointer that we just used. Zero it.
*p = 0;
return p;
}
// Allocate large object of np pages.
void*
alloclarge(int32 np)
{
Span *s;
lock(&central);
//printf("Alloc span %d\n", np);
s = allocspan(np);
unlock(&central);
s->state = SpanInUse;
s->cl = -1;
s->ref = 0;
return s->base;
}
// Allocate object of n bytes.
void*
alloc(int32 n)
{
int32 cl, np;
if(n < LargeSize) {
cl = sizetoclass(n);
if(cl < 0 || cl >= SmallFreeClasses) {
printf("%d -> %d\n", n, cl);
throw("alloc - logic error");
}
allocator·allocated += classtosize[cl];
return allocsmall(cl);
}
// count number of pages; careful about overflow for big n.
np = (n>>PageShift) + (((n&PageMask)+PageMask)>>PageShift);
allocator·allocated += (uint64)np<<PageShift;
return alloclarge(np);
}
void
allocator·malloc(int32 n, byte *out)
{
out = alloc(n);
FLUSH(&out);
}
// Check whether v points into a known memory block.
// If so, return true with
// *obj = base pointer of object (can pass to free)
// *size = size of object
// *ref = pointer to ref count for object
// Object might already be freed, in which case *ref == RefFree.
bool
findobj(void *v, void **obj, int64 *size, int32 **ref)
{
Span *s;
int32 siz, off, indx;
s = spanofptr(v);
if(s == nil || s->state != SpanInUse)
return false;
// Big object
if(s->cl < 0) {
if(obj)
*obj = s->base;
if(size)
*size = s->length<<PageShift;
if(ref)
*ref = &s->ref;
return true;
}
// Small object
if((byte*)v >= (byte*)s->refbase)
return false;
siz = classtosize[s->cl];
off = (byte*)v - (byte*)s->base;
indx = off/siz;
if(obj)
*obj = s->base + indx*siz;
if(size)
*size = siz;
if(ref)
*ref = s->refbase + indx;
return true;
}
void
allocator·find(uint64 ptr, byte *obj, int64 siz, int32 *ref, bool ok)
{
ok = findobj((void*)ptr, &obj, &siz, &ref);
FLUSH(&ok);
}
// Free object with base pointer v.
void
free(void *v)
{
void **p;
Span *s;
int32 siz, off, n;
s = spanofptr(v);
if(s->state != SpanInUse)
throw("free - invalid pointer1");
// Big object should be s->base.
if(s->cl < 0) {
if(v != s->base)
throw("free - invalid pointer2");
// TODO: For large spans, maybe just return the
// memory to the operating system and let it zero it.
if(s->ref != 0 && s->ref != RefManual && s->ref != RefStack)
throw("free - bad ref count");
s->ref = RefFree;
sys·memclr(s->base, s->length << PageShift);
//printf("Free big %D\n", s->length);
allocator·allocated -= s->length << PageShift;
lock(&central);
freespan(s);
unlock(&central);
return;
}
// Small object should be aligned properly.
if((byte*)v >= (byte*)s->refbase)
throw("free - invalid pointer4");
siz = classtosize[s->cl];
off = (byte*)v - (byte*)s->base;
if(off%siz)
throw("free - invalid pointer3");
n = off/siz;
if(s->refbase[n] != 0 && s->refbase[n] != RefManual && s->refbase[n] != RefStack)
throw("free - bad ref count1");
s->refbase[n] = RefFree;
// Zero and add to free list.
sys·memclr(v, siz);
allocator·allocated -= siz;
p = v;
*p = m->freelist[s->cl];
m->freelist[s->cl] = p;
//printf("Free siz %d cl %d\n", siz, s->cl);
}
void
allocator·free(byte *v)
{
free(v);
}
void
allocator·memset(byte *v, int32 c, int32 n)
{
int32 i;
for(i=0; i<n; i++)
v[i] = c;
}
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
#include "../../../src/runtime/runtime.h"
typedef struct PageMap PageMap;
enum
{
PageShift = 12,
PageMask = (1<<PageShift) - 1,
};
#define RefFree 0xffffffffU
#define RefManual 0xfffffffeU
#define RefStack 0xfffffffdU
enum {
PMBits = 64 - PageShift,
PMLevels = 4,
PMLevelBits = 13,
PMLevelSize = 1<<PMLevelBits,
PMLevelMask = PMLevelSize - 1,
};
struct PageMap
{
void *level0[PMLevelSize];
};
typedef struct Span Span;
typedef struct Central Central;
// A Span contains metadata about a range of pages.
enum {
SpanInUse = 0, // span has been handed out by allocator
SpanFree = 1, // span is in central free list
};
struct Span
{
Span *aprev; // in list of all spans
Span *anext;
Span *next; // in free lists
byte *base; // first byte in span
uintptr length; // number of pages in span
int32 cl;
int32 state; // state (enum above)
union {
int32 ref; // reference count if state == SpanInUse (for GC)
int32 *refbase; // ptr to packed ref counts
};
// void *type; // object type if state == SpanInUse (for GC)
};
// The Central cache contains a list of free spans,
// as well as free lists of small blocks.
struct Central
{
Lock;
Span *free[256];
Span *large; // free spans >= MaxPage pages
};
extern int64 allocator·allocated;
extern int64 allocator·footprint;
extern bool allocator·frozen;
void* trivalloc(int32);
void* pmlookup(PageMap*, uintptr);
void* pminsert(PageMap*, uintptr, void*);
void* alloc(int32);
void free(void*);
bool findobj(void*, void**, int64*, int32**);
extern Central central;
extern PageMap spanmap;
extern int32 classtosize[SmallFreeClasses];
extern Span *spanfirst, *spanlast;
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
#include "malloc.h"
void*
stackalloc(uint32 n)
{
void *v;
int32 *ref;
v = alloc(n);
//printf("stackalloc %d = %p\n", n, v);
ref = nil;
findobj(v, nil, nil, &ref);
*ref = RefStack;
return v;
}
void
stackfree(void *v)
{
//printf("stackfree %p\n", v);
free(v);
}
void*
mal(uint32 n)
{
return alloc(n);
}
void
sys·mal(uint32 n, uint8 *ret)
{
ret = alloc(n);
FLUSH(&ret);
}
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
#include "malloc.h"
// A PageMap maps page numbers to void* pointers.
// The AMD64 has 64-bit addresses and 4096-byte pages, so
// the page numbers are 52 bits. We use a four-level radix tree,
// with 13 bits for each level. This requires 32 kB per level or
// 128 kB for a table with one entry. Moving to three levels of 18 bits
// would require 3 MB for a table with one entry, which seems too expensive.
// This is easy to change.
// It may be that a balanced tree would be better anyway.
// Return the entry for page number pn in m.
void*
pmlookup(PageMap *m, uintptr pn)
{
int32 i, x;
void **v;
v = &m->level0[0];
for(i=0; i<PMLevels; i++) {
// Pick off top PMLevelBits bits as index and shift up.
x = (pn >> (PMBits - PMLevelBits)) & PMLevelMask;
pn <<= PMLevelBits;
// Walk down using index.
v = v[x];
if(v == nil)
return nil;
}
return v;
}
// Set the entry for page number pn in m to s.
// Return the old value.
void*
pminsert(PageMap *m, uintptr pn, void *value)
{
int32 i, x;
void **v, **l;
l = nil; // shut up 6c
v = &m->level0[0];
for(i=0; i<PMLevels; i++) {
// Pick off top PMLevelBits bits as index and shift up.
x = (pn >> (PMBits - PMLevelBits)) & PMLevelMask;
pn <<= PMLevelBits;
// Walk down using index, but remember location of pointer.
l = &v[x];
v = *l;
// Allocate new level if needed.
if(v == nil && i < PMLevels-1) {
v = trivalloc(PMLevelSize * sizeof v[0]);
*l = v;
}
}
// Record new value and return old.
*l = value;
return v;
}
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package main
import (
"allocator";
"rand";
"syscall"
)
var footprint int64;
var allocated int64;
func bigger() {
if footprint < allocator.footprint {
footprint = allocator.footprint;
println("Footprint", footprint, " for ", allocated);
if footprint > 1e9 {
panicln("too big");
}
}
}
// Prime the data structures by allocating one of
// each block in order. After this, there should be
// little reason to ask for more memory from the OS.
func prime() {
for i := 0; i < 16; i++ {
b := allocator.malloc(1<<uint(i));
allocator.free(b);
}
for i := 0; i < 256; i++ {
b := allocator.malloc(i<<12);
allocator.free(b);
}
}
func main() {
// prime();
var blocks [1] struct { base *byte; siz int; };
for i := 0; i < 1 << 20; i++ {
if i%(1<<10) == 0 {
println(i);
}
b := rand.rand() % len(blocks);
if blocks[b].base != nil {
// println("Free", blocks[b].siz, blocks[b].base);
allocator.free(blocks[b].base);
blocks[b].base = nil;
allocated -= int64(blocks[b].siz);
continue
}
siz := rand.rand() >> (11 + rand.urand32() % 20);
base := allocator.malloc(siz);
ptr := uint64(syscall.BytePtr(base))+uint64(siz/2);
obj, size, ref, ok := allocator.find(ptr);
if obj != base || *ref != 0 || !ok {
panicln("find", siz, obj, ref, ok);
}
blocks[b].base = base;
blocks[b].siz = siz;
allocated += int64(siz);
// println("Alloc", siz, base);
allocator.memset(base, 0xbb, siz);
bigger();
}
}
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package main
import (
"allocator"
)
var footprint int64
func bigger() {
if footprint < allocator.footprint {
footprint = allocator.footprint;
println("Footprint", footprint);
}
}
func main() {
for i := 0; i < 1<<16; i++ {
for j := 1; j <= 1<<22; j<<=1 {
if i == 0 {
println("First alloc:", j);
}
b := allocator.malloc(j);
allocator.free(b);
bigger();
}
if i%(1<<10) == 0 {
println(i);
}
if i == 0 {
println("Primed", i);
allocator.frozen = true;
}
}
}
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package main
import "allocator"
func main() {
allocator.testsizetoclass()
}
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Trivial base allocator.
#include "malloc.h"
// TODO: The call to sys·mmap should be a call to an assembly
// function sys·mmapnew that takes only a size parameter.
enum
{
PROT_NONE = 0x00,
PROT_READ = 0x01,
PROT_WRITE = 0x02,
PROT_EXEC = 0x04,
MAP_FILE = 0x0000,
MAP_SHARED = 0x0001,
MAP_PRIVATE = 0x0002,
MAP_FIXED = 0x0010,
MAP_ANON = 0x1000,
};
// Allocate and return zeroed memory.
// Simple allocator for small things like Span structures,
// and also used to grab large amounts of memory for
// the real allocator to hand out.
enum
{
Round = 15,
};
void*
trivalloc(int32 size)
{
static byte *p;
static int32 n;
byte *v;
uint64 oldfoot;
if(allocator·frozen)
throw("allocator frozen");
//prints("Newmem: ");
//sys·printint(size);
//prints("\n");
oldfoot = allocator·footprint;
if(size < 4096) { // TODO: Tune constant.
size = (size + Round) & ~Round;
if(size > n) {
n = 1<<20; // TODO: Tune constant.
p = sys·mmap(nil, n, PROT_READ|PROT_WRITE, MAP_ANON|MAP_PRIVATE, 0, 0);
allocator·footprint += n;
}
v = p;
p += size;
goto out;
}
if(size & PageMask)
size += (1<<PageShift) - (size & PageMask);
v = sys·mmap(nil, size, PROT_READ|PROT_WRITE, MAP_ANON|MAP_PRIVATE, 0, 0);
allocator·footprint += size;
out:
if((oldfoot>>24) != (allocator·footprint>>24))
printf("memory footprint = %D MB for %D MB\n", allocator·footprint>>20, allocator·allocated>>20);
if(allocator·footprint >= 2LL<<30) {
prints("out of memory\n");
sys·exit(1);
}
return v;
}
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment