Skip to content

M
MariaDB
Commit e9323991 authored by Bradley C. Kuszmaul's avatar Bradley C. Kuszmaul
Browse files
Options

{{{ 

svn delete tokudb.872
cd tokudb
svn merge -r4187:4233 https://svn.tokutek.com/tokudb/tokudb.872
}}}

Fixes #849, #872.
Addresses #820.


git-svn-id: file:///svn/tokudb@4234 c7de825b-a66e-492c-adef-691d508d4ae1
parent ed37a2cc
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 288 additions and 375 deletions
newbrt/brt-internal.h
View file @ e9323991
......@@ -220,7 +220,6 @@ int toku_cmd_leafval_bessel (OMTVALUE leafentry, void *extra);
int toku_brt_root_put_cmd(BRT brt, BRT_CMD cmd, TOKULOGGER logger);
int toku_cachefile_root_put_cmd (CACHEFILE cf, BRT_CMD cmd, TOKULOGGER logger);
int toku_omt_compress_kvspace (OMT omt, struct mempool *memp);
void *mempool_malloc_from_omt(OMT omt, struct mempool *mp, size_t size);
void toku_verify_all_in_mempool(BRTNODE node);
......
newbrt/brt-serialize.c
View file @ e9323991
......@@ -379,37 +379,35 @@ int toku_deserialize_brtnode_from (int fd, DISKOFF off, BRTNODE *brtnode) {
} else {
int n_in_buf = rbuf_int(&rc);
result->u.l.n_bytes_in_buffer = 0;
r=toku_omt_create(&result->u.l.buffer);
if (r!=0) {
if (0) { died_21: toku_omt_destroy(&result->u.l.buffer); }
goto died1;
}
//printf("%s:%d r PMA= %p\n", __FILE__, __LINE__, result->u.l.buffer);
{
int mpsize = result->nodesize + result->nodesize/4;
void *mp = toku_malloc(mpsize);
if (mp==0) return ENOMEM; // TODO cleanup
toku_mempool_init(&result->u.l.buffer_mempool, mp, mpsize);
}
toku_mempool_init(&result->u.l.buffer_mempool, rc.buf, datasize);
u_int32_t actual_sum = 0;
//printf("%s:%d node %lld, reading %d items\n", __FILE__, __LINE__, off, n_in_buf);
u_int32_t start_of_data = rc.ndone;
OMTVALUE *MALLOC_N(n_in_buf, array);
for (i=0; i<n_in_buf; i++) {
LEAFENTRY tmp_le;
//printf("%s:%d reading %dth item\n", __FILE__, __LINE__, i);
u_int32_t memsize, disksize;
rbuf_LEAFENTRY(&rc, &memsize, &disksize, &tmp_le);
LEAFENTRY le = mempool_malloc_from_omt(result->u.l.buffer, &result->u.l.buffer_mempool, memsize);
assert(le);
memcpy(le, tmp_le, memsize);
toku_free(tmp_le);
assert(disksize==leafentry_disksize(le));
result->u.l.n_bytes_in_buffer += disksize + OMT_ITEM_OVERHEAD;
toku_omt_insert_at(result->u.l.buffer, le, i);
actual_sum += result->rand4fingerprint*toku_le_crc(le);
//printf("%s:%d rand4=%08x fp=%08x \n", __FILE__, __LINE__, result->rand4fingerprint, actual_sum);
LEAFENTRY le = (LEAFENTRY)(&rc.buf[rc.ndone]);
u_int32_t disksize = leafentry_disksize(le);
rc.ndone += disksize;
assert(rc.ndone<=rc.size);
array[i]=(OMTVALUE)le;
actual_sum += toku_crc32(toku_null_crc, le, disksize);
}
u_int32_t end_of_data = rc.ndone;
result->u.l.n_bytes_in_buffer += end_of_data-start_of_data + n_in_buf*OMT_ITEM_OVERHEAD;
actual_sum *= result->rand4fingerprint;
r = toku_omt_create_from_sorted_array(&result->u.l.buffer, array, n_in_buf);
toku_free(array);
if (r!=0) {
if (0) { died_21: toku_omt_destroy(&result->u.l.buffer); }
goto died1;
}
result->u.l.buffer_mempool.frag_size = start_of_data;
result->u.l.buffer_mempool.free_offset = end_of_data;
if (r!=0) goto died_21;
if (actual_sum!=result->local_fingerprint) {
//fprintf(stderr, "%s:%d Corrupted checksum stored=%08x rand=%08x actual=%08x height=%d n_keys=%d\n", __FILE__, __LINE__, result->rand4fingerprint, result->local_fingerprint, actual_sum, result->height, n_in_buf);
......@@ -436,7 +434,10 @@ int toku_deserialize_brtnode_from (int fd, DISKOFF off, BRTNODE *brtnode) {
}
}
//printf("%s:%d Ok got %lld n_children=%d\n", __FILE__, __LINE__, result->thisnodename, result->n_children);
toku_free(rc.buf);
if (result->height>0) {
// For height==0 we used the buf inside the OMT
toku_free(rc.buf);
}
*brtnode = result;
//toku_verify_counts(result);
return 0;
......
newbrt/brt.c
View file @ e9323991
......@@ -3334,9 +3334,12 @@ static int move_it (OMTVALUE lev, u_int32_t idx, void *v) {
return 0;
}
int toku_omt_compress_kvspace (OMT omt, struct mempool *memp) {
if (toku_mempool_get_frag_size(memp) == 0)
return -1;
// Compress things, and grow the mempool if needed.
static int omt_compress_kvspace (OMT omt, struct mempool *memp, size_t added_size) {
u_int32_t total_size_needed = memp->free_offset-memp->frag_size + added_size;
if (total_size_needed+total_size_needed/4 >= memp->size) {
memp->size = total_size_needed+total_size_needed/4;
}
void *newmem = toku_malloc(memp->size);
if (newmem == 0)
return -2;
......@@ -3353,7 +3356,7 @@ int toku_omt_compress_kvspace (OMT omt, struct mempool *memp) {
void *mempool_malloc_from_omt(OMT omt, struct mempool *mp, size_t size) {
void *v = toku_mempool_malloc(mp, size, 1);
if (v==0) {
if (0 == toku_omt_compress_kvspace(omt, mp)) {
if (0 == omt_compress_kvspace(omt, mp, size)) {
v = toku_mempool_malloc(mp, size, 1);
assert(v);
}
......
newbrt/leafentry.c
View file @ e9323991
This diff is collapsed.
newbrt/leafentry.h
View file @ e9323991
......@@ -29,6 +29,7 @@
#include "brttypes.h"
#include "rbuf.h"
#include <arpa/inet.h>
u_int32_t toku_le_crc(LEAFENTRY v);
......@@ -45,61 +46,68 @@ enum le_state { LE_COMMITTED=1, // A committed pair.
LE_PROVDEL, // A committed pair that has been provisionally deleted
LE_PROVPAIR }; // No committed value, but a provisional pair.
struct contents_committed;
struct contents_both;
struct contents_provdelorpair;
u_int32_t leafentry_memsize (LEAFENTRY);
enum le_state get_le_state(LEAFENTRY);
void *get_le_contents(LEAFENTRY);
enum typ_tag get_le_tag(LEAFENTRY);
u_int32_t committed_keylen (void*cev);
void* committed_key (void*cev);
u_int32_t committed_vallen (struct contents_committed *ce);
void* committed_val (struct contents_committed *ce);
TXNID both_xid (struct contents_both *ce);
u_int32_t both_keylen (struct contents_both *ce);
u_int32_t both_committed_vallen (struct contents_both *ce);
u_int32_t both_prov_vallen (struct contents_both *ce);
void* both_key (struct contents_both *ce);
void* both_committed_val (struct contents_both *ce);
void* both_prov_val (struct contents_both*ce);
TXNID provdelorpair_xid (struct contents_provdelorpair *ce);
u_int32_t provdelorpair_keylen (struct contents_provdelorpair *ce);
u_int32_t provdelorpair_vallen (struct contents_provdelorpair *ce);
void* provdelorpair_key (struct contents_provdelorpair *ce);
void* provdelorpair_val (struct contents_provdelorpair *ce);
#define LESWITCHCALL(le,funname, ...) ({ \
assert(get_le_tag(le)==TYP_LEAFENTRY); \
switch(get_le_state(le)) { \
case LE_COMMITTED: return funname ## _le_committed( committed_keylen((struct contents_committed*)(get_le_contents(le))), \
committed_key((struct contents_committed*)(get_le_contents(le))), \
committed_vallen((struct contents_committed*)(get_le_contents(le))), \
committed_val((struct contents_committed*)(get_le_contents(le))), \
## __VA_ARGS__); \
case LE_BOTH: return funname ## _le_both( both_xid((struct contents_both*)(get_le_contents(le))), \
both_keylen((struct contents_both*)(get_le_contents(le))), \
both_key((struct contents_both*)(get_le_contents(le))), \
both_committed_vallen((struct contents_both*)(get_le_contents(le))), \
both_committed_val((struct contents_both*)(get_le_contents(le))), \
both_prov_vallen((struct contents_both*)(get_le_contents(le))), \
both_prov_val((struct contents_both*)(get_le_contents(le))), \
## __VA_ARGS__); \
case LE_PROVDEL: return funname ## _le_provdel ( provdelorpair_xid((struct contents_provdelorpair*)(get_le_contents(le))), \
provdelorpair_keylen((struct contents_provdelorpair*)(get_le_contents(le))), \
provdelorpair_key((struct contents_provdelorpair*)(get_le_contents(le))), \
provdelorpair_vallen((struct contents_provdelorpair*)(get_le_contents(le))), \
provdelorpair_val((struct contents_provdelorpair*)(get_le_contents(le))), \
## __VA_ARGS__); \
case LE_PROVPAIR: return funname ## _le_provpair(provdelorpair_xid((struct contents_provdelorpair*)(get_le_contents(le))), \
provdelorpair_keylen((struct contents_provdelorpair*)(get_le_contents(le))), \
provdelorpair_key((struct contents_provdelorpair*)(get_le_contents(le))), \
provdelorpair_vallen((struct contents_provdelorpair*)(get_le_contents(le))), \
provdelorpair_val((struct contents_provdelorpair*)(get_le_contents(le))), \
## __VA_ARGS__); \
static inline enum le_state get_le_state(LEAFENTRY le) {
return *(unsigned char *)le;
}
static inline void putint (unsigned char *p, u_int32_t i) {
#if 1
*(u_int32_t*)p = htonl(i);
#else
p[0]=(i>>24)&0xff;
p[1]=(i>>16)&0xff;
p[2]=(i>> 8)&0xff;
p[3]=(i>> 0)&0xff;
#endif
}
static inline void putint64 (unsigned char *p, u_int64_t i) {
putint(p, (u_int32_t)(i>>32));
putint(p+4, (u_int32_t)(i&0xffffffff));
}
static inline u_int32_t getint (unsigned char *p) {
#if 1
return ntohl(*(u_int32_t*)p);
#else
return (p[0]<<24)+(p[1]<<16)+(p[2]<<8)+(p[3]);
#endif
}
static inline u_int64_t getint64 (unsigned char *p) {
return (((u_int64_t)getint(p))<<32) + getint(p+4);
}
#define LESWITCHCALL(le,funname, ...) ({ \
switch(get_le_state(le)) { \
case LE_COMMITTED: { \
unsigned char* __klenaddr = 1+(unsigned char*)le; u_int32_t __klen = getint(__klenaddr); \
unsigned char* __kvaladdr = 4 + __klenaddr; \
unsigned char* __clenaddr = __klen + __kvaladdr; u_int32_t __clen = getint(__clenaddr); \
unsigned char* __cvaladdr = 4 + __clenaddr; \
return funname ## _le_committed(__klen, __kvaladdr, __clen, __cvaladdr, ## __VA_ARGS__); } \
case LE_BOTH: { \
unsigned char* __xidaddr = 1+(unsigned char*)le; u_int64_t __xid = getint64(__xidaddr); \
unsigned char* __klenaddr = 8 + __xidaddr; u_int32_t __klen = getint(__klenaddr); \
unsigned char* __kvaladdr = 4 + __klenaddr; \
unsigned char* __clenaddr = __klen + __kvaladdr; u_int32_t __clen = getint(__clenaddr); \
unsigned char* __cvaladdr = 4 + __clenaddr; \
unsigned char* __plenaddr = __clen + __cvaladdr; u_int32_t __plen = getint(__plenaddr); \
unsigned char* __pvaladdr = 4 + __plenaddr; \
return funname ## _le_both(__xid, __klen, __kvaladdr, __clen, __cvaladdr, __plen, __pvaladdr, ## __VA_ARGS__); } \
case LE_PROVDEL: { \
unsigned char* __xidaddr = 1+(unsigned char*)le; u_int64_t __xid = getint64(__xidaddr); \
unsigned char* __klenaddr = 8 + __xidaddr; u_int32_t __klen = getint(__klenaddr); \
unsigned char* __kvaladdr = 4 + __klenaddr; \
unsigned char* __dlenaddr = __klen + __kvaladdr; u_int32_t __dlen = getint(__dlenaddr); \
unsigned char* __dvaladdr = 4 + __dlenaddr; \
return funname ## _le_provdel(__xid, __klen, __kvaladdr, __dlen, __dvaladdr, ## __VA_ARGS__); } \
case LE_PROVPAIR: { \
unsigned char* __xidaddr = 1+(unsigned char*)le; u_int64_t __xid = getint64(__xidaddr); \
unsigned char* __klenaddr = 8 + __xidaddr; u_int32_t __klen = getint(__klenaddr); \
unsigned char* __kvaladdr = 4 + __klenaddr; \
unsigned char* __plenaddr = __klen + __kvaladdr; u_int32_t __plen = getint(__plenaddr); \
unsigned char* __pvaladdr = 4 + __plenaddr; \
return funname ## _le_provpair(__xid, __klen, __kvaladdr, __plen, __pvaladdr, ## __VA_ARGS__); } \
} abort(); })
......
newbrt/rbuf.h
View file @ e9323991
......@@ -5,6 +5,7 @@
#include "toku_assert.h"
#include "memory.h"
#include <arpa/inet.h>
struct rbuf {
unsigned char *buf;
......@@ -12,12 +13,18 @@ struct rbuf {
unsigned int ndone;
};
static unsigned int rbuf_char (struct rbuf *r) {
static inline unsigned int rbuf_char (struct rbuf *r) {
assert(r->ndone<r->size);
return r->buf[r->ndone++];
}
static unsigned int rbuf_int (struct rbuf *r) {
#if 1
assert(r->ndone+4 <= r->size);
u_int32_t result = ntohl(*(u_int32_t*)(r->buf+r->ndone)); // This only works on machines where unaligned loads are OK.
r->ndone+=4;
return result;
#else
unsigned char c0 = rbuf_char(r);
unsigned char c1 = rbuf_char(r);
unsigned char c2 = rbuf_char(r);
......@@ -26,6 +33,7 @@ static unsigned int rbuf_int (struct rbuf *r) {
(c1<<16)|
(c2<<8)|
(c3<<0));
#endif
}
static inline void rbuf_literal_bytes (struct rbuf *r, bytevec *bytes, unsigned int n_bytes) {
......
newbrt/tests/Makefile
View file @ e9323991
......@@ -74,6 +74,7 @@ REGRESSION_TESTS = \
test-brt-overflow \
test-del-inorder \
test-inc-split \
test-leafentry \
test-primes \
test_oexcl \
test_toku_malloc_plain_free \
......
newbrt/tests/test-leafentry.c 0 → 100644
View file @ e9323991
#include <string.h>
#include "leafentry.h"
static void test_leafentry_1 (void) {
LEAFENTRY l;
int r;
u_int32_t msize, dsize;
r = le_committed(4, "abc", 3, "xy", &msize, &dsize, &l);
assert(r==0);
char expect[] = {LE_COMMITTED,
0, 0, 0, 4,
'a', 'b', 'c', 0,
0, 0, 0, 3,
'x', 'y', 0};
assert(sizeof(expect)==msize);
assert(msize==dsize);
assert(memcmp(l, expect, msize)==0);
toku_free(l);
}
static void test_leafentry_2 (void) {
LEAFENTRY l;
int r;
u_int32_t msize, dsize;
r = le_both(0x0123456789abcdef0LL, 3, "ab", 4, "xyz", 5, "lmno", &msize, &dsize, &l);
assert(r==0);
char expect[] = {LE_BOTH,
0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc, 0xde, 0xf0,
0, 0, 0, 3, 'a', 'b', 0,
0, 0, 0, 4, 'x', 'y', 'z', 0,
0, 0, 0, 5, 'l', 'm', 'n', 'o', 0};
assert(sizeof(expect)==msize);
assert(msize==dsize);
assert(memcmp(l, expect, msize)==0);
toku_free(l);
}
static void test_leafentry_3 (void) {
LEAFENTRY l;
int r;
u_int32_t msize, dsize;
r = le_provdel(0x0123456789abcdef0LL, 3, "ab", 5, "lmno", &msize, &dsize, &l);
assert(r==0);
char expect[] = {LE_PROVDEL,
0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc, 0xde, 0xf0,
0, 0, 0, 3, 'a', 'b', 0,
0, 0, 0, 5, 'l', 'm', 'n', 'o', 0};
assert(sizeof(expect)==msize);
assert(msize==dsize);
assert(memcmp(l, expect, msize)==0);
toku_free(l);
}
static void test_leafentry_4 (void) {
LEAFENTRY l;
int r;
u_int32_t msize, dsize;
r = le_provpair(0x0123456789abcdef0LL, 3, "ab", 5, "lmno", &msize, &dsize, &l);
assert(r==0);
char expect[] = {LE_PROVPAIR,
0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc, 0xde, 0xf0,
0, 0, 0, 3, 'a', 'b', 0,
0, 0, 0, 5, 'l', 'm', 'n', 'o', 0};
assert(sizeof(expect)==msize);
assert(msize==dsize);
assert(memcmp(l, expect, msize)==0);
toku_free(l);
}
char zeros[1026];
#define n5zeros 0,0,0,0,0
#define n10zeros n5zeros,n5zeros
#define n25zeros n5zeros,n10zeros,n10zeros
#define n75zeros n25zeros,n25zeros,n25zeros
#define n125zeros n75zeros,n25zeros,n25zeros
#define n150zeros n75zeros,n75zeros
#define n300zeros n150zeros,n150zeros
#define n301zeros 0,n300zeros
#define n1025zeros n300zeros,n300zeros,n300zeros,n125zeros
char expect_3long[] = {LE_PROVDEL,
0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc, 0xde, 0xf0,
0, 0, 1, 45, n301zeros,
0, 0, 4, 1, n1025zeros};
static void test_leafentry_3long (void) {
LEAFENTRY l;
int r;
u_int32_t msize, dsize;
r = le_provdel(0x0123456789abcdef0LL, 301, zeros, 1025, zeros, &msize, &dsize, &l);
assert(r==0);
assert(sizeof(expect_3long)==msize);
assert(msize==dsize);
assert(memcmp(l, expect_3long, msize)==0);
toku_free(l);
}
int main (int argc __attribute__((__unused__)), char *argv[] __attribute__((__unused__))) {
test_leafentry_1();
test_leafentry_2();
test_leafentry_3();
test_leafentry_4();
test_leafentry_3long();
return 0;
}
newbrt/wbuf.h
View file @ e9323991
......@@ -55,10 +55,14 @@ static void wbuf_int (struct wbuf *w, int32_t i) {
wbuf_char(w, i>>0);
#else
assert(w->ndone + 4 <= w->size);
#if 0
w->buf[w->ndone+0] = i>>24;
w->buf[w->ndone+1] = i>>16;
w->buf[w->ndone+2] = i>>8;
w->buf[w->ndone+3] = i>>0;
#else
*(u_int32_t*)(&w->buf[w->ndone]) = htonl(i);
#endif
#ifdef CRC_INCR
w->crc32 = toku_crc32(w->crc32, &w->buf[w->ndone], 4);
#endif
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7