Skip to content

M
MariaDB
Commit e9323991 authored by Bradley C. Kuszmaul's avatar Bradley C. Kuszmaul
Browse files
Options

{{{ 

svn delete tokudb.872
cd tokudb
svn merge -r4187:4233 https://svn.tokutek.com/tokudb/tokudb.872
}}}

Fixes #849, #872.
Addresses #820.


git-svn-id: file:///svn/tokudb@4234 c7de825b-a66e-492c-adef-691d508d4ae1
parent ed37a2cc
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 288 additions and 375 deletions
newbrt/brt-internal.h
View file @ e9323991
...@@ -220,7 +220,6 @@ int toku_cmd_leafval_bessel (OMTVALUE leafentry, void *extra); ...@@ -220,7 +220,6 @@ int toku_cmd_leafval_bessel (OMTVALUE leafentry, void *extra);
int toku_brt_root_put_cmd(BRT brt, BRT_CMD cmd, TOKULOGGER logger); int toku_brt_root_put_cmd(BRT brt, BRT_CMD cmd, TOKULOGGER logger);
int toku_cachefile_root_put_cmd (CACHEFILE cf, BRT_CMD cmd, TOKULOGGER logger); int toku_cachefile_root_put_cmd (CACHEFILE cf, BRT_CMD cmd, TOKULOGGER logger);
int toku_omt_compress_kvspace (OMT omt, struct mempool *memp);
void *mempool_malloc_from_omt(OMT omt, struct mempool *mp, size_t size); void *mempool_malloc_from_omt(OMT omt, struct mempool *mp, size_t size);
void toku_verify_all_in_mempool(BRTNODE node); void toku_verify_all_in_mempool(BRTNODE node);
......
newbrt/brt-serialize.c
View file @ e9323991
...@@ -379,37 +379,35 @@ int toku_deserialize_brtnode_from (int fd, DISKOFF off, BRTNODE *brtnode) { ...@@ -379,37 +379,35 @@ int toku_deserialize_brtnode_from (int fd, DISKOFF off, BRTNODE *brtnode) {
} else { } else {
int n_in_buf = rbuf_int(&rc); int n_in_buf = rbuf_int(&rc);
result->u.l.n_bytes_in_buffer = 0; result->u.l.n_bytes_in_buffer = 0;
r=toku_omt_create(&result->u.l.buffer);
if (r!=0) {
if (0) { died_21: toku_omt_destroy(&result->u.l.buffer); }
goto died1;
}
//printf("%s:%d r PMA= %p\n", __FILE__, __LINE__, result->u.l.buffer); //printf("%s:%d r PMA= %p\n", __FILE__, __LINE__, result->u.l.buffer);
{ toku_mempool_init(&result->u.l.buffer_mempool, rc.buf, datasize);
int mpsize = result->nodesize + result->nodesize/4;
void *mp = toku_malloc(mpsize);
if (mp==0) return ENOMEM; // TODO cleanup
toku_mempool_init(&result->u.l.buffer_mempool, mp, mpsize);
}
u_int32_t actual_sum = 0; u_int32_t actual_sum = 0;
//printf("%s:%d node %lld, reading %d items\n", __FILE__, __LINE__, off, n_in_buf); u_int32_t start_of_data = rc.ndone;
OMTVALUE *MALLOC_N(n_in_buf, array);
for (i=0; i<n_in_buf; i++) { for (i=0; i<n_in_buf; i++) {
LEAFENTRY tmp_le; LEAFENTRY le = (LEAFENTRY)(&rc.buf[rc.ndone]);
//printf("%s:%d reading %dth item\n", __FILE__, __LINE__, i); u_int32_t disksize = leafentry_disksize(le);
u_int32_t memsize, disksize; rc.ndone += disksize;
rbuf_LEAFENTRY(&rc, &memsize, &disksize, &tmp_le); assert(rc.ndone<=rc.size);
LEAFENTRY le = mempool_malloc_from_omt(result->u.l.buffer, &result->u.l.buffer_mempool, memsize);
assert(le); array[i]=(OMTVALUE)le;
memcpy(le, tmp_le, memsize); actual_sum += toku_crc32(toku_null_crc, le, disksize);
toku_free(tmp_le); }
assert(disksize==leafentry_disksize(le)); u_int32_t end_of_data = rc.ndone;
result->u.l.n_bytes_in_buffer += disksize + OMT_ITEM_OVERHEAD; result->u.l.n_bytes_in_buffer += end_of_data-start_of_data + n_in_buf*OMT_ITEM_OVERHEAD;
toku_omt_insert_at(result->u.l.buffer, le, i); actual_sum *= result->rand4fingerprint;
actual_sum += result->rand4fingerprint*toku_le_crc(le); r = toku_omt_create_from_sorted_array(&result->u.l.buffer, array, n_in_buf);
//printf("%s:%d rand4=%08x fp=%08x \n", __FILE__, __LINE__, result->rand4fingerprint, actual_sum); toku_free(array);
if (r!=0) {
if (0) { died_21: toku_omt_destroy(&result->u.l.buffer); }
goto died1;
} }
result->u.l.buffer_mempool.frag_size = start_of_data;
result->u.l.buffer_mempool.free_offset = end_of_data;
if (r!=0) goto died_21; if (r!=0) goto died_21;
if (actual_sum!=result->local_fingerprint) { if (actual_sum!=result->local_fingerprint) {
//fprintf(stderr, "%s:%d Corrupted checksum stored=%08x rand=%08x actual=%08x height=%d n_keys=%d\n", __FILE__, __LINE__, result->rand4fingerprint, result->local_fingerprint, actual_sum, result->height, n_in_buf); //fprintf(stderr, "%s:%d Corrupted checksum stored=%08x rand=%08x actual=%08x height=%d n_keys=%d\n", __FILE__, __LINE__, result->rand4fingerprint, result->local_fingerprint, actual_sum, result->height, n_in_buf);
...@@ -436,7 +434,10 @@ int toku_deserialize_brtnode_from (int fd, DISKOFF off, BRTNODE *brtnode) { ...@@ -436,7 +434,10 @@ int toku_deserialize_brtnode_from (int fd, DISKOFF off, BRTNODE *brtnode) {
} }
} }
//printf("%s:%d Ok got %lld n_children=%d\n", __FILE__, __LINE__, result->thisnodename, result->n_children); //printf("%s:%d Ok got %lld n_children=%d\n", __FILE__, __LINE__, result->thisnodename, result->n_children);
toku_free(rc.buf); if (result->height>0) {
// For height==0 we used the buf inside the OMT
toku_free(rc.buf);
}
*brtnode = result; *brtnode = result;
//toku_verify_counts(result); //toku_verify_counts(result);
return 0; return 0;
......
newbrt/brt.c
View file @ e9323991
...@@ -3334,9 +3334,12 @@ static int move_it (OMTVALUE lev, u_int32_t idx, void *v) { ...@@ -3334,9 +3334,12 @@ static int move_it (OMTVALUE lev, u_int32_t idx, void *v) {
return 0; return 0;
} }
int toku_omt_compress_kvspace (OMT omt, struct mempool *memp) { // Compress things, and grow the mempool if needed.
if (toku_mempool_get_frag_size(memp) == 0) static int omt_compress_kvspace (OMT omt, struct mempool *memp, size_t added_size) {
return -1; u_int32_t total_size_needed = memp->free_offset-memp->frag_size + added_size;
if (total_size_needed+total_size_needed/4 >= memp->size) {
memp->size = total_size_needed+total_size_needed/4;
}
void *newmem = toku_malloc(memp->size); void *newmem = toku_malloc(memp->size);
if (newmem == 0) if (newmem == 0)
return -2; return -2;
...@@ -3353,7 +3356,7 @@ int toku_omt_compress_kvspace (OMT omt, struct mempool *memp) { ...@@ -3353,7 +3356,7 @@ int toku_omt_compress_kvspace (OMT omt, struct mempool *memp) {
void *mempool_malloc_from_omt(OMT omt, struct mempool *mp, size_t size) { void *mempool_malloc_from_omt(OMT omt, struct mempool *mp, size_t size) {
void *v = toku_mempool_malloc(mp, size, 1); void *v = toku_mempool_malloc(mp, size, 1);
if (v==0) { if (v==0) {
if (0 == toku_omt_compress_kvspace(omt, mp)) { if (0 == omt_compress_kvspace(omt, mp, size)) {
v = toku_mempool_malloc(mp, size, 1); v = toku_mempool_malloc(mp, size, 1);
assert(v); assert(v);
} }
......
newbrt/leafentry.c
View file @ e9323991
This diff is collapsed.
newbrt/leafentry.h
View file @ e9323991
...@@ -29,6 +29,7 @@ ...@@ -29,6 +29,7 @@
#include "brttypes.h" #include "brttypes.h"
#include "rbuf.h" #include "rbuf.h"
#include <arpa/inet.h>
u_int32_t toku_le_crc(LEAFENTRY v); u_int32_t toku_le_crc(LEAFENTRY v);
...@@ -45,61 +46,68 @@ enum le_state { LE_COMMITTED=1, // A committed pair. ...@@ -45,61 +46,68 @@ enum le_state { LE_COMMITTED=1, // A committed pair.
LE_PROVDEL, // A committed pair that has been provisionally deleted LE_PROVDEL, // A committed pair that has been provisionally deleted
LE_PROVPAIR }; // No committed value, but a provisional pair. LE_PROVPAIR }; // No committed value, but a provisional pair.
struct contents_committed;
struct contents_both;
struct contents_provdelorpair;
u_int32_t leafentry_memsize (LEAFENTRY); u_int32_t leafentry_memsize (LEAFENTRY);
enum le_state get_le_state(LEAFENTRY); static inline enum le_state get_le_state(LEAFENTRY le) {
void *get_le_contents(LEAFENTRY); return *(unsigned char *)le;
enum typ_tag get_le_tag(LEAFENTRY); }
u_int32_t committed_keylen (void*cev); static inline void putint (unsigned char *p, u_int32_t i) {
void* committed_key (void*cev); #if 1
u_int32_t committed_vallen (struct contents_committed *ce); *(u_int32_t*)p = htonl(i);
void* committed_val (struct contents_committed *ce); #else
TXNID both_xid (struct contents_both *ce); p[0]=(i>>24)&0xff;
u_int32_t both_keylen (struct contents_both *ce); p[1]=(i>>16)&0xff;
u_int32_t both_committed_vallen (struct contents_both *ce); p[2]=(i>> 8)&0xff;
u_int32_t both_prov_vallen (struct contents_both *ce); p[3]=(i>> 0)&0xff;
void* both_key (struct contents_both *ce); #endif
void* both_committed_val (struct contents_both *ce); }
void* both_prov_val (struct contents_both*ce); static inline void putint64 (unsigned char *p, u_int64_t i) {
TXNID provdelorpair_xid (struct contents_provdelorpair *ce); putint(p, (u_int32_t)(i>>32));
u_int32_t provdelorpair_keylen (struct contents_provdelorpair *ce); putint(p+4, (u_int32_t)(i&0xffffffff));
u_int32_t provdelorpair_vallen (struct contents_provdelorpair *ce); }
void* provdelorpair_key (struct contents_provdelorpair *ce); static inline u_int32_t getint (unsigned char *p) {
void* provdelorpair_val (struct contents_provdelorpair *ce); #if 1
return ntohl(*(u_int32_t*)p);
#define LESWITCHCALL(le,funname, ...) ({ \ #else
assert(get_le_tag(le)==TYP_LEAFENTRY); \ return (p[0]<<24)+(p[1]<<16)+(p[2]<<8)+(p[3]);
switch(get_le_state(le)) { \ #endif
case LE_COMMITTED: return funname ## _le_committed( committed_keylen((struct contents_committed*)(get_le_contents(le))), \ }
committed_key((struct contents_committed*)(get_le_contents(le))), \ static inline u_int64_t getint64 (unsigned char *p) {
committed_vallen((struct contents_committed*)(get_le_contents(le))), \ return (((u_int64_t)getint(p))<<32) + getint(p+4);
committed_val((struct contents_committed*)(get_le_contents(le))), \ }
## __VA_ARGS__); \
case LE_BOTH: return funname ## _le_both( both_xid((struct contents_both*)(get_le_contents(le))), \ #define LESWITCHCALL(le,funname, ...) ({ \
both_keylen((struct contents_both*)(get_le_contents(le))), \ switch(get_le_state(le)) { \
both_key((struct contents_both*)(get_le_contents(le))), \ case LE_COMMITTED: { \
both_committed_vallen((struct contents_both*)(get_le_contents(le))), \ unsigned char* __klenaddr = 1+(unsigned char*)le; u_int32_t __klen = getint(__klenaddr); \
both_committed_val((struct contents_both*)(get_le_contents(le))), \ unsigned char* __kvaladdr = 4 + __klenaddr; \
both_prov_vallen((struct contents_both*)(get_le_contents(le))), \ unsigned char* __clenaddr = __klen + __kvaladdr; u_int32_t __clen = getint(__clenaddr); \
both_prov_val((struct contents_both*)(get_le_contents(le))), \ unsigned char* __cvaladdr = 4 + __clenaddr; \
## __VA_ARGS__); \ return funname ## _le_committed(__klen, __kvaladdr, __clen, __cvaladdr, ## __VA_ARGS__); } \
case LE_PROVDEL: return funname ## _le_provdel ( provdelorpair_xid((struct contents_provdelorpair*)(get_le_contents(le))), \ case LE_BOTH: { \
provdelorpair_keylen((struct contents_provdelorpair*)(get_le_contents(le))), \ unsigned char* __xidaddr = 1+(unsigned char*)le; u_int64_t __xid = getint64(__xidaddr); \
provdelorpair_key((struct contents_provdelorpair*)(get_le_contents(le))), \ unsigned char* __klenaddr = 8 + __xidaddr; u_int32_t __klen = getint(__klenaddr); \
provdelorpair_vallen((struct contents_provdelorpair*)(get_le_contents(le))), \ unsigned char* __kvaladdr = 4 + __klenaddr; \
provdelorpair_val((struct contents_provdelorpair*)(get_le_contents(le))), \ unsigned char* __clenaddr = __klen + __kvaladdr; u_int32_t __clen = getint(__clenaddr); \
## __VA_ARGS__); \ unsigned char* __cvaladdr = 4 + __clenaddr; \
case LE_PROVPAIR: return funname ## _le_provpair(provdelorpair_xid((struct contents_provdelorpair*)(get_le_contents(le))), \ unsigned char* __plenaddr = __clen + __cvaladdr; u_int32_t __plen = getint(__plenaddr); \
provdelorpair_keylen((struct contents_provdelorpair*)(get_le_contents(le))), \ unsigned char* __pvaladdr = 4 + __plenaddr; \
provdelorpair_key((struct contents_provdelorpair*)(get_le_contents(le))), \ return funname ## _le_both(__xid, __klen, __kvaladdr, __clen, __cvaladdr, __plen, __pvaladdr, ## __VA_ARGS__); } \
provdelorpair_vallen((struct contents_provdelorpair*)(get_le_contents(le))), \ case LE_PROVDEL: { \
provdelorpair_val((struct contents_provdelorpair*)(get_le_contents(le))), \ unsigned char* __xidaddr = 1+(unsigned char*)le; u_int64_t __xid = getint64(__xidaddr); \
## __VA_ARGS__); \ unsigned char* __klenaddr = 8 + __xidaddr; u_int32_t __klen = getint(__klenaddr); \
unsigned char* __kvaladdr = 4 + __klenaddr; \
unsigned char* __dlenaddr = __klen + __kvaladdr; u_int32_t __dlen = getint(__dlenaddr); \
unsigned char* __dvaladdr = 4 + __dlenaddr; \
return funname ## _le_provdel(__xid, __klen, __kvaladdr, __dlen, __dvaladdr, ## __VA_ARGS__); } \
case LE_PROVPAIR: { \
unsigned char* __xidaddr = 1+(unsigned char*)le; u_int64_t __xid = getint64(__xidaddr); \
unsigned char* __klenaddr = 8 + __xidaddr; u_int32_t __klen = getint(__klenaddr); \
unsigned char* __kvaladdr = 4 + __klenaddr; \
unsigned char* __plenaddr = __klen + __kvaladdr; u_int32_t __plen = getint(__plenaddr); \
unsigned char* __pvaladdr = 4 + __plenaddr; \
return funname ## _le_provpair(__xid, __klen, __kvaladdr, __plen, __pvaladdr, ## __VA_ARGS__); } \
} abort(); }) } abort(); })
......
newbrt/rbuf.h
View file @ e9323991
...@@ -5,6 +5,7 @@ ...@@ -5,6 +5,7 @@
#include "toku_assert.h" #include "toku_assert.h"
#include "memory.h" #include "memory.h"
#include <arpa/inet.h>
struct rbuf { struct rbuf {
unsigned char *buf; unsigned char *buf;
...@@ -12,12 +13,18 @@ struct rbuf { ...@@ -12,12 +13,18 @@ struct rbuf {
unsigned int ndone; unsigned int ndone;
}; };
static unsigned int rbuf_char (struct rbuf *r) { static inline unsigned int rbuf_char (struct rbuf *r) {
assert(r->ndone<r->size); assert(r->ndone<r->size);
return r->buf[r->ndone++]; return r->buf[r->ndone++];
} }
static unsigned int rbuf_int (struct rbuf *r) { static unsigned int rbuf_int (struct rbuf *r) {
#if 1
assert(r->ndone+4 <= r->size);
u_int32_t result = ntohl(*(u_int32_t*)(r->buf+r->ndone)); // This only works on machines where unaligned loads are OK.
r->ndone+=4;
return result;
#else
unsigned char c0 = rbuf_char(r); unsigned char c0 = rbuf_char(r);
unsigned char c1 = rbuf_char(r); unsigned char c1 = rbuf_char(r);
unsigned char c2 = rbuf_char(r); unsigned char c2 = rbuf_char(r);
...@@ -26,6 +33,7 @@ static unsigned int rbuf_int (struct rbuf *r) { ...@@ -26,6 +33,7 @@ static unsigned int rbuf_int (struct rbuf *r) {
(c1<<16)| (c1<<16)|
(c2<<8)| (c2<<8)|
(c3<<0)); (c3<<0));
#endif
} }
static inline void rbuf_literal_bytes (struct rbuf *r, bytevec *bytes, unsigned int n_bytes) { static inline void rbuf_literal_bytes (struct rbuf *r, bytevec *bytes, unsigned int n_bytes) {
......
newbrt/tests/Makefile
View file @ e9323991
...@@ -74,6 +74,7 @@ REGRESSION_TESTS = \ ...@@ -74,6 +74,7 @@ REGRESSION_TESTS = \
test-brt-overflow \ test-brt-overflow \
test-del-inorder \ test-del-inorder \
test-inc-split \ test-inc-split \
test-leafentry \
test-primes \ test-primes \
test_oexcl \ test_oexcl \
test_toku_malloc_plain_free \ test_toku_malloc_plain_free \
......
newbrt/tests/test-leafentry.c 0 → 100644
View file @ e9323991
#include <string.h>
#include "leafentry.h"
static void test_leafentry_1 (void) {
LEAFENTRY l;
int r;
u_int32_t msize, dsize;
r = le_committed(4, "abc", 3, "xy", &msize, &dsize, &l);
assert(r==0);
char expect[] = {LE_COMMITTED,
0, 0, 0, 4,
'a', 'b', 'c', 0,
0, 0, 0, 3,
'x', 'y', 0};
assert(sizeof(expect)==msize);
assert(msize==dsize);
assert(memcmp(l, expect, msize)==0);
toku_free(l);
}
static void test_leafentry_2 (void) {
LEAFENTRY l;
int r;
u_int32_t msize, dsize;
r = le_both(0x0123456789abcdef0LL, 3, "ab", 4, "xyz", 5, "lmno", &msize, &dsize, &l);
assert(r==0);
char expect[] = {LE_BOTH,
0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc, 0xde, 0xf0,
0, 0, 0, 3, 'a', 'b', 0,
0, 0, 0, 4, 'x', 'y', 'z', 0,
0, 0, 0, 5, 'l', 'm', 'n', 'o', 0};
assert(sizeof(expect)==msize);
assert(msize==dsize);
assert(memcmp(l, expect, msize)==0);
toku_free(l);
}
static void test_leafentry_3 (void) {
LEAFENTRY l;
int r;
u_int32_t msize, dsize;
r = le_provdel(0x0123456789abcdef0LL, 3, "ab", 5, "lmno", &msize, &dsize, &l);
assert(r==0);
char expect[] = {LE_PROVDEL,
0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc, 0xde, 0xf0,
0, 0, 0, 3, 'a', 'b', 0,
0, 0, 0, 5, 'l', 'm', 'n', 'o', 0};
assert(sizeof(expect)==msize);
assert(msize==dsize);
assert(memcmp(l, expect, msize)==0);
toku_free(l);
}
static void test_leafentry_4 (void) {
LEAFENTRY l;
int r;
u_int32_t msize, dsize;
r = le_provpair(0x0123456789abcdef0LL, 3, "ab", 5, "lmno", &msize, &dsize, &l);
assert(r==0);
char expect[] = {LE_PROVPAIR,
0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc, 0xde, 0xf0,
0, 0, 0, 3, 'a', 'b', 0,
0, 0, 0, 5, 'l', 'm', 'n', 'o', 0};
assert(sizeof(expect)==msize);
assert(msize==dsize);
assert(memcmp(l, expect, msize)==0);
toku_free(l);
}
char zeros[1026];
#define n5zeros 0,0,0,0,0
#define n10zeros n5zeros,n5zeros
#define n25zeros n5zeros,n10zeros,n10zeros
#define n75zeros n25zeros,n25zeros,n25zeros
#define n125zeros n75zeros,n25zeros,n25zeros
#define n150zeros n75zeros,n75zeros
#define n300zeros n150zeros,n150zeros
#define n301zeros 0,n300zeros
#define n1025zeros n300zeros,n300zeros,n300zeros,n125zeros
char expect_3long[] = {LE_PROVDEL,
0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc, 0xde, 0xf0,
0, 0, 1, 45, n301zeros,
0, 0, 4, 1, n1025zeros};
static void test_leafentry_3long (void) {
LEAFENTRY l;
int r;
u_int32_t msize, dsize;
r = le_provdel(0x0123456789abcdef0LL, 301, zeros, 1025, zeros, &msize, &dsize, &l);
assert(r==0);
assert(sizeof(expect_3long)==msize);
assert(msize==dsize);
assert(memcmp(l, expect_3long, msize)==0);
toku_free(l);
}
int main (int argc __attribute__((__unused__)), char *argv[] __attribute__((__unused__))) {
test_leafentry_1();
test_leafentry_2();
test_leafentry_3();
test_leafentry_4();
test_leafentry_3long();
return 0;
}
newbrt/wbuf.h
View file @ e9323991
...@@ -55,10 +55,14 @@ static void wbuf_int (struct wbuf *w, int32_t i) { ...@@ -55,10 +55,14 @@ static void wbuf_int (struct wbuf *w, int32_t i) {
wbuf_char(w, i>>0); wbuf_char(w, i>>0);
#else #else
assert(w->ndone + 4 <= w->size); assert(w->ndone + 4 <= w->size);
#if 0
w->buf[w->ndone+0] = i>>24; w->buf[w->ndone+0] = i>>24;
w->buf[w->ndone+1] = i>>16; w->buf[w->ndone+1] = i>>16;
w->buf[w->ndone+2] = i>>8; w->buf[w->ndone+2] = i>>8;
w->buf[w->ndone+3] = i>>0; w->buf[w->ndone+3] = i>>0;
#else
*(u_int32_t*)(&w->buf[w->ndone]) = htonl(i);
#endif
#ifdef CRC_INCR #ifdef CRC_INCR
w->crc32 = toku_crc32(w->crc32, &w->buf[w->ndone], 4); w->crc32 = toku_crc32(w->crc32, &w->buf[w->ndone], 4);
#endif #endif
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7