Commit 60b6abe1 authored by Bradley C. Kuszmaul's avatar Bradley C. Kuszmaul

{{{test_log8}}} recovers now. Fixes #544.

git-svn-id: file:///svn/tokudb@2880 c7de825b-a66e-492c-adef-691d508d4ae1
parent c24c5f99
...@@ -359,8 +359,9 @@ int toku_deserialize_brtnode_from (int fd, DISKOFF off, BRTNODE *brtnode, int fl ...@@ -359,8 +359,9 @@ int toku_deserialize_brtnode_from (int fd, DISKOFF off, BRTNODE *brtnode, int fl
} }
} else { } else {
int n_in_buf = rbuf_int(&rc); int n_in_buf = rbuf_int(&rc);
int index_limit = rbuf_int(&rc);
result->u.l.n_bytes_in_buffer = 0; result->u.l.n_bytes_in_buffer = 0;
r=toku_pma_create(&result->u.l.buffer, bt_compare, db, filenum, nodesize); r=toku_pma_create(&result->u.l.buffer, bt_compare, db, filenum, nodesize, index_limit);
if (r!=0) { if (r!=0) {
if (0) { died_21: toku_pma_free(&result->u.l.buffer); } if (0) { died_21: toku_pma_free(&result->u.l.buffer); }
goto died1; goto died1;
...@@ -369,38 +370,20 @@ int toku_deserialize_brtnode_from (int fd, DISKOFF off, BRTNODE *brtnode, int fl ...@@ -369,38 +370,20 @@ int toku_deserialize_brtnode_from (int fd, DISKOFF off, BRTNODE *brtnode, int fl
toku_pma_set_dup_compare(result->u.l.buffer, dup_compare); toku_pma_set_dup_compare(result->u.l.buffer, dup_compare);
//printf("%s:%d r PMA= %p\n", __FILE__, __LINE__, result->u.l.buffer); //printf("%s:%d r PMA= %p\n", __FILE__, __LINE__, result->u.l.buffer);
toku_verify_counts(result); toku_verify_counts(result);
#define BRT_USE_PMA_BULK_INSERT 1
#if BRT_USE_PMA_BULK_INSERT u_int32_t actual_sum = 0;
int index_limit __attribute__((__unused__))= rbuf_int(&rc);
if (n_in_buf > 0) {
#define BRT_BULK_INSERT_MALLOC 1
#if BRT_BULK_INSERT_MALLOC
/* some applications run with small stacks so we malloc the
keys and vals structs */
size_t n = 2 * n_in_buf * sizeof (DBT);
DBT *keys = toku_malloc(n);
if (keys == 0) goto died_21;
DBT *vals = &keys[n_in_buf];
#else
DBT keys[n_in_buf], vals[n_in_buf];
#endif
for (i=0; i<n_in_buf; i++) { for (i=0; i<n_in_buf; i++) {
bytevec key; ITEMLEN keylen; bytevec key; ITEMLEN keylen;
bytevec val; ITEMLEN vallen; bytevec val; ITEMLEN vallen;
// The counts are wrong here int idx = rbuf_int(&rc);
int idx __attribute__((__unused__)) = rbuf_int(&rc); DBT keydbt, datadbt;
rbuf_bytes(&rc, &key, &keylen); /* Returns a pointer into the rbuf. */ rbuf_bytes(&rc, &key, &keylen); /* Returns a pointer into the rbuf. */
toku_fill_dbt(&keys[i], key, keylen);
rbuf_bytes(&rc, &val, &vallen); rbuf_bytes(&rc, &val, &vallen);
toku_fill_dbt(&vals[i], val, vallen);
result->u.l.n_bytes_in_buffer += keylen + vallen + KEY_VALUE_OVERHEAD + PMA_ITEM_OVERHEAD; result->u.l.n_bytes_in_buffer += keylen + vallen + KEY_VALUE_OVERHEAD + PMA_ITEM_OVERHEAD;
r = toku_pma_set_at_index(result->u.l.buffer, idx, toku_fill_dbt(&keydbt, key, keylen), toku_fill_dbt(&datadbt, val, vallen));
actual_sum += result->rand4fingerprint*toku_calccrc32_kvpair(key, keylen, val, vallen);
} }
u_int32_t actual_sum = 0;
r = toku_pma_bulk_insert((TOKULOGGER)0, (FILENUM){0}, (DISKOFF)0, result->u.l.buffer, keys, vals, n_in_buf, result->rand4fingerprint, &actual_sum, 0);
#if BRT_BULK_INSERT_MALLOC
toku_free_n(keys, n);
#endif
if (r!=0) goto died_21; if (r!=0) goto died_21;
if (actual_sum!=result->local_fingerprint) { if (actual_sum!=result->local_fingerprint) {
//fprintf(stderr, "%s:%d Corrupted checksum stored=%08x rand=%08x actual=%08x height=%d n_keys=%d\n", __FILE__, __LINE__, result->rand4fingerprint, result->local_fingerprint, actual_sum, result->height, n_in_buf); //fprintf(stderr, "%s:%d Corrupted checksum stored=%08x rand=%08x actual=%08x height=%d n_keys=%d\n", __FILE__, __LINE__, result->rand4fingerprint, result->local_fingerprint, actual_sum, result->height, n_in_buf);
...@@ -409,22 +392,7 @@ int toku_deserialize_brtnode_from (int fd, DISKOFF off, BRTNODE *brtnode, int fl ...@@ -409,22 +392,7 @@ int toku_deserialize_brtnode_from (int fd, DISKOFF off, BRTNODE *brtnode, int fl
} else { } else {
//fprintf(stderr, "%s:%d Good checksum=%08x height=%d\n", __FILE__, __LINE__, actual_sum, result->height); //fprintf(stderr, "%s:%d Good checksum=%08x height=%d\n", __FILE__, __LINE__, actual_sum, result->height);
} }
}
#else
for (i=0; i<n_in_buf; i++) {
bytevec key; ITEMLEN keylen;
bytevec val; ITEMLEN vallen;
toku_verify_counts(result);
rbuf_bytes(&rc, &key, &keylen); /* Returns a pointer into the rbuf. */
rbuf_bytes(&rc, &val, &vallen);
{
DBT k,v;
r = toku_pma_insert(result->u.l.buffer, toku_fill_dbt(&k, key, keylen), toku_fill_dbt(&v, val, vallen), 0);
if (r!=0) goto died_21;
}
result->u.l.n_bytes_in_buffer += keylen + vallen + KEY_VALUE_OVERHEAD + PMA_ITEM_OVERHEAD;
}
#endif
toku_verify_counts(result); toku_verify_counts(result);
} }
{ {
......
...@@ -267,7 +267,7 @@ static void initialize_brtnode (BRT t, BRTNODE n, DISKOFF nodename, int height) ...@@ -267,7 +267,7 @@ static void initialize_brtnode (BRT t, BRTNODE n, DISKOFF nodename, int height)
n->u.n.childinfos=0; n->u.n.childinfos=0;
n->u.n.childkeys=0; n->u.n.childkeys=0;
} else { } else {
int r = toku_pma_create(&n->u.l.buffer, t->compare_fun, t->db, toku_cachefile_filenum(t->cf), n->nodesize); int r = toku_pma_create(&n->u.l.buffer, t->compare_fun, t->db, toku_cachefile_filenum(t->cf), n->nodesize, 0);
assert(r==0); assert(r==0);
toku_pma_set_dup_mode(n->u.l.buffer, t->flags & (TOKU_DB_DUP+TOKU_DB_DUPSORT)); toku_pma_set_dup_mode(n->u.l.buffer, t->flags & (TOKU_DB_DUP+TOKU_DB_DUPSORT));
toku_pma_set_dup_compare(n->u.l.buffer, t->dup_compare); toku_pma_set_dup_compare(n->u.l.buffer, t->dup_compare);
......
...@@ -48,6 +48,7 @@ void dump_node (int f, DISKOFF off, struct brt_header *h) { ...@@ -48,6 +48,7 @@ void dump_node (int f, DISKOFF off, struct brt_header *h) {
toku_default_compare_fun, toku_default_compare_fun, toku_default_compare_fun, toku_default_compare_fun,
(DB*)0, (FILENUM){0}); (DB*)0, (FILENUM){0});
assert(r==0); assert(r==0);
assert(n!=0);
printf("brtnode\n"); printf("brtnode\n");
printf(" nodesize =%u\n", n->nodesize); printf(" nodesize =%u\n", n->nodesize);
printf(" sizeonddisk =%d\n", toku_serialize_brtnode_size(n)); printf(" sizeonddisk =%d\n", toku_serialize_brtnode_size(n));
...@@ -107,6 +108,7 @@ void dump_node (int f, DISKOFF off, struct brt_header *h) { ...@@ -107,6 +108,7 @@ void dump_node (int f, DISKOFF off, struct brt_header *h) {
printf(" items_in_buffer =%d\n", toku_pma_n_entries(n->u.l.buffer)); printf(" items_in_buffer =%d\n", toku_pma_n_entries(n->u.l.buffer));
PMA_ITERATE_IDX(n->u.l.buffer, idx, key, keylen, data, datalen, PMA_ITERATE_IDX(n->u.l.buffer, idx, key, keylen, data, datalen,
({ ({
printf("%d: ", idx);
print_item(key, keylen); print_item(key, keylen);
printf(" "); printf(" ");
print_item(data, datalen); print_item(data, datalen);
......
...@@ -45,7 +45,7 @@ static void test_make_space_at (void) { ...@@ -45,7 +45,7 @@ static void test_make_space_at (void) {
key = "B"; key = "B";
key_B = kv_pair_malloc(key, strlen(key)+1, 0, 0); key_B = kv_pair_malloc(key, strlen(key)+1, 0, 0);
r=toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0); r=toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0, 0);
assert(r==0); assert(r==0);
assert(toku_pma_n_entries(pma)==0); assert(toku_pma_n_entries(pma)==0);
r=toku_pmainternal_make_space_at(null_logger, null_filenum, null_diskoff, pma, 2, &newi, (LSN*)0); r=toku_pmainternal_make_space_at(null_logger, null_filenum, null_diskoff, pma, 2, &newi, (LSN*)0);
...@@ -320,7 +320,7 @@ static int do_delete_both (PMA pma, const void *key, int keylen, const void *dat ...@@ -320,7 +320,7 @@ static int do_delete_both (PMA pma, const void *key, int keylen, const void *dat
static void test_pma_random_pick (void) { static void test_pma_random_pick (void) {
PMA pma; PMA pma;
int r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0); int r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0, 0);
bytevec key,val; bytevec key,val;
ITEMLEN keylen,vallen; ITEMLEN keylen,vallen;
DBT k; DBT k;
...@@ -395,7 +395,7 @@ static void test_find_insert (void) { ...@@ -395,7 +395,7 @@ static void test_find_insert (void) {
u_int32_t expect_fingerprint = 0; u_int32_t expect_fingerprint = 0;
toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0); toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0, 0);
r=toku_pma_lookup(pma, toku_fill_dbt(&k, "aaa", 3), &v); r=toku_pma_lookup(pma, toku_fill_dbt(&k, "aaa", 3), &v);
assert(r==DB_NOTFOUND); assert(r==DB_NOTFOUND);
...@@ -427,7 +427,7 @@ static void test_find_insert (void) { ...@@ -427,7 +427,7 @@ static void test_find_insert (void) {
assert((unsigned long)pma->pairs[toku_pma_index_limit(pma)]==0xdeadbeefL); assert((unsigned long)pma->pairs[toku_pma_index_limit(pma)]==0xdeadbeefL);
r=toku_pma_free(&pma); assert(r==0); assert(pma==0); r=toku_pma_free(&pma); assert(r==0); assert(pma==0);
toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0); assert(pma!=0); toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0, 0); assert(pma!=0);
rand4fingerprint = random(); rand4fingerprint = random();
sum = expect_fingerprint = 0; sum = expect_fingerprint = 0;
...@@ -470,7 +470,7 @@ static void test_pma_iterate (void) { ...@@ -470,7 +470,7 @@ static void test_pma_iterate (void) {
u_int32_t sum = 0; u_int32_t sum = 0;
u_int32_t expect_fingerprint = 0; u_int32_t expect_fingerprint = 0;
toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0); toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0, 0);
do_insert(pma, "42", 3, "-19", 4, rand4fingerprint, &sum, &expect_fingerprint); do_insert(pma, "42", 3, "-19", 4, rand4fingerprint, &sum, &expect_fingerprint);
test_pma_iterate_internal(pma, 42, -19); test_pma_iterate_internal(pma, 42, -19);
...@@ -493,8 +493,8 @@ static void test_pma_iterate2 (void) { ...@@ -493,8 +493,8 @@ static void test_pma_iterate2 (void) {
u_int32_t sum1 = 0; u_int32_t sum1 = 0;
u_int32_t expect_fingerprint1 = 0; u_int32_t expect_fingerprint1 = 0;
r=toku_pma_create(&pma0, toku_default_compare_fun, null_db, null_filenum, 0); assert(r==0); r=toku_pma_create(&pma0, toku_default_compare_fun, null_db, null_filenum, 0, 0); assert(r==0);
r=toku_pma_create(&pma1, toku_default_compare_fun, null_db, null_filenum, 0); assert(r==0); r=toku_pma_create(&pma1, toku_default_compare_fun, null_db, null_filenum, 0, 0); assert(r==0);
do_insert(pma0, "a", 2, "aval", 5, rand4fingerprint0, &sum0, &expect_fingerprint0); do_insert(pma0, "a", 2, "aval", 5, rand4fingerprint0, &sum0, &expect_fingerprint0);
do_insert(pma0, "b", 2, "bval", 5, rand4fingerprint0, &sum0, &expect_fingerprint0); do_insert(pma0, "b", 2, "bval", 5, rand4fingerprint0, &sum0, &expect_fingerprint0);
do_insert(pma1, "x", 2, "xval", 5, rand4fingerprint1, &sum1, &expect_fingerprint1); do_insert(pma1, "x", 2, "xval", 5, rand4fingerprint1, &sum1, &expect_fingerprint1);
...@@ -511,7 +511,7 @@ static void test_pma_cursor_0 (void) { ...@@ -511,7 +511,7 @@ static void test_pma_cursor_0 (void) {
PMA pma; PMA pma;
PMA_CURSOR c=0; PMA_CURSOR c=0;
int r; int r;
r=toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0); assert(r==0); r=toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0, 0); assert(r==0);
r=toku_pma_cursor(pma, &c, &skey, &sval); assert(r==0); assert(c!=0); r=toku_pma_cursor(pma, &c, &skey, &sval); assert(r==0); assert(c!=0);
if (verbose) printf("%s:%d\n", __FILE__, __LINE__); if (verbose) printf("%s:%d\n", __FILE__, __LINE__);
#if OLDCURSORS #if OLDCURSORS
...@@ -564,7 +564,7 @@ static void test_pma_cursor_2 (void) { ...@@ -564,7 +564,7 @@ static void test_pma_cursor_2 (void) {
DBT key,val; DBT key,val;
toku_init_dbt(&key); key.flags=DB_DBT_REALLOC; toku_init_dbt(&key); key.flags=DB_DBT_REALLOC;
toku_init_dbt(&val); val.flags=DB_DBT_REALLOC; toku_init_dbt(&val); val.flags=DB_DBT_REALLOC;
r=toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0); assert(r==0); r=toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0, 0); assert(r==0);
r=toku_pma_cursor(pma, &c, &skey, &sval); assert(r==0); assert(c!=0); r=toku_pma_cursor(pma, &c, &skey, &sval); assert(r==0); assert(c!=0);
r=toku_pma_cursor_set_position_first(c); assert(r==DB_NOTFOUND); r=toku_pma_cursor_set_position_first(c); assert(r==DB_NOTFOUND);
r=toku_pma_cursor_set_position_last(c); assert(r==DB_NOTFOUND); r=toku_pma_cursor_set_position_last(c); assert(r==DB_NOTFOUND);
...@@ -582,7 +582,7 @@ static void test_pma_cursor_3 (void) { ...@@ -582,7 +582,7 @@ static void test_pma_cursor_3 (void) {
u_int32_t sum = 0; u_int32_t sum = 0;
u_int32_t expect_fingerprint = 0; u_int32_t expect_fingerprint = 0;
r=toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0); assert(r==0); r=toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0, 0); assert(r==0);
do_insert(pma, "x", 2, "xx", 3, rand4fingerprint, &sum, &expect_fingerprint); do_insert(pma, "x", 2, "xx", 3, rand4fingerprint, &sum, &expect_fingerprint);
do_insert(pma, "m", 2, "mm", 3, rand4fingerprint, &sum, &expect_fingerprint); do_insert(pma, "m", 2, "mm", 3, rand4fingerprint, &sum, &expect_fingerprint);
do_insert(pma, "aa", 3, "a", 2, rand4fingerprint, &sum, &expect_fingerprint); do_insert(pma, "aa", 3, "a", 2, rand4fingerprint, &sum, &expect_fingerprint);
...@@ -654,7 +654,7 @@ static void test_pma_cursor_4 (void) { ...@@ -654,7 +654,7 @@ static void test_pma_cursor_4 (void) {
if (verbose) printf("test_pma_cursor_4\n"); if (verbose) printf("test_pma_cursor_4\n");
r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0); r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0, 0);
assert(r == 0); assert(r == 0);
for (i=1; i<=4; i += 1) { for (i=1; i<=4; i += 1) {
...@@ -723,7 +723,7 @@ static void test_pma_cursor_delete(int n) { ...@@ -723,7 +723,7 @@ static void test_pma_cursor_delete(int n) {
u_int32_t sum = 0; u_int32_t sum = 0;
u_int32_t expect_fingerprint = 0; u_int32_t expect_fingerprint = 0;
r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0); r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0, 0);
assert(r == 0); assert(r == 0);
/* insert 1 -> 42 */ /* insert 1 -> 42 */
...@@ -834,7 +834,8 @@ static void test_pma_compare_fun (int wrong_endian_p) { ...@@ -834,7 +834,8 @@ static void test_pma_compare_fun (int wrong_endian_p) {
r = toku_pma_create(&pma, wrong_endian_p ? wrong_endian_compare_fun : toku_default_compare_fun, r = toku_pma_create(&pma, wrong_endian_p ? wrong_endian_compare_fun : toku_default_compare_fun,
null_db, null_filenum, null_db, null_filenum,
0); assert(r==0); 0, 0);
assert(r==0);
do_insert(pma, "10", 3, "10v", 4, rand4fingerprint, &sum, &expect_fingerprint); do_insert(pma, "10", 3, "10v", 4, rand4fingerprint, &sum, &expect_fingerprint);
do_insert(pma, "00", 3, "00v", 4, rand4fingerprint, &sum, &expect_fingerprint); do_insert(pma, "00", 3, "00v", 4, rand4fingerprint, &sum, &expect_fingerprint);
do_insert(pma, "01", 3, "01v", 4, rand4fingerprint, &sum, &expect_fingerprint); do_insert(pma, "01", 3, "01v", 4, rand4fingerprint, &sum, &expect_fingerprint);
...@@ -898,12 +899,12 @@ static void test_pma_dup_split_n(int n, int dup_mode) { ...@@ -898,12 +899,12 @@ static void test_pma_dup_split_n(int n, int dup_mode) {
} tuples[n]; } tuples[n];
PMA pmaa; PMA pmaa;
r = toku_pma_create(&pmaa, toku_default_compare_fun, null_db, null_filenum, n*(sizeof (struct tuple) + 2*sizeof (int))); assert(r == 0); r = toku_pma_create(&pmaa, toku_default_compare_fun, null_db, null_filenum, n*(sizeof (struct tuple) + 2*sizeof (int)), 0); assert(r == 0);
toku_pma_set_dup_mode(pmaa, dup_mode); toku_pma_set_dup_mode(pmaa, dup_mode);
toku_pma_set_dup_compare(pmaa, toku_default_compare_fun); toku_pma_set_dup_compare(pmaa, toku_default_compare_fun);
PMA pmac; PMA pmac;
r = toku_pma_create(&pmac, toku_default_compare_fun, null_db, null_filenum, n*(sizeof (struct tuple) + 2*sizeof (int))); assert(r == 0); r = toku_pma_create(&pmac, toku_default_compare_fun, null_db, null_filenum, n*(sizeof (struct tuple) + 2*sizeof (int)), 0); assert(r == 0);
toku_pma_set_dup_mode(pmac, dup_mode); toku_pma_set_dup_mode(pmac, dup_mode);
toku_pma_set_dup_compare(pmac, toku_default_compare_fun); toku_pma_set_dup_compare(pmac, toku_default_compare_fun);
...@@ -1035,9 +1036,9 @@ static void test_pma_split_varkey(void) { ...@@ -1035,9 +1036,9 @@ static void test_pma_split_varkey(void) {
if (verbose) printf("test_pma_split_varkey\n"); if (verbose) printf("test_pma_split_varkey\n");
r = toku_pma_create(&pmaa, toku_default_compare_fun, null_db, null_filenum, 0); r = toku_pma_create(&pmaa, toku_default_compare_fun, null_db, null_filenum, 0, 0);
assert(r == 0); assert(r == 0);
r = toku_pma_create(&pmac, toku_default_compare_fun, null_db, null_filenum, 0); r = toku_pma_create(&pmac, toku_default_compare_fun, null_db, null_filenum, 0, 0);
assert(r == 0); assert(r == 0);
/* insert some kv pairs */ /* insert some kv pairs */
...@@ -1165,7 +1166,7 @@ static void test_pma_bulk_insert_n(int n) { ...@@ -1165,7 +1166,7 @@ static void test_pma_bulk_insert_n(int n) {
if (verbose) printf("test_pma_bulk_insert_n: %d\n", n); if (verbose) printf("test_pma_bulk_insert_n: %d\n", n);
r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0); r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0, 0);
assert(r == 0); assert(r == 0);
/* init n kv pairs */ /* init n kv pairs */
...@@ -1248,7 +1249,7 @@ static void test_pma_insert_or_replace(void) { ...@@ -1248,7 +1249,7 @@ static void test_pma_insert_or_replace(void) {
u_int32_t rand4fingerprint = random(); u_int32_t rand4fingerprint = random();
u_int32_t sum = 0; u_int32_t sum = 0;
u_int32_t expect_fingerprint = 0; u_int32_t expect_fingerprint = 0;
r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0); r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0, 0);
assert(r==0); assert(r==0);
r = toku_pma_insert_or_replace(pma, toku_fill_dbt(&dbtk, "aaa", 4), toku_fill_dbt(&dbtv, "zzz", 4), &n_diff, NULL_ARGS, rand4fingerprint, &sum, (LSN*)0); r = toku_pma_insert_or_replace(pma, toku_fill_dbt(&dbtk, "aaa", 4), toku_fill_dbt(&dbtv, "zzz", 4), &n_diff, NULL_ARGS, rand4fingerprint, &sum, (LSN*)0);
assert(r==0); assert(n_diff==-1); assert(r==0); assert(n_diff==-1);
...@@ -1297,7 +1298,7 @@ static void test_pma_delete_shrink(int n) { ...@@ -1297,7 +1298,7 @@ static void test_pma_delete_shrink(int n) {
if (verbose) printf("test_pma_delete_shrink:%d\n", n); if (verbose) printf("test_pma_delete_shrink:%d\n", n);
r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, n*(8 + 11 + sizeof (int))); r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, n*(8 + 11 + sizeof (int)), 0);
assert(r == 0); assert(r == 0);
/* insert */ /* insert */
...@@ -1341,7 +1342,7 @@ static void test_pma_delete_random(int n) { ...@@ -1341,7 +1342,7 @@ static void test_pma_delete_random(int n) {
if (verbose) printf("test_pma_delete_random:%d\n", n); if (verbose) printf("test_pma_delete_random:%d\n", n);
r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, n * (8 + 11 + sizeof (int))); r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, n * (8 + 11 + sizeof (int)), 0);
assert(r == 0); assert(r == 0);
for (i=0; i<n; i++) { for (i=0; i<n; i++) {
...@@ -1415,7 +1416,7 @@ static void test_pma_delete_cursor(int n) { ...@@ -1415,7 +1416,7 @@ static void test_pma_delete_cursor(int n) {
u_int32_t sum = 0; u_int32_t sum = 0;
u_int32_t expect_fingerprint = 0; u_int32_t expect_fingerprint = 0;
r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0); r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0, 0);
assert(r == 0); assert(r == 0);
int i; int i;
...@@ -1478,7 +1479,7 @@ static void test_pma_delete_insert() { ...@@ -1478,7 +1479,7 @@ static void test_pma_delete_insert() {
u_int32_t sum = 0; u_int32_t sum = 0;
u_int32_t expect_fingerprint = 0; u_int32_t expect_fingerprint = 0;
r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0); r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0, 0);
assert(r == 0); assert(r == 0);
PMA_CURSOR pmacursor = 0; PMA_CURSOR pmacursor = 0;
...@@ -1531,7 +1532,7 @@ static void test_pma_double_delete() { ...@@ -1531,7 +1532,7 @@ static void test_pma_double_delete() {
u_int32_t sum = 0; u_int32_t sum = 0;
u_int32_t expect_fingerprint = 0; u_int32_t expect_fingerprint = 0;
r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0); r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0, 0);
assert(r == 0); assert(r == 0);
PMA_CURSOR pmacursor = 0; PMA_CURSOR pmacursor = 0;
...@@ -1578,7 +1579,7 @@ static void test_pma_cursor_first_delete_last() { ...@@ -1578,7 +1579,7 @@ static void test_pma_cursor_first_delete_last() {
u_int32_t sum = 0; u_int32_t sum = 0;
u_int32_t expect_fingerprint = 0; u_int32_t expect_fingerprint = 0;
r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0); r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0, 0);
assert(r == 0); assert(r == 0);
int k, v; int k, v;
...@@ -1626,7 +1627,7 @@ static void test_pma_cursor_last_delete_first() { ...@@ -1626,7 +1627,7 @@ static void test_pma_cursor_last_delete_first() {
u_int32_t sum = 0; u_int32_t sum = 0;
u_int32_t expect_fingerprint = 0; u_int32_t expect_fingerprint = 0;
r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0); r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0, 0);
assert(r == 0); assert(r == 0);
int k, v; int k, v;
...@@ -1675,7 +1676,7 @@ void test_pma_delete_both(int n) { ...@@ -1675,7 +1676,7 @@ void test_pma_delete_both(int n) {
u_int32_t sum = 0; u_int32_t sum = 0;
u_int32_t expect_fingerprint = 0; u_int32_t expect_fingerprint = 0;
r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, n * 4 * sizeof (int)); assert(r == 0); r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, n * 4 * sizeof (int), 0); assert(r == 0);
toku_pma_set_dup_mode(pma, TOKU_DB_DUP + TOKU_DB_DUPSORT); toku_pma_set_dup_mode(pma, TOKU_DB_DUP + TOKU_DB_DUPSORT);
toku_pma_set_dup_compare(pma, toku_default_compare_fun); toku_pma_set_dup_compare(pma, toku_default_compare_fun);
...@@ -1713,7 +1714,7 @@ void test_pma_delete_both_dup(int n) { ...@@ -1713,7 +1714,7 @@ void test_pma_delete_both_dup(int n) {
u_int32_t sum = 0; u_int32_t sum = 0;
u_int32_t expect_fingerprint = 0; u_int32_t expect_fingerprint = 0;
r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, n * 4 * sizeof (int)); assert(r == 0); r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, n * 4 * sizeof (int), 0); assert(r == 0);
toku_pma_set_dup_mode(pma, TOKU_DB_DUP + TOKU_DB_DUPSORT); toku_pma_set_dup_mode(pma, TOKU_DB_DUP + TOKU_DB_DUPSORT);
toku_pma_set_dup_compare(pma, toku_default_compare_fun); toku_pma_set_dup_compare(pma, toku_default_compare_fun);
...@@ -1762,7 +1763,7 @@ static void test_pma_already_there() { ...@@ -1762,7 +1763,7 @@ static void test_pma_already_there() {
u_int32_t rand4fingerprint = random(); u_int32_t rand4fingerprint = random();
u_int32_t sum = 0; u_int32_t sum = 0;
r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0); r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0, 0);
assert(r == 0); assert(r == 0);
DBT key, val; DBT key, val;
...@@ -1791,7 +1792,7 @@ static void test_pma_cursor_first(int n) { ...@@ -1791,7 +1792,7 @@ static void test_pma_cursor_first(int n) {
int r; int r;
PMA pma; PMA pma;
r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0); assert(r == 0); r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0, 0); assert(r == 0);
PMA_CURSOR cursor = 0; PMA_CURSOR cursor = 0;
r = toku_pma_cursor(pma, &cursor, &skey, &sval); assert(r == 0); r = toku_pma_cursor(pma, &cursor, &skey, &sval); assert(r == 0);
...@@ -1824,7 +1825,7 @@ static void test_pma_cursor_set_key() { ...@@ -1824,7 +1825,7 @@ static void test_pma_cursor_set_key() {
int r; int r;
PMA pma; PMA pma;
r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0); r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0, 0);
assert(r == 0); assert(r == 0);
DBT key, val; DBT key, val;
...@@ -1884,7 +1885,7 @@ static void test_pma_cursor_set_range() { ...@@ -1884,7 +1885,7 @@ static void test_pma_cursor_set_range() {
u_int32_t sum = 0; u_int32_t sum = 0;
u_int32_t expect_fingerprint = 0; u_int32_t expect_fingerprint = 0;
r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0); r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0, 0);
assert(r == 0); assert(r == 0);
DBT key, val; DBT key, val;
...@@ -1944,7 +1945,7 @@ static void test_pma_cursor_delete_under() { ...@@ -1944,7 +1945,7 @@ static void test_pma_cursor_delete_under() {
const int n = 1000; const int n = 1000;
r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, n * (8 + sizeof (int) + sizeof (int))); r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, n * (8 + sizeof (int) + sizeof (int)), 0);
assert(r == 0); assert(r == 0);
PMA_CURSOR cursor = 0; PMA_CURSOR cursor = 0;
...@@ -2016,7 +2017,7 @@ static void test_pma_cursor_delete_under_mode(int n, int dup_mode) { ...@@ -2016,7 +2017,7 @@ static void test_pma_cursor_delete_under_mode(int n, int dup_mode) {
u_int32_t sum = 0; u_int32_t sum = 0;
u_int32_t expect_fingerprint = 0; u_int32_t expect_fingerprint = 0;
r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, n * (8 + sizeof (int) + sizeof (int))); r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, n * (8 + sizeof (int) + sizeof (int)), 0);
assert(r == 0); assert(r == 0);
r = toku_pma_set_dup_mode(pma, dup_mode); assert(r == 0); r = toku_pma_set_dup_mode(pma, dup_mode); assert(r == 0);
...@@ -2099,7 +2100,7 @@ static void test_pma_cursor_set_both() { ...@@ -2099,7 +2100,7 @@ static void test_pma_cursor_set_both() {
const int n = 1000; const int n = 1000;
r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, n * (8 + sizeof (int) + sizeof (int))); r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, n * (8 + sizeof (int) + sizeof (int)), 0);
assert(r == 0); assert(r == 0);
r = toku_pma_set_dup_compare(pma, toku_default_compare_fun); assert(r == 0); r = toku_pma_set_dup_compare(pma, toku_default_compare_fun); assert(r == 0);
...@@ -2175,7 +2176,7 @@ static void test_nodup_key_insert(int n) { ...@@ -2175,7 +2176,7 @@ static void test_nodup_key_insert(int n) {
u_int32_t sum = 0; u_int32_t sum = 0;
u_int32_t expect_fingerprint = 0; u_int32_t expect_fingerprint = 0;
r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, n * (8 + sizeof (int) + sizeof (int))); r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, n * (8 + sizeof (int) + sizeof (int)), 0);
assert(r == 0); assert(r == 0);
/* insert 0->0, 0->1, .. 0->n-1 */ /* insert 0->0, 0->1, .. 0->n-1 */
...@@ -2212,7 +2213,7 @@ static void test_dup_key_delete(int n, int mode) { ...@@ -2212,7 +2213,7 @@ static void test_dup_key_delete(int n, int mode) {
u_int32_t sum = 0; u_int32_t sum = 0;
u_int32_t expect_fingerprint = 0; u_int32_t expect_fingerprint = 0;
r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, (n + 2) * (8 + sizeof (int) + sizeof (int))); r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, (n + 2) * (8 + sizeof (int) + sizeof (int)), 0);
assert(r == 0); assert(r == 0);
toku_pma_verify(pma); toku_pma_verify(pma);
...@@ -2315,7 +2316,7 @@ static void test_dupsort_key_insert(int n, int dup_data) { ...@@ -2315,7 +2316,7 @@ static void test_dupsort_key_insert(int n, int dup_data) {
u_int32_t sum = 0; u_int32_t sum = 0;
u_int32_t expect_fingerprint = 0; u_int32_t expect_fingerprint = 0;
r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, (n + 2) * (8 + sizeof (int) + sizeof (int))); r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, (n + 2) * (8 + sizeof (int) + sizeof (int)), 0);
assert(r == 0); assert(r == 0);
toku_pma_verify(pma); toku_pma_verify(pma);
...@@ -2409,7 +2410,7 @@ static void test_dup_key_lookup(int n, int mode) { ...@@ -2409,7 +2410,7 @@ static void test_dup_key_lookup(int n, int mode) {
u_int32_t sum = 0; u_int32_t sum = 0;
u_int32_t expect_fingerprint = 0; u_int32_t expect_fingerprint = 0;
r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, (n + 2) * (8 + sizeof (int) + sizeof (int))); r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, (n + 2) * (8 + sizeof (int) + sizeof (int)), 0);
assert(r == 0); assert(r == 0);
toku_pma_verify(pma); toku_pma_verify(pma);
......
...@@ -477,7 +477,7 @@ static unsigned int pma_array_size(PMA pma __attribute__((unused)), int asksize) ...@@ -477,7 +477,7 @@ static unsigned int pma_array_size(PMA pma __attribute__((unused)), int asksize)
return n; return n;
} }
int toku_pma_create(PMA *pma, pma_compare_fun_t compare_fun, DB *db, FILENUM filenum, int maxsize) { int toku_pma_create(PMA *pma, pma_compare_fun_t compare_fun, DB *db, FILENUM filenum, int maxsize, int initial_n_pairs /* set to zero to get default. */) {
int error; int error;
TAGMALLOC(PMA, result); TAGMALLOC(PMA, result);
if (result==0) return -1; if (result==0) return -1;
...@@ -490,7 +490,7 @@ int toku_pma_create(PMA *pma, pma_compare_fun_t compare_fun, DB *db, FILENUM fil ...@@ -490,7 +490,7 @@ int toku_pma_create(PMA *pma, pma_compare_fun_t compare_fun, DB *db, FILENUM fil
result->filenum = filenum; result->filenum = filenum;
result->skey = 0; result->skey = 0;
result->sval = 0; result->sval = 0;
result->N = PMA_MIN_ARRAY_SIZE; result->N = initial_n_pairs ? initial_n_pairs : PMA_MIN_ARRAY_SIZE;
result->pairs = 0; result->pairs = 0;
{ {
unsigned int n = pma_array_size(result, result->N); unsigned int n = pma_array_size(result, result->N);
......
...@@ -21,7 +21,7 @@ typedef struct pma *PMA; ...@@ -21,7 +21,7 @@ typedef struct pma *PMA;
/* compare 2 DBT's. return a value < 0, = 0, > 0 if a < b, a == b, a > b respectively */ /* compare 2 DBT's. return a value < 0, = 0, > 0 if a < b, a == b, a > b respectively */
typedef int (*pma_compare_fun_t)(DB *, const DBT *a, const DBT *b); typedef int (*pma_compare_fun_t)(DB *, const DBT *a, const DBT *b);
int toku_pma_create(PMA *, pma_compare_fun_t compare_fun, DB *, FILENUM filenum, int maxsize); int toku_pma_create(PMA *, pma_compare_fun_t compare_fun, DB *, FILENUM filenum, int maxsize, int initial_n_pairs);
int toku_pma_set_compare(PMA pma, pma_compare_fun_t compare_fun); int toku_pma_set_compare(PMA pma, pma_compare_fun_t compare_fun);
......
...@@ -153,7 +153,7 @@ void toku_recover_newbrtnode (LSN lsn, FILENUM filenum,DISKOFF diskoff,u_int32_t ...@@ -153,7 +153,7 @@ void toku_recover_newbrtnode (LSN lsn, FILENUM filenum,DISKOFF diskoff,u_int32_t
n->local_fingerprint = 0; // nothing there yet n->local_fingerprint = 0; // nothing there yet
n->dirty = 1; n->dirty = 1;
if (height==0) { if (height==0) {
r=toku_pma_create(&n->u.l.buffer, toku_dont_call_this_compare_fun, null_db, filenum, nodesize); r=toku_pma_create(&n->u.l.buffer, toku_dont_call_this_compare_fun, null_db, filenum, nodesize, 0);
assert(r==0); assert(r==0);
n->u.l.n_bytes_in_buffer=0; n->u.l.n_bytes_in_buffer=0;
} else { } else {
...@@ -348,6 +348,9 @@ void toku_recover_insertinleaf (LSN lsn, TXNID UU(txnid), FILENUM filenum, DISKO ...@@ -348,6 +348,9 @@ void toku_recover_insertinleaf (LSN lsn, TXNID UU(txnid), FILENUM filenum, DISKO
node->local_fingerprint += node->rand4fingerprint*toku_calccrc32_kvpair(keybs.data, keybs.len, databs.data, databs.len); node->local_fingerprint += node->rand4fingerprint*toku_calccrc32_kvpair(keybs.data, keybs.len, databs.data, databs.len);
node->u.l.n_bytes_in_buffer += PMA_ITEM_OVERHEAD + KEY_VALUE_OVERHEAD + keybs.len + databs.len; node->u.l.n_bytes_in_buffer += PMA_ITEM_OVERHEAD + KEY_VALUE_OVERHEAD + keybs.len + databs.len;
// PMA_ITERATE_IDX(node->u.l.buffer, idx, skey, keylen __attribute__((__unused__)), sdata, datalen __attribute__((__unused__)),
// printf("%d: %s %s\n", idx, (char*)skey, (char*)sdata));
VERIFY_COUNTS(node); VERIFY_COUNTS(node);
node->log_lsn = lsn; node->log_lsn = lsn;
...@@ -415,10 +418,13 @@ void toku_recover_pmadistribute (LSN lsn, FILENUM filenum, DISKOFF old_diskoff, ...@@ -415,10 +418,13 @@ void toku_recover_pmadistribute (LSN lsn, FILENUM filenum, DISKOFF old_diskoff,
BRTNODE nodeb = node_vb; assert(nodeb->height==0); BRTNODE nodeb = node_vb; assert(nodeb->height==0);
{ {
unsigned int i; unsigned int i;
//printf("{");
for (i=0; i<fromto.size; i++) { for (i=0; i<fromto.size; i++) {
//printf(" {%d %d}", fromto.array[i].a, fromto.array[i].b);
assert(fromto.array[i].a < toku_pma_index_limit(nodea->u.l.buffer)); assert(fromto.array[i].a < toku_pma_index_limit(nodea->u.l.buffer));
assert(fromto.array[i].b < toku_pma_index_limit(nodeb->u.l.buffer)); assert(fromto.array[i].b < toku_pma_index_limit(nodeb->u.l.buffer));
} }
//printf("}\n");
} }
r = toku_pma_move_indices (nodea->u.l.buffer, nodeb->u.l.buffer, fromto, r = toku_pma_move_indices (nodea->u.l.buffer, nodeb->u.l.buffer, fromto,
nodea->rand4fingerprint, &nodea->local_fingerprint, nodea->rand4fingerprint, &nodea->local_fingerprint,
...@@ -427,6 +433,9 @@ void toku_recover_pmadistribute (LSN lsn, FILENUM filenum, DISKOFF old_diskoff, ...@@ -427,6 +433,9 @@ void toku_recover_pmadistribute (LSN lsn, FILENUM filenum, DISKOFF old_diskoff,
); );
// The bytes in buffer and fingerprint shouldn't change // The bytes in buffer and fingerprint shouldn't change
// PMA_ITERATE_IDX(nodeb->u.l.buffer, idx, key, keylen __attribute__((__unused__)), data, datalen __attribute__((__unused__)),
// printf("%d: %s %s\n", idx, (char*)key, (char*)data));
VERIFY_COUNTS(nodea); VERIFY_COUNTS(nodea);
VERIFY_COUNTS(nodeb); VERIFY_COUNTS(nodeb);
...@@ -514,12 +523,13 @@ int tokudb_recover(const char *data_dir, const char *log_dir) { ...@@ -514,12 +523,13 @@ int tokudb_recover(const char *data_dir, const char *log_dir) {
FILE *f = fopen(logfiles[i], "r"); FILE *f = fopen(logfiles[i], "r");
struct log_entry le; struct log_entry le;
u_int32_t version; u_int32_t version;
//printf("Reading file %s\n", logfiles[i]);
r=toku_read_and_print_logmagic(f, &version); r=toku_read_and_print_logmagic(f, &version);
assert(r==0 && version==0); assert(r==0 && version==0);
r=chdir(data_wd); r=chdir(data_wd);
assert(r==0); assert(r==0);
while ((r = toku_log_fread(f, &le))==0) { while ((r = toku_log_fread(f, &le))==0) {
//printf("%lld: Got cmd %c\n", le.u.commit.lsn.lsn, le.cmd); //printf("%lld: Got cmd %c\n", (long long)le.u.commit.lsn.lsn, le.cmd);
logtype_dispatch_args(&le, toku_recover_); logtype_dispatch_args(&le, toku_recover_);
entrycount++; entrycount++;
} }
......
...@@ -35,7 +35,9 @@ static void make_db (void) { ...@@ -35,7 +35,9 @@ static void make_db (void) {
system("rm -rf " ENVDIR); system("rm -rf " ENVDIR);
r=mkdir(ENVDIR, 0777); assert(r==0); r=mkdir(ENVDIR, 0777); assert(r==0);
r=db_env_create(&env, 0); assert(r==0); r=db_env_create(&env, 0); assert(r==0);
#ifdef TOKUDB
r=env->set_lk_max_locks(env, 2*maxcount); CKERR(r); r=env->set_lk_max_locks(env, 2*maxcount); CKERR(r);
#endif
r=env->open(env, ENVDIR, DB_INIT_LOCK|DB_INIT_LOG|DB_INIT_MPOOL|DB_INIT_TXN|DB_CREATE|DB_PRIVATE, 0777); CKERR(r); r=env->open(env, ENVDIR, DB_INIT_LOCK|DB_INIT_LOG|DB_INIT_MPOOL|DB_INIT_TXN|DB_CREATE|DB_PRIVATE, 0777); CKERR(r);
r=db_create(&db, env, 0); CKERR(r); r=db_create(&db, env, 0); CKERR(r);
r=env->txn_begin(env, 0, &tid, 0); assert(r==0); r=env->txn_begin(env, 0, &tid, 0); assert(r==0);
...@@ -58,6 +60,13 @@ static void make_db (void) { ...@@ -58,6 +60,13 @@ static void make_db (void) {
key.data = hello; key.size=strlen(hello)+1; key.data = hello; key.size=strlen(hello)+1;
data.data = there; data.size=strlen(there)+1; data.data = there; data.size=strlen(there)+1;
r=db->put(db, tid, &key, &data, 0); assert(r==0); r=db->put(db, tid, &key, &data, 0); assert(r==0);
#ifndef TOKUDB
// BDB cannot handle this huge transaction even with a lot of locks.
if (i%1000==599) {
r=tid->commit(tid, 0); assert(r==0);
r=env->txn_begin(env, 0, &tid, 0); assert(r==0);
}
#endif
} }
r=tid->commit(tid, 0); assert(r==0); r=tid->commit(tid, 0); assert(r==0);
r=db->close(db, 0); assert(r==0); r=db->close(db, 0); assert(r==0);
......
...@@ -82,7 +82,7 @@ static void make_db (void) { ...@@ -82,7 +82,7 @@ static void make_db (void) {
r=db->close(db, 0); CKERR(r); r=db->close(db, 0); CKERR(r);
r=env->close(env, 0); CKERR(r); r=env->close(env, 0); CKERR(r);
for (i=0; i<2; i++) for (i=0; i<1; i++)
insert_some(i); insert_some(i);
while (items) { while (items) {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment