Skip to content

M
mariadb
Commit 8bfd0ff4 authored by Bradley C. Kuszmaul's avatar Bradley C. Kuszmaul
Browse files
Options

Break up brt-test some more. Addresses #475. Also make the fanout flexible. Fixes #126. 

git-svn-id: file:///svn/tokudb@2593 c7de825b-a66e-492c-adef-691d508d4ae1
parent 23d5d500
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 294 additions and 179 deletions
newbrt/Makefile
View file @ 8bfd0ff4
......@@ -46,14 +46,15 @@ default: bins libs recover tdb_logprint
# Put these one-per-line so that if we insert a new one the svn diff can understand it better.
# Also keep them sorted.
REGRESSION_TESTS = \
ybt-test \
pma-test \
brt-serialize-test \
cachetable-test \
cachetable-test2 \
fifo-test \
ybt-test \
pma-test \
brt-serialize-test \
cachetable-test \
cachetable-test2 \
fifo-test \
test-brt-delete-both \
brt-test \
brt-test \
brt-test3 \
brt-test4 \
brt-test-cursor \
test_oexcl \
......@@ -99,6 +100,7 @@ CHECKS = \
test-brt-delete-both \
brt-test \
brt-test-cursor \
brt-test3 \
brt-test4 \
fifo-test \
test_toku_malloc_plain_free \
......@@ -106,7 +108,8 @@ CHECKS = \
list-test \
# This line intentially kept commented so I can have a \ on the previous line
check: bins $(patsubst %,check_%,$(CHECKS)) check_benchmarktest_256
# Put check_benchmarktest_256 first because it is long-running (and therefore on the critical path, so get it started)
check: bins check_benchmarktest_256 $(patsubst %,check_%,$(CHECKS))
check_benchmarktest_256: benchmark-test
$(DTOOL) ./benchmark-test $(VERBVERBOSE) --valsize 256 --verify 1
......@@ -121,7 +124,7 @@ check_test-assert: test-assert
@# one argument, "ok" should not error
$(DTOOL) ./test-assert ok
check_%: %
$(DTOOL) ./$< $(VERBVERBOSE)
time $(DTOOL) ./$< $(VERBVERBOSE)
check-fanout:
let BRT_FANOUT=4; \
......@@ -130,7 +133,7 @@ check-fanout:
let BRT_FANOUT=BRT_FANOUT+1; \
done
pma-test benchmark-test brt-test brt-test4 brt-test-cursor test-brt-delete-both brt-serialize-test brtdump test-inc-split test-del-inorder: LDFLAGS+=-lz
pma-test benchmark-test brt-test brt-test3 brt-test4 brt-test-cursor test-brt-delete-both brt-serialize-test brtdump test-inc-split test-del-inorder: LDFLAGS+=-lz
# pma: PROF_FLAGS=-fprofile-arcs -ftest-coverage
BRT_INTERNAL_H_INCLUDES = brt-internal.h cachetable.h fifo.h pma.h brt.h brttypes.h yerror.h ybt.h log.h ../include/db.h kv-pair.h memory.h crc.h
......@@ -146,10 +149,10 @@ ybt.o: ybt.h brttypes.h ../include/db.h
ybt-test: ybt-test.o ybt.o memory.o toku_assert.o
ybt-test.o: ybt.h ../include/db.h
cachetable.o: cachetable.h hashfun.h memory.h
brt-test4 brt-test-cursor brt-test: ybt.o brt.o fifo.o pma.o memory.o brt-serialize.o cachetable.o ybt.o key.o primes.o toku_assert.o log.o mempool.o brt-verify.o fingerprint.o log_code.o roll.o
brt-test3 brt-test4 brt-test-cursor brt-test: ybt.o brt.o fifo.o pma.o memory.o brt-serialize.o cachetable.o ybt.o key.o primes.o toku_assert.o log.o mempool.o brt-verify.o fingerprint.o log_code.o roll.o
log.o: log_header.h log-internal.h log.h wbuf.h crc.h brttypes.h $(BRT_INTERNAL_H_INCLUDES)
logformat: logformat.o toku_assert.o
brt-test4.o brt-test-cursor.o brt-test.o brt.o: brt.h ../include/db.h fifo.h pma.h brttypes.h cachetable.h memory.h
brt-test3.o brt-test4.o brt-test-cursor.o brt-test.o brt.o: brt.h ../include/db.h fifo.h pma.h brttypes.h cachetable.h memory.h
brt-serialize-test.o: $(BRT_INTERNAL_H_INCLUDES)
brt.o: $(BRT_INTERNAL_H_INCLUDES) key.h log_header.h
fifo.o: fifo.h brttypes.h
......
newbrt/brt-internal.h
View file @ 8bfd0ff4
......@@ -64,14 +64,14 @@ struct brtnode {
unsigned int totalchildkeylens;
unsigned int n_bytes_in_buffers;
struct brtnode_nonleaf_childinfo childinfos[TREE_FANOUT+1]; /* One extra so we can grow */
struct brtnode_nonleaf_childinfo *childinfos; /* One extra so we can grow */
#define BNC_SUBTREE_FINGERPRINT(node,i) ((node)->u.n.childinfos[i].subtree_fingerprint)
#define BNC_DISKOFF(node,i) ((node)->u.n.childinfos[i].diskoff)
#define BNC_BUFFER(node,i) ((node)->u.n.childinfos[i].buffer)
#define BNC_NBYTESINBUF(node,i) ((node)->u.n.childinfos[i].n_bytes_in_buffer)
struct kv_pair *childkeys[TREE_FANOUT]; /* Pivot keys. Child 0's keys are <= childkeys[0]. Child 1's keys are <= childkeys[1].
struct kv_pair **childkeys; /* Pivot keys. Child 0's keys are <= childkeys[0]. Child 1's keys are <= childkeys[1].
Note: It is possible that Child 1's keys are == to child 0's key's, so it is
not necessarily true that child 1's keys are > childkeys[0].
However, in the absense of duplicate keys, child 1's keys *are* > childkeys[0]. */
......@@ -175,7 +175,7 @@ struct brt_cursor {
DBT val;
};
void toku_create_new_brtnode (BRT t, BRTNODE *result, int height, TOKULOGGER logger);
int toku_create_new_brtnode (BRT t, BRTNODE *result, int height, TOKULOGGER logger);
int toku_unpin_brtnode (BRT brt, BRTNODE node) ;
unsigned int toku_brtnode_which_child (BRTNODE node , DBT *k, DBT *d, BRT t);
......
newbrt/brt-serialize-test.c
View file @ 8bfd0ff4
......@@ -32,6 +32,8 @@ static void test_serialize(void) {
sn.local_fingerprint = 0;
sn.u.n.n_children = 2;
hello_string = toku_strdup("hello");
MALLOC_N(2, sn.u.n.childinfos);
MALLOC_N(1, sn.u.n.childkeys);
sn.u.n.childkeys[0] = kv_pair_malloc(hello_string, 6, 0, 0);
sn.u.n.totalchildkeylens = 6;
BNC_DISKOFF(&sn, 0) = sn.nodesize*30;
......@@ -45,11 +47,6 @@ static void test_serialize(void) {
r = toku_fifo_enq(BNC_BUFFER(&sn,1), "x", 2, "xval", 5, BRT_NONE, (TXNID)234); assert(r==0); sn.local_fingerprint += randval*toku_calccrc32_cmd(BRT_NONE, (TXNID)234, "x", 2, "xval", 5);
BNC_NBYTESINBUF(&sn, 0) = 2*(BRT_CMD_OVERHEAD+KEY_VALUE_OVERHEAD+2+5);
BNC_NBYTESINBUF(&sn, 1) = 1*(BRT_CMD_OVERHEAD+KEY_VALUE_OVERHEAD+2+5);
{
int i;
for (i=2; i<TREE_FANOUT+1; i++)
BNC_NBYTESINBUF(&sn, i)=0;
}
sn.u.n.n_bytes_in_buffers = 3*(BRT_CMD_OVERHEAD+KEY_VALUE_OVERHEAD+2+5);
toku_serialize_brtnode_to(fd, sn.nodesize*20, sn.nodesize, &sn); assert(r==0);
......
newbrt/brt-serialize.c
View file @ 8bfd0ff4
......@@ -278,18 +278,11 @@ int toku_deserialize_brtnode_from (int fd, DISKOFF off, BRTNODE *brtnode, int fl
//printf("height==%d\n", result->height);
if (result->height>0) {
result->u.n.totalchildkeylens=0;
for (i=0; i<TREE_FANOUT; i++) {
result->u.n.childkeys[i]=0;
}
for (i=0; i<TREE_FANOUT+1; i++) {
BNC_SUBTREE_FINGERPRINT(result, i)=0;
BNC_DISKOFF(result,i)=0;
BNC_BUFFER(result,i)=0;
BNC_NBYTESINBUF(result,i)=0;
}
u_int32_t subtree_fingerprint = rbuf_int(&rc);
u_int32_t check_subtree_fingerprint = 0;
result->u.n.n_children = rbuf_int(&rc);
MALLOC_N(result->u.n.n_children, result->u.n.childinfos);
MALLOC_N(result->u.n.n_children-1, result->u.n.childkeys);
//printf("n_children=%d\n", result->n_children);
assert(result->u.n.n_children>=0 && result->u.n.n_children<=TREE_FANOUT);
for (i=0; i<result->u.n.n_children; i++) {
......@@ -315,11 +308,9 @@ int toku_deserialize_brtnode_from (int fd, DISKOFF off, BRTNODE *brtnode, int fl
}
for (i=0; i<result->u.n.n_children; i++) {
BNC_DISKOFF(result,i) = rbuf_diskoff(&rc);
BNC_NBYTESINBUF(result,i) = 0;
//printf("Child %d at %lld\n", i, result->children[i]);
}
for (i=0; i<TREE_FANOUT+1; i++) {
BNC_NBYTESINBUF(result,i)=0;
}
result->u.n.n_bytes_in_buffers = 0;
for (i=0; i<result->u.n.n_children; i++) {
r=toku_fifo_create(&BNC_BUFFER(result,i));
......@@ -472,9 +463,6 @@ void toku_verify_counts (BRTNODE node) {
sum += BNC_NBYTESINBUF(node,i);
// We don't rally care of the later buffers have garbage in them. Valgrind would do a better job noticing if we leave it uninitialized.
// But for now the code always initializes the later tables so they are 0.
for (; i<TREE_FANOUT+1; i++) {
assert(BNC_NBYTESINBUF(node,i)==0);
}
assert(sum==node->u.n.n_bytes_in_buffers);
}
}
......
newbrt/brt-test.c
View file @ 8bfd0ff4
......@@ -108,37 +108,6 @@ static void test2 (int memcheck) {
if (verbose) printf("test2 ok\n");
}
static void test3 (int nodesize, int count, int memcheck) {
BRT t;
int r;
struct timeval t0,t1;
int i;
CACHETABLE ct;
char fname[]="testbrt.brt";
toku_memory_check=memcheck;
toku_memory_check_all_free();
r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER); assert(r==0);
gettimeofday(&t0, 0);
unlink(fname);
r = toku_open_brt(fname, 0, 1, &t, nodesize, ct, null_txn, toku_default_compare_fun, null_db);
assert(r==0);
for (i=0; i<count; i++) {
char key[100],val[100];
DBT k,v;
snprintf(key,100,"hello%d",i);
snprintf(val,100,"there%d",i);
toku_brt_insert(t, toku_fill_dbt(&k, key, 1+strlen(key)), toku_fill_dbt(&v, val, 1+strlen(val)), null_txn);
}
r = toku_close_brt(t); assert(r==0);
r = toku_cachetable_close(&ct); assert(r==0);
toku_memory_check_all_free();
gettimeofday(&t1, 0);
{
double tdiff = (t1.tv_sec-t0.tv_sec)+1e-6*(t1.tv_usec-t0.tv_usec);
if (verbose) printf("serial insertions: blocksize=%d %d insertions in %.3f seconds, %.2f insertions/second\n", nodesize, count, tdiff, count/tdiff);
}
}
static void test5 (void) {
int r;
BRT t;
......@@ -1689,17 +1658,6 @@ static void brt_blackbox_test (void) {
test5();
if (verbose) printf("test_multiple_files\n");
test_multiple_files();
if (verbose) printf("test3 slow\n");
toku_memory_check=0;
test3(2048, 1<<15, 1);
if (verbose) printf("test3 fast\n");
if (verbose) toku_pma_show_stats();
test3(1<<15, 1024, 1);
if (verbose) printf("test3 fast\n");
test3(1<<18, 1<<20, 0);
toku_memory_check = 1;
......
newbrt/brt-test3.c 0 → 100644
View file @ 8bfd0ff4
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2007, 2008 Tokutek Inc. All rights reserved."
#include "brt.h"
#include "key.h"
#include "pma.h"
#include "brt-internal.h"
#include "memory.h"
#include "toku_assert.h"
#include <fcntl.h>
#include <stdio.h>
#include <string.h>
#include <sys/time.h>
#include <stdlib.h>
#include <unistd.h>
#include <arpa/inet.h>
#include "test.h"
static const char fname[]= __FILE__ ".brt";
static TOKUTXN const null_txn = 0;
static DB * const null_db = 0;
static void test3 (int nodesize, int count, int memcheck) {
BRT t;
int r;
struct timeval t0,t1;
int i;
CACHETABLE ct;
toku_memory_check=memcheck;
toku_memory_check_all_free();
r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER); assert(r==0);
gettimeofday(&t0, 0);
unlink(fname);
r = toku_open_brt(fname, 0, 1, &t, nodesize, ct, null_txn, toku_default_compare_fun, null_db);
assert(r==0);
for (i=0; i<count; i++) {
char key[100],val[100];
DBT k,v;
snprintf(key,100,"hello%d",i);
snprintf(val,100,"there%d",i);
toku_brt_insert(t, toku_fill_dbt(&k, key, 1+strlen(key)), toku_fill_dbt(&v, val, 1+strlen(val)), null_txn);
}
r = toku_close_brt(t); assert(r==0);
r = toku_cachetable_close(&ct); assert(r==0);
toku_memory_check_all_free();
gettimeofday(&t1, 0);
{
double tdiff = (t1.tv_sec-t0.tv_sec)+1e-6*(t1.tv_usec-t0.tv_usec);
if (verbose) printf("serial insertions: blocksize=%d %d insertions in %.3f seconds, %.2f insertions/second\n", nodesize, count, tdiff, count/tdiff);
}
}
static void brt_blackbox_test (void) {
if (verbose) printf("test3 slow\n");
toku_memory_check=0;
test3(2048, 1<<15, 1);
if (verbose) printf("test3 fast\n");
if (verbose) toku_pma_show_stats();
test3(1<<15, 1024, 1);
if (verbose) printf("test3 fast\n");
test3(1<<18, 1<<20, 0);
toku_memory_check = 1;
// test3(1<<19, 1<<20, 0);
// test3(1<<20, 1<<20, 0);
// test3(1<<20, 1<<21, 0);
// test3(1<<20, 1<<22, 0);
}
int main (int argc , const char *argv[]) {
default_parse_args(argc, argv);
brt_blackbox_test();
toku_malloc_cleanup();
if (verbose) printf("test ok\n");
return 0;
}
newbrt/brt.c
View file @ 8bfd0ff4
......@@ -49,13 +49,15 @@ void toku_brtnode_free (BRTNODE *nodep) {
//printf("%s:%d %p->mdict[0]=%p\n", __FILE__, __LINE__, node, node->mdicts[0]);
if (node->height>0) {
for (i=0; i<node->u.n.n_children-1; i++) {
toku_free((void*)node->u.n.childkeys[i]);
toku_free(node->u.n.childkeys[i]);
}
for (i=0; i<node->u.n.n_children; i++) {
if (BNC_BUFFER(node,i)) {
toku_fifo_free(&BNC_BUFFER(node,i));
}
}
toku_free(node->u.n.childkeys);
toku_free(node->u.n.childinfos);
} else {
if (node->u.l.buffer) // The buffer may have been freed already, in some cases.
toku_pma_free(&node->u.l.buffer);
......@@ -246,7 +248,6 @@ int malloc_diskblock (DISKOFF *res, BRT brt, int size, TOKULOGGER logger) {
}
static void initialize_brtnode (BRT t, BRTNODE n, DISKOFF nodename, int height) {
int i;
n->tag = TYP_BRTNODE;
n->nodesize = t->h->nodesize;
n->flags = t->h->flags;
......@@ -261,18 +262,10 @@ static void initialize_brtnode (BRT t, BRTNODE n, DISKOFF nodename, int height)
assert(height>=0);
if (height>0) {
n->u.n.n_children = 0;
for (i=0; i<TREE_FANOUT; i++) {
// n->u.n.childkeys[i] = 0;
// n->u.n.childkeylens[i] = 0;
}
n->u.n.totalchildkeylens = 0;
for (i=0; i<TREE_FANOUT+1; i++) {
BNC_SUBTREE_FINGERPRINT(n, i) = 0;
// n->u.n.children[i] = 0;
// n->u.n.buffers[i] = 0;
BNC_NBYTESINBUF(n,i) = 0;
}
n->u.n.n_bytes_in_buffers = 0;
n->u.n.childinfos=0;
n->u.n.childkeys=0;
} else {
int r = toku_pma_create(&n->u.l.buffer, t->compare_fun, t->db, toku_cachefile_filenum(t->cf), n->nodesize);
assert(r==0);
......@@ -285,7 +278,7 @@ static void initialize_brtnode (BRT t, BRTNODE n, DISKOFF nodename, int height)
}
}
void toku_create_new_brtnode (BRT t, BRTNODE *result, int height, TOKULOGGER logger) {
int toku_create_new_brtnode (BRT t, BRTNODE *result, int height, TOKULOGGER logger) {
TAGMALLOC(BRTNODE, n);
int r;
DISKOFF name;
......@@ -305,6 +298,7 @@ void toku_create_new_brtnode (BRT t, BRTNODE *result, int height, TOKULOGGER log
r=toku_log_newbrtnode(logger, toku_cachefile_filenum(t->cf), n->thisnodename, height, n->nodesize, (t->flags&TOKU_DB_DUPSORT)!=0, n->rand4fingerprint);
assert(r==0);
toku_update_brtnode_loggerlsn(n, logger);
return 0;
}
static int insert_to_buffer_in_nonleaf (BRTNODE node, int childnum, DBT *k, DBT *v, int type, TXNID xid) {
......@@ -360,6 +354,8 @@ static int brt_nonleaf_split (BRT t, BRTNODE node, BRTNODE *nodea, BRTNODE *node
assert(node->u.n.n_children>=2); // Otherwise, how do we split? We need at least two children to split. */
assert(t->h->nodesize>=node->nodesize); /* otherwise we might be in trouble because the nodesize shrank. */
toku_create_new_brtnode(t, &B, node->height, logger);
MALLOC_N(n_children_in_b+1, B->u.n.childinfos);
MALLOC_N(n_children_in_b, B->u.n.childkeys);
B->u.n.n_children =n_children_in_b;
//printf("%s:%d %p (%lld) becomes %p and %p\n", __FILE__, __LINE__, node, node->thisnodename, A, B);
//printf("%s:%d A is at %lld\n", __FILE__, __LINE__, A->thisnodename);
......@@ -372,6 +368,8 @@ static int brt_nonleaf_split (BRT t, BRTNODE node, BRTNODE *nodea, BRTNODE *node
for (i=0; i<n_children_in_b; i++) {
int r = toku_fifo_create(&BNC_BUFFER(B,i));
if (r!=0) return r;
BNC_NBYTESINBUF(B,i)=0;
BNC_SUBTREE_FINGERPRINT(B,i)=0;
}
for (i=n_children_in_a; i<old_n_children; i++) {
......@@ -453,7 +451,9 @@ static int brt_nonleaf_split (BRT t, BRTNODE node, BRTNODE *nodea, BRTNODE *node
splitk->data = (void*)(node->u.n.childkeys[n_children_in_a-1]);
splitk->size = toku_brt_pivot_key_len(t, node->u.n.childkeys[n_children_in_a-1]);
node->u.n.totalchildkeylens -= toku_brt_pivot_key_len(t, node->u.n.childkeys[n_children_in_a-1]);
node->u.n.childkeys[n_children_in_a-1]=0;
REALLOC_N(n_children_in_a+1, node->u.n.childinfos);
REALLOC_N(n_children_in_a, node->u.n.childkeys);
verify_local_fingerprint_nonleaf(node);
verify_local_fingerprint_nonleaf(B);
......@@ -618,6 +618,8 @@ static int handle_split_of_child (BRT t, BRTNODE node, int childnum,
//verify_local_fingerprint_nonleaf(node);
REALLOC_N(node->u.n.n_children+2, node->u.n.childinfos);
REALLOC_N(node->u.n.n_children+1, node->u.n.childkeys);
// Slide the children over.
for (cnum=node->u.n.n_children; cnum>childnum+1; cnum--) {
node->u.n.childinfos[cnum] = node->u.n.childinfos[cnum-1];
......@@ -625,6 +627,8 @@ static int handle_split_of_child (BRT t, BRTNODE node, int childnum,
r = toku_log_addchild(logger, toku_cachefile_filenum(t->cf), node->thisnodename, childnum+1, childb->thisnodename, 0);
assert(BNC_DISKOFF(node, childnum)==childa->thisnodename);
BNC_DISKOFF(node, childnum+1) = childb->thisnodename;
BNC_SUBTREE_FINGERPRINT(node, childnum)=0;
BNC_SUBTREE_FINGERPRINT(node, childnum+1)=0;
fixup_child_fingerprint(node, childnum, childa, t, logger);
fixup_child_fingerprint(node, childnum+1, childb, t, logger);
r=toku_fifo_create(&BNC_BUFFER(node,childnum)); assert(r==0); // ??? SHould handle this error case
......@@ -1625,6 +1629,8 @@ static int brt_init_new_root(BRT brt, BRTNODE nodea, BRTNODE nodeb, DBT splitk,
initialize_brtnode (brt, newroot, newroot_diskoff, new_height);
//printf("new_root %lld %d %lld %lld\n", newroot_diskoff, newroot->height, nodea->thisnodename, nodeb->thisnodename);
newroot->u.n.n_children=2;
MALLOC_N(3, newroot->u.n.childinfos);
MALLOC_N(2, newroot->u.n.childkeys);
//printf("%s:%d Splitkey=%p %s\n", __FILE__, __LINE__, splitkey, splitkey);
newroot->u.n.childkeys[0] = splitk.data;
newroot->u.n.totalchildkeylens=splitk.size;
......@@ -1632,6 +1638,10 @@ static int brt_init_new_root(BRT brt, BRTNODE nodea, BRTNODE nodeb, DBT splitk,
BNC_DISKOFF(newroot,1)=nodeb->thisnodename;
r=toku_fifo_create(&BNC_BUFFER(newroot,0)); if (r!=0) return r;
r=toku_fifo_create(&BNC_BUFFER(newroot,1)); if (r!=0) return r;
BNC_NBYTESINBUF(newroot, 0)=0;
BNC_NBYTESINBUF(newroot, 1)=0;
BNC_SUBTREE_FINGERPRINT(newroot, 0)=0;
BNC_SUBTREE_FINGERPRINT(newroot, 1)=0;
toku_verify_counts(newroot);
//verify_local_fingerprint_nonleaf(nodea);
//verify_local_fingerprint_nonleaf(nodeb);
......
newbrt/brt2.c
View file @ 8bfd0ff4
This diff is collapsed.
newbrt/memory.h
View file @ 8bfd0ff4
......@@ -42,6 +42,8 @@ void *toku_realloc(void *, size_t size);
*/
#define MALLOC_N(n,v) v = toku_malloc((n)*sizeof(*v))
#define REALLOC_N(n,v) v = toku_realloc(v, (n)*sizeof(*v))
/* If you have a type such as
* struct pma *PMA;
* and you define a corresponding int constant, such as
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7