Break up brt-test some more. Addresses #475. Also make the fanout flexible. Fixes #126.

git-svn-id: file:///svn/tokudb@2593 c7de825b-a66e-492c-adef-691d508d4ae1

newbrt/Makefile

@@ -54,6 +54,7 @@ REGRESSION_TESTS = \
 	fifo-test \
 	test-brt-delete-both \
 	brt-test \
+	brt-test3 \
 	brt-test4 \
 	brt-test-cursor \
 	test_oexcl \
@@ -99,6 +100,7 @@ CHECKS = \
  test-brt-delete-both \
  brt-test \
  brt-test-cursor \
+ brt-test3 \
  brt-test4 \
  fifo-test \
  test_toku_malloc_plain_free \
@@ -106,7 +108,8 @@ CHECKS = \
  list-test \
 # This line intentially kept commented so I can have a \ on the previous line
 
-check: bins $(patsubst %,check_%,$(CHECKS)) check_benchmarktest_256
+# Put check_benchmarktest_256 first because it is long-running (and therefore on the critical path, so get it started)
+check: bins check_benchmarktest_256 $(patsubst %,check_%,$(CHECKS))
 check_benchmarktest_256: benchmark-test
 	$(DTOOL) ./benchmark-test $(VERBVERBOSE) --valsize 256 --verify 1
 
@@ -121,7 +124,7 @@ check_test-assert: test-assert
 	@# one argument, "ok" should not error
 	$(DTOOL) ./test-assert ok
 check_%: %
-	$(DTOOL)  ./$< $(VERBVERBOSE) 
+	time $(DTOOL)  ./$< $(VERBVERBOSE) 
 
 check-fanout:
 	let BRT_FANOUT=4; \
@@ -130,7 +133,7 @@ check-fanout:
 		let BRT_FANOUT=BRT_FANOUT+1; \
 	done
 
-pma-test benchmark-test brt-test brt-test4 brt-test-cursor test-brt-delete-both brt-serialize-test brtdump test-inc-split test-del-inorder: LDFLAGS+=-lz
+pma-test benchmark-test brt-test brt-test3 brt-test4 brt-test-cursor test-brt-delete-both brt-serialize-test brtdump test-inc-split test-del-inorder: LDFLAGS+=-lz
 # pma: PROF_FLAGS=-fprofile-arcs -ftest-coverage
 
 BRT_INTERNAL_H_INCLUDES = brt-internal.h cachetable.h fifo.h pma.h brt.h brttypes.h yerror.h ybt.h log.h ../include/db.h kv-pair.h memory.h crc.h
@@ -146,10 +149,10 @@ ybt.o: ybt.h brttypes.h ../include/db.h
 ybt-test: ybt-test.o ybt.o memory.o toku_assert.o
 ybt-test.o: ybt.h ../include/db.h
 cachetable.o: cachetable.h hashfun.h memory.h
-brt-test4 brt-test-cursor brt-test: ybt.o brt.o fifo.o pma.o memory.o brt-serialize.o cachetable.o ybt.o key.o primes.o toku_assert.o log.o mempool.o brt-verify.o fingerprint.o log_code.o roll.o
+brt-test3 brt-test4 brt-test-cursor brt-test: ybt.o brt.o fifo.o pma.o memory.o brt-serialize.o cachetable.o ybt.o key.o primes.o toku_assert.o log.o mempool.o brt-verify.o fingerprint.o log_code.o roll.o
 log.o: log_header.h log-internal.h log.h wbuf.h crc.h brttypes.h $(BRT_INTERNAL_H_INCLUDES)
 logformat: logformat.o toku_assert.o
-brt-test4.o brt-test-cursor.o brt-test.o brt.o: brt.h ../include/db.h fifo.h pma.h brttypes.h cachetable.h memory.h
+brt-test3.o brt-test4.o brt-test-cursor.o brt-test.o brt.o: brt.h ../include/db.h fifo.h pma.h brttypes.h cachetable.h memory.h
 brt-serialize-test.o: $(BRT_INTERNAL_H_INCLUDES)
 brt.o: $(BRT_INTERNAL_H_INCLUDES) key.h log_header.h
 fifo.o: fifo.h brttypes.h 

newbrt/brt-internal.h

@@ -64,14 +64,14 @@ struct brtnode {
 	    unsigned int    totalchildkeylens;
 	    unsigned int    n_bytes_in_buffers;
 
-	    struct brtnode_nonleaf_childinfo childinfos[TREE_FANOUT+1]; /* One extra so we can grow */
+	    struct brtnode_nonleaf_childinfo *childinfos; /* One extra so we can grow */
 
 #define BNC_SUBTREE_FINGERPRINT(node,i) ((node)->u.n.childinfos[i].subtree_fingerprint)
 #define BNC_DISKOFF(node,i) ((node)->u.n.childinfos[i].diskoff)
 #define BNC_BUFFER(node,i) ((node)->u.n.childinfos[i].buffer)
 #define BNC_NBYTESINBUF(node,i) ((node)->u.n.childinfos[i].n_bytes_in_buffer)
 
-	    struct kv_pair *childkeys[TREE_FANOUT];   /* Pivot keys.  Child 0's keys are <= childkeys[0].  Child 1's keys are <= childkeys[1].
+	    struct kv_pair **childkeys;   /* Pivot keys.  Child 0's keys are <= childkeys[0].  Child 1's keys are <= childkeys[1].
 							 Note: It is possible that Child 1's keys are == to child 0's key's, so it is
 							 not necessarily true that child 1's keys are > childkeys[0].
 						         However, in the absense of duplicate keys, child 1's keys *are* > childkeys[0]. */
@@ -175,7 +175,7 @@ struct brt_cursor {
     DBT val;
 };
 
-void toku_create_new_brtnode (BRT t, BRTNODE *result, int height, TOKULOGGER logger);
+int toku_create_new_brtnode (BRT t, BRTNODE *result, int height, TOKULOGGER logger);
 int toku_unpin_brtnode (BRT brt, BRTNODE node) ;
 unsigned int toku_brtnode_which_child (BRTNODE node , DBT *k, DBT *d, BRT t);
 

newbrt/brt-serialize-test.c

@@ -32,6 +32,8 @@ static void test_serialize(void) {
     sn.local_fingerprint = 0;
     sn.u.n.n_children = 2;
     hello_string = toku_strdup("hello");
+    MALLOC_N(2, sn.u.n.childinfos);
+    MALLOC_N(1, sn.u.n.childkeys);
     sn.u.n.childkeys[0] = kv_pair_malloc(hello_string, 6, 0, 0); 
     sn.u.n.totalchildkeylens = 6;
     BNC_DISKOFF(&sn, 0) = sn.nodesize*30;
@@ -45,11 +47,6 @@ static void test_serialize(void) {
     r = toku_fifo_enq(BNC_BUFFER(&sn,1), "x", 2, "xval", 5, BRT_NONE, (TXNID)234); assert(r==0);    sn.local_fingerprint += randval*toku_calccrc32_cmd(BRT_NONE, (TXNID)234, "x", 2, "xval", 5);
     BNC_NBYTESINBUF(&sn, 0) = 2*(BRT_CMD_OVERHEAD+KEY_VALUE_OVERHEAD+2+5);
     BNC_NBYTESINBUF(&sn, 1) = 1*(BRT_CMD_OVERHEAD+KEY_VALUE_OVERHEAD+2+5);
-    {
-	int i;
-	for (i=2; i<TREE_FANOUT+1; i++)
-	    BNC_NBYTESINBUF(&sn, i)=0;
-    }
     sn.u.n.n_bytes_in_buffers = 3*(BRT_CMD_OVERHEAD+KEY_VALUE_OVERHEAD+2+5);
 
     toku_serialize_brtnode_to(fd, sn.nodesize*20, sn.nodesize, &sn);  assert(r==0);

newbrt/brt-serialize.c

@@ -278,18 +278,11 @@ int toku_deserialize_brtnode_from (int fd, DISKOFF off, BRTNODE *brtnode, int fl
     //printf("height==%d\n", result->height);
     if (result->height>0) {
 	result->u.n.totalchildkeylens=0;
-	for (i=0; i<TREE_FANOUT; i++) { 
-            result->u.n.childkeys[i]=0; 
-        }
-	for (i=0; i<TREE_FANOUT+1; i++) { 
-	    BNC_SUBTREE_FINGERPRINT(result, i)=0;
-            BNC_DISKOFF(result,i)=0; 
-            BNC_BUFFER(result,i)=0; 
-            BNC_NBYTESINBUF(result,i)=0;
-        }
 	u_int32_t subtree_fingerprint = rbuf_int(&rc);
 	u_int32_t check_subtree_fingerprint = 0;
 	result->u.n.n_children = rbuf_int(&rc);
+	MALLOC_N(result->u.n.n_children,   result->u.n.childinfos);
+	MALLOC_N(result->u.n.n_children-1, result->u.n.childkeys);
 	//printf("n_children=%d\n", result->n_children);
 	assert(result->u.n.n_children>=0 && result->u.n.n_children<=TREE_FANOUT);
 	for (i=0; i<result->u.n.n_children; i++) {
@@ -315,11 +308,9 @@ int toku_deserialize_brtnode_from (int fd, DISKOFF off, BRTNODE *brtnode, int fl
 	}
 	for (i=0; i<result->u.n.n_children; i++) {
 	    BNC_DISKOFF(result,i) = rbuf_diskoff(&rc);
+	    BNC_NBYTESINBUF(result,i) = 0;
 	    //printf("Child %d at %lld\n", i, result->children[i]);
 	}
-	for (i=0; i<TREE_FANOUT+1; i++) {
-	    BNC_NBYTESINBUF(result,i)=0;
-	}
 	result->u.n.n_bytes_in_buffers = 0; 
 	for (i=0; i<result->u.n.n_children; i++) {
 	    r=toku_fifo_create(&BNC_BUFFER(result,i));
@@ -472,9 +463,6 @@ void toku_verify_counts (BRTNODE node) {
 	    sum += BNC_NBYTESINBUF(node,i);
 	// We don't rally care of the later buffers have garbage in them.  Valgrind would do a better job noticing if we leave it uninitialized.
 	// But for now the code always initializes the later tables so they are 0.
-	for (; i<TREE_FANOUT+1; i++) {
-	    assert(BNC_NBYTESINBUF(node,i)==0);
-        }
 	assert(sum==node->u.n.n_bytes_in_buffers);
     }
 }

newbrt/brt-test.c

@@ -108,37 +108,6 @@ static void test2 (int memcheck) {
     if (verbose) printf("test2 ok\n");
 }
 
-static void test3 (int nodesize, int count, int memcheck) {
-    BRT t;
-    int r;
-    struct timeval t0,t1;
-    int i;
-    CACHETABLE ct;
-    char fname[]="testbrt.brt";
-    toku_memory_check=memcheck;
-    toku_memory_check_all_free();
-    r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER); assert(r==0);
-    gettimeofday(&t0, 0);
-    unlink(fname);
-    r = toku_open_brt(fname, 0, 1, &t, nodesize, ct, null_txn, toku_default_compare_fun, null_db);
-    assert(r==0);
-    for (i=0; i<count; i++) {
-	char key[100],val[100];
-	DBT k,v;
-	snprintf(key,100,"hello%d",i);
-	snprintf(val,100,"there%d",i);
-	toku_brt_insert(t, toku_fill_dbt(&k, key, 1+strlen(key)), toku_fill_dbt(&v, val, 1+strlen(val)), null_txn);
-    }
-    r = toku_close_brt(t);              assert(r==0);
-    r = toku_cachetable_close(&ct);     assert(r==0);
-    toku_memory_check_all_free();
-    gettimeofday(&t1, 0);
-    {
-	double tdiff = (t1.tv_sec-t0.tv_sec)+1e-6*(t1.tv_usec-t0.tv_usec);
-	if (verbose) printf("serial insertions: blocksize=%d %d insertions in %.3f seconds, %.2f insertions/second\n", nodesize, count, tdiff, count/tdiff);
-    }
-}
-
 static void test5 (void) {
     int r;
     BRT t;
@@ -1689,17 +1658,6 @@ static void brt_blackbox_test (void) {
     test5();
     if (verbose) printf("test_multiple_files\n");
     test_multiple_files();
-    if (verbose) printf("test3 slow\n");
-    toku_memory_check=0;
-    test3(2048, 1<<15, 1);
-    if (verbose) printf("test3 fast\n");
-
-    if (verbose) toku_pma_show_stats();
-
-    test3(1<<15, 1024, 1);
-    if (verbose) printf("test3 fast\n");
-
-    test3(1<<18, 1<<20, 0);
 
     toku_memory_check = 1;
 

newbrt/brt-test3.c

+/* -*- mode: C; c-basic-offset: 4 -*- */
+#ident "Copyright (c) 2007, 2008 Tokutek Inc.  All rights reserved."
+
+#include "brt.h"
+#include "key.h"
+#include "pma.h"
+#include "brt-internal.h"
+#include "memory.h"
+#include "toku_assert.h"
+
+#include <fcntl.h>
+#include <stdio.h>
+#include <string.h>
+#include <sys/time.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <arpa/inet.h>
+
+#include "test.h"
+
+static const char fname[]= __FILE__ ".brt";
+
+static TOKUTXN const null_txn = 0;
+static DB * const null_db = 0;
+
+static void test3 (int nodesize, int count, int memcheck) {
+    BRT t;
+    int r;
+    struct timeval t0,t1;
+    int i;
+    CACHETABLE ct;
+    toku_memory_check=memcheck;
+    toku_memory_check_all_free();
+    r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER); assert(r==0);
+    gettimeofday(&t0, 0);
+    unlink(fname);
+    r = toku_open_brt(fname, 0, 1, &t, nodesize, ct, null_txn, toku_default_compare_fun, null_db);
+    assert(r==0);
+    for (i=0; i<count; i++) {
+	char key[100],val[100];
+	DBT k,v;
+	snprintf(key,100,"hello%d",i);
+	snprintf(val,100,"there%d",i);
+	toku_brt_insert(t, toku_fill_dbt(&k, key, 1+strlen(key)), toku_fill_dbt(&v, val, 1+strlen(val)), null_txn);
+    }
+    r = toku_close_brt(t);              assert(r==0);
+    r = toku_cachetable_close(&ct);     assert(r==0);
+    toku_memory_check_all_free();
+    gettimeofday(&t1, 0);
+    {
+	double tdiff = (t1.tv_sec-t0.tv_sec)+1e-6*(t1.tv_usec-t0.tv_usec);
+	if (verbose) printf("serial insertions: blocksize=%d %d insertions in %.3f seconds, %.2f insertions/second\n", nodesize, count, tdiff, count/tdiff);
+    }
+}
+
+static void brt_blackbox_test (void) {
+    if (verbose) printf("test3 slow\n");
+    toku_memory_check=0;
+    test3(2048, 1<<15, 1);
+    if (verbose) printf("test3 fast\n");
+
+    if (verbose) toku_pma_show_stats();
+
+    test3(1<<15, 1024, 1);
+    if (verbose) printf("test3 fast\n");
+
+    test3(1<<18, 1<<20, 0);
+
+    toku_memory_check = 1;
+
+//    test3(1<<19, 1<<20, 0);
+
+//    test3(1<<20, 1<<20, 0);
+
+//    test3(1<<20, 1<<21, 0);
+
+//    test3(1<<20, 1<<22, 0);
+
+}
+
+int main (int argc , const char *argv[]) {
+    default_parse_args(argc, argv);
+
+    brt_blackbox_test();
+    toku_malloc_cleanup();
+    if (verbose) printf("test ok\n");
+    return 0;
+}

newbrt/brt.c

@@ -49,13 +49,15 @@ void toku_brtnode_free (BRTNODE *nodep) {
     //printf("%s:%d %p->mdict[0]=%p\n", __FILE__, __LINE__, node, node->mdicts[0]);
     if (node->height>0) {
 	for (i=0; i<node->u.n.n_children-1; i++) {
-	    toku_free((void*)node->u.n.childkeys[i]);
+	    toku_free(node->u.n.childkeys[i]);
 	}
 	for (i=0; i<node->u.n.n_children; i++) {
 	    if (BNC_BUFFER(node,i)) {
 		toku_fifo_free(&BNC_BUFFER(node,i));
 	    }
 	}
+	toku_free(node->u.n.childkeys);
+	toku_free(node->u.n.childinfos);
     } else {
 	if (node->u.l.buffer) // The buffer may have been freed already, in some cases.
 	    toku_pma_free(&node->u.l.buffer);
@@ -246,7 +248,6 @@ int malloc_diskblock (DISKOFF *res, BRT brt, int size, TOKULOGGER logger) {
 }
 
 static void initialize_brtnode (BRT t, BRTNODE n, DISKOFF nodename, int height) {
-    int i;
     n->tag = TYP_BRTNODE;
     n->nodesize = t->h->nodesize;
     n->flags = t->h->flags;
@@ -261,18 +262,10 @@ static void initialize_brtnode (BRT t, BRTNODE n, DISKOFF nodename, int height)
     assert(height>=0);
     if (height>0) {
 	n->u.n.n_children   = 0;
-	for (i=0; i<TREE_FANOUT; i++) {
-//	    n->u.n.childkeys[i] = 0;
-//	    n->u.n.childkeylens[i] = 0;
-	}
 	n->u.n.totalchildkeylens = 0;
-	for (i=0; i<TREE_FANOUT+1; i++) {
-	    BNC_SUBTREE_FINGERPRINT(n, i) = 0;
-//	    n->u.n.children[i] = 0;
-//	    n->u.n.buffers[i] = 0;
-	    BNC_NBYTESINBUF(n,i) = 0;
-	}
 	n->u.n.n_bytes_in_buffers = 0;
+	n->u.n.childinfos=0;
+	n->u.n.childkeys=0;
     } else {
 	int r = toku_pma_create(&n->u.l.buffer, t->compare_fun, t->db, toku_cachefile_filenum(t->cf), n->nodesize);
         assert(r==0);
@@ -285,7 +278,7 @@ static void initialize_brtnode (BRT t, BRTNODE n, DISKOFF nodename, int height)
     }
 }
 
-void toku_create_new_brtnode (BRT t, BRTNODE *result, int height, TOKULOGGER logger) {
+int toku_create_new_brtnode (BRT t, BRTNODE *result, int height, TOKULOGGER logger) {
     TAGMALLOC(BRTNODE, n);
     int r;
     DISKOFF name;
@@ -305,6 +298,7 @@ void toku_create_new_brtnode (BRT t, BRTNODE *result, int height, TOKULOGGER log
     r=toku_log_newbrtnode(logger, toku_cachefile_filenum(t->cf), n->thisnodename, height, n->nodesize, (t->flags&TOKU_DB_DUPSORT)!=0, n->rand4fingerprint);
     assert(r==0);
     toku_update_brtnode_loggerlsn(n, logger);
+    return 0;
 }
 
 static int insert_to_buffer_in_nonleaf (BRTNODE node, int childnum, DBT *k, DBT *v, int type, TXNID xid) {
@@ -360,6 +354,8 @@ static int brt_nonleaf_split (BRT t, BRTNODE node, BRTNODE *nodea, BRTNODE *node
     assert(node->u.n.n_children>=2); // Otherwise, how do we split?  We need at least two children to split. */
     assert(t->h->nodesize>=node->nodesize); /* otherwise we might be in trouble because the nodesize shrank. */
     toku_create_new_brtnode(t, &B, node->height, logger);
+    MALLOC_N(n_children_in_b+1, B->u.n.childinfos);
+    MALLOC_N(n_children_in_b, B->u.n.childkeys);
     B->u.n.n_children   =n_children_in_b;
     //printf("%s:%d %p (%lld) becomes %p and %p\n", __FILE__, __LINE__, node, node->thisnodename, A, B);
     //printf("%s:%d A is at %lld\n", __FILE__, __LINE__, A->thisnodename);
@@ -372,6 +368,8 @@ static int brt_nonleaf_split (BRT t, BRTNODE node, BRTNODE *nodea, BRTNODE *node
 	for (i=0; i<n_children_in_b; i++) {
 	    int r = toku_fifo_create(&BNC_BUFFER(B,i));
 	    if (r!=0) return r;
+	    BNC_NBYTESINBUF(B,i)=0;
+	    BNC_SUBTREE_FINGERPRINT(B,i)=0;
 	}
 
 	for (i=n_children_in_a; i<old_n_children; i++) {
@@ -453,7 +451,9 @@ static int brt_nonleaf_split (BRT t, BRTNODE node, BRTNODE *nodea, BRTNODE *node
 	splitk->data = (void*)(node->u.n.childkeys[n_children_in_a-1]);
 	splitk->size = toku_brt_pivot_key_len(t, node->u.n.childkeys[n_children_in_a-1]);
 	node->u.n.totalchildkeylens -= toku_brt_pivot_key_len(t, node->u.n.childkeys[n_children_in_a-1]);
-	node->u.n.childkeys[n_children_in_a-1]=0;
+
+	REALLOC_N(n_children_in_a+1,   node->u.n.childinfos);
+	REALLOC_N(n_children_in_a, node->u.n.childkeys);
 
 	verify_local_fingerprint_nonleaf(node);
 	verify_local_fingerprint_nonleaf(B);
@@ -618,6 +618,8 @@ static int handle_split_of_child (BRT t, BRTNODE node, int childnum,
 
     //verify_local_fingerprint_nonleaf(node);
 
+    REALLOC_N(node->u.n.n_children+2, node->u.n.childinfos);
+    REALLOC_N(node->u.n.n_children+1, node->u.n.childkeys);
     // Slide the children over.
     for (cnum=node->u.n.n_children; cnum>childnum+1; cnum--) {
 	node->u.n.childinfos[cnum] = node->u.n.childinfos[cnum-1];
@@ -625,6 +627,8 @@ static int handle_split_of_child (BRT t, BRTNODE node, int childnum,
     r = toku_log_addchild(logger, toku_cachefile_filenum(t->cf), node->thisnodename, childnum+1, childb->thisnodename, 0);
     assert(BNC_DISKOFF(node, childnum)==childa->thisnodename);
     BNC_DISKOFF(node, childnum+1) = childb->thisnodename;
+    BNC_SUBTREE_FINGERPRINT(node, childnum)=0;
+    BNC_SUBTREE_FINGERPRINT(node, childnum+1)=0;
     fixup_child_fingerprint(node, childnum,   childa, t, logger);
     fixup_child_fingerprint(node, childnum+1, childb, t, logger);
     r=toku_fifo_create(&BNC_BUFFER(node,childnum));   assert(r==0); // ??? SHould handle this error case
@@ -1625,6 +1629,8 @@ static int brt_init_new_root(BRT brt, BRTNODE nodea, BRTNODE nodeb, DBT splitk,
     initialize_brtnode (brt, newroot, newroot_diskoff, new_height);
     //printf("new_root %lld %d %lld %lld\n", newroot_diskoff, newroot->height, nodea->thisnodename, nodeb->thisnodename);
     newroot->u.n.n_children=2;
+    MALLOC_N(3, newroot->u.n.childinfos);
+    MALLOC_N(2, newroot->u.n.childkeys);
     //printf("%s:%d Splitkey=%p %s\n", __FILE__, __LINE__, splitkey, splitkey);
     newroot->u.n.childkeys[0] = splitk.data;
     newroot->u.n.totalchildkeylens=splitk.size;
@@ -1632,6 +1638,10 @@ static int brt_init_new_root(BRT brt, BRTNODE nodea, BRTNODE nodeb, DBT splitk,
     BNC_DISKOFF(newroot,1)=nodeb->thisnodename;
     r=toku_fifo_create(&BNC_BUFFER(newroot,0)); if (r!=0) return r;
     r=toku_fifo_create(&BNC_BUFFER(newroot,1)); if (r!=0) return r;
+    BNC_NBYTESINBUF(newroot, 0)=0;
+    BNC_NBYTESINBUF(newroot, 1)=0;
+    BNC_SUBTREE_FINGERPRINT(newroot, 0)=0; 
+    BNC_SUBTREE_FINGERPRINT(newroot, 1)=0; 
     toku_verify_counts(newroot);
     //verify_local_fingerprint_nonleaf(nodea);
     //verify_local_fingerprint_nonleaf(nodeb);

newbrt/memory.h

@@ -42,6 +42,8 @@ void *toku_realloc(void *, size_t size);
  */
 #define MALLOC_N(n,v) v = toku_malloc((n)*sizeof(*v))
 
+#define REALLOC_N(n,v) v = toku_realloc(v, (n)*sizeof(*v))
+
 /* If you have a type such as 
  *    struct pma *PMA;
  * and you define a corresponding int constant, such as