Things compile again after fiddling with the compare function. There's a malloc bug, however

git-svn-id: file:///svn/tokudb@32 c7de825b-a66e-492c-adef-691d508d4ae1

newbrt/Makefile

 # GCOV_FLAGS = -fprofile-arcs -ftest-coverage
 #PROF_FLAGS = -pg
-#OPTFLAGS = -O2
+OPTFLAGS = -O2
 CFLAGS = -Wall -W $(OPTFLAGS) -g $(GCOV_FLAGS) $(PROF_FLAGS) -Werror -fPIC
 LDFLAGS = $(OPTFLAGS) -g $(GCOV_FLAGS) $(PROF_FLAGS) 
 
@@ -24,7 +24,7 @@ pma.o: pma.h yerror.h pma-internal.h memory.h key.h ybt.h brttypes.h  ../include
 ybt.o: ybt.h brttypes.h
 ybt-test: ybt-test.o ybt.o memory.o
 cachetable.o: cachetable.h
-brt-test: brt.o hashtable.o pma.o memory.o brt-serialize.o cachetable.o header-io.o ybt.o key.o
+brt-test: ybt.o brt.o hashtable.o pma.o memory.o brt-serialize.o cachetable.o header-io.o ybt.o key.o
 brt-test.o brt.o: brt.h cachetable.h brttypes.h
 brt-serialize-test.o: pma.h yerror.h brt.h memory.h hashtable.h brttypes.h brt-internal.h
 brt.o: brt.h mdict.h pma.h brttypes.h memory.h brt-internal.h cachetable.h

newbrt/brt-internal.h

@@ -63,13 +63,16 @@ struct brt {
     struct brt_header *h;
 
     BRT_CURSOR cursors_head, cursors_tail;
+
+    int (*compare_fun)(DB*,DBT*,DBT*);
+
+    void *skey,*sval; /* Used for DBT return values. */
 };
 
 /* serialization code */
 void serialize_brtnode_to(int fd, diskoff off, diskoff size, BRTNODE node);
 int deserialize_brtnode_from (int fd, diskoff off, BRTNODE *brtnode, int nodesize);
 unsigned int serialize_brtnode_size(BRTNODE node); /* How much space will it take? */
-unsigned int brtnode_which_child (BRTNODE node, bytevec key, ITEMLEN keylen);
 int keycompare (bytevec key1, ITEMLEN key1len, bytevec key2, ITEMLEN key2len);
 
 void verify_counts(BRTNODE);
@@ -77,7 +80,7 @@ void verify_counts(BRTNODE);
 int serialize_brt_header_to (int fd, struct brt_header *h);
 int deserialize_brtheader_from (int fd, diskoff off, struct brt_header **brth);
 
-static inline int brtnode_n_hashtables(BRTNODE node) { if (node->height==0) return 1; else return node->u.n.n_children; }
+//static inline int brtnode_n_hashtables(BRTNODE node) { if (node->height==0) return 1; else return node->u.n.n_children; }
 
 //int write_brt_header (int fd, struct brt_header *header);
 

newbrt/brt-serialize.c

@@ -4,6 +4,7 @@
 #include "memory.h"
 //#include "pma.h"
 #include "brt-internal.h"
+#include "key.h"
 
 #include <assert.h>
 #include <unistd.h>
@@ -87,7 +88,7 @@ static unsigned int serialize_brtnode_size_slow(BRTNODE node) {
 	for (i=0; i<node->u.n.n_children; i++) {
 	    size+=8;
 	}
-	int n_hashtables = brtnode_n_hashtables(node);
+	int n_hashtables = node->u.n.n_bytes_in_hashtables;
 	size+=4; /* n_entries */
 	for (i=0; i< n_hashtables; i++) {
 	    HASHTABLE_ITERATE(node->u.n.htables[i],
@@ -118,8 +119,7 @@ unsigned int serialize_brtnode_size (BRTNODE node) {
 	result+=4; /* n_children */
 	result+=4*(node->u.n.n_children-1); /* key lengths */
 	result+=node->u.n.totalchildkeylens; /* the lengths of the pivot keys, without their key lengths. */
-	result+=8*(node->u.n.n_children); /* child offsets. */
-	result+=4; /* n_entries in hash table. */
+	result+=(8+4)*(node->u.n.n_children); /* For each child, a child offset and a count for the number of hash table entries. */
 	result+=node->u.n.n_bytes_in_hashtables;
     } else {
 	result+=4; /* n_entries in buffer table. */
@@ -158,15 +158,10 @@ void serialize_brtnode_to(int fd, diskoff off, diskoff size, BRTNODE node) {
 	}
 
 	{
-	    int n_entries=0;
-	    int n_hash_tables = brtnode_n_hashtables(node);
+	    int n_hash_tables = node->u.n.n_children;
 	    for (i=0; i< n_hash_tables; i++) {
 		//printf("%s:%d p%d=%p n_entries=%d\n", __FILE__, __LINE__, i, node->mdicts[i], mdict_n_entries(node->mdicts[i]));
-		n_entries += hashtable_n_entries(node->u.n.htables[i]);
-	    }
-	    //printf("%s:%d n_entries=%d\n", __FILE__, __LINE__, n_entries);
-	    wbuf_int(&w, n_entries);
-	    for (i=0; i< n_hash_tables; i++) {
+		wbuf_int(&w, hashtable_n_entries(node->u.n.htables[i]));
 		HASHTABLE_ITERATE(node->u.n.htables[i], key, keylen, data, datalen,
 				  (wbuf_bytes(&w, key, keylen),
 				   wbuf_bytes(&w, data, datalen)));
@@ -206,7 +201,7 @@ int deserialize_brtnode_from (int fd, diskoff off, BRTNODE *brtnode, int nodesiz
     }
     {
 	uint32_t datasize_n;
-	int r = pread(fd, &datasize_n, sizeof(datasize_n), off);
+	r = pread(fd, &datasize_n, sizeof(datasize_n), off);
 	//printf("%s:%d r=%d the datasize=%d\n", __FILE__, __LINE__, r, ntohl(datasize_n));
 	if (r!=sizeof(datasize_n)) {
 	    if (r==-1) r=errno;
@@ -261,42 +256,43 @@ int deserialize_brtnode_from (int fd, diskoff off, BRTNODE *brtnode, int nodesiz
 	    result->u.n.n_bytes_in_hashtable[i] = 0;
 	}
 	result->u.n.n_bytes_in_hashtables = 0; 
-	for (i=0; i<brtnode_n_hashtables(result); i++) {
+	for (i=0; i<result->u.n.n_children; i++) {
 	    int r=hashtable_create(&result->u.n.htables[i]);
 	    if (r!=0) {
 		int j;
-		if (0) { died_12: j=brtnode_n_hashtables(result); }
+		if (0) { died_12: j=result->u.n.n_bytes_in_hashtables; }
 		for (j=0; j<i; j++) hashtable_free(&result->u.n.htables[j]);
 		goto died1;
 	    }
 	}
 	{
-	    int n_in_hash = rbuf_int(&rc);
-	    //printf("%d in hash\n", n_in_hash);
-
-	    for (i=0; i<n_in_hash; i++) {
-		int childnum, diff;
-		bytevec key; ITEMLEN keylen; 
-		bytevec val; ITEMLEN vallen;
-		verify_counts(result);
-		rbuf_bytes(&rc, &key, &keylen); /* Returns a pointer into the rbuf. */
-		rbuf_bytes(&rc, &val, &vallen);
-		//printf("Found %s,%s\n", key, val);
-		childnum = brtnode_which_child(result, key, keylen);
-		{
-		    int r=hash_insert(result->u.n.htables[childnum], key, keylen, val, vallen); /* Copies the data into the hash table. */
-		    if (r!=0) { goto died_12; }
+	    int cnum;
+	    for (cnum=0; cnum<result->u.n.n_children; cnum++) {
+		int n_in_this_hash = rbuf_int(&rc);
+		//printf("%d in hash\n", n_in_hash);
+		for (i=0; i<n_in_this_hash; i++) {
+		    int diff;
+		    bytevec key; ITEMLEN keylen; 
+		    bytevec val; ITEMLEN vallen;
+		    verify_counts(result);
+		    rbuf_bytes(&rc, &key, &keylen); /* Returns a pointer into the rbuf. */
+		    rbuf_bytes(&rc, &val, &vallen);
+		    //printf("Found %s,%s\n", key, val);
+		    {
+			int r=hash_insert(result->u.n.htables[cnum], key, keylen, val, vallen); /* Copies the data into the hash table. */
+			if (r!=0) { goto died_12; }
+		    }
+		    diff =  keylen + vallen + KEY_VALUE_OVERHEAD;
+		    result->u.n.n_bytes_in_hashtables += diff;
+		    result->u.n.n_bytes_in_hashtable[cnum] += diff;
+		    //printf("Inserted\n");
 		}
-		diff =  keylen + vallen + KEY_VALUE_OVERHEAD;
-		result->u.n.n_bytes_in_hashtables += diff;
-		result->u.n.n_bytes_in_hashtable[childnum] += diff;
-	    //printf("Inserted\n");
 	    }
 	}
     } else {
 	int n_in_buf = rbuf_int(&rc);
 	result->u.l.n_bytes_in_buffer = 0;
-	int r=pma_create(&result->u.l.buffer);
+	int r=pma_create(&result->u.l.buffer, default_compare_fun);
 	if (r!=0) {
 	    if (0) { died_21: pma_free(&result->u.l.buffer); }
 	    goto died1;
@@ -309,7 +305,8 @@ int deserialize_brtnode_from (int fd, diskoff off, BRTNODE *brtnode, int nodesiz
 	    rbuf_bytes(&rc, &key, &keylen); /* Returns a pointer into the rbuf. */
 	    rbuf_bytes(&rc, &val, &vallen);
 	    {
-		int r = pma_insert(result->u.l.buffer, key, keylen, val, vallen);
+		DBT k,v;
+		int r = pma_insert(result->u.l.buffer, fill_dbt(&k, key, keylen), fill_dbt(&v, val, vallen), 0);
 		if (r!=0) goto died_21;
 	    }
 	    result->u.l.n_bytes_in_buffer += keylen + vallen + KEY_VALUE_OVERHEAD;
@@ -322,17 +319,6 @@ int deserialize_brtnode_from (int fd, diskoff off, BRTNODE *brtnode, int nodesiz
     return 0;
 }
 
-unsigned int brtnode_which_child (BRTNODE node, bytevec key, ITEMLEN keylen) {
-    int i;
-    assert(node->height>0);
-    for (i=0; i<node->u.n.n_children-1; i++) {
-	if (keycompare(key, keylen, node->u.n.childkeys[i], node->u.n.childkeylens[i])<=0) {
-	    return i;
-	}
-    }
-    return node->u.n.n_children-1;
-}
-
 void verify_counts (BRTNODE node) {
     if (node->height==0) {
 	assert(node->u.l.buffer);

newbrt/brt.h

@@ -9,11 +9,11 @@
 #include "../include/ydb-constants.h"
 #include "cachetable.h"
 typedef struct brt *BRT;
-int open_brt (const char *fname, const char *dbname, int is_create, BRT *, int nodesize, CACHETABLE);
+int open_brt (const char *fname, const char *dbname, int is_create, BRT *, int nodesize, CACHETABLE, int(*)(DB*,DBT*,DBT*));
 //int brt_create (BRT **, int nodesize, int n_nodes_in_cache); /* the nodesize and n_nodes in cache really should be separately configured. */
 //int brt_open (BRT *, char *fname, char *dbname);
-int brt_insert (BRT brt, bytevec key, ITEMLEN keylen, bytevec val, ITEMLEN vallen);
-int brt_lookup (BRT brt, bytevec key, ITEMLEN keylen, bytevec*val, ITEMLEN *vallen);
+int brt_insert (BRT brt, DBT *k, DBT *v, DB*db);
+int brt_lookup (BRT brt, DBT *k, DBT *v, DB*db);
 int close_brt (BRT);
 int dump_brt (BRT brt);
 void brt_fsync (BRT); /* fsync, but don't clear the caches. */

newbrt/key.h

+#include "ybt.h"
 #include "brttypes.h"
 
 int keycompare (bytevec key1, ITEMLEN key1len, bytevec key2, ITEMLEN key2len);

newbrt/pma-test.c

@@ -268,7 +268,7 @@ static void test_find_insert (void) {
     r=pma_insert(pma, fill_dbt(&k, "aaa", 3), fill_dbt(&v, "aaadata", 7), 0);
     assert(r==BRT_OK);
 
-    ybt_init(&v);
+    init_dbt(&v);
     r=pma_lookup(pma, fill_dbt(&k, "aaa", 3), &v, 0);
     assert(r==BRT_OK);
     assert(v.size==7);
@@ -278,12 +278,12 @@ static void test_find_insert (void) {
     r=pma_insert(pma, fill_dbt(&k, "bbb", 4), fill_dbt(&v, "bbbdata", 8), 0);
     assert(r==BRT_OK);
 
-    ybt_init(&v);
+    init_dbt(&v);
     r=pma_lookup(pma, fill_dbt(&k, "aaa", 3), &v, 0);
     assert(r==BRT_OK);
     assert(keycompare(v.data,v.size,"aaadata", 7)==0);
 
-    ybt_init(&v);
+    init_dbt(&v);
     r=pma_lookup(pma, fill_dbt(&k, "bbb", 4), &v, 0);
     assert(r==BRT_OK);
     assert(keycompare(v.data,v.size,"bbbdata", 8)==0);
@@ -415,8 +415,8 @@ void test_pma_cursor_2 (void) {
     PMA_CURSOR c=0;
     int r;
     DBT key,val;
-    ybt_init(&key); key.flags=DB_DBT_REALLOC;
-    ybt_init(&val); val.flags=DB_DBT_REALLOC;
+    init_dbt(&key); key.flags=DB_DBT_REALLOC;
+    init_dbt(&val); val.flags=DB_DBT_REALLOC;
     r=pma_create(&pma, default_compare_fun); assert(r==0);
     r=pma_cursor(pma, &c); assert(r==0); assert(c!=0);
     r=pma_cursor_set_position_last(c); assert(r==DB_NOTFOUND);
@@ -434,8 +434,8 @@ void test_pma_cursor_3 (void) {
     r=pma_insert(pma, fill_dbt(&k, "x", 2),  fill_dbt(&v, "xx", 3), 0); assert(r==BRT_OK);
     r=pma_insert(pma, fill_dbt(&k, "m", 2),  fill_dbt(&v, "mm", 3), 0); assert(r==BRT_OK);
     r=pma_insert(pma, fill_dbt(&k, "aa", 3), fill_dbt(&v,"a", 2),   0); assert(r==BRT_OK);
-    ybt_init(&key); key.flags=DB_DBT_REALLOC;
-    ybt_init(&val); val.flags=DB_DBT_REALLOC;
+    init_dbt(&key); key.flags=DB_DBT_REALLOC;
+    init_dbt(&val); val.flags=DB_DBT_REALLOC;
     r=pma_cursor(pma, &c); assert(r==0); assert(c!=0);
 
     r=pma_cursor_set_position_first(c); assert(r==0);
@@ -508,8 +508,8 @@ void test_pma_compare_fun (int wrong_endian_p) {
     r = pma_insert(pma, fill_dbt(&k, "00", 3), fill_dbt(&v, "00v", 4), 0); assert(r==BRT_OK);
     r = pma_insert(pma, fill_dbt(&k, "01", 3), fill_dbt(&v, "01v", 4), 0); assert(r==BRT_OK);
     r = pma_insert(pma, fill_dbt(&k, "11", 3), fill_dbt(&v, "11v", 4), 0); assert(r==BRT_OK);
-    ybt_init(&key); key.flags=DB_DBT_REALLOC;
-    ybt_init(&val); val.flags=DB_DBT_REALLOC;
+    init_dbt(&key); key.flags=DB_DBT_REALLOC;
+    init_dbt(&val); val.flags=DB_DBT_REALLOC;
     r=pma_cursor(pma, &c); assert(r==0); assert(c!=0);
     
     for (i=0; i<4; i++) {

newbrt/ybt-test.c

@@ -8,8 +8,8 @@
 void ybt_test0 (void) {
     void *v0=0,*v1=0;
     DBT  t0,t1;
-    ybt_init(&t0);
-    ybt_init(&t1);
+    init_dbt(&t0);
+    init_dbt(&t1);
     ybt_set_value(&t0, "hello", 6, &v0);
     ybt_set_value(&t1, "foo",   4, &v1);
     assert(t0.size==6);
@@ -25,7 +25,7 @@ void ybt_test0 (void) {
     memory_check_all_free();
 
     /* See if we can probe to find out how big something is by setting ulen=0 with YBT_USERMEM */
-    ybt_init(&t0);
+    init_dbt(&t0);
     t0.flags = DB_DBT_USERMEM;
     t0.ulen  = 0;
     ybt_set_value(&t0, "hello", 6, 0);
@@ -33,7 +33,7 @@ void ybt_test0 (void) {
     assert(t0.size==6);
 
     /* Check realloc. */
-    ybt_init(&t0);
+    init_dbt(&t0);
     t0.flags = DB_DBT_REALLOC;
     v0 = 0;
     ybt_set_value(&t0, "internationalization", 21, &v0);

newbrt/ybt.c

@@ -4,13 +4,13 @@
 #include <errno.h>
 #include <string.h>
 
-int ybt_init (DBT *ybt) {
+DBT *init_dbt (DBT *ybt) {
     memset(ybt, 0, sizeof(*ybt));
-    return 0;
+    return ybt;
 }
 
 DBT *fill_dbt(DBT *dbt, bytevec k, ITEMLEN len) {
-    ybt_init(dbt);
+    init_dbt(dbt);
     dbt->size=len;
     dbt->data=(char*)k;
     return dbt;

newbrt/ybt.h

@@ -6,7 +6,7 @@
 #include "../include/db.h"
 
 
-int ybt_init (DBT *);
+DBT* init_dbt (DBT *);
 DBT *fill_dbt(DBT *dbt, bytevec k, ITEMLEN len);
 int ybt_set_value (DBT *, bytevec val, ITEMLEN vallen, void **staticptrp);