Change to implement delete by key. A BRT_DELETE command is injected into the

tree.  It replaces any INSERT or DELETE command with the same key at all
interior nodes.  It is translated into a PMA delete operation at a leaf
node.  

The database file format was changed for the contents of interior node
buffers to include a 1 byte type field.  There are currently 2 types:
BRT_INSERT and BRT_DELETE.




git-svn-id: file:///svn/tokudb@278 c7de825b-a66e-492c-adef-691d508d4ae1

newbrt/brt-internal.h

@@ -11,9 +11,10 @@ typedef long long diskoff;  /* Offset in a disk. -1 is the NULL pointer. */
 #endif
 enum { TREE_FANOUT = BRT_FANOUT }; //, NODESIZE=1<<20 };
 enum { KEY_VALUE_OVERHEAD = 8 }; /* Must store the two lengths. */
+enum { BRT_CMD_OVERHEAD = 1 };
+
 struct nodeheader_in_file {
     int n_in_buffer;
-    
 };
 enum { BUFFER_HEADER_SIZE = (4 // height//
 			     + 4 // n_children
@@ -140,3 +141,23 @@ void brt_update_cursors_new_root(BRT t, BRTNODE newroot, BRTNODE left, BRTNODE r
 void brt_update_cursors_leaf_split(BRT t, BRTNODE oldnode, BRTNODE left, BRTNODE right);
 void brt_update_cursors_nonleaf_expand(BRT t, BRTNODE oldnode, int childnum, BRTNODE left, BRTNODE right);
 void brt_update_cursors_nonleaf_split(BRT t, BRTNODE oldnode, BRTNODE left, BRTNODE right);
+
+enum brt_cmd_type {
+    BRT_NONE = 0,
+    BRT_INSERT = 1,
+    BRT_DELETE = 2,
+};
+
+struct brt_cmd {
+    enum brt_cmd_type type;
+    union {
+        /* insert or delete */
+        struct brt_cmd_insert_delete {
+            DBT *key;
+            DBT *val;
+            DB *db;
+        } id;
+    } u;
+};
+typedef struct brt_cmd BRT_CMD;
+

newbrt/brt-serialize-test.c

@@ -27,10 +27,10 @@ void test_serialize(void) {
     sn.u.n.children[1] = sn.nodesize*35;
     r = toku_hashtable_create(&sn.u.n.htables[0]); assert(r==0);
     r = toku_hashtable_create(&sn.u.n.htables[1]); assert(r==0);
-    r = toku_hash_insert(sn.u.n.htables[0], "a", 2, "aval", 5); assert(r==0);
-    r = toku_hash_insert(sn.u.n.htables[0], "b", 2, "bval", 5); assert(r==0);
-    r = toku_hash_insert(sn.u.n.htables[1], "x", 2, "xval", 5); assert(r==0);
-    sn.u.n.n_bytes_in_hashtables = 3*(KEY_VALUE_OVERHEAD+2+5);
+    r = toku_hash_insert(sn.u.n.htables[0], "a", 2, "aval", 5, BRT_NONE); assert(r==0);
+    r = toku_hash_insert(sn.u.n.htables[0], "b", 2, "bval", 5, BRT_NONE); assert(r==0);
+    r = toku_hash_insert(sn.u.n.htables[1], "x", 2, "xval", 5, BRT_NONE); assert(r==0);
+    sn.u.n.n_bytes_in_hashtables = 3*(BRT_CMD_OVERHEAD+KEY_VALUE_OVERHEAD+2+5);
 
     deserialize_brtnode_from(fd, nodesize*20, &dn, nodesize);
 
@@ -46,24 +46,26 @@ void test_serialize(void) {
     assert(dn->u.n.children[0]==nodesize*30);
     assert(dn->u.n.children[1]==nodesize*35);
     {
-	bytevec data; ITEMLEN datalen;
-	int r = toku_hash_find(dn->u.n.htables[0], "a", 2, &data, &datalen);
+	bytevec data; ITEMLEN datalen; int type;
+	int r = toku_hash_find(dn->u.n.htables[0], "a", 2, &data, &datalen, &type);
 	assert(r==0);
 	assert(strcmp(data,"aval")==0);
 	assert(datalen==5);
+        assert(type == BRT_NONE);
 
-	r=toku_hash_find(dn->u.n.htables[0], "b", 2, &data, &datalen);
+	r=toku_hash_find(dn->u.n.htables[0], "b", 2, &data, &datalen, &type);
 	assert(r==0);
 	assert(strcmp(data,"bval")==0);
 	assert(datalen==5);
+        assert(type == BRT_NONE);
 
-	r=toku_hash_find(dn->u.n.htables[1], "x", 2, &data, &datalen);
+	r=toku_hash_find(dn->u.n.htables[1], "x", 2, &data, &datalen, &type);
 	assert(r==0);
 	assert(strcmp(data,"xval")==0);
 	assert(datalen==5);
-
+        assert(type == BRT_NONE);
     }
-    brtnode_free(&dn);
+    //    brtnode_free(&dn);
 
     toku_free(hello_string);
     toku_hashtable_free(&sn.u.n.htables[0]);

newbrt/brt-serialize.c

@@ -34,7 +34,8 @@ static unsigned int serialize_brtnode_size_slow(BRTNODE node) {
 	    HASHTABLE_ITERATE(node->u.n.htables[i],
 			      key __attribute__((__unused__)), keylen,
 			      data __attribute__((__unused__)), datalen,
-			      (hsize+=8+keylen+datalen));
+                              type __attribute__((__unused__)),
+			      (hsize+=BRT_CMD_OVERHEAD+KEY_VALUE_OVERHEAD+keylen+datalen));
 	}
 	assert(hsize==node->u.n.n_bytes_in_hashtables);
 	assert(csize==node->u.n.totalchildkeylens);
@@ -44,12 +45,11 @@ static unsigned int serialize_brtnode_size_slow(BRTNODE node) {
 	PMA_ITERATE(node->u.l.buffer,
 		    key __attribute__((__unused__)), keylen,
 		    data __attribute__((__unused__)), datalen,
-		    (hsize+=8+keylen+datalen));
+		    (hsize+=KEY_VALUE_OVERHEAD+keylen+datalen));
 	assert(hsize==node->u.l.n_bytes_in_buffer);
 	hsize+=4; /* add n entries in buffer table. */
 	return size+hsize;
     }
-
 }
 
 unsigned int serialize_brtnode_size (BRTNODE node) {
@@ -101,8 +101,8 @@ int serialize_brtnode_to(int fd, diskoff off, diskoff size, BRTNODE node) {
 	    for (i=0; i< n_hash_tables; i++) {
 		//printf("%s:%d p%d=%p n_entries=%d\n", __FILE__, __LINE__, i, node->mdicts[i], mdict_n_entries(node->mdicts[i]));
 		wbuf_int(&w, toku_hashtable_n_entries(node->u.n.htables[i]));
-		HASHTABLE_ITERATE(node->u.n.htables[i], key, keylen, data, datalen,
-				  (wbuf_bytes(&w, key, keylen),
+		HASHTABLE_ITERATE(node->u.n.htables[i], key, keylen, data, datalen, type,
+				  (wbuf_char(&w, type), wbuf_bytes(&w, key, keylen),
 				   wbuf_bytes(&w, data, datalen)));
 	    }
 	}
@@ -220,17 +220,19 @@ int deserialize_brtnode_from (int fd, diskoff off, BRTNODE *brtnode, int nodesiz
 		//printf("%d in hash\n", n_in_hash);
 		for (i=0; i<n_in_this_hash; i++) {
 		    int diff;
+                    int type;
 		    bytevec key; ITEMLEN keylen; 
 		    bytevec val; ITEMLEN vallen;
 		    verify_counts(result);
+                    type = rbuf_char(&rc);
 		    rbuf_bytes(&rc, &key, &keylen); /* Returns a pointer into the rbuf. */
 		    rbuf_bytes(&rc, &val, &vallen);
-		    //printf("Found %s,%s\n", key, val);
+		    //printf("Found %s,%s\n", (char*)key, (char*)val);
 		    {
-			int r=toku_hash_insert(result->u.n.htables[cnum], key, keylen, val, vallen); /* Copies the data into the hash table. */
+			int r=toku_hash_insert(result->u.n.htables[cnum], key, keylen, val, vallen, type); /* Copies the data into the hash table. */
 			if (r!=0) { goto died_12; }
 		    }
-		    diff =  keylen + vallen + KEY_VALUE_OVERHEAD;
+		    diff =  keylen + vallen + KEY_VALUE_OVERHEAD + BRT_CMD_OVERHEAD;
 		    result->u.n.n_bytes_in_hashtables += diff;
 		    result->u.n.n_bytes_in_hashtable[cnum] += diff;
 		    //printf("Inserted\n");

newbrt/brt-test.c

@@ -1328,8 +1328,264 @@ void test_brt_cursor() {
     }
 }
 
+void test_large_kv(int bsize, int ksize, int vsize) {
+    BRT t;
+    int r;
+    CACHETABLE ct;
+    char fname[]="testbrt.brt";
+
+    printf("test_large_kv: %d %d %d\n", bsize, ksize, vsize);
+
+    r = brt_create_cachetable(&ct, 0);
+    assert(r==0);
+    unlink(fname);
+    r = open_brt(fname, 0, 1, &t, bsize, ct, default_compare_fun);
+    assert(r==0);
+
+    DBT key, val;
+    char *k, *v;
+    k = toku_malloc(ksize); assert(k); memset(k, 0, ksize);
+    v = toku_malloc(vsize); assert(v); memset(v, 0, vsize);
+    fill_dbt(&key, k, ksize);
+    fill_dbt(&val, v, vsize);
+
+    r = brt_insert(t, &key, &val, 0);
+    assert(r == 0);
+
+    toku_free(k);
+    toku_free(v);
+
+    r = close_brt(t);              assert(r==0);
+    r = cachetable_close(&ct);     assert(r==0);
+}
+
+/*
+ * test the key and value limits
+ * the current implementation crashes when kvsize == bsize/2 rather than fails
+ */
+void test_brt_limits() {
+    int bsize = 1024;
+    int kvsize = 4;
+    while (kvsize < bsize/2) {
+        test_large_kv(bsize, kvsize, kvsize);        memory_check_all_free();
+        kvsize *= 2;
+    }
+}
+
+/*
+ * verify that a delete on an empty tree fails
+ */
+void test_brt_delete_empty() {
+    printf("test_brt_delete_empty\n");
+
+    BRT t;
+    int r;
+    CACHETABLE ct;
+    char fname[]="testbrt.brt";
+
+    r = brt_create_cachetable(&ct, 0);
+    assert(r==0);
+    unlink(fname);
+    r = open_brt(fname, 0, 1, &t, 4096, ct, default_compare_fun);
+    assert(r==0);
+
+    DBT key;
+    int k = 1;
+    fill_dbt(&key, &k, sizeof k);
+    r = brt_delete(t, &key, 0);
+    assert(r != 0);
+
+    r = close_brt(t);              assert(r==0);
+    r = cachetable_close(&ct);     assert(r==0);
+}
+
+/*
+ * insert n keys, delete all n keys, verify that lookups for all the keys fail,
+ * verify that a cursor walk of the tree finds nothing
+ */
+void test_brt_delete_present(int n) {
+    printf("test_brt_delete_present:%d\n", n);
+
+    BRT t;
+    int r;
+    CACHETABLE ct;
+    char fname[]="testbrt.brt";
+    int i;
+
+    r = brt_create_cachetable(&ct, 0);
+    assert(r==0);
+    unlink(fname);
+    r = open_brt(fname, 0, 1, &t, 4096, ct, default_compare_fun);
+    assert(r==0);
+
+    DBT key, val;
+    int k, v;
+
+    for (i=0; i<n; i++) {
+        k = i; v = n + i;
+        fill_dbt(&key, &k, sizeof k);
+        fill_dbt(&val, &v, sizeof v);
+        r = brt_insert(t, &key, &val, 0);
+        assert(r == 0);
+    }
+
+    for (i=0; i<n; i++) {
+        k = i;
+        fill_dbt(&key, &k, sizeof k);
+        r = brt_delete(t, &key, 0);
+        assert(r == 0);
+    }
+
+    /* lookups should all fail */
+    for (i=0; i<n; i++) {
+        k = i;
+        fill_dbt(&key, &k, sizeof k);
+        init_dbt(&val); val.flags = DB_DBT_MALLOC;
+        r = brt_lookup(t, &key, &val, 0);
+        assert(r == DB_NOTFOUND);
+    }
+
+    /* cursor should not find anything */
+    BRT_CURSOR cursor;
+
+    r = brt_cursor(t, &cursor);
+    assert(r == 0);
+
+    init_dbt(&key); key.flags = DB_DBT_MALLOC;
+    init_dbt(&val); val.flags = DB_DBT_MALLOC;
+    r = brt_c_get(cursor, &key, &val, DB_FIRST);
+    assert(r != 0);
+
+    r = brt_cursor_close(cursor);
+    assert(r == 0);
+
+    r = close_brt(t);              assert(r==0);
+    r = cachetable_close(&ct);     assert(r==0);
+}
+
+void test_brt_delete_not_present(int n) {
+    printf("test_brt_delete_not_present:%d\n", n);
+
+    BRT t;
+    int r;
+    CACHETABLE ct;
+    char fname[]="testbrt.brt";
+    int i;
+
+    r = brt_create_cachetable(&ct, 0);
+    assert(r==0);
+    unlink(fname);
+    r = open_brt(fname, 0, 1, &t, 4096, ct, default_compare_fun);
+    assert(r==0);
+
+    DBT key, val;
+    int k, v;
+
+    for (i=0; i<n; i++) {
+        k = i; v = n + i;
+        fill_dbt(&key, &k, sizeof k);
+        fill_dbt(&val, &v, sizeof v);
+        r = brt_insert(t, &key, &val, 0);
+        assert(r == 0);
+    }
+
+    for (i=0; i<n; i++) {
+        k = i;
+        fill_dbt(&key, &k, sizeof k);
+        r = brt_delete(t, &key, 0);
+        assert(r == 0);
+    }
+
+    k = n+1;
+    fill_dbt(&key, &k, sizeof k);
+    r = brt_delete(t, &key, 0);
+    printf("brt_delete k=%d %d\n", k, r);
+
+    r = close_brt(t);              assert(r==0);
+    r = cachetable_close(&ct);     assert(r==0);
+}
+
+void test_brt_delete_cursor_first(int n) {
+    printf("test_brt_delete_cursor_first:%d\n", n);
+
+    BRT t;
+    int r;
+    CACHETABLE ct;
+    char fname[]="testbrt.brt";
+    int i;
+
+    r = brt_create_cachetable(&ct, 0);
+    assert(r==0);
+    unlink(fname);
+    r = open_brt(fname, 0, 1, &t, 4096, ct, default_compare_fun);
+    assert(r==0);
+
+    DBT key, val;
+    int k, v;
+
+    for (i=0; i<n; i++) {
+        k = i; v = ~i;
+        fill_dbt(&key, &k, sizeof k);
+        fill_dbt(&val, &v, sizeof v);
+        r = brt_insert(t, &key, &val, 0);
+        assert(r == 0);
+    }
+
+    for (i=0; i<n-1; i++) {
+        k = i;
+        fill_dbt(&key, &k, sizeof k);
+        r = brt_delete(t, &key, 0);
+        assert(r == 0);
+    }
+
+    /* lookups should all fail */
+    for (i=0; i<n-1; i++) {
+        k = i;
+        fill_dbt(&key, &k, sizeof k);
+        init_dbt(&val); val.flags = DB_DBT_MALLOC;
+        r = brt_lookup(t, &key, &val, 0);
+        assert(r == DB_NOTFOUND);
+    }
+
+    /* cursor should find the last key */
+    BRT_CURSOR cursor;
+
+    r = brt_cursor(t, &cursor);
+    assert(r == 0);
+
+    init_dbt(&key); key.flags = DB_DBT_MALLOC;
+    init_dbt(&val); val.flags = DB_DBT_MALLOC;
+    r = brt_c_get(cursor, &key, &val, DB_FIRST);
+    assert(r == 0);
+    int vv;
+    assert(val.size == sizeof vv);
+    memcpy(&vv, val.data, val.size);
+    assert(vv == ~(n-1));
+    toku_free(key.data);
+    toku_free(val.data);
+
+    r = brt_cursor_close(cursor);
+    assert(r == 0);
+
+    r = close_brt(t);              assert(r==0);
+    r = cachetable_close(&ct);     assert(r==0);
+}
+
+void test_brt_delete() {
+    test_brt_delete_empty(); memory_check_all_free();
+    test_brt_delete_present(1); memory_check_all_free();
+    test_brt_delete_present(100); memory_check_all_free();
+    test_brt_delete_present(500); memory_check_all_free();
+    test_brt_delete_not_present(1); memory_check_all_free();
+    test_brt_delete_not_present(100); memory_check_all_free();
+    test_brt_delete_not_present(500); memory_check_all_free();
+    test_brt_delete_cursor_first(1); memory_check_all_free();
+    test_brt_delete_cursor_first(100); memory_check_all_free();
+    test_brt_delete_cursor_first(500); memory_check_all_free();
+}
+
 static void brt_blackbox_test (void) {
-    test_brt_cursor();
+    memory_check = 1;
     test_wrongendian_compare(0, 2);          memory_check_all_free();
     test_wrongendian_compare(1, 2);          memory_check_all_free();
     test_wrongendian_compare(1, 257);        memory_check_all_free();
@@ -1377,7 +1633,11 @@ static void brt_blackbox_test (void) {
     // Once upon a time srandom(8) caused this test to fail.
     srandom(8); test4(2048, 1<<15, 1);
 
+    memory_check = 1;
 
+    test_brt_limits();
+    test_brt_cursor();
+    test_brt_delete();
 
 //    test3(1<<19, 1<<20, 0);
 //    test4(1<<19, 1<<20, 0);

newbrt/brt.h

@@ -14,6 +14,7 @@ int open_brt (const char *fname, const char *dbname, int is_create, BRT *, int n
 //int brt_open (BRT *, char *fname, char *dbname);
 int brt_insert (BRT brt, DBT *k, DBT *v, DB*db);
 int brt_lookup (BRT brt, DBT *k, DBT *v, DB*db);
+int brt_delete (BRT brt, DBT *k, DB *db);
 int close_brt (BRT);
 int dump_brt (BRT brt);
 void brt_fsync (BRT); /* fsync, but don't clear the caches. */

newbrt/hashtable.c

@@ -2,7 +2,7 @@
 #include "hashtable.h"
 #include "memory.h"
 #include "primes.h"
-#include "../include/db.h"
+// #include "../include/ydb-constants.h"
 #include <assert.h>
 #include <string.h>
 #include <stdlib.h>
@@ -41,7 +41,7 @@ static void hash_find_internal (HASHTABLE tab, unsigned int hash, const unsigned
     *hashelt = 0;
 }
     
-int toku_hash_find (HASHTABLE tab, bytevec key, ITEMLEN keylen, bytevec *data, ITEMLEN *datalen) {
+int toku_hash_find (HASHTABLE tab, bytevec key, ITEMLEN keylen, bytevec *data, ITEMLEN *datalen, int *type) {
     HASHELT he, *prev_ptr;
     hash_find_internal(tab, hash_key (key, keylen), key, keylen, &he, &prev_ptr);
     if (he==0) {
@@ -49,6 +49,7 @@ int toku_hash_find (HASHTABLE tab, bytevec key, ITEMLEN keylen, bytevec *data, I
     } else {
 	*data = &he->keyval[he->keylen];
 	*datalen = he->vallen;
+        *type = he->type;
 	return 0;
     }
 }
@@ -82,7 +83,7 @@ int toku_hash_rehash_everything (HASHTABLE tab, unsigned int primeindexdelta) {
     return 0;
 }
 
-int toku_hash_insert (HASHTABLE tab, const void *key, ITEMLEN keylen, const void *val, ITEMLEN vallen)
+int toku_hash_insert (HASHTABLE tab, const void *key, ITEMLEN keylen, const void *val, ITEMLEN vallen, int type)
 {
     unsigned int hk = hash_key (key,keylen);
     unsigned int h = hk%tab->arraysize;
@@ -97,6 +98,7 @@ int toku_hash_insert (HASHTABLE tab, const void *key, ITEMLEN keylen, const void
 	/* Otherwise the key is not already present, so we need to add it. */
 	HASHELT he=toku_malloc(sizeof(*he)+keylen+vallen);
 	assert(he); // ?????
+        he->type = type;
 	he->keylen = keylen;
 	he->vallen = vallen;
 	memmove(&he->keyval[0], key, keylen);
@@ -134,7 +136,7 @@ int toku_hash_delete (HASHTABLE tab, const void *key, ITEMLEN keylen) {
 }
 
 
-int toku_hashtable_random_pick(HASHTABLE h, bytevec *key, ITEMLEN *keylen, bytevec *data, ITEMLEN *datalen, long int *randomnumber) {
+int toku_hashtable_random_pick(HASHTABLE h, bytevec *key, ITEMLEN *keylen, bytevec *data, ITEMLEN *datalen, int *type, long int *randomnumber) {
     unsigned int i;
     unsigned int usei = (*randomnumber)%h->arraysize;
     for (i=0; i<h->arraysize; i++, usei++) {
@@ -145,6 +147,7 @@ int toku_hashtable_random_pick(HASHTABLE h, bytevec *key, ITEMLEN *keylen, bytev
 	    *keylen = he->keylen;
 	    *data = &he->keyval[he->keylen];
 	    *datalen = he->vallen;
+            *type = he->type;
 	    *randomnumber = usei;
 	    return 0;
 	}
@@ -177,7 +180,7 @@ int hashtable_find_last(HASHTABLE h, bytevec *key, ITEMLEN *keylen, bytevec *dat
 }
 #endif
 
-void toku_hashtable_iterate (HASHTABLE tab, void(*f)(bytevec key, ITEMLEN keylen, bytevec data, ITEMLEN datalen, void*args), void* args) {
+void toku_hashtable_iterate (HASHTABLE tab, void(*f)(bytevec key, ITEMLEN keylen, bytevec data, ITEMLEN datalen, int type, void*args), void* args) {
   /*
     int i;
     for (i=0; i<tab->arraysize; i++) {
@@ -187,7 +190,7 @@ void toku_hashtable_iterate (HASHTABLE tab, void(*f)(bytevec key, ITEMLEN keylen
 	}
     }
   */
-    HASHTABLE_ITERATE(tab, key, keylen, val, vallen, f(key,keylen,val,vallen,args));
+    HASHTABLE_ITERATE(tab, key, keylen, val, vallen, type, f(key,keylen,val,vallen,type,args));
 }
 
 int toku_hashtable_n_entries(HASHTABLE tab) {

newbrt/hashtable.h

@@ -12,10 +12,10 @@ int toku_hashtable_create (HASHTABLE*);
 
 /* Return 0 if the key is found in the hashtable, -1 otherwise. */
 /* Warning: The data returned points to the internals of the hashtable.  It is set to "const" to try to prevent you from messing it up. */
-int toku_hash_find (HASHTABLE tab, bytevec key, ITEMLEN keylen, bytevec*data, ITEMLEN *datalen);
+int toku_hash_find (HASHTABLE tab, bytevec key, ITEMLEN keylen, bytevec*data, ITEMLEN *datalen, int *type);
 
 /* Replace the key if it was already there. */
-int toku_hash_insert (HASHTABLE tab, const void *key, ITEMLEN keylen, const void *data, ITEMLEN datalen);
+int toku_hash_insert (HASHTABLE tab, const void *key, ITEMLEN keylen, const void *data, ITEMLEN datalen, int type);
 
 /* It is OK to delete something that isn't there. */
 int toku_hash_delete (HASHTABLE tab, const void *key, ITEMLEN keylen);
@@ -24,15 +24,16 @@ int toku_hashtable_n_entries(HASHTABLE);
   
 void toku_hashtable_clear(HASHTABLE);
 
-int toku_hashtable_random_pick(HASHTABLE h, bytevec *key, ITEMLEN *keylen, bytevec *data, ITEMLEN *datalen, long int *randomnumber);
+int toku_hashtable_random_pick(HASHTABLE h, bytevec *key, ITEMLEN *keylen, bytevec *data, ITEMLEN *datalen, int *type, long int *randomnumber);
 //int hashtable_find_last(HASHTABLE h, bytevec *key, ITEMLEN *keylen, bytevec *data, ITEMLEN *datalen);
 
 typedef struct hashelt *HASHELT;
 struct hashelt {
-    ITEMLEN keylen;  
-    ITEMLEN vallen;
     unsigned int hash;
     HASHELT next;
+    int type;
+    ITEMLEN keylen;  
+    ITEMLEN vallen;
     char keyval[]; /* the first KEYLEN bytes are the key.  The next bytes are the value. */
 };
 
@@ -44,9 +45,9 @@ struct hashtable {
 };
 
 /* You cannot add or delete elements from the hashtable while iterating. */
-void toku_hashtable_iterate (HASHTABLE tab, void(*f)(bytevec key,ITEMLEN keylen,bytevec data,ITEMLEN datalen,void*), void*);
+void toku_hashtable_iterate (HASHTABLE tab, void(*f)(bytevec key,ITEMLEN keylen,bytevec data,ITEMLEN datalen,int type, void*), void*);
 // If you don't want to use something, do something like use "key __attribute__((__unused__))" for keyvar.
-#define HASHTABLE_ITERATE(table,keyvar,keylenvar,datavar,datalenvar,body) ({ \
+#define HASHTABLE_ITERATE(table,keyvar,keylenvar,datavar,datalenvar,typevar,body) ({ \
   unsigned int hi_counter;                                                   \
   for (hi_counter=0; hi_counter<table->arraysize; hi_counter++) {            \
     HASHELT hi_he;                                                           \
@@ -55,6 +56,7 @@ void toku_hashtable_iterate (HASHTABLE tab, void(*f)(bytevec key,ITEMLEN keylen,
       ITEMLEN     keylenvar  = hi_he->keylen;                                \
       const char *datavar    = &hi_he->keyval[hi_he->keylen];                \
       ITEMLEN     datalenvar = hi_he->vallen;                                \
+      int         typevar    = hi_he->type;                                  \
       body;                                                                  \
   }}})
 

newbrt/hashtest.c

@@ -28,7 +28,7 @@ void verify_hash_instance (bytevec kv_v, ITEMLEN kl, bytevec dv_v, ITEMLEN dl,
     fprintf(stderr, "%s isn't there\n", kv); abort();
 }
 
-void verify_htable_instance (bytevec kv_v, ITEMLEN kl, bytevec dv_v, ITEMLEN dl,
+void verify_htable_instance (bytevec kv_v, ITEMLEN kl, bytevec dv_v, ITEMLEN dl, int type,
 			    int N, int *data, char *saw) {
     char *kv = (char*)kv_v;
     char *dv = (char*)dv_v;
@@ -38,6 +38,7 @@ void verify_htable_instance (bytevec kv_v, ITEMLEN kl, bytevec dv_v, ITEMLEN dl,
     assert(strcmp(kv+1, dv+1)==0);
     assert(strlen(kv)+1==kl);
     assert(strlen(dv)+1==dl);
+    assert(type == 0);
     num = atoi(kv+1);
     for (k=0; k<N; k++) {
 	if (data[k]==num) {
@@ -54,8 +55,8 @@ void verify_htable (HASHTABLE htable, int N, int *data, char *saw) {
     for (j=0; j<N; j++) {
 	saw[j]=0;
     }
-    HASHTABLE_ITERATE(htable, kv, kl, dv, dl,
-		      verify_htable_instance (kv, kl, dv, dl,
+    HASHTABLE_ITERATE(htable, kv, kl, dv, dl, type,
+		      verify_htable_instance (kv, kl, dv, dl, type,
 					      N, data, saw));
     for (j=0; j<N; j++) {
 	assert(saw[j]);
@@ -99,7 +100,7 @@ void test0 (void) {
 		}
 		snprintf(kv, 99, "k%d", ra);
 		snprintf(dv, 99, "d%d", ra);
-		toku_hash_insert(htable, kv, strlen(kv)+1, dv, strlen(dv)+1);
+		toku_hash_insert(htable, kv, strlen(kv)+1, dv, strlen(dv)+1, 0);
 		data[data_n++]=ra;
 	    }
 	} else {
@@ -122,13 +123,14 @@ void test1(void) {
 	for (j=0; j<4; j++) {
 	    snprintf(keys[j], 100, "k%ld", (long)(random()));
 	    snprintf(vals[j], 100, "v%d", j);
-	    toku_hash_insert(table, keys[j], strlen(keys[j])+1, vals[j], strlen(vals[j])+1);
+	    toku_hash_insert(table, keys[j], strlen(keys[j])+1, vals[j], strlen(vals[j])+1, 0);
 	}
 	for (j=0; j<4; j++) {
 	    bytevec key, val;
 	    ITEMLEN keylen, vallen;
+            int type;
 	    long int randnum=random();
-	    r = toku_hashtable_random_pick(table, &key, &keylen, &val, &vallen, &randnum);
+	    r = toku_hashtable_random_pick(table, &key, &keylen, &val, &vallen, &type, &randnum);
 	    assert(r==0);
 	    r = toku_hash_delete(table, key, keylen);
 	    assert(r==0);