Did

{{{
svn merge -r4556:4568 https://svn.tokutek.com/tokudb/tokudb.906
}}}
to incorporate the hash improvements.

Addresses #906.


git-svn-id: file:///svn/tokudb@4596 c7de825b-a66e-492c-adef-691d508d4ae1

newbrt/brt-internal.h

@@ -67,6 +67,7 @@ struct brtnode {
     u_int32_t rand4fingerprint;
     u_int32_t local_fingerprint; /* For leaves this is everything in the buffer.  For nonleaves, this is everything in the buffers, but does not include child subtree fingerprints. */
     int    dirty;
+    u_int32_t fullhash;
     union node {
 	struct nonleaf {
 	    // Don't actually store the subree fingerprint in the in-memory data structure.
@@ -103,6 +104,7 @@ enum {
 
 struct brt_header {
     int dirty;
+    u_int32_t fullhash;
     int layout_version;
     unsigned int nodesize;
     DISKOFF freelist;
@@ -140,7 +142,7 @@ struct brt {
 
 /* serialization code */
 void toku_serialize_brtnode_to(int fd, DISKOFF off, BRTNODE node);
-int toku_deserialize_brtnode_from (int fd, DISKOFF off, BRTNODE *brtnode);
+int toku_deserialize_brtnode_from (int fd, DISKOFF off, u_int32_t /*fullhash*/, BRTNODE *brtnode);
 unsigned int toku_serialize_brtnode_size(BRTNODE node); /* How much space will it take? */
 int toku_keycompare (bytevec key1, ITEMLEN key1len, bytevec key2, ITEMLEN key2len);
 
@@ -149,7 +151,7 @@ void toku_verify_counts(BRTNODE);
 int toku_serialize_brt_header_size (struct brt_header *h);
 int toku_serialize_brt_header_to (int fd, struct brt_header *h);
 int toku_serialize_brt_header_to_wbuf (struct wbuf *, struct brt_header *h);
-int toku_deserialize_brtheader_from (int fd, DISKOFF off, struct brt_header **brth);
+int toku_deserialize_brtheader_from (int fd, DISKOFF off, u_int32_t fullhash, struct brt_header **brth);
 
 int toku_serialize_fifo_at (int fd, off_t freeoff, FIFO fifo); // Write a fifo into a disk, without worrying about fitting it into a block.  This write is done at the end of the file.
 int toku_deserialize_fifo_at (int fd, off_t at, FIFO *fifo);
@@ -200,7 +202,7 @@ struct brt_cursor {
 
 // logs the memory allocation, but not the creation of the new node
 int toku_create_new_brtnode (BRT t, BRTNODE *result, int height, TOKULOGGER logger);
-int toku_unpin_brtnode (BRT brt, BRTNODE node) ;
+int toku_unpin_brtnode (BRT brt, BRTNODE node);
 unsigned int toku_brtnode_which_child (BRTNODE node , DBT *k, DBT *d, BRT t);
 
 /* Stuff for testing */

newbrt/brt-serialize.c

@@ -215,7 +215,7 @@ void toku_serialize_brtnode_to (int fd, DISKOFF off, BRTNODE node) {
     toku_free(buf);
 }
 
-int toku_deserialize_brtnode_from (int fd, DISKOFF off, BRTNODE *brtnode) {
+int toku_deserialize_brtnode_from (int fd, DISKOFF off, u_int32_t fullhash, BRTNODE *brtnode) {
     TAGMALLOC(BRTNODE, result);
     struct rbuf rc;
     int i;
@@ -294,6 +294,7 @@ int toku_deserialize_brtnode_from (int fd, DISKOFF off, BRTNODE *brtnode) {
     result->local_fingerprint = rbuf_int(&rc);
 //    printf("%s:%d read %08x\n", __FILE__, __LINE__, result->local_fingerprint);
     result->dirty = 0;
+    result->fullhash = fullhash;
     //printf("height==%d\n", result->height);
     if (result->height>0) {
 	result->u.n.totalchildkeylens=0;
@@ -552,10 +553,11 @@ int toku_serialize_brt_header_to (int fd, struct brt_header *h) {
     return r;
 }
 
-static int deserialize_brtheader_6_or_earlier (int fd, DISKOFF off, struct brt_header **brth) {
+static int deserialize_brtheader_6_or_earlier (int fd, DISKOFF off, struct brt_header **brth, u_int32_t fullhash) {
     // Deserialize a brt header from version 6 or earlier.
     struct brt_header *MALLOC(h);
     if (h==0) return errno;
+    h->fullhash = fullhash;
     int ret=-1;
     if (0) { died0: toku_free(h); return ret; }
     int size;
@@ -632,11 +634,12 @@ static int deserialize_brtheader_6_or_earlier (int fd, DISKOFF off, struct brt_h
     return 0;
 }
 
-int deserialize_brtheader_7_or_later(u_int32_t size, int fd, DISKOFF off, struct brt_header **brth) {
+int deserialize_brtheader_7_or_later(u_int32_t size, int fd, DISKOFF off, struct brt_header **brth, u_int32_t fullhash) {
     // We already know the first 8 bytes are "tokudata", and we read in the size.
     struct brt_header *MALLOC(h);
     if (h==0) return errno;
     int ret=-1;
+    h->fullhash = fullhash;
     if (0) { died0: toku_free(h); return ret; }
     struct rbuf rc;
     rc.buf = toku_malloc(size-12); // we can skip the first 12 bytes.
@@ -685,7 +688,7 @@ int deserialize_brtheader_7_or_later(u_int32_t size, int fd, DISKOFF off, struct
     return 0;
 }
 
-int toku_deserialize_brtheader_from (int fd, DISKOFF off, struct brt_header **brth) {
+int toku_deserialize_brtheader_from (int fd, DISKOFF off, u_int32_t fullhash, struct brt_header **brth) {
     //printf("%s:%d calling MALLOC\n", __FILE__, __LINE__);
     assert(off==0);
     //printf("%s:%d malloced %p\n", __FILE__, __LINE__, h);
@@ -697,9 +700,9 @@ int toku_deserialize_brtheader_from (int fd, DISKOFF off, struct brt_header **br
     if (r!=12) return EINVAL;
     if (memcmp(magic,"tokudata",8)==0) {
 	// It's version 7 or later
-	return deserialize_brtheader_7_or_later(ntohl(*(int*)(&magic[8])), fd, off, brth);
+	return deserialize_brtheader_7_or_later(ntohl(*(int*)(&magic[8])), fd, off, brth, fullhash);
     } else {
-	return deserialize_brtheader_6_or_earlier(fd, off, brth);
+	return deserialize_brtheader_6_or_earlier(fd, off, brth, fullhash);
     }
 }
 

newbrt/brt-test-helpers.c

@@ -58,7 +58,7 @@ int toku_testsetup_root(BRT brt, DISKOFF diskoff) {
 int toku_testsetup_get_sersize(BRT brt, DISKOFF diskoff) // Return the size on disk
 {
     void *node_v;
-    int r  = toku_cachetable_get_and_pin(brt->cf, diskoff, &node_v, NULL,
+    int r  = toku_cachetable_get_and_pin(brt->cf, diskoff, toku_cachetable_hash(brt->cf, diskoff), &node_v, NULL,
 					 toku_brtnode_flush_callback, toku_brtnode_fetch_callback, brt);
     assert(r==0);
     int size = toku_serialize_brtnode_size(node_v);
@@ -70,7 +70,7 @@ int toku_testsetup_get_sersize(BRT brt, DISKOFF diskoff) // Return the size on d
 int toku_testsetup_insert_to_leaf (BRT brt, DISKOFF diskoff, char *key, int keylen, char *val, int vallen, u_int32_t *subtree_fingerprint) {
     void *node_v;
     int r;
-    r = toku_cachetable_get_and_pin(brt->cf, diskoff, &node_v, NULL,
+    r = toku_cachetable_get_and_pin(brt->cf, diskoff, toku_cachetable_hash(brt->cf, diskoff), &node_v, NULL,
 				    toku_brtnode_flush_callback, toku_brtnode_fetch_callback, brt);
     if (r!=0) return r;
     BRTNODE node=node_v;
@@ -122,7 +122,7 @@ int toku_testsetup_insert_to_leaf (BRT brt, DISKOFF diskoff, char *key, int keyl
 int toku_testsetup_insert_to_nonleaf (BRT brt, DISKOFF diskoff, enum brt_cmd_type cmdtype, char *key, int keylen, char *val, int vallen, u_int32_t *subtree_fingerprint) {
     void *node_v;
     int r;
-    r = toku_cachetable_get_and_pin(brt->cf, diskoff, &node_v, NULL,
+    r = toku_cachetable_get_and_pin(brt->cf, diskoff, toku_cachetable_hash(brt->cf, diskoff), &node_v, NULL,
 				    toku_brtnode_flush_callback, toku_brtnode_fetch_callback, brt);
     if (r!=0) return r;
     BRTNODE node=node_v;

newbrt/brt-verify.c

@@ -64,11 +64,13 @@ int toku_verify_brtnode (BRT brt, DISKOFF off, bytevec lorange, ITEMLEN lolen, b
     BRTNODE node;
     void *node_v;
     int r;
-    if ((r = toku_cachetable_get_and_pin(brt->cf, off, &node_v, NULL,
+    u_int32_t fullhash = toku_cachetable_hash(brt->cf, off);
+    if ((r = toku_cachetable_get_and_pin(brt->cf, off, fullhash, &node_v, NULL,
 					 toku_brtnode_flush_callback, toku_brtnode_fetch_callback, (void*)(long)brt->h->nodesize)))
 	return r;
     //printf("%s:%d pin %p\n", __FILE__, __LINE__, node_v);
     node=node_v;
+    assert(node->fullhash==fullhash);
     verify_local_fingerprint(node);
     if (node->height>0) {
 	int i;
@@ -138,7 +140,7 @@ int toku_verify_brtnode (BRT brt, DISKOFF off, bytevec lorange, ITEMLEN lolen, b
 	LEAFENTRY prev=0;
 	toku_omt_iterate(node->u.l.buffer, check_increasing, &prev);
     }
-    if ((r = toku_cachetable_unpin(brt->cf, off, 0, 0))) return r;
+    if ((r = toku_cachetable_unpin(brt->cf, off, fullhash, 0, 0))) return r;
     return result;
 }
 

newbrt/brtdump.c

@@ -26,7 +26,7 @@ void print_item (bytevec val, ITEMLEN len) {
 void dump_header (int f, struct brt_header **header) {
     struct brt_header *h;
     int r;
-    r = toku_deserialize_brtheader_from (f, 0, &h); assert(r==0);
+    r = toku_deserialize_brtheader_from (f, 0, 0/*pass 0 for hash.  It doesn't matter.*/, &h); assert(r==0);
     printf("brtheader:\n");
     if (h->layout_version==BRT_LAYOUT_VERSION_6) printf(" layout_version<=6\n");
     else printf(" layout_version=%d\n", h->layout_version);
@@ -78,7 +78,7 @@ void dump_header (int f, struct brt_header **header) {
 
 void dump_node (int f, DISKOFF off) {
     BRTNODE n;
-    int r = toku_deserialize_brtnode_from (f, off, &n);
+    int r = toku_deserialize_brtnode_from (f, off, 0 /*pass zero for hash, it doesn't matter*/, &n);
     assert(r==0);
     assert(n!=0);
     printf("brtnode\n");

newbrt/cachetable.h

@@ -32,25 +32,27 @@ typedef void (cachetable_flush_func_t)(CACHEFILE, CACHEKEY key, void*value, long
 typedef cachetable_flush_func_t *CACHETABLE_FLUSH_FUNC_T;
 
 /* If we are asked to fetch something, get it by calling this back. */
-typedef int (cachetable_fetch_func_t)(CACHEFILE, CACHEKEY key, void **value, long *sizep, void *extraargs, LSN *written_lsn);
+typedef int (cachetable_fetch_func_t)(CACHEFILE, CACHEKEY key, u_int32_t fullhash, void **value, long *sizep, void *extraargs, LSN *written_lsn);
 typedef cachetable_fetch_func_t *CACHETABLE_FETCH_FUNC_T;
 
 /* Error if already present.  On success, pin the value. */
-int toku_cachetable_put(CACHEFILE cf, CACHEKEY key, void* value, long size,
+int toku_cachetable_put(CACHEFILE cf, CACHEKEY key, u_int32_t fullhash,
+			void* value, long size,
 			cachetable_flush_func_t flush_callback, cachetable_fetch_func_t fetch_callback, void *extraargs);
 
-int toku_cachetable_get_and_pin(CACHEFILE, CACHEKEY, void**/*value*/, long *sizep,
+int toku_cachetable_get_and_pin(CACHEFILE, CACHEKEY, u_int32_t /*fullhash*/,
+				void**/*value*/, long *sizep,
 				cachetable_flush_func_t flush_callback, cachetable_fetch_func_t fetch_callback, void *extraargs);
 
 /* If the the item is already in memory, then return 0 and store it in the void**.
  * If the item is not in memory, then return nonzero. */
-int toku_cachetable_maybe_get_and_pin (CACHEFILE, CACHEKEY, void**);
+int toku_cachetable_maybe_get_and_pin (CACHEFILE, CACHEKEY, u_int32_t /*fullhash*/, void**);
 
 /* cachetable object state wrt external memory */
 #define CACHETABLE_CLEAN 0
 #define CACHETABLE_DIRTY 1
 
-int toku_cachetable_unpin(CACHEFILE, CACHEKEY, int dirty, long size); /* Note whether it is dirty when we unpin it. */
+int toku_cachetable_unpin(CACHEFILE, CACHEKEY, u_int32_t fullhash, int dirty, long size); /* Note whether it is dirty when we unpin it. */
 int toku_cachetable_remove (CACHEFILE, CACHEKEY, int /*write_me*/); /* Removing something already present is OK. */
 int toku_cachetable_assert_all_unpinned (CACHETABLE);
 int toku_cachefile_count_pinned (CACHEFILE, int /*printthem*/ );
@@ -90,4 +92,9 @@ int toku_cachefile_of_filenum (CACHETABLE t, FILENUM filenum, CACHEFILE *cf);
 
 int toku_cachetable_checkpoint (CACHETABLE ct);
 
+u_int32_t toku_cachetable_hash (CACHEFILE cachefile, CACHEKEY key);
+// Effect: Return a 32-bit hash key.  The hash key shall be suitable for using with bitmasking for a table of size power-of-two.
+
+u_int32_t toku_cachefile_fullhash_of_header (CACHEFILE cachefile);
+
 #endif

newbrt/tests/brt-serialize-test.c

@@ -55,7 +55,7 @@ static void test_serialize(void) {
 
     toku_serialize_brtnode_to(fd, sn.nodesize*(DISKOFF)20, &sn);  assert(r==0);
 
-    r = toku_deserialize_brtnode_from(fd, nodesize*(DISKOFF)20, &dn);
+    r = toku_deserialize_brtnode_from(fd, nodesize*(DISKOFF)20, 0/*pass zero for hash*/, &dn);
     assert(r==0);
 
     assert(dn->thisnodename==nodesize*20);

newbrt/tests/cachetable-test2.c

@@ -80,7 +80,8 @@ static void flush_forchain (CACHEFILE f            __attribute__((__unused__)),
     //print_ints();
 }
 
-static int fetch_forchain (CACHEFILE f __attribute__((__unused__)), CACHEKEY key, void**value, long *sizep __attribute__((__unused__)), void*extraargs, LSN *written_lsn) {
+static int fetch_forchain (CACHEFILE f, CACHEKEY key, u_int32_t fullhash, void**value, long *sizep __attribute__((__unused__)), void*extraargs, LSN *written_lsn) {
+    assert(toku_cachetable_hash(f, key)==fullhash);
     assert((long)extraargs==(long)key);
     *value = (void*)(long)key;
     written_lsn->lsn = 0;
@@ -91,11 +92,13 @@ static void verify_cachetable_against_present (void) {
     int i;
     for (i=0; i<n_present; i++) {
 	void *v;
-	int r;
-	assert(toku_cachetable_maybe_get_and_pin(present_items[i].cf,
-						 present_items[i].key,
-						 &v)==0);
-	r = toku_cachetable_unpin(present_items[i].cf, present_items[i].key, CACHETABLE_CLEAN, test_object_size);
+	u_int32_t fullhash = toku_cachetable_hash(present_items[i].cf, present_items[i].key);
+	int r=toku_cachetable_maybe_get_and_pin(present_items[i].cf,
+						present_items[i].key,
+						toku_cachetable_hash(present_items[i].cf, present_items[i].key),
+						&v);
+	assert(r==0);
+	r = toku_cachetable_unpin(present_items[i].cf, present_items[i].key, fullhash, CACHETABLE_CLEAN, test_object_size);
     }
 }
 
@@ -117,9 +120,10 @@ static void test_chaining (void) {
     for (i=0; i<N_PRESENT_LIMIT; i++) {
 	int fnum = i%N_FILES;
 	//printf("%s:%d Add %d\n", __FILE__, __LINE__, i);
-	r = toku_cachetable_put(f[fnum], i, (void*)i, test_object_size, flush_forchain, fetch_forchain, (void*)i); assert(r==0);
+	u_int32_t fhash = toku_cachetable_hash(f[fnum], i);
+	r = toku_cachetable_put(f[fnum], i, fhash, (void*)i, test_object_size, flush_forchain, fetch_forchain, (void*)i); assert(r==0);
 	item_becomes_present(f[fnum], i);
-	r = toku_cachetable_unpin(f[fnum], i, CACHETABLE_CLEAN, test_object_size);                                                assert(r==0);
+	r = toku_cachetable_unpin(f[fnum], i, fhash, CACHETABLE_CLEAN, test_object_size);                                 assert(r==0);
 	//print_ints();
     }
     for (trial=0; trial<TRIALS; trial++) {
@@ -128,8 +132,10 @@ static void test_chaining (void) {
 	    int whichone = random()%n_present;
 	    void *value;
 	    //printf("Touching %d (%lld, %p)\n", whichone, present_items[whichone].key, present_items[whichone].cf);
+	    u_int32_t fhash = toku_cachetable_hash(present_items[whichone].cf, present_items[whichone].key);
 	    r = toku_cachetable_get_and_pin(present_items[whichone].cf,
 					    present_items[whichone].key,
+					    fhash,
 					    &value,
 					    NULL,
 					    flush_forchain,
@@ -139,6 +145,7 @@ static void test_chaining (void) {
 	    assert(r==0);
 	    r = toku_cachetable_unpin(present_items[whichone].cf,
 				      present_items[whichone].key,
+				      fhash,
 				      CACHETABLE_CLEAN, test_object_size);
 	    assert(r==0);
 	}
@@ -147,11 +154,12 @@ static void test_chaining (void) {
 	int fnum = i%N_FILES;
 	// i is always incrementing, so we need not worry about inserting a duplicate
 	//printf("%s:%d Add {%d,%p}\n", __FILE__, __LINE__, i, f[fnum]);
-	r = toku_cachetable_put(f[fnum], i, (void*)i, test_object_size, flush_forchain, fetch_forchain, (void*)i); assert(r==0);
+	u_int32_t fhash = toku_cachetable_hash(f[fnum], i);
+	r = toku_cachetable_put(f[fnum], i, fhash, (void*)i, test_object_size, flush_forchain, fetch_forchain, (void*)i); assert(r==0);
 	item_becomes_present(f[fnum], i);
 	//print_ints();
 	//cachetable_print_state(ct);
-	r = toku_cachetable_unpin(f[fnum], i, CACHETABLE_CLEAN, test_object_size);                                                assert(r==0);
+	r = toku_cachetable_unpin(f[fnum], i, fhash, CACHETABLE_CLEAN, test_object_size);                                                assert(r==0);
 	verify_cachetable_against_present();
 
 	if (random()%10==0) {