#3588 merge tokudb.3588 to main refs[t:3588]

git-svn-id: file:///svn/toku/tokudb@32669 c7de825b-a66e-492c-adef-691d508d4ae1

Makefile

@@ -32,6 +32,9 @@ db-benchmark-test-cxx.dir: cxx.dir
 
 build: $(patsubst %,%.dir, $(BUILDDIRS))
 
+%.build:
+	cd $(patsubst %.build, %,$@) && $(MAKE) build
+
 %.local:
 	cd $(patsubst %.local, %,$@) && $(MAKE) local
 

newbrt/brt-internal.h

@@ -435,10 +435,19 @@ toku_brt_search_which_child(
     BRTNODE node, 
     brt_search_t *search
     );
+
 u_int8_t 
 toku_brtnode_partition_state (struct brtnode_fetch_extra* bfe, int childnum);
-// logs the memory allocation, but not the creation of the new node
+
+// allocate a block number
+// allocate and initialize a brtnode
+// put the brtnode into the cache table
 void toku_create_new_brtnode (BRT t, BRTNODE *result, int height, int n_children);
+
+// Effect: Fill in N as an empty brtnode.
+void toku_initialize_empty_brtnode (BRTNODE n, BLOCKNUM nodename, int height, int num_children, 
+                                    int layout_version, unsigned int nodesize, unsigned int flags);
+
 int toku_pin_brtnode (BRT brt, BLOCKNUM blocknum, u_int32_t fullhash,
 		      UNLOCKERS unlockers,
 		      ANCESTORS ancestors, struct pivot_bounds const * const pbounds,

newbrt/brt-serialize.c

@@ -602,11 +602,6 @@ toku_serialize_brtnode_to_memory (BRTNODE node,
                           /*out*/ size_t *n_bytes_to_write,
                           /*out*/ char  **bytes_to_write)
 {
-
-    
-
-    char* data = NULL;
-    char* curr_ptr = NULL;
     toku_assert_entire_node_in_memory(node);
 
     if (node->height == 0) {
@@ -658,8 +653,8 @@ toku_serialize_brtnode_to_memory (BRTNODE node,
     //
     node->bp_offset = serialize_node_header_size(node) + sb_node_info.compressed_size + 4;
     
-    data = toku_xmalloc(total_node_size);
-    curr_ptr = data;
+    char *data = toku_xmalloc(total_node_size);
+    char *curr_ptr = data;
     // now create the final serialized node
 
     // write the header

newbrt/brt.c

@@ -881,33 +881,32 @@ toku_brtheader_free (struct brt_header *h) {
     brtheader_free(h);
 }
 
-static void
-initialize_empty_brtnode (BRT t, BRTNODE n, BLOCKNUM nodename, int height, int num_children)
+void
+toku_initialize_empty_brtnode (BRTNODE n, BLOCKNUM nodename, int height, int num_children, int layout_version, unsigned int nodesize, unsigned int flags)
 // Effect: Fill in N as an empty brtnode.
 {
+    assert(layout_version != 0);
+    assert(height >= 0);
+
     n->max_msn_applied_to_node_on_disk = MIN_MSN;    // correct value for root node, harmless for others
     n->max_msn_applied_to_node_in_memory = MIN_MSN;  // correct value for root node, harmless for others
-    n->nodesize = t->h->nodesize;
-    n->flags = t->flags;
+    n->nodesize = nodesize;
+    n->flags = flags;
     n->thisnodename = nodename;
-    assert(t->h->layout_version != 0);
-    n->layout_version	       = t->h->layout_version;
-    n->layout_version_original = t->h->layout_version;
-    n->layout_version_read_from_disk = t->h->layout_version;
+    n->layout_version	       = layout_version;
+    n->layout_version_original = layout_version;
+    n->layout_version_read_from_disk = layout_version;
     n->height = height;
     n->dirty = 1;
-    assert(height>=0);
     n->totalchildkeylens = 0;
-    n->childkeys=0;
+    n->childkeys = 0;
     n->bp = 0;
     n->n_children = num_children; 
     n->bp_offset = 0;
 
     if (num_children > 0) {
-	MALLOC_N(num_children-1, n->childkeys);
-	assert(n->childkeys);
-        MALLOC_N(num_children, n->bp);
-        assert(n->bp);
+        XMALLOC_N(num_children-1, n->childkeys);
+        XMALLOC_N(num_children, n->bp);
 	for (int i = 0; i < num_children; i++) {
             BP_FULLHASH(n,i)=0;
             BP_HAVE_FULLHASH(n,i)=FALSE;
@@ -921,11 +920,11 @@ initialize_empty_brtnode (BRT t, BRTNODE n, BLOCKNUM nodename, int height, int n
                 n->bp[i].ptr = toku_malloc(sizeof(struct brtnode_nonleaf_childinfo));
                 memset(n->bp[i].ptr, 0, sizeof(struct brtnode_nonleaf_childinfo));
                 int r = toku_fifo_create(&BNC_BUFFER(n,i));
-                assert(r==0);
+                assert_zero(r);
                 BNC_NBYTESINBUF(n,i) = 0;
             }
             else {
-                n->bp[i].ptr = toku_malloc(sizeof(struct brtnode_leaf_basement_node));
+                n->bp[i].ptr = toku_xmalloc(sizeof(struct brtnode_leaf_basement_node));
                 BASEMENTNODE bn = (BASEMENTNODE)n->bp[i].ptr;
                 memset(bn, 0, sizeof(struct brtnode_leaf_basement_node));
                 toku_setup_empty_bn(bn);
@@ -941,14 +940,14 @@ brt_init_new_root(BRT brt, BRTNODE nodea, BRTNODE nodeb, DBT splitk, CACHEKEY *r
 //  Unpin nodea and nodeb.
 //  Leave the new root pinned.
 {
-    BRTNODE MALLOC(newroot);
+    BRTNODE XMALLOC(newroot);
     int new_height = nodea->height+1;
     BLOCKNUM newroot_diskoff;
     toku_allocate_blocknum(brt->h->blocktable, &newroot_diskoff, brt->h);
     assert(newroot);
     *rootp=newroot_diskoff;
     assert(new_height > 0);
-    initialize_empty_brtnode (brt, newroot, newroot_diskoff, new_height, 2);
+    toku_initialize_empty_brtnode (newroot, newroot_diskoff, new_height, 2, brt->h->layout_version, brt->h->nodesize, brt->flags);
     //printf("new_root %lld %d %lld %lld\n", newroot_diskoff, newroot->height, nodea->thisnodename, nodeb->thisnodename);
     //printf("%s:%d Splitkey=%p %s\n", __FILE__, __LINE__, splitkey, splitkey);
     newroot->childkeys[0] = splitk.data;
@@ -978,26 +977,27 @@ brt_init_new_root(BRT brt, BRTNODE nodea, BRTNODE nodeb, DBT splitk, CACHEKEY *r
     *newrootp = newroot;
 }
 
-// logs the memory allocation, but not the creation of the new node
-void toku_create_new_brtnode (BRT t, BRTNODE *result, int height, int n_children) {
-    BRTNODE MALLOC(n);
-    int r;
+void 
+toku_create_new_brtnode (BRT t, BRTNODE *result, int height, int n_children) {
+    assert(t->h->nodesize > 0);
+    if (height == 0) 
+        assert(n_children > 0);
+
     BLOCKNUM name;
     toku_allocate_blocknum(t->h->blocktable, &name, t->h);
-    if (height == 0) { assert(n_children > 0); }
-    assert(n);
-    assert(t->h->nodesize>0);
-    initialize_empty_brtnode(t, n, name, height, n_children);
-    *result = n;
-    assert(n->nodesize>0);
-    //	  n->brt	    = t;
-    //printf("%s:%d putting %p (%lld)\n", __FILE__, __LINE__, n, n->thisnodename);
+
+    BRTNODE XMALLOC(n);
+    toku_initialize_empty_brtnode(n, name, height, n_children, t->h->layout_version, t->h->nodesize, t->flags);
+    assert(n->nodesize > 0);
+
     u_int32_t fullhash = toku_cachetable_hash(t->cf, n->thisnodename);
     n->fullhash = fullhash;
-    r=toku_cachetable_put(t->cf, n->thisnodename, fullhash,
+    int r = toku_cachetable_put(t->cf, n->thisnodename, fullhash,
                             n, brtnode_memory_size(n),
                             toku_brtnode_flush_callback, toku_brtnode_pe_callback, t->h);
     assert_zero(r);
+
+    *result = n;
 }
 
 static void
@@ -3391,23 +3391,20 @@ int toku_open_brt (const char *fname, int is_create, BRT *newbrt, int nodesize,
 }
 
 static int setup_initial_brt_root_node (BRT t, BLOCKNUM blocknum) {
-    int r;
-    BRTNODE MALLOC(node);
-    assert(node);
-    //printf("%s:%d\n", __FILE__, __LINE__);
-    initialize_empty_brtnode(t, node, blocknum, 0, 1);
+    BRTNODE XMALLOC(node);
+    toku_initialize_empty_brtnode(node, blocknum, 0, 1, t->h->layout_version, t->h->nodesize, t->flags);
     BP_STATE(node,0) = PT_AVAIL;
+
     u_int32_t fullhash = toku_cachetable_hash(t->cf, blocknum);
     node->fullhash = fullhash;
-    r=toku_cachetable_put(t->cf, blocknum, fullhash,
+    int r = toku_cachetable_put(t->cf, blocknum, fullhash,
                                 node, brtnode_memory_size(node),
                                 toku_brtnode_flush_callback, toku_brtnode_pe_callback, t->h);
-    if (r!=0) {
+    if (r != 0)
 	toku_free(node);
-	return r;
-    }
+    else
         toku_unpin_brtnode(t, node);
-    return 0;
+    return r;
 }
 
 // open a file for use by the brt

newbrt/brtdump.c

@@ -143,24 +143,23 @@ dump_node (int f, BLOCKNUM blocknum, struct brt_header *h) {
     printf(" n_children=%d\n", n->n_children);
     printf(" total_childkeylens=%u\n", n->totalchildkeylens);
 
-    int i;
     printf(" subleafentry_estimates={");
-    for (i=0; i<n->n_children; i++) {
+    for (int i=0; i<n->n_children; i++) {
         if (i>0) printf(" ");
         struct subtree_estimates *est = &BP_SUBTREE_EST(n,i);
         printf("{nkey=%" PRIu64 " ndata=%" PRIu64 " dsize=%" PRIu64 " %s }", est->nkeys, est->ndata, est->dsize, est->exact ? "T" : "F");
     }
     printf("}\n");
     printf(" pivots:\n");
-    for (i=0; i<n->n_children-1; i++) {
+    for (int i=0; i<n->n_children-1; i++) {
         struct kv_pair *piv = n->childkeys[i];
-        printf("  pivot %d:", i);
+        printf("  pivot %2d:", i);
         assert(n->flags == 0);
         print_item(kv_pair_key_const(piv), kv_pair_keylen(piv));
         printf("\n");
     }
     printf(" children:\n");
-    for (i=0; i<n->n_children; i++) {
+    for (int i=0; i<n->n_children; i++) {
         if (n->height > 0) {
             printf("   child %d: %" PRId64 "\n", i, BP_BLOCKNUM(n, i).b);
             unsigned int n_bytes = BNC_NBYTESINBUF(n, i); 
@@ -203,9 +202,9 @@ dump_node (int f, BLOCKNUM blocknum, struct brt_header *h) {
 			     );
 	    }
 	} else {
-	    printf(" optimized_for_upgrade=%u\n", BLB_OPTIMIZEDFORUPGRADE(n, i));
-	    printf(" n_bytes_in_buffer=%u\n", BLB_NBYTESINBUF(n, i));
-	    printf(" items_in_buffer  =%u\n", toku_omt_size(BLB_BUFFER(n, i)));
+	    printf("  bn %2d: optimized_for_upgrade=%u", i, BLB_OPTIMIZEDFORUPGRADE(n, i));
+	    printf(" n_bytes_in_buffer=%u", BLB_NBYTESINBUF(n, i));
+	    printf(" items_in_buffer=%u\n", toku_omt_size(BLB_BUFFER(n, i)));
 	    if (dump_data) toku_omt_iterate(BLB_BUFFER(n, i), print_le, 0);
 	}
     }

newbrt/brtloader-internal.h

@@ -246,7 +246,7 @@ int toku_brt_loader_internal_init (/* out */ BRTLOADER *blp,
 
 void toku_brtloader_internal_destroy (BRTLOADER bl, BOOL is_error);
 
-enum { disksize_row_overhead = 9 }; // how much overhead for a row in the fractal tree
+enum { disksize_row_overhead = 9 }; // how much overhead for a row in the fractal tree (#3588, 9 = cmd + keylen + vallen?)
 
 // For test purposes only.  (In production, the rowset size is determined by negotation with the cachetable for some memory.  See #2613.)
 uint64_t toku_brtloader_get_rowset_budget_for_testing (void);

newbrt/tests/brtloader-test-writer.c

@@ -195,25 +195,25 @@ int test_main (int argc, const char *argv[]) {
     const char *progname=argv[0];
     argc--; argv++;
     while (argc>0) {
-	if (strcmp(argv[0],"-h")==0) {
+	if (strcmp(argv[0], "-h") == 0 || strcmp(argv[0], "--help") == 0) {
 	    return usage(progname);
-        } else if (strcmp(argv[0],"-v")==0) {
+        } else if (strcmp(argv[0], "-v") == 0 || strcmp(argv[0], "--verbose") == 0) {
 	    verbose=1;
-	} else if (strcmp(argv[0],"-q")==0) {
+	} else if (strcmp(argv[0], "-q") == 0) {
 	    verbose=0;
-        } else if (strcmp(argv[0],"-r") == 0) {
+        } else if (strcmp(argv[0], "-r") == 0) {
             argc--; argv++;
             nrows = atoi(argv[0]);
-        } else if (strcmp(argv[0],"-s") == 0) {
+        } else if (strcmp(argv[0], "-s") == 0) {
             toku_brtloader_set_size_factor(1);
-	} else if (argc!=1) {
+	} else if (argv[0][0] == '-' || argc != 1) {
             return usage(progname);
 	} else {
             break;
         }
 	argc--; argv++;
     }
-    assert(argc==1); // argv[1] is the directory in which to do the test.
+    assert(argc == 1); // argv[1] is the directory in which to do the test.
     const char* directory = argv[0];
     char unlink_all[strlen(directory)+20];
     snprintf(unlink_all, strlen(directory)+20, "rm -rf %s", directory);
@@ -234,9 +234,10 @@ int test_main (int argc, const char *argv[]) {
 
     test_write_dbfile(template, nrows, output_name);
 
+#if 0
     r = system(unlink_all);
     CKERR(r);
-
+#endif
     return 0;
 }
 

newbrt/tests/verify-unsorted-leaf.c

@@ -20,8 +20,8 @@ static void
 append_leaf(BRTNODE leafnode, void *key, size_t keylen, void *val, size_t vallen) {
     assert(leafnode->height == 0);
 
-    DBT thekey; toku_fill_dbt(&thekey, key, keylen);
-    DBT theval; toku_fill_dbt(&theval, val, vallen);
+    DBT thekey = { .data = key, .size = keylen }; 
+    DBT theval = { .data = val, .size = vallen };
 
     // get an index that we can use to create a new leaf entry
     uint32_t idx = toku_omt_size(BLB_BUFFER(leafnode, 0));