[t:4502] Merged new Garbage Collection to main, one test, stat64-root-changes...

[t:4502] Merged new Garbage Collection to main, one test, stat64-root-changes fails, but just needs to be updated now that GC is removed from message application.

git-svn-id: file:///svn/toku/tokudb@40096 c7de825b-a66e-492c-adef-691d508d4ae1

newbrt/brt-internal.h

@@ -912,8 +912,6 @@ brt_leaf_apply_cmd_once (
     const BRT_MSG cmd,
     u_int32_t idx,
     LEAFENTRY le,
-    OMT snapshot_txnids,
-    OMT live_list_reverse,
     uint64_t *workdonep
     );
 
@@ -927,9 +925,7 @@ brt_leaf_put_cmd (
     BASEMENTNODE bn, 
     BRT_MSG cmd, 
     bool* made_change,
-    uint64_t *workdone,
-    OMT snapshot_txnids,
-    OMT live_list_reverse
+    uint64_t *workdone
     );
 
 void toku_apply_cmd_to_leaf(
@@ -939,9 +935,7 @@ void toku_apply_cmd_to_leaf(
     BRTNODE node, 
     BRT_MSG cmd, 
     bool *made_change, 
-    uint64_t *workdone, 
-    OMT snapshot_txnids, 
-    OMT live_list_reverse
+    uint64_t *workdone
     );
 
 // FIXME needs toku prefix
@@ -951,9 +945,7 @@ void brtnode_put_cmd (
     DESCRIPTOR desc,
     BRTNODE node, 
     BRT_MSG cmd, 
-    bool is_fresh, 
-    OMT snapshot_txnids, 
-    OMT live_list_reverse
+    bool is_fresh
     );
 
 void toku_reset_root_xid_that_created(BRT brt, TXNID new_root_xid_that_created);

newbrt/brt-test-helpers.c

@@ -143,9 +143,7 @@ int toku_testsetup_insert_to_leaf (BRT brt, BLOCKNUM blocknum, char *key, int ke
         &brt->h->descriptor,
         node,
         &cmd,
-        true, 
-        NULL, 
-        NULL
+        true
         );    
 
     toku_verify_or_set_counts(node);

newbrt/brtloader.c

@@ -2788,7 +2788,7 @@ static void add_pair_to_leafnode (struct leaf_buf *lbuf, unsigned char *key, int
     DBT theval = { .data = val, .size = vallen };
     BRT_MSG_S cmd = { BRT_INSERT, ZERO_MSN, lbuf->xids, .u.id = { &thekey, &theval } };
     uint64_t workdone=0;
-    brt_leaf_apply_cmd_once(leafnode, BLB(leafnode,0), &cmd, idx, NULL, NULL, NULL, &workdone);
+    brt_leaf_apply_cmd_once(leafnode, BLB(leafnode,0), &cmd, idx, NULL, &workdone);
 }
 
 static int write_literal(struct dbout *out, void*data,  size_t len) {

newbrt/tests/make-tree.c

@@ -39,7 +39,7 @@ append_leaf(BRTNODE leafnode, void *key, size_t keylen, void *val, size_t vallen
 
     // apply an insert to the leaf node
     BRT_MSG_S cmd = { BRT_INSERT, msn, xids_get_root_xids(), .u.id = { &thekey, &theval } };
-    brt_leaf_apply_cmd_once(leafnode, BLB(leafnode,0), &cmd, idx, NULL, NULL, NULL, NULL);
+    brt_leaf_apply_cmd_once(leafnode, BLB(leafnode,0), &cmd, idx, NULL, NULL);
 
     leafnode->max_msn_applied_to_node_on_disk = msn;
 

newbrt/tests/msnfilter.c

@@ -48,7 +48,7 @@ append_leaf(BRT brt, BRTNODE leafnode, void *key, size_t keylen, void *val, size
 
     bool made_change;
     u_int64_t workdone=0;
-    toku_apply_cmd_to_leaf(brt->compare_fun, brt->update_fun, &brt->h->descriptor, leafnode, &cmd, &made_change, &workdone, NULL, NULL);
+    toku_apply_cmd_to_leaf(brt->compare_fun, brt->update_fun, &brt->h->descriptor, leafnode, &cmd, &made_change, &workdone);
     {
 	int r = toku_brt_lookup(brt, &thekey, lookup_checkf, &pair);
 	assert(r==0);
@@ -56,7 +56,7 @@ append_leaf(BRT brt, BRTNODE leafnode, void *key, size_t keylen, void *val, size
     }
 
     BRT_MSG_S badcmd = { BRT_INSERT, msn, xids_get_root_xids(), .u.id = { &thekey, &badval } };
-    toku_apply_cmd_to_leaf(brt->compare_fun, brt->update_fun, &brt->h->descriptor, leafnode, &badcmd, &made_change, &workdone, NULL, NULL);
+    toku_apply_cmd_to_leaf(brt->compare_fun, brt->update_fun, &brt->h->descriptor, leafnode, &badcmd, &made_change, &workdone);
 
     
     // message should be rejected for duplicate msn, row should still have original val
@@ -69,7 +69,7 @@ append_leaf(BRT brt, BRTNODE leafnode, void *key, size_t keylen, void *val, size
     // now verify that message with proper msn gets through
     msn = next_dummymsn();
     BRT_MSG_S cmd2 = { BRT_INSERT, msn, xids_get_root_xids(), .u.id = { &thekey, &val2 } };
-    toku_apply_cmd_to_leaf(brt->compare_fun, brt->update_fun, &brt->h->descriptor, leafnode, &cmd2, &made_change, &workdone, NULL, NULL);
+    toku_apply_cmd_to_leaf(brt->compare_fun, brt->update_fun, &brt->h->descriptor, leafnode, &cmd2, &made_change, &workdone);
     
     // message should be accepted, val should have new value
     {
@@ -81,7 +81,7 @@ append_leaf(BRT brt, BRTNODE leafnode, void *key, size_t keylen, void *val, size
     // now verify that message with lesser (older) msn is rejected
     msn.msn = msn.msn - 10;
     BRT_MSG_S cmd3 = { BRT_INSERT, msn, xids_get_root_xids(), .u.id = { &thekey, &badval } };
-    toku_apply_cmd_to_leaf(brt->compare_fun, brt->update_fun, &brt->h->descriptor, leafnode, &cmd3, &made_change, &workdone, NULL, NULL);
+    toku_apply_cmd_to_leaf(brt->compare_fun, brt->update_fun, &brt->h->descriptor, leafnode, &cmd3, &made_change, &workdone);
     
     // message should be rejected, val should still have value in pair2
     {

newbrt/tests/orthopush-flush.c

@@ -125,10 +125,10 @@ insert_random_message_to_leaf(BRT t, BRTNODE leafnode, BASEMENTNODE blb, LEAFENT
     msg.u.id.val = valdbt;
     size_t memsize;
     int64_t numbytes;
-    int r = apply_msg_to_leafentry(&msg, NULL, &memsize, save, NULL, NULL, NULL, NULL, NULL, &numbytes);
+    int r = apply_msg_to_leafentry(&msg, NULL, &memsize, save, NULL, NULL, NULL, &numbytes);
     assert_zero(r);
     bool made_change;
-    brt_leaf_put_cmd(t->compare_fun, t->update_fun, NULL, leafnode, blb, &msg, &made_change, NULL, NULL, NULL);
+    brt_leaf_put_cmd(t->compare_fun, t->update_fun, NULL, leafnode, blb, &msg, &made_change, NULL);
     if (msn.msn > blb->max_msn_applied.msn) {
         blb->max_msn_applied = msn;
     }
@@ -167,14 +167,14 @@ insert_same_message_to_leaves(BRT t, BRTNODE child1, BASEMENTNODE blb1, BRTNODE
     msg.u.id.val = valdbt;
     size_t memsize;
     int64_t numbytes;
-    int r = apply_msg_to_leafentry(&msg, NULL, &memsize, save, NULL, NULL, NULL, NULL, NULL, &numbytes);
+    int r = apply_msg_to_leafentry(&msg, NULL, &memsize, save, NULL, NULL, NULL, &numbytes);
     assert_zero(r);
     bool made_change;
-    brt_leaf_put_cmd(t->compare_fun, t->update_fun, NULL, child1, blb1, &msg, &made_change, NULL, NULL, NULL);
+    brt_leaf_put_cmd(t->compare_fun, t->update_fun, NULL, child1, blb1, &msg, &made_change, NULL);
     if (msn.msn > blb1->max_msn_applied.msn) {
         blb1->max_msn_applied = msn;
     }
-    brt_leaf_put_cmd(t->compare_fun, t->update_fun, NULL, child2, blb2, &msg, &made_change, NULL, NULL, NULL);
+    brt_leaf_put_cmd(t->compare_fun, t->update_fun, NULL, child2, blb2, &msg, &made_change, NULL);
     if (msn.msn > blb2->max_msn_applied.msn) {
         blb2->max_msn_applied = msn;
     }
@@ -587,7 +587,7 @@ flush_to_leaf(BRT t, bool make_leaf_up_to_date, bool use_flush) {
         for (i = 0; i < num_parent_messages; ++i) {
             if (!parent_messages_is_fresh[i]) {
                 bool made_change;
-                toku_apply_cmd_to_leaf(t->compare_fun, t->update_fun, &t->h->descriptor, child, parent_messages[i], &made_change, NULL, NULL, NULL);
+                toku_apply_cmd_to_leaf(t->compare_fun, t->update_fun, &t->h->descriptor, child, parent_messages[i], &made_change, NULL);
             }
         }
         for (i = 0; i < 8; ++i) {
@@ -811,7 +811,7 @@ flush_to_leaf_with_keyrange(BRT t, bool make_leaf_up_to_date) {
             if (dummy_cmp(NULL, parent_messages[i]->u.id.key, &childkeys[7]) <= 0 &&
                 !parent_messages_is_fresh[i]) {
                 bool made_change;
-                toku_apply_cmd_to_leaf(t->compare_fun, t->update_fun, &t->h->descriptor, child, parent_messages[i], &made_change, NULL, NULL, NULL);
+                toku_apply_cmd_to_leaf(t->compare_fun, t->update_fun, &t->h->descriptor, child, parent_messages[i], &made_change, NULL);
             }
         }
         for (i = 0; i < 8; ++i) {
@@ -998,8 +998,8 @@ compare_apply_and_flush(BRT t, bool make_leaf_up_to_date) {
         for (i = 0; i < num_parent_messages; ++i) {
             if (!parent_messages_is_fresh[i]) {
                 bool made_change;
-                toku_apply_cmd_to_leaf(t->compare_fun, t->update_fun, &t->h->descriptor, child1, parent_messages[i], &made_change, NULL, NULL, NULL);
-                toku_apply_cmd_to_leaf(t->compare_fun, t->update_fun, &t->h->descriptor, child2, parent_messages[i], &made_change, NULL, NULL, NULL);
+                toku_apply_cmd_to_leaf(t->compare_fun, t->update_fun, &t->h->descriptor, child1, parent_messages[i], &made_change, NULL);
+                toku_apply_cmd_to_leaf(t->compare_fun, t->update_fun, &t->h->descriptor, child2, parent_messages[i], &made_change, NULL);
             }
         }
         for (i = 0; i < 8; ++i) {

newbrt/tests/test-leafentry-nested.c

@@ -398,7 +398,7 @@ test_le_apply(ULE ule_initial, BRT_MSG msg, ULE ule_expected) {
                                le_initial,
                                &result_memsize,
                                &le_result,
-                               NULL, NULL, NULL, 
+                               NULL, 
                                NULL, NULL, &ignoreme);
     CKERR(r);
 

newbrt/tests/verify-bad-msn.c

@@ -42,7 +42,7 @@ append_leaf(BRTNODE leafnode, void *key, size_t keylen, void *val, size_t vallen
 
     // apply an insert to the leaf node
     BRT_MSG_S cmd = { BRT_INSERT, msn, xids_get_root_xids(), .u.id = { &thekey, &theval } };
-    brt_leaf_apply_cmd_once(leafnode, BLB(leafnode, 0), &cmd, idx, NULL, NULL, NULL, NULL);
+    brt_leaf_apply_cmd_once(leafnode, BLB(leafnode, 0), &cmd, idx, NULL, NULL);
 
     // Create bad tree (don't do following):
     // leafnode->max_msn_applied_to_node = msn;

newbrt/tests/verify-bad-pivots.c

@@ -30,7 +30,7 @@ append_leaf(BRTNODE leafnode, void *key, size_t keylen, void *val, size_t vallen
     // apply an insert to the leaf node
     MSN msn = next_dummymsn();
     BRT_MSG_S cmd = { BRT_INSERT, msn, xids_get_root_xids(), .u.id = { &thekey, &theval } };
-    brt_leaf_apply_cmd_once(leafnode, BLB(leafnode, 0), &cmd, idx, NULL, NULL, NULL, NULL);
+    brt_leaf_apply_cmd_once(leafnode, BLB(leafnode, 0), &cmd, idx, NULL, NULL);
 
     // dont forget to dirty the node
     leafnode->dirty = 1;

newbrt/tests/verify-dup-in-leaf.c

@@ -31,7 +31,7 @@ append_leaf(BRTNODE leafnode, void *key, size_t keylen, void *val, size_t vallen
     // apply an insert to the leaf node
     MSN msn = next_dummymsn();
     BRT_MSG_S cmd = { BRT_INSERT, msn, xids_get_root_xids(), .u.id = { &thekey, &theval } };
-    brt_leaf_apply_cmd_once(leafnode, BLB(leafnode, 0), &cmd, idx, NULL, NULL, NULL, NULL);
+    brt_leaf_apply_cmd_once(leafnode, BLB(leafnode, 0), &cmd, idx, NULL, NULL);
 
     // dont forget to dirty the node
     leafnode->dirty = 1;

newbrt/tests/verify-dup-pivots.c

@@ -30,7 +30,7 @@ append_leaf(BRTNODE leafnode, void *key, size_t keylen, void *val, size_t vallen
     // apply an insert to the leaf node
     MSN msn = next_dummymsn();
     BRT_MSG_S cmd = { BRT_INSERT, msn, xids_get_root_xids(), .u.id = { &thekey, &theval } };
-    brt_leaf_apply_cmd_once(leafnode, BLB(leafnode, 0), &cmd, idx, NULL, NULL, NULL, NULL);
+    brt_leaf_apply_cmd_once(leafnode, BLB(leafnode, 0), &cmd, idx, NULL, NULL);
 
     // dont forget to dirty the node
     leafnode->dirty = 1;

newbrt/tests/verify-misrouted-msgs.c

@@ -31,7 +31,7 @@ append_leaf(BRTNODE leafnode, void *key, size_t keylen, void *val, size_t vallen
     // apply an insert to the leaf node
     MSN msn = next_dummymsn();
     BRT_MSG_S cmd = { BRT_INSERT, msn, xids_get_root_xids(), .u.id = { &thekey, &theval } };
-    brt_leaf_apply_cmd_once(leafnode, BLB(leafnode,0), &cmd, idx, NULL, NULL, NULL, NULL);
+    brt_leaf_apply_cmd_once(leafnode, BLB(leafnode,0), &cmd, idx, NULL, NULL);
 
     // dont forget to dirty the node
     leafnode->dirty = 1;

newbrt/tests/verify-unsorted-leaf.c

@@ -31,7 +31,7 @@ append_leaf(BRTNODE leafnode, void *key, size_t keylen, void *val, size_t vallen
     // apply an insert to the leaf node
     MSN msn = next_dummymsn();
     BRT_MSG_S cmd = { BRT_INSERT, msn, xids_get_root_xids(), .u.id = { &thekey, &theval } };
-    brt_leaf_apply_cmd_once(leafnode, BLB(leafnode, 0), &cmd, idx, NULL, NULL, NULL, NULL);
+    brt_leaf_apply_cmd_once(leafnode, BLB(leafnode, 0), &cmd, idx, NULL, NULL);
 
     // dont forget to dirty the node
     leafnode->dirty = 1;

newbrt/tests/verify-unsorted-pivots.c

@@ -30,7 +30,7 @@ append_leaf(BRTNODE leafnode, void *key, size_t keylen, void *val, size_t vallen
     // apply an insert to the leaf node
     MSN msn = next_dummymsn();
     BRT_MSG_S cmd = { BRT_INSERT, msn, xids_get_root_xids(), .u.id = { &thekey, &theval } };
-    brt_leaf_apply_cmd_once(leafnode, BLB(leafnode, 0), &cmd, idx, NULL, NULL, NULL, NULL);
+    brt_leaf_apply_cmd_once(leafnode, BLB(leafnode, 0), &cmd, idx, NULL, NULL);
 
     // dont forget to dirty the node
     leafnode->dirty = 1;

newbrt/ule.c

@@ -336,8 +336,6 @@ apply_msg_to_leafentry(BRT_MSG   msg,		// message to apply to leafentry
 		       OMT omt, 
 		       struct mempool *mp, 
 		       void **maybe_free,
-                       OMT snapshot_xids,
-                       OMT live_list_reverse,
                        int64_t * numbytes_delta_p) {  // change in total size of key and val, not including any overhead
     ULE_S ule;
     int rval;
@@ -351,9 +349,6 @@ apply_msg_to_leafentry(BRT_MSG   msg,		// message to apply to leafentry
 	oldnumbytes = ule_get_innermost_numbytes(&ule);
     }
     msg_modify_ule(&ule, msg);          // modify unpacked leafentry
-    if (snapshot_xids && live_list_reverse) {
-        garbage_collection(&ule, snapshot_xids, live_list_reverse);
-    }
     rval = le_pack(&ule,                // create packed leafentry
 		   new_leafentry_memorysize, 
 		   new_leafentry_p,
@@ -368,6 +363,49 @@ apply_msg_to_leafentry(BRT_MSG   msg,		// message to apply to leafentry
 }
 
 
+// Garbage collect one leaf entry, using the given OMT's.
+// Parameters:
+// -- old_leaf_entry : the leaf we intend to clean up through garbage
+// collecting.
+// -- new_leaf_entry (OUTPUT) : a pointer to the leaf entry after
+// garbage collection.
+// -- new_leaf_entry_memory_size : after this call, our leaf entry
+// should be empty or smaller.  This number represents that and is
+// used in a previous call to truncate the existing size.
+// -- omt : the memory where our leaf entry resides.
+// -- mp : our memory pool.
+// -- maybe_free (OUTPUT) : in a previous call, we may be able to free
+// the memory completely, if we removed the leaf entry.
+// -- snapshot_xids : we use these in memory transaction ids to
+// determine what to garbage collect.
+// -- live_list_reverse : list of in memory active transactions.
+// NOTE: it is not a good idea to garbage collect a leaf
+// entry with only one committed value.
+int
+garbage_collect_leafentry(LEAFENTRY old_leaf_entry,
+                          LEAFENTRY *new_leaf_entry,
+                          size_t *new_leaf_entry_memory_size,
+                          OMT omt,
+                          struct mempool *mp,
+                          void **maybe_free,
+                          OMT snapshot_xids,
+                          OMT live_list_reverse) {
+    int r = 0;
+    ULE_S ule;
+    le_unpack(&ule, old_leaf_entry);
+    assert(snapshot_xids);
+    assert(live_list_reverse);
+    garbage_collection(&ule, snapshot_xids, live_list_reverse);
+    r = le_pack(&ule,
+                new_leaf_entry_memory_size,
+                new_leaf_entry,
+                omt,
+                mp,
+                maybe_free);
+    assert(r == 0);
+    ule_cleanup(&ule);
+    return r;
+}
 
 /////////////////////////////////////////////////////////////////////////////////
 // This layer of abstraction (msg_xxx)

newbrt/ule.h

@@ -59,10 +59,16 @@ int apply_msg_to_leafentry(BRT_MSG   msg,
 			   OMT omt, 
 			   struct mempool *mp, 
 			   void **maybe_free,
-                           OMT snapshot_xids,
-                           OMT live_list_reverse,
                            int64_t * numbytes_delta_p);
 
+int garbage_collect_leafentry(LEAFENTRY old_leaf_entry,
+                              LEAFENTRY *new_leaf_entry,
+                              size_t *new_leaf_entry_memory_size,
+                              OMT omt,
+                              struct mempool *mp,
+                              void **maybe_free,
+                              OMT snapshot_xids,
+                              OMT live_list_reverse);
 
 TXNID toku_get_youngest_live_list_txnid_for(TXNID xc, OMT live_list_reverse);