Dramatic speedups: For {{{db-benchmark-test -x}}} (4 interations)

||                                    ||      DB size ||  logsize ||  rate    ||
|| Before (with transactions)         ||      .33GB   ||   3.0GB  ||  17477/s ||
|| No Transactions (with this fix)    ||      .43GB   ||   0GB    || 104124/s ||
|| After removing undo info (#775)    ||      .33GB   ||   1.4GB  ||  25696/s ||
|| After optimizing commit (#725)     ||      .43GB   ||   0.6GB  ||  39909/s ||

The file size is a larger, but the logs are a lot smaller.

Addresses #725, #775.

Manipulations:
{{{
$ (cd tokudb; svn merge -r3763:3778 https://svn.tokutek.com/tokudb/tokudb.725)
$ svn delete tokudb.725
}}}


git-svn-id: file:///svn/tokudb@3779 c7de825b-a66e-492c-adef-691d508d4ae1

newbrt/brt.c

@@ -1153,12 +1153,21 @@ static int apply_cmd_to_le_committed (u_int32_t klen, void *kval,
     return 0;
 }
 
-static int apply_cmd_to_le_both (TXNID xid __attribute__((__unused__)),
+static int apply_cmd_to_le_both (TXNID xid,
 				 u_int32_t klen, void *kval,
 				 u_int32_t clen, void *cval,
 				 u_int32_t plen, void *pval,
 				 BRT_CMD cmd,
 				 u_int32_t *newlen, u_int32_t *disksize, LEAFENTRY *new_data) {
+    u_int32_t prev_len;
+    void     *prev_val;
+    if (xid==cmd->xid) {
+	// The xids match, so throw away the provisional value.
+	prev_len = clen;  prev_val = cval;
+    } else {
+	// If the xids don't match, then we are moving the provisional value to committed status.
+	prev_len = plen;  prev_val = pval;
+    }
     // keep the committed value for rollback.
     assert(cmd->u.id.key->size == klen);
     assert(memcmp(cmd->u.id.key->data, kval, klen)==0);
@@ -1166,22 +1175,24 @@ static int apply_cmd_to_le_both (TXNID xid __attribute__((__unused__)),
     case BRT_INSERT:
 	return le_both(cmd->xid,
 		       klen, kval,
-		       clen, cval,
+		       prev_len, prev_val,
 		       cmd->u.id.val->size, cmd->u.id.val->data,
 		       newlen, disksize, new_data);
     case BRT_DELETE_ANY:
     case BRT_DELETE_BOTH:
 	return le_provdel(cmd->xid,
 			  klen, kval,
-			  clen, cval,
+			  prev_len, prev_val,
 			  newlen, disksize, new_data);
     case BRT_ABORT_BOTH:
     case BRT_ABORT_ANY:
+	// I don't see how you could have an abort where the xids don't match.  But do it anyway.
 	return le_committed(klen, kval,
-			    clen, cval,
+			    prev_len, prev_val,
 			    newlen, disksize, new_data);
     case BRT_COMMIT_BOTH:
     case BRT_COMMIT_ANY:
+	// In the future we won't even have these commit messages.
 	return le_committed(klen, kval,
 			    plen, pval,
 			    newlen, disksize, new_data);
@@ -1190,7 +1201,8 @@ static int apply_cmd_to_le_both (TXNID xid __attribute__((__unused__)),
     assert(0);
     return 0;
 }
-static int apply_cmd_to_le_provdel (TXNID xid __attribute__((__unused__)),
+
+static int apply_cmd_to_le_provdel (TXNID xid,
 				    u_int32_t klen, void *kval,
 				    u_int32_t clen, void *cval,
 				    BRT_CMD cmd,
@@ -1200,21 +1212,36 @@ static int apply_cmd_to_le_provdel (TXNID xid __attribute__((__unused__)),
     assert(memcmp(cmd->u.id.key->data, kval, klen)==0);
     switch (cmd->type) {
     case BRT_INSERT:
-	return le_both(cmd->xid,
-		       klen, kval,
-		       clen, cval,
-		       cmd->u.id.val->size, cmd->u.id.val->data,
-		       newlen, disksize, new_data);
+	if (cmd->xid == xid) {
+	    return le_both(cmd->xid,
+			   klen, kval,
+			   clen, cval,
+			   cmd->u.id.val->size, cmd->u.id.val->data,
+			   newlen, disksize, new_data);
+	} else {
+	    // It's an insert, but the committed value is deleted (since the xids don't match, we assume the delete took effect)
+	    return le_provpair(cmd->xid,
+			       klen, kval,
+			       cmd->u.id.val->size, cmd->u.id.val->data,
+			       newlen, disksize, new_data);
+	}
     case BRT_DELETE_ANY:
     case BRT_DELETE_BOTH:
-	// A delete of a delete could conceivably return the same item, but to simplify things we just reallocate it
-	// because othewise we have to notice not to free() the olditem.
-	return le_provdel(cmd->xid,
-			  klen, kval,
-			  clen, cval,
-			  newlen, disksize, new_data);
+	if (cmd->xid == xid) {
+	    // A delete of a delete could conceivably return the identical value, saving a malloc and a free, but to simplify things we just reallocate it
+	    // because othewise we have to notice not to free() the olditem.
+	    return le_provdel(cmd->xid,
+			      klen, kval,
+			      clen, cval,
+			      newlen, disksize, new_data);
+	} else {
+	    // The commited value is deleted, and we are deleting, so treat as a delete.
+	    *new_data = 0;
+	    return 0;
+	}
     case BRT_ABORT_BOTH:
     case BRT_ABORT_ANY:
+	// I don't see how the xids could not match...
 	return le_committed(klen, kval,
 			    clen, cval,
 			    newlen, disksize, new_data);
@@ -1228,7 +1255,7 @@ static int apply_cmd_to_le_provdel (TXNID xid __attribute__((__unused__)),
     return 0;
 }
 
-static int apply_cmd_to_le_provpair (TXNID xid __attribute__((__unused__)),
+static int apply_cmd_to_le_provpair (TXNID xid,
 				     u_int32_t klen, void *kval,
 				     u_int32_t plen , void *pval,
 				     BRT_CMD cmd,
@@ -1237,16 +1264,36 @@ static int apply_cmd_to_le_provpair (TXNID xid __attribute__((__unused__)),
     assert(memcmp(cmd->u.id.key->data, kval, klen)==0);
     switch (cmd->type) {
     case BRT_INSERT:
-	// it's still a provpair (the old prov value is lost)
-	return le_provpair(cmd->xid,
+	if (cmd->xid == xid) {
+	    // it's still a provpair (the old prov value is lost)
+	    return le_provpair(cmd->xid,
+			       klen, kval,
+			       cmd->u.id.val->size, cmd->u.id.val->data,
+			       newlen, disksize, new_data);
+	} else {
+	    // the old prov was actually committed.
+	    return le_both(cmd->xid,
 			   klen, kval,
+			   plen, pval,
 			   cmd->u.id.val->size, cmd->u.id.val->data,
 			   newlen, disksize, new_data);
+	}
     case BRT_DELETE_BOTH:
     case BRT_DELETE_ANY:
+	if (cmd->xid == xid) {
+	    // A delete of a provisional pair is nothign
+	    *new_data = 0;
+	    return 0;
+	} else {
+	    // The prov pair is actually a committed value.
+	    return le_provdel(cmd->xid,
+			      klen, kval,
+			      plen, pval,
+			      newlen, disksize, new_data);
+	}
     case BRT_ABORT_BOTH:
     case BRT_ABORT_ANY:
-	// A delete or abort of a provisional pair is nothing.
+	// An abort of a provisional pair is nothing.
 	*new_data = 0;
 	return 0;
     case BRT_COMMIT_ANY:

newbrt/roll.c

@@ -70,7 +70,16 @@ static int do_insertion (enum brt_cmd_type type, TXNID xid, FILENUM filenum, BYT
 }
 
 
+static int do_nothing_with_filenum(TOKUTXN txn, FILENUM filenum) {
+    CACHEFILE cf;
+    int r = toku_cachefile_of_filenum(txn->logger->ct, filenum, &cf);
+    assert(r==0);
+    return toku_cachefile_close(&cf, toku_txn_logger(txn));
+}
+
+
 int toku_commit_cmdinsert (TXNID xid, FILENUM filenum, BYTESTRING key,BYTESTRING data,TOKUTXN txn) {
+    return do_nothing_with_filenum(txn, filenum);
     return do_insertion (BRT_COMMIT_BOTH, xid, filenum, key, &data, txn);
 }
 
@@ -79,6 +88,7 @@ int toku_rollback_cmdinsert (TXNID xid, FILENUM filenum, BYTESTRING key,BYTESTRI
 }
 
 int toku_commit_cmddeleteboth (TXNID xid, FILENUM filenum, BYTESTRING key,BYTESTRING data,TOKUTXN txn) {
+    return do_nothing_with_filenum(txn, filenum);
     return do_insertion (BRT_COMMIT_BOTH, xid, filenum, key, &data, txn);
 }
 
@@ -87,6 +97,7 @@ int toku_rollback_cmddeleteboth (TXNID xid, FILENUM filenum, BYTESTRING key,BYTE
 }
 
 int toku_commit_cmddelete (TXNID xid, FILENUM filenum, BYTESTRING key,TOKUTXN txn) {
+    return do_nothing_with_filenum(txn, filenum);
     return do_insertion (BRT_COMMIT_ANY, xid, filenum, key, 0, txn);
 }
 

src/lock_tree/db_id.c

@@ -59,7 +59,7 @@ int toku_db_id_create(toku_db_id** pdbid, const char* path,
         goto cleanup;
     }
     char* tmp = (char*)toku_realloc(db_id->absolute_path,
-                             (strlen(db_id->absolute_path) + 1) * sizeof(char));
+				    (strlen(db_id->absolute_path) + 1) * sizeof(char));
     if (!tmp) { r = ENOMEM; goto cleanup; }
     db_id->absolute_path = tmp;
                     

src/tests/test_archive1.c

@@ -23,7 +23,7 @@ int main (int argc, const char *argv[]) {
     r=txn->commit(txn, 0);    CKERR(r);
 
     int i;
-    for (i=0; i<100; i++) {
+    for (i=0; i<200; i++) {
 	DBT key,data;
 	char hello[30],there[30];
 	snprintf(hello, sizeof(hello), "hello%d", i);