closes[t:2449] [t:2484] Merge #2449 changes to main.

Rollback logs are now checkpointed.  There are no rolltmp files.


git-svn-id: file:///svn/toku/tokudb@19167 c7de825b-a66e-492c-adef-691d508d4ae1

buildheader/db.h_4_1

@@ -379,7 +379,7 @@ typedef struct __toku_txn_progress {
 } *TOKU_TXN_PROGRESS, TOKU_TXN_PROGRESS_S;
 typedef void(*TXN_PROGRESS_POLL_FUNCTION)(TOKU_TXN_PROGRESS, void*);
 struct txn_stat {
-  u_int64_t rolltmp_raw_count;
+  u_int64_t rollback_raw_count;
 };
 struct __toku_db_txn {
   DB_ENV *mgrp /*In TokuDB, mgrp is a DB_ENV not a DB_TXNMGR*/; /* 32-bit offset=0 size=4, 64=bit offset=0 size=8 */

buildheader/db.h_4_3

@@ -395,7 +395,7 @@ typedef struct __toku_txn_progress {
 } *TOKU_TXN_PROGRESS, TOKU_TXN_PROGRESS_S;
 typedef void(*TXN_PROGRESS_POLL_FUNCTION)(TOKU_TXN_PROGRESS, void*);
 struct txn_stat {
-  u_int64_t rolltmp_raw_count;
+  u_int64_t rollback_raw_count;
 };
 struct __toku_db_txn {
   DB_ENV *mgrp /*In TokuDB, mgrp is a DB_ENV not a DB_TXNMGR*/; /* 32-bit offset=0 size=4, 64=bit offset=0 size=8 */

buildheader/db.h_4_4

@@ -403,7 +403,7 @@ typedef struct __toku_txn_progress {
 } *TOKU_TXN_PROGRESS, TOKU_TXN_PROGRESS_S;
 typedef void(*TXN_PROGRESS_POLL_FUNCTION)(TOKU_TXN_PROGRESS, void*);
 struct txn_stat {
-  u_int64_t rolltmp_raw_count;
+  u_int64_t rollback_raw_count;
 };
 struct __toku_db_txn {
   DB_ENV *mgrp /*In TokuDB, mgrp is a DB_ENV not a DB_TXNMGR*/; /* 32-bit offset=0 size=4, 64=bit offset=0 size=8 */

buildheader/db.h_4_5

@@ -403,7 +403,7 @@ typedef struct __toku_txn_progress {
 } *TOKU_TXN_PROGRESS, TOKU_TXN_PROGRESS_S;
 typedef void(*TXN_PROGRESS_POLL_FUNCTION)(TOKU_TXN_PROGRESS, void*);
 struct txn_stat {
-  u_int64_t rolltmp_raw_count;
+  u_int64_t rollback_raw_count;
 };
 struct __toku_db_txn {
   DB_ENV *mgrp /*In TokuDB, mgrp is a DB_ENV not a DB_TXNMGR*/; /* 32-bit offset=0 size=4, 64=bit offset=0 size=8 */

buildheader/db.h_4_6

@@ -407,7 +407,7 @@ typedef struct __toku_txn_progress {
 } *TOKU_TXN_PROGRESS, TOKU_TXN_PROGRESS_S;
 typedef void(*TXN_PROGRESS_POLL_FUNCTION)(TOKU_TXN_PROGRESS, void*);
 struct txn_stat {
-  u_int64_t rolltmp_raw_count;
+  u_int64_t rollback_raw_count;
 };
 struct __toku_db_txn {
   DB_ENV *mgrp /*In TokuDB, mgrp is a DB_ENV not a DB_TXNMGR*/; /* 32-bit offset=0 size=4, 64=bit offset=0 size=8 */

buildheader/make_db_h.c

@@ -585,7 +585,7 @@ int main (int argc __attribute__((__unused__)), char *const argv[] __attribute__
         printf("} *TOKU_TXN_PROGRESS, TOKU_TXN_PROGRESS_S;\n");
         printf("typedef void(*TXN_PROGRESS_POLL_FUNCTION)(TOKU_TXN_PROGRESS, void*);\n");
 
-	printf("struct txn_stat {\n  u_int64_t rolltmp_raw_count;\n};\n");
+	printf("struct txn_stat {\n  u_int64_t rollback_raw_count;\n};\n");
 	const char *extra[] = {
             "int (*txn_stat)(DB_TXN *, struct txn_stat **)", 
             "struct { void *next, *prev; } open_txns",

buildheader/tdb.h

@@ -354,7 +354,7 @@ typedef struct __toku_txn_progress {
 } *TOKU_TXN_PROGRESS, TOKU_TXN_PROGRESS_S;
 typedef void(*TXN_PROGRESS_POLL_FUNCTION)(TOKU_TXN_PROGRESS, void*);
 struct txn_stat {
-  u_int64_t rolltmp_raw_count;
+  u_int64_t rollback_raw_count;
 };
 struct __toku_db_txn {
   DB_ENV *mgrp /*In TokuDB, mgrp is a DB_ENV not a DB_TXNMGR*/;

db-benchmark-test/Makefile

@@ -64,7 +64,7 @@ build: build.tdb build.bdb
 build.bdb: $(TARGET_BDB) $(SCANSCAN_BDB) $(WINDOWS_BDB_LIB_NAME)
 build.tdb: $(TARGET_TDB) $(SCANSCAN_TDB)
 
-check: check-default check-rowsize-dup check-rowsize check-xfast check-x check-no-rolltmp check-4G
+check: check-default check-rowsize-dup check-rowsize check-xfast check-x check-no-rollback check-4G child.benchmark.dir
 
 SUPPORT_KEYSIZE=$$((3*1024)) 	# at least 3KiB
 SUPPORT_ROWSIZE=$$((80*1024))  	# at least 80KiB
@@ -96,8 +96,8 @@ check-xfast: $(TARGET_TDB)
 	./$(TARGET_TDB) $(VERBVERBOSE) --noserial -x --valsize 1000  --cachesize 8000000 --xcount 1000 --periter 20000 --env xfast.dir 1  $(SUMMARIZE_CMD)
 
 # A relatively fast test that detects #853 (don't log changes to a dictionary created in the same txn)
-check-no-rolltmp: $(TARGET_TDB)
-	./$(TARGET_TDB) $(VERBVERBOSE) --env no-rolltmp.dir --singlex --nolog --check_small_rolltmp $(SUMMARIZE_CMD)
+check-no-rollback: $(TARGET_TDB)
+	./$(TARGET_TDB) $(VERBVERBOSE) --env no-rollback.dir --singlex --nolog --check_small_rollback $(SUMMARIZE_CMD)
 
 # Check to make sure that if we make a file that's bigger than 4GB that we can read the file back out and get all the rows.
 ifeq ($(TOKU_SKIP_4G),1)

db-benchmark-test/db-benchmark-test.c

@@ -53,7 +53,7 @@ int singlex_child = 0;  // Do a single transaction, but do all work with a child
 int singlex = 0;  // Do a single transaction
 int singlex_create = 0;  // Create the db using the single transaction (only valid if singlex)
 int insert1first = 0;  // insert 1 before doing the rest
-int check_small_rolltmp = 0; // verify that the rollback logs are small (only valid if singlex)
+int check_small_rollback = 0; // verify that the rollback logs are small (only valid if singlex)
 int do_transactions = 0;
 int if_transactions_do_logging = DB_INIT_LOG; // set this to zero if we want no logging when transactions are used
 int do_abort = 0;
@@ -294,14 +294,14 @@ static void benchmark_shutdown (void) {
 #endif
     if (do_transactions && singlex && !insert1first && (singlex_create || prelock)) {
 #if defined(TOKUDB)
-        //There should be a single 'truncate' in the rolltmp instead of many 'insert' entries.
+        //There should be a single 'truncate' in the rollback instead of many 'insert' entries.
 	struct txn_stat *s;
 	r = tid->txn_stat(tid, &s);
 	assert(r==0);
         //TODO: #1125 Always do the test after performance testing is done.
-        if (singlex_child) fprintf(stderr, "SKIPPED 'small rolltmp' test for child txn\n");
+        if (singlex_child) fprintf(stderr, "SKIPPED 'small rollback' test for child txn\n");
         else
-            assert(s->rolltmp_raw_count < 100);  // gross test, not worth investigating details
+            assert(s->rollback_raw_count < 100);  // gross test, not worth investigating details
 	os_free(s);
 	//system("ls -l bench.tokudb");
 #endif
@@ -487,7 +487,7 @@ static int print_usage (const char *argv0) {
     fprintf(stderr, "    --singlex-child       (implies -x) Run the whole job as a single transaction, do all work a child of that transaction.\n");
     fprintf(stderr, "    --finish-child-first  Commit/abort child before doing so to parent (no effect if no child).\n");
     fprintf(stderr, "    --singlex-create      (implies --singlex)  Create the file using the single transaction (Default is to use a different transaction to create.)\n");
-    fprintf(stderr, "    --check_small_rolltmp (Only valid in --singlex mode)  Verify that very little data was saved in the rollback logs.\n");
+    fprintf(stderr, "    --check_small_rollback (Only valid in --singlex mode)  Verify that very little data was saved in the rollback logs.\n");
     fprintf(stderr, "    --prelock             Prelock the database.\n");
     fprintf(stderr, "    --prelockflag         Prelock the database and send the DB_PRELOCKED_WRITE flag.\n");
     fprintf(stderr, "    --abort               Abort the singlex after the transaction is over. (Requires --singlex.)\n");
@@ -589,8 +589,8 @@ int main (int argc, const char *const argv[]) {
 	    singlex = 1;
 	} else if (strcmp(arg, "--insert1first") == 0) {
 	    insert1first = 1;
-	} else if (strcmp(arg, "--check_small_rolltmp") == 0) {
-	    check_small_rolltmp = 1;
+	} else if (strcmp(arg, "--check_small_rollback") == 0) {
+	    check_small_rollback = 1;
 	} else if (strcmp(arg, "--xcount") == 0) {
             if (i+1 >= argc) return print_usage(argv[0]);
             items_per_transaction = strtoll(argv[++i], &endptr, 10); assert(*endptr == 0);
@@ -685,8 +685,8 @@ int main (int argc, const char *const argv[]) {
 	fprintf(stderr, "--insert_multiple only works on the TokuDB (not BDB)\n");
 	return print_usage(argv[0]);
     }
-    if (check_small_rolltmp) {
-	fprintf(stderr, "--check_small_rolltmp only works on the TokuDB (not BDB)\n");
+    if (check_small_rollback) {
+	fprintf(stderr, "--check_small_rollback only works on the TokuDB (not BDB)\n");
 	return print_usage(argv[0]);
     }
 #endif
@@ -697,8 +697,8 @@ int main (int argc, const char *const argv[]) {
             put_flagss[i] = put_flags;
         }
     }
-    if (check_small_rolltmp && !singlex) {
-	fprintf(stderr, "--check_small_rolltmp requires --singlex\n");
+    if (check_small_rollback && !singlex) {
+	fprintf(stderr, "--check_small_rollback requires --singlex\n");
 	return print_usage(argv[0]);
     }
     if (!do_transactions && insert_multiple) {

include/db.h

@@ -354,7 +354,7 @@ typedef struct __toku_txn_progress {
 } *TOKU_TXN_PROGRESS, TOKU_TXN_PROGRESS_S;
 typedef void(*TXN_PROGRESS_POLL_FUNCTION)(TOKU_TXN_PROGRESS, void*);
 struct txn_stat {
-  u_int64_t rolltmp_raw_count;
+  u_int64_t rollback_raw_count;
 };
 struct __toku_db_txn {
   DB_ENV *mgrp /*In TokuDB, mgrp is a DB_ENV not a DB_TXNMGR*/;

newbrt/Makefile

@@ -41,7 +41,6 @@ local: bins libs $(TEST_NEWBRT);
 BRT_SOURCES = \
   block_allocator \
   block_table \
-  bread \
   brt-serialize \
   brt-verify \
   brt \

newbrt/block_table.c

@@ -596,6 +596,19 @@ toku_block_verify_no_free_blocknums(BLOCK_TABLE bt) {
     assert(bt->current.blocknum_freelist_head.b == freelist_null.b);
 }
 
+//Verify there are no data blocks except root.
+void
+toku_block_verify_no_data_blocks_except_root_unlocked(BLOCK_TABLE bt, BLOCKNUM root) {
+    //Relies on checkpoint having used optimize_translation
+    assert(root.b >= RESERVED_BLOCKNUMS);
+    assert(bt->current.smallest_never_used_blocknum.b == root.b + 1);
+    int64_t i;
+    for (i=RESERVED_BLOCKNUMS; i < root.b; i++) {
+        BLOCKNUM b = make_blocknum(i);
+        assert(bt->current.block_translation[b.b].size == size_is_free);
+    }
+}
+
 //Verify a blocknum is currently allocated.
 void
 toku_verify_blocknum_allocated(BLOCK_TABLE bt, BLOCKNUM b) {

newbrt/block_table.h

@@ -35,6 +35,7 @@ void toku_allocate_blocknum(BLOCK_TABLE bt, BLOCKNUM *res, struct brt_header * h
 void toku_allocate_blocknum_unlocked(BLOCK_TABLE bt, BLOCKNUM *res, struct brt_header * h);
 void toku_free_blocknum(BLOCK_TABLE bt, BLOCKNUM *b, struct brt_header * h);
 void toku_verify_blocknum_allocated(BLOCK_TABLE bt, BLOCKNUM b);
+void toku_block_verify_no_data_blocks_except_root_unlocked(BLOCK_TABLE bt, BLOCKNUM root);
 void toku_block_verify_no_free_blocknums(BLOCK_TABLE bt);
 void toku_realloc_descriptor_on_disk(BLOCK_TABLE bt, DISKOFF size, DISKOFF *offset, struct brt_header * h);
 void toku_get_descriptor_offset_size(BLOCK_TABLE bt, DISKOFF *offset, DISKOFF *size);

newbrt/bread.c

-/* Buffered read. */
-
-#ident "$Id$"
-#ident "Copyright (c) 2007, 2008, 2009 Tokutek Inc.  All rights reserved."
-#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
-
-#include "includes.h"
-
-struct bread {
-    int64_t fileoff;        // The byte before this offset is the next byte we will read (since we are reading backward)
-    int fd;
-    int bufoff;             // The current offset in the buf.   The next byte we will read is buf[bufoff-1] (assuming that bufoff>0).
-    char *buf;              // A buffer with at least bufoff bytes in it.
-};
-
-BREAD create_bread_from_fd_initialize_at(int fd) {
-    BREAD XMALLOC(result);
-    int r = toku_os_get_file_size(fd, &result->fileoff); 
-    assert(r==0);
-    result->fd=fd;
-    result->bufoff=0;
-    result->buf = 0;
-    return result;
-}
-
-int close_bread_without_closing_fd(BREAD br) {
-    toku_free(br->buf);
-    toku_free(br);
-    return 0;
-}
-
-
-ssize_t bread_backwards(BREAD br, void *vbuf, size_t nbytes) {
-    char *buf=vbuf;
-    ssize_t result=0;
-    const int i4 = sizeof(u_int32_t);
-    while (nbytes > 0) {
-	// read whatever we can out of the buffer.
-	if (br->bufoff>0) {
-	    size_t to_copy = ((size_t)br->bufoff >= nbytes) ? nbytes : (size_t)br->bufoff;
-	    memcpy(buf+nbytes-to_copy, &br->buf[br->bufoff-to_copy], to_copy);
-	    nbytes             -= to_copy;
-	    result             += to_copy;
-	    br->bufoff         -= to_copy;
-	}
-	if (nbytes>0) {
-	    assert(br->bufoff==0);
-	    u_int32_t compressed_length_n, uncompressed_length_n;
-	    assert(br->fileoff>=i4); // there better be the three lengths plus the compressed data.
-	    { ssize_t r = pread(br->fd, &compressed_length_n,   i4, br->fileoff-  i4); assert(r==i4); }
-	    u_int32_t compressed_length = toku_dtoh32(compressed_length_n);
-	    assert(br->fileoff >= compressed_length + 3*i4);
-	    { ssize_t r = pread(br->fd, &uncompressed_length_n, i4, br->fileoff-2*i4); assert(r==i4); }
-	    u_int32_t uncompressed_length = toku_dtoh32(uncompressed_length_n);
-	    char *XMALLOC_N(compressed_length, zbuf);
-	    {
-		ssize_t r = pread(br->fd, zbuf,               compressed_length, br->fileoff- compressed_length -2*i4);
-		assert(r==(ssize_t)compressed_length);
-	    }
-	    {
-		u_int32_t compressed_length_n_again;
-		ssize_t r = pread(br->fd, &compressed_length_n_again,   i4, br->fileoff-compressed_length-3*i4); assert(r==i4);
-		assert(compressed_length_n_again == compressed_length_n);
-	    }
-	    uLongf destlen = uncompressed_length;
-	    XREALLOC_N(uncompressed_length, br->buf);
-	    uncompress((Bytef*)br->buf, &destlen, (Bytef*)zbuf, compressed_length);
-	    assert(destlen==uncompressed_length);
-	    toku_free(zbuf);
-	    
-	    br->bufoff = uncompressed_length;
-	    br->fileoff -= (compressed_length + 3*i4);
-	}
-    }
-    return result;
-}
-
-int bread_has_more(BREAD br) {
-    return (br->fileoff>0) || (br->bufoff>0);
-}

newbrt/bread.h

-#ifndef BREAD_H
-#define BREAD_H
-#ident "$Id$"
-#ident "Copyright (c) 2007, 2008, 2009 Tokutek Inc.  All rights reserved."
-#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
-
-// A BREAD reads a file backwards using buffered I/O.  BREAD stands for Buffered Read or Backwards Read.
-// Conceivably, we could read forward too.
-// The buffered I/O is buffered using a large buffer (e.g., something like a megabyte).
-// Furthermore, data is compressed into blocks.  Each block is a 4-byte compressed length (in network order), followed by compressed data, followed by a 4-byte uncompressed-length (in network order), followed by  a 4-byte compressed length
-// The compressed-length appears twice so that the file can be read backward or forward.
-// If not for the large-buffer requirement, as well as compression, as well as reading backward, we could have used a FILE.
-
-#include <sys/types.h>
-typedef struct bread *BREAD;
-
-BREAD create_bread_from_fd_initialize_at(int fd);
-// Effect:  Given a file descriptor, fd, create a BREAD.
-// Requires: The fd must be an open fd.
-
-int close_bread_without_closing_fd(BREAD);
-// Effect: Close the BREAD, but don't close the underlying fd.
-
-ssize_t bread_backwards(BREAD, void *buf, size_t nbytes);
-// Read nbytes into buf, reading backwards.
-
-int bread_has_more(BREAD);
-// Is there more to read?
-
-#endif

newbrt/brt-internal.h

@@ -217,6 +217,10 @@ int toku_serialize_brtnode_to_memory (BRTNODE node, int n_workitems, int n_threa
 				      /*out*/ size_t *n_bytes_to_write,
 				      /*out*/ char  **bytes_to_write);
 int toku_serialize_brtnode_to(int fd, BLOCKNUM, BRTNODE node, struct brt_header *h, int n_workitems, int n_threads, BOOL for_checkpoint);
+int toku_serialize_rollback_log_to (int fd, BLOCKNUM blocknum, ROLLBACK_LOG_NODE log,
+                                    struct brt_header *h, int n_workitems, int n_threads,
+                                    BOOL for_checkpoint);
+int toku_deserialize_rollback_log_from (int fd, BLOCKNUM blocknum, u_int32_t fullhash, ROLLBACK_LOG_NODE *logp, TOKUTXN txn, struct brt_header *h);
 int toku_deserialize_brtnode_from (int fd, BLOCKNUM off, u_int32_t /*fullhash*/, BRTNODE *brtnode, struct brt_header *h);
 unsigned int toku_serialize_brtnode_size(BRTNODE node); /* How much space will it take? */
 int toku_keycompare (bytevec key1, ITEMLEN key1len, bytevec key2, ITEMLEN key2len);

newbrt/brt.h

@@ -52,7 +52,7 @@ int brt_set_cachetable(BRT, CACHETABLE);
 int toku_brt_open(BRT, const char *fname_in_env,
 		  int is_create, int only_create, CACHETABLE ct, TOKUTXN txn, DB *db);
 int toku_brt_open_recovery(BRT, const char *fname_in_env,
-			   int is_create, int only_create, CACHETABLE ct, TOKUTXN txn, DB *db, int recovery_force_fcreate, FILENUM use_filenum);
+			   int is_create, int only_create, CACHETABLE ct, TOKUTXN txn, DB *db, FILENUM use_filenum);
 
 int toku_brt_remove_subdb(BRT brt, const char *dbname, u_int32_t flags);
 
@@ -206,12 +206,14 @@ void toku_maybe_truncate_cachefile (CACHEFILE cf, int fd, u_int64_t size_used);
 int maybe_preallocate_in_file (int fd, u_int64_t size);
 // Effect: If file size is less than SIZE, make it bigger by either doubling it or growing by 16MB whichever is less.
 
-int toku_brt_note_table_lock (BRT brt, TOKUTXN txn);
+int toku_brt_note_table_lock (BRT brt, TOKUTXN txn, BOOL ignore_not_empty);
 // Effect: Record the fact that the BRT has a table lock (and thus no other txn will modify it until this txn completes.  As a result, we can limit the amount of information in the rollback data structure.
 
 int toku_brt_zombie_needed (BRT brt);
 
 int toku_brt_get_fragmentation(BRT brt, TOKU_DB_FRAGMENTATION report);
+int toku_brt_header_set_panic(struct brt_header *h, int panic, char *panic_string);
+BOOL toku_brt_is_empty (BRT brt);
 
 double get_tdiff(void) __attribute__((__visibility__("default")));
 

newbrt/brttypes.h

@@ -33,6 +33,7 @@ typedef u_int64_t TXNID;
 #define TXNID_NONE        ((TXNID)0)
 
 typedef struct s_blocknum { int64_t b; } BLOCKNUM; // make a struct so that we will notice type problems.
+#define ROLLBACK_NONE     ((BLOCKNUM){0})
 
 static inline BLOCKNUM make_blocknum(int64_t b) { BLOCKNUM result={b}; return result; }
 
@@ -70,6 +71,7 @@ typedef enum __toku_bool { FALSE=0, TRUE=1} BOOL;
 typedef struct tokulogger *TOKULOGGER;
 #define NULL_LOGGER ((TOKULOGGER)0)
 typedef struct tokutxn    *TOKUTXN;
+typedef struct txninfo    *TXNINFO;
 #define NULL_TXN ((TOKUTXN)0)
 
 struct logged_btt_pair {
@@ -121,5 +123,8 @@ typedef int (*generate_row_for_del_func)(DB *dest_db, DB *src_db, DBT *dest_val,
 
 #define UU(x) x __attribute__((__unused__))
 
+typedef struct memarena *MEMARENA;
+typedef struct rollback_log_node *ROLLBACK_LOG_NODE;
+
 #endif
 

newbrt/cachetable.h

@@ -123,6 +123,7 @@ typedef int (*CACHETABLE_FETCH_CALLBACK)(CACHEFILE, int fd, CACHEKEY key, u_int3
 
 void toku_cachefile_set_userdata(CACHEFILE cf, void *userdata,
     int (*log_fassociate_during_checkpoint)(CACHEFILE, void*),
+    int (*log_suppress_rollback_during_checkpoint)(CACHEFILE, void*),
     int (*close_userdata)(CACHEFILE, int, void*, char **/*error_string*/, BOOL, LSN),
     int (*checkpoint_userdata)(CACHEFILE, int, void*),
     int (*begin_checkpoint_userdata)(CACHEFILE, int, LSN, void*),

newbrt/checkpoint.c

@@ -218,7 +218,6 @@ toku_checkpoint(CACHETABLE ct, TOKULOGGER logger,
     checkpoint_footprint = 40;
     time_last_checkpoint_begin = time(NULL);
     r = toku_cachetable_begin_checkpoint(ct, logger);
-    LSN oldest_live_lsn = toku_logger_get_oldest_living_lsn(logger);
 
     multi_operation_checkpoint_unlock();
     ydb_unlock();
@@ -230,7 +229,7 @@ toku_checkpoint(CACHETABLE ct, TOKULOGGER logger,
 	r = toku_cachetable_end_checkpoint(ct, logger, ydb_lock, ydb_unlock, callback2_f, extra2);
     }
     if (r==0 && logger) {
-        LSN trim_lsn = (oldest_live_lsn.lsn < logger->checkpoint_lsn.lsn) ? oldest_live_lsn : logger->checkpoint_lsn;
+        LSN trim_lsn = logger->last_completed_checkpoint_lsn;
         r = toku_logger_maybe_trim_log(logger, trim_lsn);
     }
 

newbrt/log-internal.h

@@ -85,7 +85,7 @@ struct tokulogger {
     // To access these, you must have the output condition lock.
     LSN written_lsn; // the last lsn written
     LSN fsynced_lsn; // What is the LSN of the highest fsynced log entry  (accessed only while holding the output lock, and updated only when the output lock and output permission are held)
-    LSN checkpoint_lsn;     // What is the LSN of the most recent completed checkpoint.
+    LSN last_completed_checkpoint_lsn;     // What is the LSN of the most recent completed checkpoint.
     long long next_log_file_number;
     struct logbuf outbuf; // data being written to the file
     int  n_in_file; // The amount of data in the current file
@@ -101,6 +101,7 @@ struct tokulogger {
     u_int64_t swap_ctr;                   // how many times have input/output log buffers been swapped
     void (*remove_finalize_callback) (DICTIONARY_ID, void*);  // ydb-level callback to be called when a transaction that ...
     void * remove_finalize_callback_extra;                    // ... deletes a file is committed or when one that creates a file is aborted.
+    CACHEFILE rollback_cachefile;
 };
 
 int toku_logger_find_next_unused_log_file(const char *directory, long long *result);
@@ -116,25 +117,36 @@ struct tokutxn {
     u_int64_t txnid64; /* this happens to be the first lsn */
     TOKULOGGER logger;
     TOKUTXN    parent;
-    LSN        last_lsn; /* Everytime anything is logged, update the LSN.  (We need to atomically record the LSN along with writing into the log.) */
-    LSN        first_lsn; /* The first lsn in the transaction. */
-    struct roll_entry *oldest_logentry,*newest_logentry; /* Only logentries with rollbacks are here. There is a list going from newest to oldest. */
 
-    MEMARENA   rollentry_arena;
-
-    size_t     rollentry_resident_bytecount; // How many bytes for the rollentries that are stored in main memory.
-    char      *rollentry_filename;
-    int        rollentry_fd;         // If we spill the roll_entries, we write them into this fd.
-    toku_off_t      rollentry_filesize;   // How many bytes are in the rollentry file (this is the uncompressed bytes.  If the file is compressed it may actually be smaller (or even larger with header information))
     u_int64_t  rollentry_raw_count;  // the total count of every byte in the transaction and all its children.
     OMT        open_brts; // a collection of the brts that we touched.  Indexed by filenum.
     XIDS       xids;      //Represents the xid list
     BOOL       force_fsync_on_commit;  //This transaction NEEDS an fsync once (if) it commits.  (commit means root txn)
-    BOOL       has_done_work;          //If this transaction has not done work, there is no need to fsync.
     TXN_PROGRESS_POLL_FUNCTION progress_poll_fun;
     void *                     progress_poll_fun_extra;
+    uint64_t   num_rollback_nodes;
     uint64_t   num_rollentries;
     uint64_t   num_rollentries_processed;
+    BLOCKNUM   spilled_rollback_head;
+    uint32_t   spilled_rollback_head_hash;
+    BLOCKNUM   spilled_rollback_tail;
+    uint32_t   spilled_rollback_tail_hash;
+    BLOCKNUM   current_rollback;
+    uint32_t   current_rollback_hash;
+    BOOL       recovered_from_checkpoint;
+    ROLLBACK_LOG_NODE pinned_inprogress_rollback_log;
+};
+
+struct txninfo {
+    uint64_t   rollentry_raw_count;  // the total count of every byte in the transaction and all its children.
+    uint32_t   num_brts;
+    BRT       *open_brts;
+    BOOL       force_fsync_on_commit;  //This transaction NEEDS an fsync once (if) it commits.  (commit means root txn)
+    uint64_t   num_rollback_nodes;
+    uint64_t   num_rollentries;
+    BLOCKNUM   spilled_rollback_head;
+    BLOCKNUM   spilled_rollback_tail;
+    BLOCKNUM   current_rollback;
 };
 
 static inline int toku_logsizeof_u_int8_t (u_int32_t v __attribute__((__unused__))) {
@@ -180,5 +192,4 @@ static inline char *fixup_fname(BYTESTRING *f) {
     return fname;
 }
 
-int toku_read_rollback_backwards(BREAD, struct roll_entry **item, MEMARENA);
 #endif

newbrt/log.h

@@ -11,7 +11,6 @@
 #include "../include/db.h"
 #include "brttypes.h"
 #include "memory.h"
-#include "bread.h"
 #include "x1764.h"
 
 typedef void(*voidfp)(void);

newbrt/logger.c

@@ -55,7 +55,7 @@ int toku_logger_create (TOKULOGGER *resultp) {
     result->outbuf = (struct logbuf) {0, LOGGER_MIN_BUF_SIZE, toku_xmalloc(LOGGER_MIN_BUF_SIZE), ZERO_LSN};
     // written_lsn is uninitialized
     // fsynced_lsn is uninitialized
-    result->checkpoint_lsn = ZERO_LSN;
+    result->last_completed_checkpoint_lsn = ZERO_LSN;
     // next_log_file_number is uninitialized
     // n_in_file is uninitialized
     result->write_block_size = BRT_DEFAULT_NODE_SIZE; // default logging size is the same as the default brt block size
@@ -68,6 +68,7 @@ int toku_logger_create (TOKULOGGER *resultp) {
     result->input_lock_ctr = 0;
     result->output_condition_lock_ctr = 0;
     result->swap_ctr = 0;
+    result->rollback_cachefile = NULL;
     result->output_is_available = TRUE;
     return 0;
 
@@ -136,6 +137,68 @@ int toku_logger_open (const char *directory, TOKULOGGER logger) {
     return 0;
 }
 
+int
+toku_logger_open_rollback(TOKULOGGER logger, CACHETABLE cachetable, BOOL create) {
+    assert(logger->is_open);
+    assert(!logger->is_panicked);
+    assert(!logger->rollback_cachefile);
+    
+    int r;
+    BRT t = NULL;   // Note, there is no DB associated with this BRT.
+
+    r = toku_brt_create(&t);
+    assert(r==0);
+    r = toku_brt_open(t, ROLLBACK_CACHEFILE_NAME, create, create, cachetable, NULL_TXN, NULL);
+    assert(r==0);
+    logger->rollback_cachefile = t->cf;
+    toku_brtheader_lock(t->h);
+    //Verify it is empty
+    assert(!t->h->panic);
+    //Must have no data blocks (rollback logs or otherwise).
+    toku_block_verify_no_data_blocks_except_root_unlocked(t->h->blocktable, t->h->root);
+    toku_brtheader_unlock(t->h);
+    assert(toku_brt_is_empty(t));
+    return r;
+}
+
+
+//  Requires: Rollback cachefile can only be closed immediately after a checkpoint,
+//            so it will always be clean (!h->dirty) when about to be closed.
+//            Rollback log can only be closed when there are no open transactions,
+//            so it will always be empty (no data blocks) when about to be closed.
+int
+toku_logger_close_rollback(TOKULOGGER logger, BOOL recovery_failed) {
+    int r = 0;
+    CACHEFILE cf = logger->rollback_cachefile;  // stored in logger at rollback cachefile open
+    if (!logger->is_panicked && cf) {
+        BRT brt_to_close;
+        {   //Find "brt"
+            struct brt_header *h = toku_cachefile_get_userdata(cf);
+            toku_brtheader_lock(h);
+            if (!h->panic && recovery_failed) {
+                toku_brt_header_set_panic(h, EINVAL, "Recovery failed");
+            }
+            //Verify it is safe to close it.
+            if (!h->panic) { //If paniced, it is safe to close.
+                assert(!h->dirty);  //Must not be dirty.
+                //Must have no data blocks (rollback logs or otherwise).
+                toku_block_verify_no_data_blocks_except_root_unlocked(h->blocktable, h->root);
+            }
+            assert(!toku_list_empty(&h->live_brts));  // there is always one brt associated with the header
+	    brt_to_close = toku_list_struct(toku_list_head(&h->live_brts), struct brt, live_brt_link);
+            assert(brt_to_close);
+            toku_brtheader_unlock(h);
+            assert(toku_brt_is_empty(brt_to_close));
+        }
+
+        char *error_string_ignore = NULL;
+        r = toku_close_brt(brt_to_close, &error_string_ignore);
+        //Set as dealt with already.
+        logger->rollback_cachefile = NULL;
+    }
+    return r;
+}
+
 // No locks held on entry
 // No locks held on exit.
 // No locks are needed, since you cannot legally close the log concurrently with doing anything else.
@@ -183,7 +246,8 @@ int toku_logger_shutdown(TOKULOGGER logger) {
     if (logger->is_open) {
         if (toku_omt_size(logger->live_txns) == 0) {
             BYTESTRING comment = { strlen("shutdown"), "shutdown" };
-            r = toku_log_comment(logger, NULL, TRUE, 0, comment);
+            int r2 = toku_log_comment(logger, NULL, TRUE, 0, comment);
+            if (!r) r = r2;
         }
     }
     return r;
@@ -787,6 +851,10 @@ int toku_fread_LSN     (FILE *f, LSN *lsn, struct x1764 *checksum, u_int32_t *le
     return toku_fread_u_int64_t (f, &lsn->lsn, checksum, len);
 }
 
+int toku_fread_BLOCKNUM (FILE *f, BLOCKNUM *b, struct x1764 *checksum, u_int32_t *len) {
+    return toku_fread_u_int64_t (f, (u_int64_t*)&b->b, checksum, len);
+}
+
 int toku_fread_FILENUM (FILE *f, FILENUM *filenum, struct x1764 *checksum, u_int32_t *len) {
     return toku_fread_u_int32_t (f, &filenum->fileid, checksum, len);
 }
@@ -903,6 +971,11 @@ int toku_logprint_BYTESTRING (FILE *outf, FILE *inf, const char *fieldname, stru
     return 0;
 }
 
+int toku_logprint_BLOCKNUM (FILE *outf, FILE *inf, const char *fieldname, struct x1764 *checksum, u_int32_t *len, const char *format) {
+    return toku_logprint_u_int64_t(outf, inf, fieldname, checksum, len, format);
+
+}
+
 int toku_logprint_FILENUM (FILE *outf, FILE *inf, const char *fieldname, struct x1764 *checksum, u_int32_t *len, const char *format) {
     return toku_logprint_u_int32_t(outf, inf, fieldname, checksum, len, format);
 
@@ -982,11 +1055,6 @@ TXNID toku_txn_get_txnid (TOKUTXN txn) {
     else return txn->txnid64;
 }
 
-LSN toku_txn_get_last_lsn (TOKUTXN txn) {
-    if (txn==0) return (LSN){0};
-    return txn->last_lsn;
-}
-
 LSN toku_logger_last_lsn(TOKULOGGER logger) {
     return logger->lsn;
 }
@@ -1083,32 +1151,20 @@ int toku_logger_log_archive (TOKULOGGER logger, char ***logs_p, int flags) {
     // get them into increasing order
     qsort(all_logs, all_n_logs, sizeof(all_logs[0]), logfilenamecompare);
 
-    LSN oldest_live_txn_lsn;
-    {
-        TXNID oldest_living_xid = toku_logger_get_oldest_living_xid(logger);
-        if (oldest_living_xid == TXNID_NONE_LIVING)
-            oldest_live_txn_lsn = MAX_LSN;
-        else
-            oldest_live_txn_lsn.lsn = oldest_living_xid;
-    }
-
-    //printf("%s:%d Oldest txn is %lld\n", __FILE__, __LINE__, (long long)oldest_live_txn_lsn.lsn);
+    LSN save_lsn = logger->last_completed_checkpoint_lsn;
 
     // Now starting at the last one, look for archivable ones.
     // Count the total number of bytes, because we have to return a single big array.  (That's the BDB interface.  Bleah...)
     LSN earliest_lsn_in_logfile={(unsigned long long)(-1LL)};
     r = peek_at_log(logger, all_logs[all_n_logs-1], &earliest_lsn_in_logfile); // try to find the lsn that's in the most recent log
-    if ((earliest_lsn_in_logfile.lsn <= logger->checkpoint_lsn.lsn)&&
-	(earliest_lsn_in_logfile.lsn <= oldest_live_txn_lsn.lsn)) {
+    if (earliest_lsn_in_logfile.lsn <= save_lsn.lsn) {
 	i=all_n_logs-1;
     } else {
 	for (i=all_n_logs-2; i>=0; i--) { // start at all_n_logs-2 because we never archive the most recent log
 	    r = peek_at_log(logger, all_logs[i], &earliest_lsn_in_logfile);
 	    if (r!=0) continue; // In case of error, just keep going
 	
-	    //printf("%s:%d file=%s firstlsn=%lld checkpoint_lsns={%lld %lld}\n", __FILE__, __LINE__, all_logs[i], (long long)earliest_lsn_in_logfile.lsn, (long long)logger->checkpoint_lsns[0].lsn, (long long)logger->checkpoint_lsns[1].lsn);
-	    if ((earliest_lsn_in_logfile.lsn <= logger->checkpoint_lsn.lsn)&&
-		(earliest_lsn_in_logfile.lsn <= oldest_live_txn_lsn.lsn)) {
+	    if (earliest_lsn_in_logfile.lsn <= save_lsn.lsn) {
 		break;
 	    }
 	}
@@ -1148,7 +1204,7 @@ TOKUTXN toku_logger_txn_parent (TOKUTXN txn) {
 }
 
 void toku_logger_note_checkpoint(TOKULOGGER logger, LSN lsn) {
-    logger->checkpoint_lsn = lsn;
+    logger->last_completed_checkpoint_lsn = lsn;
 }
 
 TXNID toku_logger_get_oldest_living_xid(TOKULOGGER logger) {
@@ -1158,17 +1214,6 @@ TXNID toku_logger_get_oldest_living_xid(TOKULOGGER logger) {
     return rval;
 }
 
-LSN toku_logger_get_oldest_living_lsn(TOKULOGGER logger) {
-    LSN lsn = {0};
-    if (logger) {
-        if (logger->oldest_living_xid == TXNID_NONE_LIVING)
-            lsn = MAX_LSN;
-        else
-            lsn.lsn = logger->oldest_living_xid;
-    }
-    return lsn;
-}
-
 LSN
 toku_logger_get_next_lsn(TOKULOGGER logger) {
     return logger->lsn;

newbrt/logger.h

@@ -5,12 +5,20 @@
 #ident "Copyright (c) 2007, 2008, 2009 Tokutek Inc.  All rights reserved."
 #ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
 
-enum { TOKU_LOG_VERSION = 1 };
+enum {
+    TOKU_LOG_VERSION_1 = 1,
+    TOKU_LOG_VERSION_2 = 2,
+    TOKU_LOG_NEXT_VERSION,           // the version after the current version
+    TOKU_LOG_VERSION   = TOKU_LOG_NEXT_VERSION-1, // A hack so I don't have to change this line.
+};
+#define ROLLBACK_CACHEFILE_NAME "tokudb.rollback"
 
 int toku_logger_create (TOKULOGGER *resultp);
 int toku_logger_open (const char *directory, TOKULOGGER logger);
 int toku_logger_shutdown(TOKULOGGER logger);
 int toku_logger_close(TOKULOGGER *loggerp);
+int toku_logger_open_rollback(TOKULOGGER logger, CACHETABLE cachetable, BOOL create);
+int toku_logger_close_rollback(TOKULOGGER logger, BOOL recovery_failed);
 
 int toku_logger_fsync (TOKULOGGER logger);
 void toku_logger_panic (TOKULOGGER logger, int err);
@@ -49,6 +57,7 @@ int toku_fread_u_int32_t_nocrclen (FILE *f, u_int32_t *v);
 int toku_fread_u_int32_t (FILE *f, u_int32_t *v, struct x1764 *checksum, u_int32_t *len);
 int toku_fread_u_int64_t (FILE *f, u_int64_t *v, struct x1764 *checksum, u_int32_t *len);
 int toku_fread_LSN     (FILE *f, LSN *lsn, struct x1764 *checksum, u_int32_t *len);
+int toku_fread_BLOCKNUM (FILE *f, BLOCKNUM *lsn, struct x1764 *checksum, u_int32_t *len);
 int toku_fread_FILENUM (FILE *f, FILENUM *filenum, struct x1764 *checksum, u_int32_t *len);
 int toku_fread_TXNID   (FILE *f, TXNID *txnid, struct x1764 *checksum, u_int32_t *len);
 int toku_fread_BYTESTRING (FILE *f, BYTESTRING *bs, struct x1764 *checksum, u_int32_t *len);
@@ -58,6 +67,7 @@ int toku_logprint_LSN (FILE *outf, FILE *inf, const char *fieldname, struct x176
 int toku_logprint_TXNID (FILE *outf, FILE *inf, const char *fieldname, struct x1764 *checksum, u_int32_t *len, const char *format __attribute__((__unused__)));
 int toku_logprint_u_int8_t (FILE *outf, FILE *inf, const char *fieldname, struct x1764 *checksum, u_int32_t *len, const char *format);
 int toku_logprint_u_int32_t (FILE *outf, FILE *inf, const char *fieldname, struct x1764 *checksum, u_int32_t *len, const char *format);
+int toku_logprint_BLOCKNUM (FILE *outf, FILE *inf, const char *fieldname, struct x1764 *checksum, u_int32_t *len, const char *format);
 int toku_logprint_u_int64_t (FILE *outf, FILE *inf, const char *fieldname, struct x1764 *checksum, u_int32_t *len, const char *format);
 void toku_print_BYTESTRING (FILE *outf, u_int32_t len, char *data);
 int toku_logprint_BYTESTRING (FILE *outf, FILE *inf, const char *fieldname, struct x1764 *checksum, u_int32_t *len, const char *format __attribute__((__unused__)));
@@ -67,7 +77,6 @@ int toku_read_and_print_logmagic (FILE *f, u_int32_t *versionp);
 int toku_read_logmagic (FILE *f, u_int32_t *versionp);
 
 TXNID toku_txn_get_txnid (TOKUTXN txn);
-LSN toku_txn_get_last_lsn (TOKUTXN txn);
 LSN toku_logger_last_lsn(TOKULOGGER logger);
 TOKULOGGER toku_txn_logger (TOKUTXN txn);
 
@@ -81,7 +90,6 @@ TOKUTXN toku_logger_txn_parent (TOKUTXN txn);
 void toku_logger_note_checkpoint(TOKULOGGER logger, LSN lsn);
 
 TXNID toku_logger_get_oldest_living_xid(TOKULOGGER logger);
-LSN toku_logger_get_oldest_living_lsn(TOKULOGGER logger);
 LSN toku_logger_get_next_lsn(TOKULOGGER logger);
 void toku_logger_set_remove_finalize_callback(TOKULOGGER logger, void (*funcp)(DICTIONARY_ID, void *), void * extra);
 void toku_logger_call_remove_finalize_callback(TOKULOGGER logger, DICTIONARY_ID dict_id);

newbrt/memarena.c

@@ -12,9 +12,9 @@ struct memarena {
     int n_other_bufs;
 };
 
-MEMARENA memarena_create (void) {
-    MEMARENA MALLOC(result);  assert(result);
-    result->buf_size = 1024;
+MEMARENA memarena_create_presized (size_t initial_size) {
+    MEMARENA XMALLOC(result);
+    result->buf_size = initial_size;
     result->buf_used = 0;
     result->other_bufs = NULL;
     result->size_of_other_bufs = 0;
@@ -23,6 +23,10 @@ MEMARENA memarena_create (void) {
     return result;
 }
 
+MEMARENA memarena_create (void) {
+    return memarena_create_presized(1024);
+}
+
 void memarena_clear (MEMARENA ma) {
     // Free the other bufs.
     int i;

newbrt/memarena.h

@@ -19,10 +19,11 @@
 
 #include <sys/types.h>
 
-typedef struct memarena *MEMARENA;
+MEMARENA memarena_create_presized (size_t initial_size);
+// Effect: Create a memarena with initial size.  In case of ENOMEM, aborts.
 
 MEMARENA memarena_create (void);
-// Effect: Create a memarena.  In case of ENOMEM, aborts.
+// Effect: Create a memarena with default initial size.  In case of ENOMEM, aborts.
 
 void memarena_clear (MEMARENA ma);
 // Effect: Reset the internal state so that the allocated memory can be used again.

newbrt/rbuf.h

@@ -100,6 +100,18 @@ static inline BLOCKNUM rbuf_blocknum (struct rbuf *r) {
     BLOCKNUM result = make_blocknum(rbuf_longlong(r));
     return result;
 }
+static inline void rbuf_ma_BLOCKNUM (struct rbuf *r, MEMARENA ma __attribute__((__unused__)), BLOCKNUM *blocknum) {
+    *blocknum = rbuf_blocknum(r);
+}
+
+static inline void rbuf_ma_u_int32_t (struct rbuf *r, MEMARENA ma __attribute__((__unused__)), u_int32_t *num) {
+    *num = rbuf_int(r);
+}
+
+static inline void rbuf_ma_u_int64_t (struct rbuf *r, MEMARENA ma __attribute__((__unused__)), u_int64_t *num) {
+    *num = rbuf_ulonglong(r);
+}
+
 
 static inline void rbuf_TXNID (struct rbuf *r, TXNID *txnid) {
     *txnid = rbuf_ulonglong(r);
@@ -119,7 +131,7 @@ static inline void rbuf_ma_FILENUM (struct rbuf *r, MEMARENA ma __attribute__((_
 static inline void rbuf_BYTESTRING (struct rbuf *r, BYTESTRING *bs) {
     bs->len  = rbuf_int(r);
     u_int32_t newndone = r->ndone + bs->len;
-    assert(newndone < r->size);
+    assert(newndone <= r->size);
     bs->data = toku_memdup(&r->buf[r->ndone], (size_t)bs->len);
     assert(bs->data);
     r->ndone = newndone;
@@ -128,7 +140,7 @@ static inline void rbuf_BYTESTRING (struct rbuf *r, BYTESTRING *bs) {
 static inline void rbuf_ma_BYTESTRING  (struct rbuf *r, MEMARENA ma, BYTESTRING *bs) {
     bs->len  = rbuf_int(r);
     u_int32_t newndone = r->ndone + bs->len;
-    assert(newndone < r->size);
+    assert(newndone <= r->size);
     bs->data = memarena_memdup(ma, &r->buf[r->ndone], (size_t)bs->len);
     assert(bs->data);
     r->ndone = newndone;

newbrt/recover.h

@@ -11,7 +11,6 @@
 #include "../include/db.h"
 #include "brttypes.h"
 #include "memory.h"
-#include "bread.h"
 #include "x1764.h"
 
 // Run tokudb recovery from the log
@@ -29,10 +28,6 @@ int tokudb_recover (const char *env_dir, const char *log_dir,
 // Returns: TRUE if we need recovery, otherwise FALSE.
 int tokudb_needs_recovery(const char *logdir, BOOL ignore_empty_log);
 
-// Delete the rolltmp files
-// Ruturns 0 if success
-int tokudb_recover_delete_rolltmp_files(const char *datadir, const char *logdir);
-
 // Return 0 if recovery log exists, ENOENT if log is missing
 int tokudb_recover_log_exists(const char * log_dir);
 

newbrt/rollback.h

@@ -13,19 +13,24 @@ void toku_poll_txn_progress_function(TOKUTXN txn, uint8_t is_commit, uint8_t sta
 int toku_rollback_commit(TOKUTXN txn, YIELDF yield, void*yieldv, LSN lsn);
 int toku_rollback_abort(TOKUTXN txn, YIELDF yield, void*yieldv, LSN lsn);
 void toku_rollback_txn_close (TOKUTXN txn);
+int toku_get_and_pin_rollback_log_for_new_entry (TOKUTXN txn, ROLLBACK_LOG_NODE *result);
+int toku_get_and_pin_rollback_log(TOKUTXN txn, TXNID xid, uint64_t sequence, BLOCKNUM name, uint32_t hash, ROLLBACK_LOG_NODE *result);
+int toku_rollback_log_unpin(TOKUTXN txn, ROLLBACK_LOG_NODE log);
+int toku_delete_rollback_log(TOKUTXN txn, ROLLBACK_LOG_NODE log);
+
+typedef int(*apply_rollback_item)(TOKUTXN txn, struct roll_entry *item, YIELDF yield, void*yieldv, LSN lsn);
 
 int toku_commit_rollback_item (TOKUTXN txn, struct roll_entry *item, YIELDF yield, void*yieldv, LSN lsn);
 int toku_abort_rollback_item (TOKUTXN txn, struct roll_entry *item, YIELDF yield, void*yieldv, LSN lsn);
 
-void *toku_malloc_in_rollback(TOKUTXN txn, size_t size);
-void *toku_memdup_in_rollback(TOKUTXN txn, const void *v, size_t len);
-char *toku_strdup_in_rollback(TOKUTXN txn, const char *s);
-int toku_maybe_spill_rollbacks (TOKUTXN txn);
+void *toku_malloc_in_rollback(ROLLBACK_LOG_NODE log, size_t size);
+void *toku_memdup_in_rollback(ROLLBACK_LOG_NODE log, const void *v, size_t len);
+int toku_maybe_spill_rollbacks (TOKUTXN txn, ROLLBACK_LOG_NODE log);
 
 int toku_txn_note_brt (TOKUTXN txn, BRT brt);
 int toku_txn_note_swap_brt (BRT live, BRT zombie);
 int toku_txn_note_close_brt (BRT brt);
-int toku_logger_txn_rolltmp_raw_count(TOKUTXN txn, u_int64_t *raw_count);
+int toku_logger_txn_rollback_raw_count(TOKUTXN txn, u_int64_t *raw_count);
 
 int toku_txn_find_by_xid (BRT brt, TXNID xid, TOKUTXN *txnptr);
 
@@ -36,4 +41,23 @@ int toku_commit_fileentries (int fd, TOKUTXN txn, YIELDF yield,void *yieldv, LSN
 //Heaviside function to find a TOKUTXN by TOKUTXN (used to find the index)
 int find_xid (OMTVALUE v, void *txnv);
 
+struct rollback_log_node {
+    enum typ_tag       tag;
+    int                layout_version;
+    int                layout_version_original;
+    int                layout_version_read_from_disk;
+    int                dirty;
+    TXNID              txnid;         // Which transaction made this?
+    uint64_t           sequence;      // Which rollback log in the sequence is this?
+    BLOCKNUM           thislogname;   // Which block number is this chunk of the log?
+    uint32_t           thishash;
+    BLOCKNUM           older;         // Which block number is the next oldest chunk of the log?
+    uint32_t           older_hash;
+    struct roll_entry *oldest_logentry;
+    struct roll_entry *newest_logentry;
+    MEMARENA           rollentry_arena;
+    size_t             rollentry_resident_bytecount; // How many bytes for the rollentries that are stored in main memory.
+};
+
 #endif
+

newbrt/tests/Makefile

@@ -58,15 +58,6 @@ check_ok:
 	test 0 = 0 $(SUMMARIZE_CMD)
 
 
-ifeq ($(TOKU_SKIP_1305),1)
-check_test1305:
-	@echo SKIPPED SLOW TEST $@
-else
-# Don't run 1305 under valgrind.  It takes way too long.
-check_test1305$(BINSUF): test1305$(BINSUF)
-	./$< $(VERBVERBOSE) $(SUMMARIZE_CMD)
-endif
-
 ifeq ($(TOKU_SKIP_MINICRON),1)
 check_minicron-test$(BINSUF):
 	@echo "SKIPPED TEST $@ (Fails in win64 vm due to timing issues)"

newbrt/tests/bread-test.c

-/* Test bread by writing random data and then reading it using bread_backwards() to see if it gives the right answer.
- * See test_1305 for another bread test (testing to see if it can read 1GB files) */
-
-#include "test.h"
-#include <toku_portability.h>
-
-
-#include <fcntl.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <stdio.h>
-#include <string.h>
-#include <zlib.h>
-
-#include "../brttypes.h"
-#include "../bread.h"
-
-#define FNAME "bread-test.data"
-
-#define RECORDS 20
-#define RECORDLEN 100
-
-char buf[RECORDS][RECORDLEN];
-int sizes[RECORDS];
-int sizesn[RECORDS];
-int nwrote=0;
-char wrotedata[RECORDS*RECORDLEN];
-
-static void
-test (int seed) {
-    srandom(seed);
-    unlink(FNAME);
-    int i;
-    {
-        int fd = open(FNAME, O_CREAT+O_RDWR+O_BINARY, 0777);
-	assert(fd>=0);
-	for (i=0; i<RECORDS; i++) {
-	    sizes[i]  = 1+ random()%RECORDLEN;
-	    sizesn[i] = toku_htod32(sizes[i]);
-	    int j;
-	    for (j=0; j<sizes[i]; j++) {
-		buf[i][j] = wrotedata[nwrote++] = (char)random();
-	    }
-	    uLongf compressed_size = compressBound(sizes[i]);
-	    Bytef compressed_buf[compressed_size];
-	    { int r = compress2(compressed_buf, &compressed_size, (Bytef*)(buf[i]), sizes[i], 1); assert(r==Z_OK); }
-	    u_int32_t compressed_size_n = toku_htod32(compressed_size);
-	    { toku_os_full_write(fd, &compressed_size_n, 4); }
-	    { toku_os_full_write(fd, compressed_buf, compressed_size); }
-	    { toku_os_full_write(fd, &sizesn[i], 4); } // the uncompressed size
-	    { toku_os_full_write(fd, &compressed_size_n, 4); }
-	}
-	{ int r=close(fd); assert(r==0); }
-    }
-    int fd = open(FNAME, O_RDONLY+O_BINARY);  	assert(fd>=0);
-    // Now read it all backward
-    BREAD br = create_bread_from_fd_initialize_at(fd);
-    while (bread_has_more(br)) {
-	assert(nwrote>0);
-	int to_read = 1+(random()%RECORDLEN); // read from 1 to 100 (if RECORDLEN is 100)
-	if (to_read>nwrote) to_read=nwrote;
-	char rbuf[to_read];
-	int r = bread_backwards(br, rbuf, to_read);
-	assert(r==to_read);
-	assert(memcmp(rbuf, &wrotedata[nwrote-to_read], to_read)==0);
-	nwrote-=to_read;
-    }
-    assert(nwrote==0);
-
-    { int r=close_bread_without_closing_fd(br); assert(r==0); }
-    { int r=close(fd); assert(r==0); }
-    unlink(FNAME);
-}
-
-int
-test_main (int argc __attribute__((__unused__)), const char *argv[] __attribute__((__unused__))) {
-    int i;
-    for (i=0; i<10; i++) test(i);
-    return 0;
-}

newbrt/tests/cachetable-checkpoint-pending.c

@@ -102,7 +102,7 @@ static void checkpoint_pending(void) {
     char fname1[] = __FILE__ "test1.dat";
     r = unlink(fname1); if (r!=0) CKERR2(errno, ENOENT);
     r = toku_cachetable_openf(&cf, ct, fname1, O_RDWR|O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO); assert(r == 0);
-    toku_cachefile_set_userdata(cf, NULL, NULL, NULL, NULL, NULL, NULL,
+    toku_cachefile_set_userdata(cf, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
                                 dummy_pin_unpin, dummy_pin_unpin);
 
     // Insert items into the cachetable. All dirty.

newbrt/tests/cachetable-checkpoint-test.c

@@ -60,7 +60,7 @@ static void cachetable_checkpoint_test(int n, enum cachetable_dirty dirty) {
     unlink(fname1);
     CACHEFILE f1;
     r = toku_cachetable_openf(&f1, ct, fname1, O_RDWR|O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO); assert(r == 0);
-    toku_cachefile_set_userdata(f1, NULL, NULL, NULL, NULL, NULL, NULL,
+    toku_cachefile_set_userdata(f1, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
                                 dummy_pin_unpin, dummy_pin_unpin);
 
     // insert items into the cachetable. all should be dirty

newbrt/tests/cachetable-prefetch-checkpoint-test.c

@@ -46,7 +46,7 @@ static void cachetable_prefetch_checkpoint_test(int n, enum cachetable_dirty dir
     unlink(fname1);
     CACHEFILE f1;
     r = toku_cachetable_openf(&f1, ct, fname1, O_RDWR|O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO); assert(r == 0);
-    toku_cachefile_set_userdata(f1, NULL, NULL, NULL, NULL, NULL, NULL,
+    toku_cachefile_set_userdata(f1, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
                                 dummy_pin_unpin, dummy_pin_unpin);
 
     // prefetch block n+1. this will take 10 seconds.

newbrt/tests/recovery-bad-last-entry.c

@@ -5,7 +5,7 @@
 
 #define TESTDIR "dir." __FILE__ 
 
-static const int magic_begin_end_checkpoint_sz = 77;  // leave this many bytes in file
+static const int magic_begin_end_checkpoint_sz = 85;  // leave this many bytes in file
 
 static int 
 run_test(void) {
@@ -28,12 +28,12 @@ run_test(void) {
         LSN beginlsn;
         // all logs must contain a valid checkpoint
         r = toku_log_begin_checkpoint(logger, &beginlsn, TRUE, 0); assert(r == 0);
-        r = toku_log_end_checkpoint(logger, NULL, TRUE, beginlsn.lsn, 0); assert(r == 0);
+        r = toku_log_end_checkpoint(logger, NULL, TRUE, beginlsn.lsn, 0, 0, 0); assert(r == 0);
 
         r = toku_log_comment(logger, NULL, TRUE, 0, hello); assert(r == 0);
         r = toku_log_comment(logger, NULL, TRUE, 0, world); assert(r == 0);
         r = toku_log_begin_checkpoint(logger, &beginlsn, TRUE, 0); assert(r == 0);
-        r = toku_log_end_checkpoint(logger, NULL, TRUE, beginlsn.lsn, 0); assert(r == 0);
+        r = toku_log_end_checkpoint(logger, NULL, TRUE, beginlsn.lsn, 0, 0, 0); assert(r == 0);
         r = toku_log_comment(logger, NULL, TRUE, 0, hello); assert(r == 0);
         r = toku_log_comment(logger, NULL, TRUE, 0, there); assert(r == 0);
         r = toku_logger_close(&logger); assert(r == 0);

newbrt/tests/recovery-cbegin-cend-hello.c

@@ -21,7 +21,7 @@ run_test(void) {
     // add begin checkpoint, end checkpoint
     LSN beginlsn;
     r = toku_log_begin_checkpoint(logger, &beginlsn, FALSE, 0); assert(r == 0);
-    r = toku_log_end_checkpoint(logger, NULL, TRUE, beginlsn.lsn, 0); assert(r == 0);
+    r = toku_log_end_checkpoint(logger, NULL, TRUE, beginlsn.lsn, 0, 0, 0); assert(r == 0);
     r = toku_logger_close(&logger); assert(r == 0);
 
     // add hello

newbrt/tests/recovery-cbegin-cend.c

@@ -21,7 +21,7 @@ run_test(void) {
 
     r = toku_logger_create(&logger); assert(r == 0);
     r = toku_logger_open(TESTDIR, logger); assert(r == 0);
-    r = toku_log_end_checkpoint(logger, NULL, TRUE, beginlsn.lsn, 0); assert(r == 0);
+    r = toku_log_end_checkpoint(logger, NULL, TRUE, beginlsn.lsn, 0, 0, 0); assert(r == 0);
     r = toku_logger_close(&logger); assert(r == 0);
 
     // run recovery

newbrt/tests/recovery-cbegin.c

@@ -17,7 +17,7 @@ run_test(void) {
     r = toku_logger_open(TESTDIR, logger); assert(r == 0);
     LSN beginlsn;
     r = toku_log_begin_checkpoint(logger, &beginlsn, TRUE, 0); assert(r == 0);
-    r = toku_log_end_checkpoint(logger, NULL, TRUE, beginlsn.lsn, 0); assert(r == 0);
+    r = toku_log_end_checkpoint(logger, NULL, TRUE, beginlsn.lsn, 0, 0, 0); assert(r == 0);
     r = toku_log_begin_checkpoint(logger, &beginlsn, TRUE, 0); assert(r == 0);
     r = toku_logger_close(&logger); assert(r == 0);
 

newbrt/tests/recovery-cend-cbegin.c

@@ -20,7 +20,7 @@ run_test(void) {
     LSN firstbegin = ZERO_LSN;
     r = toku_log_begin_checkpoint(logger, &firstbegin, TRUE, 0); assert(r == 0);
     assert(firstbegin.lsn != ZERO_LSN.lsn);
-    r = toku_log_end_checkpoint(logger, NULL, FALSE, firstbegin.lsn, 0); assert(r == 0);
+    r = toku_log_end_checkpoint(logger, NULL, FALSE, firstbegin.lsn, 0, 0, 0); assert(r == 0);
     r = toku_log_begin_checkpoint(logger, NULL, TRUE, 0); assert(r == 0);
     r = toku_logger_close(&logger); assert(r == 0);
 

newbrt/tests/recovery-fopen-missing-file.c

@@ -19,7 +19,7 @@ run_test(void) {
     r = toku_logger_open(TESTDIR, logger); assert(r == 0);
     LSN beginlsn;
     r = toku_log_begin_checkpoint(logger, &beginlsn, TRUE, 0); assert(r == 0);
-    r = toku_log_end_checkpoint(logger, NULL, TRUE, beginlsn.lsn, 0); assert(r == 0);
+    r = toku_log_end_checkpoint(logger, NULL, TRUE, beginlsn.lsn, 0, 0, 0); assert(r == 0);
 
     BYTESTRING iname  = { strlen("missing_tokudb_file"), "missing_tokudb_file" };
     FILENUM filenum = {42};

newbrt/tests/recovery-hello.c

@@ -21,7 +21,7 @@ run_test(void) {
     r = toku_log_comment(logger, NULL, TRUE, 0, hello); assert(r == 0);
     LSN beginlsn;
     r = toku_log_begin_checkpoint(logger, &beginlsn, TRUE, 0); assert(r == 0);
-    r = toku_log_end_checkpoint(logger, NULL, TRUE, beginlsn.lsn, 0); assert(r == 0);
+    r = toku_log_end_checkpoint(logger, NULL, TRUE, beginlsn.lsn, 0, 0, 0); assert(r == 0);
     r = toku_log_comment(logger, NULL, TRUE, 0, hello); assert(r == 0);
     BYTESTRING there  = { strlen("there"), "there" };
     r = toku_log_comment(logger, NULL, TRUE, 0, there); assert(r == 0);

newbrt/tests/recovery-lsn-error-during-forward-scan.c

@@ -30,7 +30,7 @@ run_test(void) {
 
     LSN beginlsn;
     r = toku_log_begin_checkpoint(logger, &beginlsn, TRUE, 0); assert(r == 0);
-    r = toku_log_end_checkpoint(logger, NULL, TRUE, beginlsn.lsn, 0); assert(r == 0);
+    r = toku_log_end_checkpoint(logger, NULL, TRUE, beginlsn.lsn, 0, 0, 0); assert(r == 0);
 
     r = toku_logger_close(&logger); assert(r == 0);
 

newbrt/tests/test1305.c

-/* Test bread_backwards to make sure it can read backwards even for large files. */
-
-#include <toku_portability.h>
-
-#include <fcntl.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <stdio.h>
-#include <string.h>
-#include <zlib.h>
-
-#include "../brttypes.h"
-#include "../bread.h"
-#include "test.h"
-
-#define FNAME "test1305.data"
-
-// THe buffer size in units of 64-bit integers.
-#define N_BIGINTS (1<<20)
-#define BIGINT_SIZE (sizeof(u_int64_t))
-// How big is the readback buffer (in 8-bit integers)?
-#define READBACK_BUFSIZE (1<<20)
-
-
-static void
-test (u_int64_t fsize) {
-    unlink(FNAME);
-    // Create a file of size fsize.  Fill it with 8-byte values which are integers, in order)
-    assert(fsize%(N_BIGINTS*sizeof(u_int64_t)) == 0); // Make sure the fsize is a multiple of the buffer size.
-    u_int64_t i = 0;
-    {
-	int fd = open(FNAME, O_CREAT+O_RDWR+O_BINARY, 0777);
-	assert(fd>=0);
-	static u_int64_t buf[N_BIGINTS]; //windows cannot handle this on the stack
-	static char compressed_buf[N_BIGINTS*2 + 1000]; // this is more than compressbound returns
-	uLongf compressed_len;
-	while (i*BIGINT_SIZE < fsize) {
-            if (verbose>0 && i % (1<<25) == 0) {
-                printf("   %s:test (%"PRIu64") forwards [%"PRIu64"%%]\n", __FILE__, fsize, 100*BIGINT_SIZE*((u_int64_t)i) / fsize);
-                fflush(stdout);
-            }
-
-	    int j;
-	    for (j=0; j<N_BIGINTS; j++) {
-		buf[j] = i++;
-	    }
-assert(sizeof(buf) == N_BIGINTS * BIGINT_SIZE);
-	    {
-		compressed_len = sizeof(compressed_buf);
-		int r = compress2((Bytef*)compressed_buf, &compressed_len, (Bytef*)buf, sizeof(buf), 1);
-		assert(r==Z_OK);
-	    }
-	    {
-		u_int32_t v = toku_htod32(compressed_len);
-		toku_os_full_write(fd, &v, sizeof(v));
-	    }
-	    {
-		toku_os_full_write(fd, compressed_buf, compressed_len);
-	    }
-	    {
-		u_int32_t v = toku_htod32(sizeof(buf));
-		toku_os_full_write(fd, &v, sizeof(v));
-	    }
-	    {
-		u_int32_t v = toku_htod32(compressed_len);
-		toku_os_full_write(fd, &v, sizeof(v));
-	    }
-	}
-	{ int r = close(fd); assert(r==0); }
-    }
-    assert(i*BIGINT_SIZE == fsize);
-    // Now read it all backward
-    {
-	int fd = open(FNAME, O_RDONLY+O_BINARY);  	assert(fd>=0);
-	BREAD br = create_bread_from_fd_initialize_at(fd);
-	while (bread_has_more(br)) {
-            if (verbose>0 && (fsize/BIGINT_SIZE - i) % (1<<25) == 0) {
-                printf("   %s:test (%"PRIu64") backwards [%"PRIu64"%%]\n", __FILE__, fsize, 100*BIGINT_SIZE*((u_int64_t)i) / fsize);
-                fflush(stdout);
-            }
-	    assert(i>0);
-	    i--;
-	    u_int64_t storedi;
-	    { int r = bread_backwards(br, &storedi, sizeof(storedi)); assert(r==sizeof(storedi)); }
-	    assert(storedi==i);
-	}
-	assert(i==0);
-	{ int r=close_bread_without_closing_fd(br); assert(r==0); }
-	{ int r=close(fd); assert(r==0); }
-    }
-    //printf("Did %" PRIu64 "\n", fsize);
-    //system("ls -l " FNAME);
-    unlink(FNAME);
-}
-
-int
-test_main (int argc, const char *argv[]) {
-    default_parse_args(argc, argv);
-    test(1LL<<23);
-    test(1LL<<30);
-    test(1LL<<31);
-    test(1LL<<32);
-    test(1LL<<33);
-    return 0;
-}
-

newbrt/tests/test_logcursor.c

@@ -154,6 +154,9 @@ int create_logfiles() {
     TXNID txnid = 0;
     TXNID cp_txnid = 0;
 
+    u_int32_t num_fassociate = 0;
+    u_int32_t num_xstillopen = 0;
+    
     bs_aname.len = 4; bs_aname.data="a.db";
     bs_bname.len = 4; bs_bname.data="b.db";
     bs_a.len = 2; bs_a.data="a";
@@ -171,31 +174,40 @@ int create_logfiles() {
     //fcreate                   'F': lsn=2 txnid=1 filenum=0 fname={len=4 data="a.db"} mode=0777 treeflags=0 crc=18a3d525 len=49
     r = toku_log_fcreate(logger, &lsn, NO_FSYNC, txnid, fn_aname, bs_aname, 0x0777, 0, 0, bs_empty); assert(r==0);
     //commit                    'C': lsn=3 txnid=1 crc=00001f1e len=29
-    r = toku_log_commit(logger, &lsn, FSYNC, txnid); assert(r==0);
+    r = toku_log_xcommit(logger, &lsn, FSYNC, txnid); assert(r==0);
     //xbegin                    'b': lsn=4 parenttxnid=0 crc=00000a1f len=29
     r = toku_log_xbegin(logger, &lsn, NO_FSYNC, 0); assert(r==0); txnid = lsn.lsn;
     //fcreate                   'F': lsn=5 txnid=4 filenum=1 fname={len=4 data="b.db"} mode=0777 treeflags=0 crc=14a47925 len=49
     r = toku_log_fcreate(logger, &lsn, NO_FSYNC, txnid, fn_bname, bs_bname, 0x0777, 0, 0, bs_empty); assert(r==0);
     //commit                    'C': lsn=6 txnid=4 crc=0000c11e len=29
-    r = toku_log_commit(logger, &lsn, FSYNC, txnid); assert(r==0);
+    r = toku_log_xcommit(logger, &lsn, FSYNC, txnid); assert(r==0);
     //xbegin                    'b': lsn=7 parenttxnid=0 crc=0000f91f len=29
     r = toku_log_xbegin(logger, &lsn, NO_FSYNC, 0); assert(r==0); txnid = lsn.lsn;
     //enq_insert                'I': lsn=8 filenum=0 xid=7 key={len=2 data="a\000"} value={len=2 data="b\000"} crc=40b863e4 len=45
     r = toku_log_enq_insert(logger, &lsn, NO_FSYNC, fn_aname, txnid, bs_a, bs_b); assert(r==0);
     //begin_checkpoint          'x': lsn=9 timestamp=1251309957584197 crc=cd067878 len=29
     r = toku_log_begin_checkpoint(logger, &lsn, NO_FSYNC, 1251309957584197); assert(r==0); cp_txnid = lsn.lsn;
-   //xstillopen                's': lsn=10 txnid=7 parent=0 crc=00061816 len=37
-    r = toku_log_xstillopen(logger, &lsn, NO_FSYNC, txnid, 0); assert(r==0);
     //fassociate                'f': lsn=11 filenum=1 fname={len=4 data="b.db"} crc=a7126035 len=33
     r = toku_log_fassociate(logger, &lsn, NO_FSYNC, fn_bname, 0, bs_bname); assert(r==0);
+    num_fassociate++;
     //fassociate                'f': lsn=12 filenum=0 fname={len=4 data="a.db"} crc=a70c5f35 len=33
     r = toku_log_fassociate(logger, &lsn, NO_FSYNC, fn_aname, 0, bs_aname); assert(r==0);
+    num_fassociate++;
+   //xstillopen                's': lsn=10 txnid=7 parent=0 crc=00061816 len=37 <- obsolete
+    {
+        FILENUMS filenums = {0, NULL};
+        r = toku_log_xstillopen(logger, &lsn, NO_FSYNC, txnid, 0,
+                                0, filenums, 0, 0, 0,
+                                ROLLBACK_NONE, ROLLBACK_NONE, ROLLBACK_NONE);
+        assert(r==0);
+    }
+    num_xstillopen++;
     //end_checkpoint            'X': lsn=13 txnid=9 timestamp=1251309957586872 crc=cd285c30 len=37
-    r = toku_log_end_checkpoint(logger, &lsn, FSYNC, cp_txnid, 1251309957586872); assert(r==0);
+    r = toku_log_end_checkpoint(logger, &lsn, FSYNC, cp_txnid, 1251309957586872, num_fassociate, num_xstillopen); assert(r==0);
     //enq_insert                'I': lsn=14 filenum=1 xid=7 key={len=2 data="b\000"} value={len=2 data="a\000"} crc=40388be4 len=45
     r = toku_log_enq_insert(logger, &lsn, NO_FSYNC, fn_bname, txnid, bs_b, bs_a); assert(r==0);
     //commit                    'C': lsn=15 txnid=7 crc=00016d1e len=29
-    r = toku_log_commit(logger, &lsn, FSYNC, txnid); assert(r==0);
+    r = toku_log_xcommit(logger, &lsn, FSYNC, txnid); assert(r==0);
 
     // close logger
     r = toku_logger_close(&logger); assert(r==0);

newbrt/txn.c

@@ -12,18 +12,20 @@ int toku_txn_begin_txn (TOKUTXN parent_tokutxn, TOKUTXN *tokutxn, TOKULOGGER log
 
 int toku_txn_begin_with_xid (TOKUTXN parent_tokutxn, TOKUTXN *tokutxn, TOKULOGGER logger, TXNID xid) {
     if (logger->is_panicked) return EINVAL;
+    assert(logger->rollback_cachefile);
     TAGMALLOC(TOKUTXN, result);
     if (result==0) 
         return errno;
     int r;
+    LSN first_lsn;
     if (xid == 0) {
-        r = toku_log_xbegin(logger, &result->first_lsn, 0, parent_tokutxn ? parent_tokutxn->txnid64 : 0);
+        r = toku_log_xbegin(logger, &first_lsn, 0, parent_tokutxn ? parent_tokutxn->txnid64 : 0);
         if (r!=0) goto died;
     } else
-        result->first_lsn.lsn = xid;
+        first_lsn.lsn = xid;
     r = toku_omt_create(&result->open_brts);
     if (r!=0) goto died;
-    result->txnid64 = result->first_lsn.lsn;
+    result->txnid64 = first_lsn.lsn;
     XIDS parent_xids;
     if (parent_tokutxn==NULL)
         parent_xids = xids_get_root_xids();
@@ -33,13 +35,19 @@ int toku_txn_begin_with_xid (TOKUTXN parent_tokutxn, TOKUTXN *tokutxn, TOKULOGGE
         goto died;
     result->logger = logger;
     result->parent = parent_tokutxn;
-    result->oldest_logentry = result->newest_logentry = 0;
 
-    result->rollentry_arena = memarena_create();
     result->num_rollentries = 0;
     result->num_rollentries_processed = 0;
     result->progress_poll_fun = NULL;
     result->progress_poll_fun_extra = NULL;
+    result->spilled_rollback_head      = ROLLBACK_NONE;
+    result->spilled_rollback_tail      = ROLLBACK_NONE;
+    result->spilled_rollback_head_hash = 0;
+    result->spilled_rollback_tail_hash = 0;
+    result->current_rollback      = ROLLBACK_NONE;
+    result->current_rollback_hash = 0;
+    result->num_rollback_nodes = 0;
+    result->pinned_inprogress_rollback_log = NULL;
 
     if (toku_omt_size(logger->live_txns) == 0) {
         assert(logger->oldest_living_xid == TXNID_NONE_LIVING);
@@ -59,13 +67,9 @@ int toku_txn_begin_with_xid (TOKUTXN parent_tokutxn, TOKUTXN *tokutxn, TOKULOGGE
             assert(idx > 0);
     }
 
-    result->rollentry_resident_bytecount=0;
     result->rollentry_raw_count = 0;
-    result->rollentry_filename = 0;
-    result->rollentry_fd = -1;
-    result->rollentry_filesize = 0;
     result->force_fsync_on_commit = FALSE;
-    result->has_done_work = FALSE;
+    result->recovered_from_checkpoint = FALSE;
     *tokutxn = result;
     return 0;
 
@@ -75,6 +79,36 @@ int toku_txn_begin_with_xid (TOKUTXN parent_tokutxn, TOKUTXN *tokutxn, TOKULOGGE
     return r; 
 }
 
+//Used on recovery to recover a transaction.
+int
+toku_txn_load_txninfo (TOKUTXN txn, TXNINFO info) {
+#define COPY_FROM_INFO(field) txn->field = info->field
+    COPY_FROM_INFO(rollentry_raw_count);
+    uint32_t i;
+    for (i = 0; i < info->num_brts; i++) {
+        BRT brt = info->open_brts[i];
+        int r = toku_txn_note_brt(txn, brt);
+        assert(r==0);
+    }
+    COPY_FROM_INFO(force_fsync_on_commit );
+    COPY_FROM_INFO(num_rollback_nodes);
+    COPY_FROM_INFO(num_rollentries);
+
+    CACHEFILE rollback_cachefile = txn->logger->rollback_cachefile;
+
+    COPY_FROM_INFO(spilled_rollback_head);
+    txn->spilled_rollback_head_hash = toku_cachetable_hash(rollback_cachefile,
+                                                           txn->spilled_rollback_head);
+    COPY_FROM_INFO(spilled_rollback_tail);
+    txn->spilled_rollback_tail_hash = toku_cachetable_hash(rollback_cachefile,
+                                                           txn->spilled_rollback_tail);
+    COPY_FROM_INFO(current_rollback);
+    txn->current_rollback_hash = toku_cachetable_hash(rollback_cachefile,
+                                                      txn->current_rollback);
+#undef COPY_FROM_INFO
+    txn->recovered_from_checkpoint = TRUE;
+    return 0;
+}
 
 // Doesn't close the txn, just performs the commit operations.
 int toku_txn_commit_txn(TOKUTXN txn, int nosync, YIELDF yield, void *yieldv,
@@ -92,13 +126,13 @@ int toku_txn_commit_with_lsn(TOKUTXN txn, int nosync, YIELDF yield, void *yieldv
     // panic handled in log_commit
 
     //Child transactions do not actually 'commit'.  They promote their changes to parent, so no need to fsync if this txn has a parent.
-    int do_fsync = !txn->parent && (txn->force_fsync_on_commit || (!nosync && txn->has_done_work));
+    int do_fsync = !txn->parent && (txn->force_fsync_on_commit || (!nosync && txn->num_rollentries>0));
 
     txn->progress_poll_fun = poll;
     txn->progress_poll_fun_extra = poll_extra;
 
     if (release_locks) release_locks(locks_thunk);
-    r = toku_log_commit(txn->logger, (LSN*)0, do_fsync, txn->txnid64); // exits holding neither of the tokulogger locks.
+    r = toku_log_xcommit(txn->logger, (LSN*)0, do_fsync, txn->txnid64); // exits holding neither of the tokulogger locks.
     if (reacquire_locks) reacquire_locks(locks_thunk);
     if (r!=0)
         return r;

newbrt/txn.h

@@ -7,6 +7,7 @@
 
 int toku_txn_begin_txn (TOKUTXN parent_tokutxn, TOKUTXN *tokutxn, TOKULOGGER logger);
 int toku_txn_begin_with_xid (TOKUTXN parent_tokutxn, TOKUTXN *tokutxn, TOKULOGGER logger, TXNID xid);
+int toku_txn_load_txninfo (TOKUTXN txn, TXNINFO info);
 
 int toku_txn_commit_txn (TOKUTXN txn, int nosync, YIELDF yield, void *yieldv,
 			 TXN_PROGRESS_POLL_FUNCTION poll, void *poll_extra,

newbrt/wbuf.h

@@ -170,7 +170,6 @@ static inline void wbuf_DISKOFF (struct wbuf *w, DISKOFF off) {
 static inline void wbuf_BLOCKNUM (struct wbuf *w, BLOCKNUM b) {
     wbuf_ulonglong(w, b.b);
 }
-
 static inline void wbuf_nocrc_BLOCKNUM (struct wbuf *w, BLOCKNUM b) {
     wbuf_nocrc_ulonglong(w, b.b);
 }

release/examples/db-insert.c

@@ -60,7 +60,7 @@ int singlex_child = 0;  // Do a single transaction, but do all work with a child
 int singlex = 0;  // Do a single transaction
 int singlex_create = 0;  // Create the db using the single transaction (only valid if singlex)
 int insert1first = 0;  // insert 1 before doing the rest
-int check_small_rolltmp = 0; // verify that the rollback logs are small (only valid if singlex)
+int check_small_rollback = 0; // verify that the rollback logs are small (only valid if singlex)
 int do_transactions = 0;
 int if_transactions_do_logging = DB_INIT_LOG; // set this to zero if we want no logging when transactions are used
 int do_abort = 0;
@@ -210,14 +210,14 @@ static void benchmark_shutdown (void) {
 #endif
     if (do_transactions && singlex && !insert1first && (singlex_create || prelock)) {
 #if defined(TOKUDB)
-        //There should be a single 'truncate' in the rolltmp instead of many 'insert' entries.
+        //There should be a single 'truncate' in the rollback instead of many 'insert' entries.
 	struct txn_stat *s;
 	r = tid->txn_stat(tid, &s);
 	assert(r==0);
         //TODO: #1125 Always do the test after performance testing is done.
-        if (singlex_child) fprintf(stderr, "SKIPPED 'small rolltmp' test for child txn\n");
+        if (singlex_child) fprintf(stderr, "SKIPPED 'small rollback' test for child txn\n");
         else
-            assert(s->rolltmp_raw_count < 100);  // gross test, not worth investigating details
+            assert(s->rollback_raw_count < 100);  // gross test, not worth investigating details
 	free(s);
 	//system("ls -l bench.tokudb");
 #endif
@@ -374,7 +374,7 @@ static int print_usage (const char *argv0) {
     fprintf(stderr, "    --singlex-child       (implies -x) Run the whole job as a single transaction, do all work a child of that transaction.\n");
     fprintf(stderr, "    --finish-child-first  Commit/abort child before doing so to parent (no effect if no child).\n");
     fprintf(stderr, "    --singlex-create      (implies --singlex)  Create the file using the single transaction (Default is to use a different transaction to create.)\n");
-    fprintf(stderr, "    --check_small_rolltmp (Only valid in --singlex mode)  Verify that very little data was saved in the rollback logs.\n");
+    fprintf(stderr, "    --check_small_rollback (Only valid in --singlex mode)  Verify that very little data was saved in the rollback logs.\n");
     fprintf(stderr, "    --prelock             Prelock the database.\n");
     fprintf(stderr, "    --prelockflag         Prelock the database and send the DB_PRELOCKED_WRITE flag.\n");
     fprintf(stderr, "    --abort               Abort the singlex after the transaction is over. (Requires --singlex.)\n");
@@ -463,8 +463,8 @@ int main (int argc, const char *argv[]) {
 	    singlex = 1;
 	} else if (strcmp(arg, "--insert1first") == 0) {
 	    insert1first = 1;
-	} else if (strcmp(arg, "--check_small_rolltmp") == 0) {
-	    check_small_rolltmp = 1;
+	} else if (strcmp(arg, "--check_small_rollback") == 0) {
+	    check_small_rollback = 1;
 	} else if (strcmp(arg, "--xcount") == 0) {
             if (i+1 >= argc) return print_usage(argv[0]);
             items_per_transaction = strtoll(argv[++i], &endptr, 10); assert(*endptr == 0);
@@ -560,13 +560,13 @@ int main (int argc, const char *argv[]) {
 	printf("insertions of %d per batch%s\n", items_per_iteration, do_transactions ? " (with transactions)" : "");
     }
 #if !defined TOKUDB
-    if (check_small_rolltmp) {
-	fprintf(stderr, "--check_small_rolltmp only works on the TokuDB (not BDB)\n");
+    if (check_small_rollback) {
+	fprintf(stderr, "--check_small_rollback only works on the TokuDB (not BDB)\n");
 	return print_usage(argv[0]);
     }
 #endif
-    if (check_small_rolltmp && !singlex) {
-	fprintf(stderr, "--check_small_rolltmp requires --singlex\n");
+    if (check_small_rollback && !singlex) {
+	fprintf(stderr, "--check_small_rollback requires --singlex\n");
 	return print_usage(argv[0]);
     }
     benchmark_setup();

src/tests/Makefile

@@ -332,6 +332,7 @@ endif
 	mkdir dir.$*.c.tdb.recover                                                              && \
 	cp dir.$*.c.tdb/tokudb.directory   dir.$*.c.tdb.recover/                                && \
 	cp dir.$*.c.tdb/tokudb.environment dir.$*.c.tdb.recover/                                && \
+	cp dir.$*.c.tdb/tokudb.rollback    dir.$*.c.tdb.recover/                                && \
 	cp dir.$*.c.tdb/*.tokulog          dir.$*.c.tdb.recover/                                && \
 	echo doing recovery &&\
 	$(VGRIND) ../../newbrt/tdb-recover dir.$*.c.tdb.recover dir.$*.c.tdb.recover            && \

src/tests/bug1381.c

@@ -60,13 +60,13 @@ static void do_1381_maybe_lock (int do_table_lock, u_int64_t *raw_count) {
 	}
 
 	r = txn->txn_stat(txn, &s2);                                      CKERR(r);
-	//printf("Raw counts = %" PRId64 ", %" PRId64 "\n", s1->rolltmp_raw_count, s2->rolltmp_raw_count);
+	//printf("Raw counts = %" PRId64 ", %" PRId64 "\n", s1->rollback_raw_count, s2->rollback_raw_count);
 
-	*raw_count = s2->rolltmp_raw_count - s1->rolltmp_raw_count;
+	*raw_count = s2->rollback_raw_count - s1->rollback_raw_count;
 	if (do_table_lock) {
-	    assert(s1->rolltmp_raw_count == s2->rolltmp_raw_count);
+	    assert(s1->rollback_raw_count == s2->rollback_raw_count);
 	} else {
-	    assert(s1->rolltmp_raw_count < s2->rolltmp_raw_count);
+	    assert(s1->rollback_raw_count < s2->rollback_raw_count);
 	}
 	
 	toku_free(s1); toku_free(s2);

src/tests/diskfull.c

@@ -91,7 +91,7 @@ do_db_work(void) {
 	}
 	if (did_fail) goto shutdown2;
 
-	// Put an extra item in so that the rolltmp file will be created. 
+	// Put an extra item in
 	r=env->txn_begin(env, 0, &tid, 0);                                         assert(r==0);
 	r=db->put(db, tid, dbt_init(&key, "a", 2), dbt_init(&data, "b", 2), 0);    DOERR(r);
 	if (did_fail) {

src/tests/stat64.c

@@ -21,7 +21,7 @@ test_stat64 (unsigned int N) {
     DB_TXN *txn;
     r = db_env_create(&env, 0);                                           CKERR(r);
 
-    r = env->set_cachesize(env, 0, 10*1000000, 1);
+    r = env->set_cachesize(env, 0, 20*1000000, 1);
     r = env->open(env, ENVDIR, DB_INIT_LOCK|DB_INIT_LOG|DB_INIT_MPOOL|DB_INIT_TXN|DB_CREATE|DB_PRIVATE, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
     r = db_create(&db, env, 0);                                           CKERR(r);
 
@@ -38,6 +38,10 @@ test_stat64 (unsigned int N) {
     unsigned int i;
     u_int64_t dsize=0;
     for (i=0; i<N; i++) {
+        if (verbose>1 && i % (1<<14) == 0) {
+            printf("%s(total=%u) inserted %u so far\n", __FILE__, N, i);
+            fflush(stdout);
+        }
 	char hello[30], there[30];
 	snprintf(hello, sizeof(hello), "hello%8d", i);
 	snprintf(there, sizeof(there), "there%d", i);

toku_include/memory.h

@@ -21,7 +21,8 @@ enum typ_tag { TYP_BRTNODE = 0xdead0001,
 	       TYP_GPMA,
                TYP_TOKULOGGER,
 	       TYP_TOKUTXN,
-	       TYP_LEAFENTRY
+	       TYP_LEAFENTRY,
+               TYP_ROLLBACK_LOG_NODE
 };
 
 /* Everything should call toku_malloc() instead of malloc(), and toku_calloc() instead of calloc() */

toku_include/toku_list.h

@@ -78,7 +78,7 @@ static inline void toku_list_move(struct toku_list *newhead, struct toku_list *o
 // Note: Need the extra level of parens in these macros so that
 //   toku_list_struct(h, foo, b)->zot
 // will work right.  Otherwise the type cast will try to include ->zot, and it will be all messed up.
-#if defined(__GNUC__) && __GNUC__ >= 4
+#if (defined(__GNUC__) && __GNUC__ >= 4) || defined(__builtin_offsetof)
 #define toku_list_struct(p, t, f) ((t*)((char*)(p) - __builtin_offsetof(t, f)))
 #else
 #define toku_list_struct(p, t, f) ((t*)((char*)(p) - ((char*)&((t*)0)->f)))

windows/misc.h

@@ -7,6 +7,7 @@ extern "C" {
 
 #include "toku_os.h"
 #include <sys/stat.h>
+#include <stddef.h>
 
 //These are functions that really exist in windows but are named
 //something else.
@@ -82,6 +83,9 @@ int mkstemp(char * ttemplate);
 
 toku_off_t ftello(FILE *stream);
 
+#define __builtin_offsetof(type, member) offsetof(type, member)
+
+
 #if defined(__cplusplus)
 };
 #endif