Some fixes for recovery (#1364). Also some temporary fixes for #1277. Addresses #1364, #1277.

Delete the 1364 branch, so I can restart it.
On the 1343 branch, I did:
{{{
svn merge -r 8586:8595 https://svn.tokutek.com/tokudb/toku/tokudb.1032b+1343+1364
}}}


git-svn-id: file:///svn/toku/tokudb.1032b+1343@8596 c7de825b-a66e-492c-adef-691d508d4ae1

newbrt/brt.c

@@ -2856,7 +2856,7 @@ int toku_brt_open(BRT t, const char *fname, const char *fname_in_env, const char
         if (r != 0) goto died0a;
         if (did_create) {
             mode_t mode = S_IRWXU|S_IRWXG|S_IRWXO;
-            r = toku_logger_log_fcreate(txn, fname_in_env, toku_cachefile_filenum(t->cf), mode);
+            r = toku_logger_log_fcreate(txn, fname_in_env, mode);
             if (r != 0) goto died_after_open;
         }
 	r = toku_logger_log_fopen(txn, fname_in_env, toku_cachefile_filenum(t->cf));

newbrt/leafentry.c

@@ -162,7 +162,6 @@ u_int32_t toku_logsizeof_LEAFENTRY (LEAFENTRY le) {
 }
 
 int toku_fread_LEAFENTRY(FILE *f, LEAFENTRY *le, struct x1764 *checksum, u_int32_t *len) {
-    assert(0);
     u_int8_t state;
     int r = toku_fread_u_int8_t (f, &state, checksum, len); if (r!=0) return r;
     TXNID xid;

newbrt/log.c

@@ -531,13 +531,13 @@ int toku_logger_txn_begin (TOKUTXN parent_tokutxn, TOKUTXN *tokutxn, TOKULOGGER
     return 0;
 }
 
-int toku_logger_log_fcreate (TOKUTXN txn, const char *fname, FILENUM filenum, int mode) {
+int toku_logger_log_fcreate (TOKUTXN txn, const char *fname, int mode) {
     if (txn==0) return 0;
     if (txn->logger->is_panicked) return EINVAL;
     BYTESTRING bs = { .len=strlen(fname), .data = toku_strdup_in_rollback(txn, fname) };
-    int r = toku_log_fcreate (txn->logger, (LSN*)0, 0, toku_txn_get_txnid(txn), filenum, bs, mode);
+    int r = toku_log_fcreate (txn->logger, (LSN*)0, 0, toku_txn_get_txnid(txn), bs, mode);
     if (r!=0) return r;
-    r = toku_logger_save_rollback_fcreate(txn, toku_txn_get_txnid(txn), filenum, bs);
+    r = toku_logger_save_rollback_fcreate(txn, toku_txn_get_txnid(txn), bs);
     return r;
 }
 

newbrt/log.h

@@ -43,7 +43,7 @@ int toku_logger_commit (TOKUTXN txn, int no_sync);
 
 int toku_logger_txn_begin (TOKUTXN /*parent*/,TOKUTXN *, TOKULOGGER /*logger*/);
 
-int toku_logger_log_fcreate (TOKUTXN, const char */*fname*/, FILENUM /*filenum*/, int /*mode*/);
+int toku_logger_log_fcreate (TOKUTXN, const char */*fname*/, int /*mode*/);
 
 int toku_logger_log_fopen (TOKUTXN, const char * /*fname*/, FILENUM);
 

newbrt/logformat.c

@@ -41,7 +41,6 @@ int logformat_version_number = 0;
 
 const struct logtype rollbacks[] = {
     {"fcreate", 'F', FA{{"TXNID", "xid", 0},
-			{"FILENUM", "filenum", 0},
 			{"BYTESTRING", "fname", 0},
 			NULLFIELD}},
     // cmdinsert is used to insert a key-value pair into a NODUP DB.  For rollback we don't need the data.
@@ -96,7 +95,6 @@ const struct logtype logtypes[] = {
 		       NULLFIELD}},
 #endif
     {"fcreate", 'F', FA{{"TXNID",      "txnid", 0},
-			{"FILENUM",    "filenum", 0},
 			{"BYTESTRING", "fname", 0},
 			{"u_int32_t",  "mode",  "0%o"},
 			NULLFIELD}},

newbrt/recover.c

@@ -83,13 +83,15 @@ create_dir_from_file (const char *fname) {
 }
 
 static void
-toku_recover_fcreate (LSN UU(lsn), TXNID UU(txnid), FILENUM UU(filenum), BYTESTRING fname,u_int32_t mode) {
+toku_recover_fcreate (LSN UU(lsn), TXNID UU(txnid), BYTESTRING fname,u_int32_t mode) {
     char *fixed_fname = fixup_fname(&fname);
     create_dir_from_file(fixed_fname);
     int fd = open(fixed_fname, O_CREAT+O_TRUNC+O_WRONLY+O_BINARY, mode);
     assert(fd>=0);
     toku_free(fixed_fname);
     toku_free_BYTESTRING(fname);
+    int r = close(fd);
+    assert(r==0);
 }
 
 static int
@@ -130,6 +132,7 @@ static void toku_recover_fheader (LSN UU(lsn), TXNID UU(txnid),FILENUM filenum,L
     struct brt_header *MALLOC(h);
     assert(h);
     h->dirty=0;
+    XMALLOC(h->flags_array);
     h->flags_array[0] = header.flags;
     h->nodesize = header.nodesize;
     h->free_blocks   = header.free_blocks;
@@ -203,7 +206,7 @@ toku_recover_newbrtnode (LSN lsn, FILENUM filenum, BLOCKNUM blocknum,u_int32_t h
     // Now put it in the cachetable
     u_int32_t fullhash = toku_cachetable_hash(pair->cf, blocknum);
     n->fullhash = fullhash;
-    toku_cachetable_put(pair->cf, blocknum, fullhash, n, toku_serialize_brtnode_size(n),  toku_brtnode_flush_callback, toku_brtnode_fetch_callback, 0);
+    toku_cachetable_put(pair->cf, blocknum, fullhash, n, toku_serialize_brtnode_size(n),  toku_brtnode_flush_callback, toku_brtnode_fetch_callback, pair->brt->h);
 
     VERIFY_COUNTS(n);
 

newbrt/roll.c

@@ -12,14 +12,12 @@
 #define TOKU_DO_COMMIT_CMD_DELETE_BOTH 1
 
 int toku_commit_fcreate (TXNID UU(xid),
-                         FILENUM UU(filenum),
 			 BYTESTRING UU(bs_fname),
 			 TOKUTXN    UU(txn)) {
     return 0;
 }
 
 int toku_rollback_fcreate (TXNID xid __attribute__((__unused__)),
-                           FILENUM filenum,
 			   BYTESTRING bs_fname,
 			   TOKUTXN    txn       __attribute__((__unused__))) {
     char *fname = fixup_fname(&bs_fname);
@@ -28,6 +26,8 @@ int toku_rollback_fcreate (TXNID xid __attribute__((__unused__)),
     char full_fname[full_len];
     int l = snprintf(full_fname,full_len, "%s/%s", directory, fname);
     assert(l<=full_len);
+#if 0
+    // I don't think this is right.  fcreate simply creates the file, and doesn't put it in the cache table.
     //Remove reference to the fd in the cachetable
     CACHEFILE cf;
     int r = toku_cachefile_of_filenum(txn->logger->ct, filenum, &cf);
@@ -35,7 +35,8 @@ int toku_rollback_fcreate (TXNID xid __attribute__((__unused__)),
         r = toku_cachefile_redirect_nullfd(cf);
         assert(r==0);
     }
-    r = unlink(full_fname);
+#endif
+    int r = unlink(full_fname);
     assert(r==0);
     toku_free(fname);
     return 0;

newbrt/threadpool.c

 #include "includes.h"
 
 // use gcc builtin fetch_and_add 0->no 1->yes
-#define DO_ATOMIC_FETCH_AND_ADD 0
+#define DO_ATOMIC_FETCH_AND_ADD 1
 
 struct threadpool {
     int max_threads;

src/tests/Makefile

@@ -43,6 +43,7 @@ BDB_DONTRUN_TESTS = \
 	test938c \
 	helgrind1 \
 	helgrind2 \
+	helgrind3 \
 #\ ends prev line
 
 BDB_TESTS = $(patsubst %.c,%.bdb$(BINSUF),$(filter-out $(patsubst %,%.c,$(BDB_DONTRUN_TESTS)),$(SRCS)))
@@ -58,8 +59,6 @@ TDB_TESTS_THAT_SHOULD_FAIL= \
 	test_truncate_subdb \
 	test_txn_nested_abort3 \
 	test_txn_nested_abort4 \
-	helgrind2 \
-	helgrind3 \
 #\ ends prev line
 
 TDB_TESTS_THAT_SHOULD_FAIL_LIT= \
@@ -123,12 +122,14 @@ SHOULD_FAIL = $(TDB_TESTS_THAT_SHOULD_FAIL_LIT) $(patsubst %,%.tdbrun,$(TDB_TEST
 $(SHOULD_FAIL): MAYBEINVERTER=$(INVERTER)
 $(SHOULD_FAIL): SUMMARIZE_CMD=$(SUMMARIZE_SHOULD_FAIL)
 
+TDBVGRIND=$(VGRIND)
+
 # Use -s on the command line to make things quiet.
 # Use -s on the command line to make things quiet.
 %.bdbrun: %.bdb$(BINSUF) $(DEPEND_COMPILE) $(DEPEND_LINK)
 	$(BDBVGRIND) ./$< $(VERBVERBOSE) $(SUMMARIZE_CMD)
 %.tdbrun: %.tdb$(BINSUF) $(DEPEND_COMPILE) $(DEPEND_LINK) $(notdir $(LIBTDB))
-	$(VGRIND)    ./$< $(VERBVERBOSE) $(MAYBEINVERTER) $(SUMMARIZE_CMD)
+	$(TDBVGRIND)    ./$< $(VERBVERBOSE) $(MAYBEINVERTER) $(SUMMARIZE_CMD)
 
 %.recover: %.tdb$(BINSUF)
 	$(VGRIND) ./$< && \
@@ -273,11 +274,11 @@ tgbr_%_c.tdbrun: test_get_both_range.tdb$(BINSUF)
 	$(VGRIND) ./$< $(VERBVERBOSE) -i $* -c $(MAYBEINVERTER) $(SUMMARIZE_CMD)
 
 # helgrind1 is supposed to fail.
-helgrind1.tdbrun: VGRIND=$(HGRIND) --log-file=helgrind1.tdb.deleteme
+helgrind1.tdbrun: TDBVGRIND=$(HGRIND) --log-file=helgrind1.tdb.deleteme
 helgrind1.tdbrun: MAYBEINVERTER=$(INVERTER)
-helgrind2.tdbrun: VGRIND=$(HGRIND)
+helgrind2.tdbrun: TDBVGRIND=$(HGRIND)
 helgrind2.bdbrun: BDBVGRIND=$(HGRIND)
-helgrind3.tdbrun: VGRIND=$(HGRIND)
+helgrind3.tdbrun: TDBVGRIND=$(HGRIND)
 helgrind3.bdbrun: BDBVGRIND=$(HGRIND)
 
 dumpit:

src/tests/test_db_remove.c

@@ -14,12 +14,17 @@ void test_db_remove (void) {
     r=toku_os_mkdir(ENVDIR, S_IRWXU+S_IRWXG+S_IRWXO); assert(r==0);
 
     // create the DB
-    r = db_create(&db1, null_env, 0);                                 assert(r == 0);
+    r = db_create(&db1, null_env, 0);                                  assert(r == 0);
     r = db1->open(db1, null_txn, fname, 0, DB_BTREE, DB_CREATE, 0666); assert(r == 0);
 
-    // Now remove it.
-    r = db_create(&db2, null_env, 0);                                 assert(r==0);
-    r = db2->remove(db2, fname, 0, 0);                                 assert(r==0);
+    // Now remove it, while it is open.
+    r = db_create(&db2, null_env, 0);                                  assert(r==0);
+    r = db2->remove(db2, fname, 0, 0);
+#ifdef USE_TDB
+    assert(r!=0);
+#else
+    assert(r==0);
+#endif
 
     r = db1->close(db1, 0);                                            assert(r==0);
 }

windows/memory.c

@@ -76,7 +76,7 @@ toku_realloc(void *p, size_t size)
 void
 toku_free(void* p)
 {
-    toku_free_counter++;
+    (void)__sync_fetch_and_add(&toku_free_counter, 1);
     if (t_free)
 	t_free(p);
     else