Addresses #1396

Merge tokudb.1396 back into main

git-svn-id: file:///svn/toku/tokudb@9110 c7de825b-a66e-492c-adef-691d508d4ae1

buildheader/db.h_4_1

@@ -102,6 +102,7 @@ typedef enum {
 /* TOKUDB specific error codes */
 #define TOKUDB_OUT_OF_LOCKS -100000
 #define TOKUDB_SUCCEEDED_EARLY -100001
+#define TOKUDB_DIRTY_DICTIONARY -100004
 /* in wrap mode, top-level function txn_begin is renamed, but the field isn't renamed, so we have to hack it here.*/
 #ifdef _TOKUDB_WRAP_H
 #undef txn_begin

buildheader/db.h_4_3

@@ -104,6 +104,7 @@ typedef enum {
 /* TOKUDB specific error codes */
 #define TOKUDB_OUT_OF_LOCKS -100000
 #define TOKUDB_SUCCEEDED_EARLY -100001
+#define TOKUDB_DIRTY_DICTIONARY -100004
 /* in wrap mode, top-level function txn_begin is renamed, but the field isn't renamed, so we have to hack it here.*/
 #ifdef _TOKUDB_WRAP_H
 #undef txn_begin

buildheader/db.h_4_4

@@ -105,6 +105,7 @@ typedef enum {
 /* TOKUDB specific error codes */
 #define TOKUDB_OUT_OF_LOCKS -100000
 #define TOKUDB_SUCCEEDED_EARLY -100001
+#define TOKUDB_DIRTY_DICTIONARY -100004
 /* in wrap mode, top-level function txn_begin is renamed, but the field isn't renamed, so we have to hack it here.*/
 #ifdef _TOKUDB_WRAP_H
 #undef txn_begin

buildheader/db.h_4_5

@@ -105,6 +105,7 @@ typedef enum {
 /* TOKUDB specific error codes */
 #define TOKUDB_OUT_OF_LOCKS -100000
 #define TOKUDB_SUCCEEDED_EARLY -100001
+#define TOKUDB_DIRTY_DICTIONARY -100004
 /* in wrap mode, top-level function txn_begin is renamed, but the field isn't renamed, so we have to hack it here.*/
 #ifdef _TOKUDB_WRAP_H
 #undef txn_begin

buildheader/db.h_4_6

@@ -11,9 +11,9 @@ extern "C" {
 #define TOKUDB 1
 #define DB_VERSION_MAJOR 4
 #define DB_VERSION_MINOR 6
-#define DB_VERSION_PATCH 21
+#define DB_VERSION_PATCH 19
 #ifndef _TOKUDB_WRAP_H
-#define DB_VERSION_STRING "Tokutek: TokuDB 4.6.21"
+#define DB_VERSION_STRING "Tokutek: TokuDB 4.6.19"
 #else
 #define DB_VERSION_STRING_ydb "Tokutek: TokuDB (wrapped bdb)"
 #endif
@@ -106,6 +106,7 @@ typedef enum {
 /* TOKUDB specific error codes */
 #define TOKUDB_OUT_OF_LOCKS -100000
 #define TOKUDB_SUCCEEDED_EARLY -100001
+#define TOKUDB_DIRTY_DICTIONARY -100004
 /* in wrap mode, top-level function txn_begin is renamed, but the field isn't renamed, so we have to hack it here.*/
 #ifdef _TOKUDB_WRAP_H
 #undef txn_begin

buildheader/make_db_h.c

@@ -31,7 +31,8 @@ void print_db_notices (void) {
 
 enum {
 	TOKUDB_OUT_OF_LOCKS     = -100000,
-        TOKUDB_SUCCEEDED_EARLY = -100001
+        TOKUDB_SUCCEEDED_EARLY  = -100001,
+        TOKUDB_DIRTY_DICTIONARY = -100004
 };
 
 void print_defines (void) {
@@ -138,6 +139,7 @@ void print_defines (void) {
     printf("/* TOKUDB specific error codes */\n");
     dodefine(TOKUDB_OUT_OF_LOCKS);
     dodefine(TOKUDB_SUCCEEDED_EARLY);
+    dodefine(TOKUDB_DIRTY_DICTIONARY);
 }
 
 //#define DECL_LIMIT 100

include/db.h

@@ -11,9 +11,9 @@ extern "C" {
 #define TOKUDB 1
 #define DB_VERSION_MAJOR 4
 #define DB_VERSION_MINOR 6
-#define DB_VERSION_PATCH 21
+#define DB_VERSION_PATCH 19
 #ifndef _TOKUDB_WRAP_H
-#define DB_VERSION_STRING "Tokutek: TokuDB 4.6.21"
+#define DB_VERSION_STRING "Tokutek: TokuDB 4.6.19"
 #else
 #define DB_VERSION_STRING_ydb "Tokutek: TokuDB (wrapped bdb)"
 #endif
@@ -106,6 +106,7 @@ typedef enum {
 /* TOKUDB specific error codes */
 #define TOKUDB_OUT_OF_LOCKS -100000
 #define TOKUDB_SUCCEEDED_EARLY -100001
+#define TOKUDB_DIRTY_DICTIONARY -100004
 /* in wrap mode, top-level function txn_begin is renamed, but the field isn't renamed, so we have to hack it here.*/
 #ifdef _TOKUDB_WRAP_H
 #undef txn_begin

newbrt/brt.c

@@ -380,7 +380,16 @@ void toku_brtnode_flush_callback (CACHEFILE cachefile, BLOCKNUM nodename, void *
 	if (!h->panic) { // if the brt panicked, stop writing, otherwise try to write it.
 	    int n_workitems, n_threads; 
 	    toku_cachefile_get_workqueue_load(cachefile, &n_workitems, &n_threads);
-	    int r = toku_serialize_brtnode_to(toku_cachefile_fd(cachefile), brtnode->thisnodename, brtnode, h, n_workitems, n_threads);
+            int r = toku_graceful_dirty(cachefile);
+	    if (r) {
+		if (h->panic==0) {
+		    char s[200];
+		    h->panic=r;
+		    snprintf(s, sizeof(s), "While creating dirty bit, error %d (%s)", r, strerror(r));
+		    h->panic_string = toku_strdup(s);
+		}
+	    }
+	    r = toku_serialize_brtnode_to(toku_cachefile_fd(cachefile), brtnode->thisnodename, brtnode, h, n_workitems, n_threads);
 	    if (r) {
 		if (h->panic==0) {
 		    char s[200];
@@ -2866,7 +2875,7 @@ int toku_brt_open(BRT t, const char *fname, const char *fname_in_env, const char
         if (r != 0) {
             t->database_name = 0; goto died0a;
         }
-	r=toku_cachetable_openfd(&t->cf, cachetable, fd, fname_in_env);
+        r=toku_cachetable_openfd(&t->cf, cachetable, fd, fname);
         if (r != 0) goto died0a;
         if (did_create) {
             mode_t mode = S_IRWXU|S_IRWXG|S_IRWXO;
@@ -3049,6 +3058,10 @@ toku_brtheader_checkpoint (CACHEFILE cachefile, void *header_v)
     //printf("%s:%d allocated_limit=%lu writing queue to %lu\n", __FILE__, __LINE__,
     //       block_allocator_allocated_limit(h->block_allocator), h->unused_blocks.b*h->nodesize);
     if (h->dirty) {
+        {
+            int r = toku_graceful_dirty(cachefile);
+            if (r!=0) return r;
+        }
 	{
 	    int r = toku_serialize_brt_header_to(toku_cachefile_fd(cachefile), h);
 	    if (r) return r;
@@ -4325,11 +4338,13 @@ int toku_brt_truncate (BRT brt) {
 static void toku_brt_lock_init(void) {
     toku_pwrite_lock_init();
     toku_logger_lock_init();
+    toku_graceful_lock_init();
 }
 
 static void toku_brt_lock_destroy(void) {
     toku_pwrite_lock_destroy();
     toku_logger_lock_destroy();
+    toku_graceful_lock_destroy();
 }
 
 void toku_brt_init(void) {

newbrt/cachetable.c

@@ -128,6 +128,7 @@ struct cachefile {
 			 * every record in the transaction, we'll be ok.  Hence we use a 64-bit counter to make sure we don't run out.
 			 */
     int fd;       /* Bug: If a file is opened read-only, then it is stuck in read-only.  If it is opened read-write, then subsequent writers can write to it too. */
+    BOOL is_dirty;      /* Has this been written to since it was closed? */
     CACHETABLE cachetable;
     struct fileid fileid;
     FILENUM filenum;
@@ -220,6 +221,13 @@ int toku_cachetable_openfd (CACHEFILE *cfptr, CACHETABLE ct, int fd, const char
 	}
     }
     {
+        BOOL was_dirty = FALSE;
+        r = toku_graceful_open(fname, &was_dirty);
+        if (r!=0) {
+            close(fd); 
+            return r;
+        }
+
         CACHEFILE MALLOC(newcf);
         newcf->cachetable = ct;
         newcf->filenum.fileid = next_filenum_to_use.fileid++;
@@ -231,6 +239,7 @@ int toku_cachetable_openfd (CACHEFILE *cfptr, CACHETABLE ct, int fd, const char
 	newcf->userdata = 0;
 	newcf->close_userdata = 0;
 	newcf->checkpoint_userdata = 0;
+        newcf->is_dirty = was_dirty;
 
 	*cfptr = newcf;
 	return 0;
@@ -314,10 +323,17 @@ int toku_cachefile_close (CACHEFILE *cfp, TOKULOGGER logger, char **error_string
     if (cf->refcount==0) {
 	int r;
 	if ((r = cachetable_flush_cachefile(ct, cf))) {
+            //This is not a graceful shutdown; do not set file as clean.
             cachetable_unlock(ct);
             return r;
         }
 	if (cf->close_userdata && (r = cf->close_userdata(cf, cf->userdata, error_string))) {
+            //This is not a graceful shutdown; do not set file as clean.
+	    cachetable_unlock(ct);
+	    return r;
+	}
+        //Graceful shutdown.  'clean' the file.
+	if ((r = toku_graceful_close(cf))) {
 	    cachetable_unlock(ct);
             return r;
         }
@@ -334,8 +350,7 @@ int toku_cachefile_close (CACHEFILE *cfp, TOKULOGGER logger, char **error_string
 	    //BYTESTRING bs = {.len=strlen(cf->fname), .data=cf->fname};
 	    //r = toku_log_cfclose(logger, 0, 0, bs, cf->filenum);
 	}
-	if (cf->fname)
-	    toku_free(cf->fname);
+	if (cf->fname) toku_free(cf->fname);
 	toku_free(cf);
 	*cfp=0;
 	return r;
@@ -1039,6 +1054,7 @@ static int cachetable_flush_cachefile(CACHETABLE ct, CACHEFILE cf) {
     // are dirty.  pairs in the READING or WRITING states are already in the
     // work queue.
     unsigned i;
+
     for (i=0; i < ct->table_size; i++) {
 	PAIR p;
 	for (p = ct->table[i]; p; p = p->hash_chain) {
@@ -1362,3 +1378,239 @@ int toku_cachefile_redirect_nullfd (CACHEFILE cf) {
     return 0;
 }
 
+static toku_pthread_mutex_t graceful_mutex = TOKU_PTHREAD_MUTEX_INITIALIZER;
+static int graceful_is_locked=0;
+
+void toku_graceful_lock_init(void) {
+    int r = toku_pthread_mutex_init(&graceful_mutex, NULL); assert(r == 0);
+}
+
+void toku_graceful_lock_destroy(void) {
+    int r = toku_pthread_mutex_destroy(&graceful_mutex); assert(r == 0);
+}
+
+static inline void
+lock_for_graceful (void) {
+    // Locks the graceful_mutex. 
+    int r = toku_pthread_mutex_lock(&graceful_mutex);
+    assert(r==0);
+    graceful_is_locked = 1;
+}
+
+static inline void
+unlock_for_graceful (void) {
+    graceful_is_locked = 0;
+    int r = toku_pthread_mutex_unlock(&graceful_mutex);
+    assert(r==0);
+}
+
+static int
+graceful_open_get_append_fd(const char *db_fname, BOOL *is_dirtyp, BOOL *create) {
+    BOOL clean_exists;
+    BOOL dirty_exists;
+    char cleanbuf[strlen(db_fname) + sizeof(".clean")];
+    char dirtybuf[strlen(db_fname) + sizeof(".dirty")];
+
+    sprintf(cleanbuf, "%s.clean", db_fname);
+    sprintf(dirtybuf, "%s.dirty", db_fname);
+
+    struct stat tmpbuf;
+    clean_exists = stat(cleanbuf, &tmpbuf) == 0;
+    dirty_exists = stat(dirtybuf, &tmpbuf) == 0;
+    mode_t mode = S_IRWXU|S_IRWXG|S_IRWXO;
+    int r = 0;
+
+    *create = FALSE;
+    if (dirty_exists && clean_exists) {
+        r = unlink(cleanbuf);
+        clean_exists = FALSE;
+    }
+    if (r==0) {
+        if (!dirty_exists && !clean_exists) {
+            *create = TRUE;
+            dirty_exists = TRUE;
+        }
+        if (dirty_exists) {
+            *is_dirtyp = TRUE;
+            r = open(dirtybuf, O_WRONLY | O_CREAT | O_BINARY | O_APPEND, mode);
+        }
+        else {
+            assert(clean_exists);
+            *is_dirtyp = FALSE;
+            r = open(cleanbuf, O_WRONLY | O_CREAT | O_BINARY | O_APPEND, mode);
+        }
+    }
+    return r;
+}
+
+static int
+graceful_close_get_append_fd(const char *db_fname, BOOL *db_missing) {
+    BOOL clean_exists;
+    BOOL dirty_exists;
+    BOOL db_exists;
+    char cleanbuf[strlen(db_fname) + sizeof(".clean")];
+    char dirtybuf[strlen(db_fname) + sizeof(".dirty")];
+
+    sprintf(cleanbuf, "%s.clean", db_fname);
+    sprintf(dirtybuf, "%s.dirty", db_fname);
+
+    struct stat tmpbuf;
+    clean_exists = stat(cleanbuf, &tmpbuf) == 0;
+    dirty_exists = stat(dirtybuf, &tmpbuf) == 0;
+    db_exists    = stat(db_fname, &tmpbuf) == 0;
+    mode_t mode = S_IRWXU|S_IRWXG|S_IRWXO;
+    int r = 0;
+
+    if (dirty_exists) {
+        if (clean_exists) r = unlink(dirtybuf);
+        else              r = rename(dirtybuf, cleanbuf);
+    }
+    if (db_exists) r = open(cleanbuf, O_WRONLY | O_CREAT | O_BINARY | O_APPEND, mode);
+    else if (clean_exists) r = unlink(cleanbuf);
+    *db_missing = !db_exists;
+    return r;
+}
+
+static int
+graceful_dirty_get_append_fd(const char *db_fname) {
+    BOOL clean_exists;
+    BOOL dirty_exists;
+    char cleanbuf[strlen(db_fname) + sizeof(".clean")];
+    char dirtybuf[strlen(db_fname) + sizeof(".dirty")];
+
+    sprintf(cleanbuf, "%s.clean", db_fname);
+    sprintf(dirtybuf, "%s.dirty", db_fname);
+
+    struct stat tmpbuf;
+    clean_exists = stat(cleanbuf, &tmpbuf) == 0;
+    dirty_exists = stat(dirtybuf, &tmpbuf) == 0;
+    mode_t mode = S_IRWXU|S_IRWXG|S_IRWXO;
+    int r = 0;
+
+    if (clean_exists) {
+        if (dirty_exists) r = unlink(cleanbuf);
+        else              r = rename(cleanbuf, dirtybuf);
+    }
+    r = open(dirtybuf, O_WRONLY | O_CREAT | O_BINARY | O_APPEND, mode);
+    return r;
+}
+
+static void
+graceful_log(int fd, char *operation, BOOL was_dirty, BOOL is_dirty) {
+    //Logging.  Ignore errors.
+    static char buf[sizeof(":-> pid= tid=  ")
+                    +7  //operation
+                    +5  //was dirty
+                    +5  //is  dirty
+                    +5  //process id
+                    +5  //thread id
+                    +26 //ctime string (including \n)
+                   ];
+    assert(graceful_is_locked); //ctime uses static buffer.  Lock must be held.
+    time_t temptime;
+    time(&temptime);
+    snprintf(buf, sizeof(buf), "%-7s:%-5s->%-5s pid=%-5d tid=%-5d  %s",
+             operation,
+             was_dirty ? "dirty" : "clean",
+             is_dirty  ? "dirty" : "clean",
+             toku_os_getpid(),
+             toku_os_gettid(),
+             ctime(&temptime));
+    write(fd, buf, strlen(buf));
+} 
+
+int
+toku_graceful_open(const char *db_fname, BOOL *is_dirtyp) {
+    int r;
+    int r2 = 0;
+    BOOL is_dirty;
+    BOOL created;
+    int fd;
+
+    lock_for_graceful();
+    fd = graceful_open_get_append_fd(db_fname, &is_dirty, &created);
+    if (fd == -1) r = errno;
+    else {
+        graceful_log(fd, created ? "Created" : "Opened", is_dirty, is_dirty);
+        *is_dirtyp = is_dirty;
+        if (created || !is_dirty) r = 0;
+        else r = TOKUDB_DIRTY_DICTIONARY;
+        r2 = close(fd);
+    }
+    unlock_for_graceful();
+    return r ? r : r2;
+}
+
+int
+toku_graceful_close(CACHEFILE cf) {
+    int r  = 0;
+    int r2 = 0;
+    int fd;
+    const char *db_fname = cf->fname;
+
+    if (db_fname) {
+        lock_for_graceful();
+        BOOL db_missing = FALSE;
+        BOOL was_dirty = cf->is_dirty;
+        fd = graceful_close_get_append_fd(db_fname, &db_missing);
+        if (fd == -1) {
+            if (!db_missing) r = errno;
+        }
+        else {
+            graceful_log(fd, "Closed", was_dirty, FALSE);
+            r2 = close(fd);
+            cf->is_dirty = FALSE;
+        }
+        unlock_for_graceful();
+    }
+    return r ? r : r2;
+
+}
+
+int
+toku_graceful_dirty(CACHEFILE cf) {
+    int r  = 0;
+    int r2 = 0;
+    int fd;
+    const char *db_fname = cf->fname;
+
+    if (!cf->is_dirty && db_fname) {
+        lock_for_graceful();
+        fd = graceful_dirty_get_append_fd(db_fname);
+        if (fd == -1) r = errno;
+        else {
+            graceful_log(fd, "Dirtied", FALSE, TRUE);
+            r2 = close(fd);
+            cf->is_dirty = TRUE;
+        }
+        unlock_for_graceful();
+    }
+    return r ? r : r2;
+}
+
+int
+toku_graceful_delete(const char *db_fname) {
+    BOOL clean_exists;
+    char cleanbuf[strlen(db_fname) + sizeof(".clean")];
+    BOOL dirty_exists;
+    char dirtybuf[strlen(db_fname) + sizeof(".dirty")];
+
+    sprintf(cleanbuf, "%s.clean", db_fname);
+    sprintf(dirtybuf, "%s.dirty", db_fname);
+
+    struct stat tmpbuf;
+    lock_for_graceful();
+    clean_exists = stat(cleanbuf, &tmpbuf) == 0;
+    dirty_exists = stat(dirtybuf, &tmpbuf) == 0;
+
+    int r = 0;
+    if (clean_exists) {
+        r = unlink(cleanbuf);
+    }
+    if (r==0 && dirty_exists) {
+        r = unlink(dirtybuf);
+    }
+    unlock_for_graceful();
+    return r;
+}
+

newbrt/cachetable.h

@@ -209,4 +209,13 @@ void toku_cachetable_verify (CACHETABLE t);
 // Not for use in production, but useful for testing.
 void print_hash_histogram (void) __attribute__((__visibility__("default")));
 
+int toku_graceful_open(const char *db_fname, BOOL *is_dirtyp);
+int toku_graceful_close(CACHEFILE cf);
+int toku_graceful_dirty(CACHEFILE cf);
+int toku_graceful_delete(const char *db_fname);
+void toku_graceful_lock_init(void);
+void toku_graceful_lock_destroy(void);
+
+
+
 #endif

newbrt/roll.c

@@ -48,6 +48,8 @@ toku_rollback_fcreate (TXNID      UU(xid),
 #endif
     int r = unlink(full_fname);
     assert(r==0);
+    r = toku_graceful_delete(full_fname);
+    assert(r==0);
     toku_free(fname);
     return 0;
 }

newbrt/tests/Makefile

@@ -143,6 +143,6 @@ clean:
 	rm -rf log-test7.c.dir_*
 	rm -rf *.dir 
 	rm -f cachetable-fd-test.ctest2.data test_oexcl.c.tmp
-	rm -f *.brt *.tdb *.dat *.data *.out
+	rm -f *.brt *brt.clean *brt.dirty *.tdb *.dat *.data *.out
 
 $(BINS):

newbrt/tests/benchmark-test.c

@@ -28,7 +28,7 @@ static BRT t;
 
 static void setup (void) {
     int r;
-    unlink(fname);
+    unlink_file_and_bit(fname);
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);         assert(r==0);
     r = toku_open_brt(fname, 0, 1, &t, nodesize, ct, NULL_TXN, toku_default_compare_fun, (DB*)0); assert(r==0);
 }

newbrt/tests/bread-test.c

@@ -29,7 +29,7 @@ char wrotedata[RECORDS*RECORDLEN];
 static void
 test (int seed) {
     srandom(seed);
-    unlink(FNAME);
+    unlink_file_and_bit(FNAME);
     int i;
     {
         int fd = open(FNAME, O_CREAT+O_RDWR+O_BINARY, 0777);
@@ -69,7 +69,7 @@ test (int seed) {
 
     { int r=close_bread_without_closing_fd(br); assert(r==0); }
     { int r=close(fd); assert(r==0); }
-    unlink(FNAME);
+    unlink_file_and_bit(FNAME);
 }
 
 int

newbrt/tests/brt-test-cursor-2.c

@@ -18,7 +18,7 @@ static void test_multiple_brt_cursor_dbts(int n, DB *db) {
     BRT brt;
     BRT_CURSOR cursors[n];
 
-    unlink(fname);
+    unlink_file_and_bit(fname);
 
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
     assert(r==0);

newbrt/tests/brt-test-cursor.c

@@ -105,7 +105,7 @@ static void test_brt_cursor_first(int n, DB *db) {
 
     if (verbose) printf("test_brt_cursor_first:%d %p\n", n, db);
 
-    unlink(fname);
+    unlink_file_and_bit(fname);
 
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
     assert(r==0);
@@ -146,7 +146,7 @@ static void test_brt_cursor_last(int n, DB *db) {
 
     if (verbose) printf("test_brt_cursor_last:%d %p\n", n, db);
 
-    unlink(fname);
+    unlink_file_and_bit(fname);
 
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
     assert(r==0);
@@ -187,7 +187,7 @@ static void test_brt_cursor_first_last(int n, DB *db) {
 
     if (verbose) printf("test_brt_cursor_first_last:%d %p\n", n, db);
 
-    unlink(fname);
+    unlink_file_and_bit(fname);
 
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
     assert(r==0);
@@ -232,7 +232,7 @@ static void test_brt_cursor_rfirst(int n, DB *db) {
 
     if (verbose) printf("test_brt_cursor_rfirst:%d %p\n", n, db);
 
-    unlink(fname);
+    unlink_file_and_bit(fname);
 
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
     assert(r==0);
@@ -306,7 +306,7 @@ static void test_brt_cursor_walk(int n, DB *db) {
 
     if (verbose) printf("test_brt_cursor_walk:%d %p\n", n, db);
 
-    unlink(fname);
+    unlink_file_and_bit(fname);
 
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
     assert(r==0);
@@ -378,7 +378,7 @@ static void test_brt_cursor_rwalk(int n, DB *db) {
 
     if (verbose) printf("test_brt_cursor_rwalk:%d %p\n", n, db);
 
-    unlink(fname);
+    unlink_file_and_bit(fname);
 
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
     assert(r==0);
@@ -456,7 +456,7 @@ static void test_brt_cursor_rand(int n, DB *db) {
 
     if (verbose) printf("test_brt_cursor_rand:%d %p\n", n, db);
 
-    unlink(fname);
+    unlink_file_and_bit(fname);
 
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
     assert(r==0);
@@ -508,7 +508,7 @@ static void test_brt_cursor_split(int n, DB *db) {
 
     if (verbose) printf("test_brt_cursor_split:%d %p\n", n, db);
 
-    unlink(fname);
+    unlink_file_and_bit(fname);
 
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
     assert(r==0);
@@ -585,7 +585,7 @@ static void test_multiple_brt_cursors(int n, DB *db) {
     BRT brt;
     BRT_CURSOR cursors[n];
 
-    unlink(fname);
+    unlink_file_and_bit(fname);
 
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
     assert(r==0);
@@ -631,7 +631,7 @@ static void test_multiple_brt_cursor_walk(int n, DB *db) {
     const int ncursors = n/cursor_gap;
     BRT_CURSOR cursors[ncursors];
 
-    unlink(fname);
+    unlink_file_and_bit(fname);
 
     int nodesize = 1<<12;
     int h = log16(n);
@@ -715,7 +715,7 @@ static void test_brt_cursor_set(int n, int cursor_op, DB *db) {
     BRT brt;
     BRT_CURSOR cursor=0;
 
-    unlink(fname);
+    unlink_file_and_bit(fname);
 
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
     assert(r==0);
@@ -787,7 +787,7 @@ static void test_brt_cursor_set_range(int n, DB *db) {
     BRT brt;
     BRT_CURSOR cursor=0;
 
-    unlink(fname);
+    unlink_file_and_bit(fname);
 
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
     assert(r==0);
@@ -853,7 +853,7 @@ static void test_brt_cursor_delete(int n, DB *db) {
     BRT brt;
     BRT_CURSOR cursor=0;
 
-    unlink(fname);
+    unlink_file_and_bit(fname);
 
     error = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
     assert(error == 0);
@@ -914,7 +914,7 @@ static void test_brt_cursor_get_both(int n, DB *db) {
     BRT brt;
     BRT_CURSOR cursor=0;
 
-    unlink(fname);
+    unlink_file_and_bit(fname);
 
     error = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
     assert(error == 0);

newbrt/tests/brt-test-named-db.c

@@ -15,7 +15,7 @@ static void test_named_db (void) {
     DBT k,v;
 
     if (verbose) printf("test_named_db\n");
-    unlink(n0);
+    unlink_file_and_bit(n0);
     toku_memory_check_all_free();
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);           assert(r==0);
     r = toku_open_brt(n0, "db1", 1, &t0, 1<<12, ct, null_txn, toku_default_compare_fun, null_db); assert(r==0);

newbrt/tests/brt-test.c

@@ -16,7 +16,7 @@ static void test_dump_empty_db (void) {
     toku_memory_check=1;
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
     assert(r==0);
-    unlink(fname);
+    unlink_file_and_bit(fname);
     r = toku_open_brt(fname, 0, 1, &t, 1024, ct, null_txn, toku_default_compare_fun, null_db);
     assert(r==0);
     if (verbose) toku_dump_brt(stdout, t);
@@ -33,8 +33,8 @@ static void test_multiple_files_of_size (int size) {
     BRT t0,t1;
     int r,i;
     if (verbose) printf("test_multiple_files_of_size(%d)\n", size);
-    unlink(n0);
-    unlink(n1);
+    unlink_file_and_bit(n0);
+    unlink_file_and_bit(n1);
     toku_memory_check_all_free();
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);   assert(r==0);
     r = toku_open_brt(n0, 0, 1, &t0, size, ct, null_txn, toku_default_compare_fun, null_db); assert(r==0);
@@ -101,8 +101,8 @@ static void test_multiple_dbs (void) {
     int r;
     DBT k,v;
     if (verbose) printf("test_multiple_dbs: ");
-    unlink(n0);
-    unlink(n1);
+    unlink_file_and_bit(n0);
+    unlink_file_and_bit(n1);
     toku_memory_check_all_free();
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);        assert(r==0);
     r = toku_open_brt(n0, "db1", 1, &t0, 1<<12, ct, null_txn, toku_default_compare_fun, null_db); assert(r==0);
@@ -156,7 +156,7 @@ static void test_multiple_dbs_many (void) {
     BRT trees[MANYN];
     if (verbose) printf("test_multiple_dbs_many:\n");
     toku_memory_check_all_free();
-    unlink(name);
+    unlink_file_and_bit(name);
     r = toku_brt_create_cachetable(&ct, (MANYN+4), ZERO_LSN, NULL_LOGGER);     assert(r==0);
     for (i=0; i<MANYN; i++) {
 	char dbname[20];
@@ -186,7 +186,7 @@ static void test_multiple_brts_one_db_one_file (void) {
     BRT trees[MANYN];
     if (verbose) printf("test_multiple_brts_one_db_one_file:");
     toku_memory_check_all_free();
-    unlink(fname);
+    unlink_file_and_bit(fname);
     r = toku_brt_create_cachetable(&ct, 32, ZERO_LSN, NULL_LOGGER); assert(r==0);
     for (i=0; i<MANYN; i++) {
 	r = toku_open_brt(fname, 0, (i==0), &trees[i], 1<<12, ct, null_txn, toku_default_compare_fun, null_db);
@@ -227,7 +227,7 @@ static void  test_read_what_was_written (void) {
 
     if (verbose) printf("test_read_what_was_written(): "); fflush(stdout);
 
-    unlink(fname);
+    unlink_file_and_bit(fname);
     toku_memory_check_all_free();
 
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);       assert(r==0);
@@ -370,7 +370,7 @@ static void test_cursor_last_empty(void) {
     int r;
     DBT kbt, vbt;
     if (verbose) printf("%s", __FUNCTION__);
-    unlink(fname);
+    unlink_file_and_bit(fname);
     toku_memory_check_all_free();
     //printf("%s:%d %d alloced\n", __FILE__, __LINE__, toku_get_n_items_malloced()); toku_print_malloced_items();
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);       assert(r==0);
@@ -402,7 +402,7 @@ static void test_cursor_next (void) {
     int r;
     DBT kbt, vbt;
 
-    unlink(fname);
+    unlink_file_and_bit(fname);
     toku_memory_check_all_free();
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);       assert(r==0);
     //printf("%s:%d %d alloced\n", __FILE__, __LINE__, toku_get_n_items_malloced()); toku_print_malloced_items();
@@ -470,7 +470,7 @@ static void test_wrongendian_compare (int wrong_p, unsigned int N) {
     DBT kbt, vbt;
     unsigned int i;
 
-    unlink(fname);
+    unlink_file_and_bit(fname);
     toku_memory_check_all_free();
 
     {
@@ -579,7 +579,7 @@ static void test_large_kv(int bsize, int ksize, int vsize) {
 
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
     assert(r==0);
-    unlink(fname);
+    unlink_file_and_bit(fname);
     r = toku_open_brt(fname, 0, 1, &t, bsize, ct, null_txn, toku_default_compare_fun, null_db);
     assert(r==0);
 
@@ -625,7 +625,7 @@ static void test_brt_delete_empty() {
 
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
     assert(r==0);
-    unlink(fname);
+    unlink_file_and_bit(fname);
     r = toku_open_brt(fname, 0, 1, &t, 4096, ct, null_txn, toku_default_compare_fun, null_db);
     assert(r==0);
 
@@ -653,7 +653,7 @@ static void test_brt_delete_present(int n) {
 
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
     assert(r==0);
-    unlink(fname);
+    unlink_file_and_bit(fname);
     r = toku_open_brt(fname, 0, 1, &t, 4096, ct, null_txn, toku_default_compare_fun, null_db);
     assert(r==0);
 
@@ -714,7 +714,7 @@ static void test_brt_delete_not_present(int n) {
 
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
     assert(r==0);
-    unlink(fname);
+    unlink_file_and_bit(fname);
     r = toku_open_brt(fname, 0, 1, &t, 4096, ct, null_txn, toku_default_compare_fun, null_db);
     assert(r==0);
 
@@ -760,7 +760,7 @@ static void test_brt_delete_cursor_first(int n) {
 
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
     assert(r==0);
-    unlink(fname);
+    unlink_file_and_bit(fname);
     r = toku_open_brt(fname, 0, 1, &t, 4096, ct, null_txn, toku_default_compare_fun, null_db);
     assert(r==0);
 
@@ -851,7 +851,7 @@ static void test_insert_delete_lookup(int n) {
 
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
     assert(r==0);
-    unlink(fname);
+    unlink_file_and_bit(fname);
     r = toku_open_brt(fname, 0, 1, &t, 4096, ct, null_txn, toku_default_compare_fun, null_db);
     assert(r==0);
 
@@ -897,7 +897,7 @@ static void test_brt_delete_both(int n) {
     int i;
 
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER); assert(r==0);
-    unlink(fname);
+    unlink_file_and_bit(fname);
     r = toku_brt_create(&t); assert(r == 0);
     r = toku_brt_set_flags(t, TOKU_DB_DUP + TOKU_DB_DUPSORT); assert(r == 0);
     r = toku_brt_set_nodesize(t, 4096); assert(r == 0);
@@ -1002,7 +1002,7 @@ static void test_new_brt_cursor_first(int n, int dup_mode) {
     int i;
 
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER); assert(r==0);
-    unlink(fname);
+    unlink_file_and_bit(fname);
     r = toku_brt_create(&t); assert(r == 0);
     r = toku_brt_set_flags(t, dup_mode); assert(r == 0);
     r = toku_brt_set_nodesize(t, 4096); assert(r == 0);
@@ -1056,7 +1056,7 @@ static void test_new_brt_cursor_last(int n, int dup_mode) {
     int i;
 
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER); assert(r==0);
-    unlink(fname);
+    unlink_file_and_bit(fname);
     r = toku_brt_create(&t); assert(r == 0);
     r = toku_brt_set_flags(t, dup_mode); assert(r == 0);
     r = toku_brt_set_nodesize(t, 4096); assert(r == 0);
@@ -1111,7 +1111,7 @@ static void test_new_brt_cursor_next(int n, int dup_mode) {
     int i;
 
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER); assert(r==0);
-    unlink(fname);
+    unlink_file_and_bit(fname);
     r = toku_brt_create(&t); assert(r == 0);
     r = toku_brt_set_flags(t, dup_mode); assert(r == 0);
     r = toku_brt_set_nodesize(t, 4096); assert(r == 0);
@@ -1163,7 +1163,7 @@ static void test_new_brt_cursor_prev(int n, int dup_mode) {
     int i;
 
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER); assert(r==0);
-    unlink(fname);
+    unlink_file_and_bit(fname);
     r = toku_brt_create(&t); assert(r == 0);
     r = toku_brt_set_flags(t, dup_mode); assert(r == 0);
     r = toku_brt_set_nodesize(t, 4096); assert(r == 0);
@@ -1215,7 +1215,7 @@ static void test_new_brt_cursor_current(int n, int dup_mode) {
     int i;
 
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER); assert(r==0);
-    unlink(fname);
+    unlink_file_and_bit(fname);
     r = toku_brt_create(&t); assert(r == 0);
     r = toku_brt_set_flags(t, dup_mode); assert(r == 0);
     r = toku_brt_set_nodesize(t, 4096); assert(r == 0);
@@ -1295,7 +1295,7 @@ static void test_new_brt_cursor_set_range(int n, int dup_mode) {
     BRT_CURSOR cursor=0;
 
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER); assert(r==0);
-    unlink(fname);
+    unlink_file_and_bit(fname);
     r = toku_brt_create(&brt); assert(r == 0);
     r = toku_brt_set_flags(brt, dup_mode); assert(r == 0);
     r = toku_brt_set_nodesize(brt, 4096); assert(r == 0);
@@ -1352,7 +1352,7 @@ static void test_new_brt_cursor_set(int n, int cursor_op, DB *db) {
     BRT brt;
     BRT_CURSOR cursor=0;
 
-    unlink(fname);
+    unlink_file_and_bit(fname);
 
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER); assert(r==0);
 

newbrt/tests/brt-test0.c

@@ -18,7 +18,7 @@ static void test0 (void) {
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
     assert(r==0);
     if (verbose) printf("%s:%d test0\n", __FILE__, __LINE__);
-    unlink(fname);
+    unlink_file_and_bit(fname);
     r = toku_open_brt(fname, 0, 1, &t, 1024, ct, null_txn, toku_default_compare_fun, null_db);
     assert(r==0);
     //printf("%s:%d test0\n", __FILE__, __LINE__);

newbrt/tests/brt-test1.c

@@ -17,7 +17,7 @@ static void test1 (void) {
     toku_memory_check_all_free();
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
     assert(r==0);
-    unlink(fname);
+    unlink_file_and_bit(fname);
     r = toku_open_brt(fname, 0, 1, &t, 1024, ct, null_txn, toku_default_compare_fun, null_db);
     assert(r==0);
     toku_brt_insert(t, toku_fill_dbt(&k, "hello", 6), toku_fill_dbt(&v, "there", 6), null_txn);

newbrt/tests/brt-test2.c

@@ -17,7 +17,7 @@ static void test2 (int memcheck, int limit) {
     if (verbose) printf("%s:%d checking\n", __FILE__, __LINE__);
     toku_memory_check_all_free();
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER); assert(r==0);
-    unlink(fname);
+    unlink_file_and_bit(fname);
     r = toku_open_brt(fname, 0, 1, &t, 1024, ct, null_txn, toku_default_compare_fun, null_db);
     if (verbose) printf("%s:%d did setup\n", __FILE__, __LINE__);
     assert(r==0);

newbrt/tests/brt-test3.c

@@ -19,7 +19,7 @@ static void test3 (int nodesize, int count, int memcheck) {
     toku_memory_check_all_free();
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER); assert(r==0);
     gettimeofday(&t0, 0);
-    unlink(fname);
+    unlink_file_and_bit(fname);
     r = toku_open_brt(fname, 0, 1, &t, nodesize, ct, null_txn, toku_default_compare_fun, null_db);
     assert(r==0);
     for (i=0; i<count; i++) {

newbrt/tests/brt-test4.c

@@ -16,7 +16,7 @@ static void test4 (int nodesize, int count, int memcheck) {
     int i;
     CACHETABLE ct;
     gettimeofday(&t0, 0);
-    unlink(fname);
+    unlink_file_and_bit(fname);
     toku_memory_check=memcheck;
     toku_memory_check_all_free();
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);         assert(r==0);

newbrt/tests/brt-test5.c

@@ -18,7 +18,7 @@ static void test5 (void) {
     toku_memory_check_all_free();
     MALLOC_N(limit,values);
     for (i=0; i<limit; i++) values[i]=-1;
-    unlink(fname);
+    unlink_file_and_bit(fname);
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);        assert(r==0);
     r = toku_open_brt(fname, 0, 1, &t, 1<<12, ct, null_txn, toku_default_compare_fun, null_db);   assert(r==0);
     for (i=0; i<limit/2; i++) {

newbrt/tests/cachetable-checkpoint-test.c

@@ -37,7 +37,7 @@ static void cachetable_checkpoint_test(int n, enum cachetable_dirty dirty) {
     CACHETABLE ct;
     r = toku_create_cachetable(&ct, test_limit, ZERO_LSN, NULL_LOGGER); assert(r == 0);
     char fname1[] = __FILE__ "test1.dat";
-    unlink(fname1);
+    unlink_file_and_bit(fname1);
     CACHEFILE f1;
     r = toku_cachetable_openf(&f1, ct, fname1, O_RDWR|O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO); assert(r == 0);
 

newbrt/tests/cachetable-count-pinned-test.c

@@ -34,7 +34,7 @@ cachetable_count_pinned_test (int n) {
     CACHETABLE ct;
     r = toku_create_cachetable(&ct, test_limit, ZERO_LSN, NULL_LOGGER); assert(r == 0);
     char fname1[] = __FILE__ "test1.dat";
-    unlink(fname1);
+    unlink_file_and_bit(fname1);
     CACHEFILE f1;
     r = toku_cachetable_openf(&f1, ct, fname1, O_RDWR|O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO); assert(r == 0);
 

newbrt/tests/cachetable-debug-test.c

@@ -34,7 +34,7 @@ cachetable_debug_test (int n) {
     CACHETABLE ct;
     r = toku_create_cachetable(&ct, test_limit, ZERO_LSN, NULL_LOGGER); assert(r == 0);
     char fname1[] = __FILE__ "test1.dat";
-    unlink(fname1);
+    unlink_file_and_bit(fname1);
     CACHEFILE f1;
     r = toku_cachetable_openf(&f1, ct, fname1, O_RDWR|O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO); assert(r == 0);
 

newbrt/tests/cachetable-fd-test.c

@@ -9,7 +9,7 @@ cachetable_fd_test (void) {
     CACHETABLE ct;
     r = toku_create_cachetable(&ct, test_limit, ZERO_LSN, NULL_LOGGER); assert(r == 0);
     char fname1[] = __FILE__ "test1.dat";
-    unlink(fname1);
+    unlink_file_and_bit(fname1);
     CACHEFILE cf;
     r = toku_cachetable_openf(&cf, ct, fname1, O_RDWR|O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO); assert(r == 0);
 
@@ -17,7 +17,7 @@ cachetable_fd_test (void) {
 
     // test set to good fd succeeds
     char fname2[] = __FILE__ "test2.data";
-    unlink(fname2);
+    unlink_file_and_bit(fname2);
     int fd2 = open(fname2, O_RDWR | O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO); assert(fd2 >= 0 && fd1 != fd2);
     r = toku_cachefile_set_fd(cf, fd2, fname2); assert(r == 0);
     assert(toku_cachefile_fd(cf) == fd2);

newbrt/tests/cachetable-flush-test.c

@@ -34,12 +34,12 @@ test_cachetable_flush (int n) {
     CACHETABLE ct;
     r = toku_create_cachetable(&ct, test_limit, ZERO_LSN, NULL_LOGGER); assert(r == 0);
     char fname1[] = __FILE__ "test1.dat";
-    unlink(fname1);
+    unlink_file_and_bit(fname1);
     CACHEFILE f1;
     r = toku_cachetable_openf(&f1, ct, fname1, O_RDWR|O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO); assert(r == 0);
 
     char fname2[] = __FILE__ "test2.dat";
-    unlink(fname2);
+    unlink_file_and_bit(fname2);
     CACHEFILE f2;
     r = toku_cachetable_openf(&f2, ct, fname2, O_RDWR|O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO); assert(r == 0);
 

newbrt/tests/cachetable-getandpin-test.c

@@ -43,7 +43,7 @@ cachetable_getandpin_test (int n) {
     CACHETABLE ct;
     r = toku_create_cachetable(&ct, test_limit, ZERO_LSN, NULL_LOGGER); assert(r == 0);
     char fname1[] = __FILE__ "test_getandpin.dat";
-    unlink(fname1);
+    unlink_file_and_bit(fname1);
     CACHEFILE f1;
     r = toku_cachetable_openf(&f1, ct, fname1, O_RDWR|O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO); assert(r == 0);
 

newbrt/tests/cachetable-put-test.c

@@ -34,7 +34,7 @@ cachetable_put_test (int n) {
     CACHETABLE ct;
     r = toku_create_cachetable(&ct, test_limit, ZERO_LSN, NULL_LOGGER); assert(r == 0);
     char fname1[] = __FILE__ "test1.dat";
-    unlink(fname1);
+    unlink_file_and_bit(fname1);
     CACHEFILE f1;
     r = toku_cachetable_openf(&f1, ct, fname1, O_RDWR|O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO); assert(r == 0);
 

newbrt/tests/cachetable-rename-test.c

@@ -82,7 +82,7 @@ static void test_rename (void) {
     test_mutex_init();
     const char fname[] = __FILE__ "rename.dat";
     r=toku_create_cachetable(&t, KEYLIMIT, ZERO_LSN, NULL_LOGGER); assert(r==0);
-    unlink(fname);
+    unlink_file_and_bit(fname);
     r = toku_cachetable_openf(&f, t, fname, O_RDWR|O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO);
     assert(r==0);
     for (i=0; i<TRIALLIMIT; i++) {

newbrt/tests/cachetable-test.c

@@ -167,7 +167,7 @@ static void test0 (void) {
 
     r=toku_create_cachetable(&t, 5, ZERO_LSN, NULL_LOGGER);
     assert(r==0);
-    unlink(fname);
+    unlink_file_and_bit(fname);
     r = toku_cachetable_openf(&f, t, fname, O_RDWR|O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO);
     assert(r==0);
 
@@ -321,7 +321,7 @@ static void test_nested_pin (void) {
     char fname[] = __FILE__ "test_ct.dat";
     r = toku_create_cachetable(&t, 1, ZERO_LSN, NULL_LOGGER);
     assert(r==0);
-    unlink(fname);
+    unlink_file_and_bit(fname);
     r = toku_cachetable_openf(&f, t, fname, O_RDWR|O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO);
     assert(r==0);
     expect_f = f;
@@ -391,8 +391,8 @@ static void test_multi_filehandles (void) {
     char fname3[]= __FILE__ "test3_ct.dat";
     int r;
     void *v;
-    unlink(fname1);
-    unlink(fname2);
+    unlink_file_and_bit(fname1);
+    unlink_file_and_bit(fname2);
 
     r = toku_create_cachetable(&t, 4, ZERO_LSN, NULL_LOGGER);          assert(r==0);
     r = toku_cachetable_openf(&f1, t, fname1, O_RDWR|O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO);   assert(r==0);
@@ -462,7 +462,7 @@ static void test_dirty() {
     assert(r == 0);
 
     char *fname = __FILE__ "test.dat";
-    unlink(fname);
+    unlink_file_and_bit(fname);
     r = toku_cachetable_openf(&f, t, fname, O_RDWR|O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO);
     assert(r == 0);
 
@@ -586,7 +586,7 @@ static void test_size_resize() {
     assert(r == 0);
 
     char *fname = __FILE__ "test.dat";
-    unlink(fname);
+    unlink_file_and_bit(fname);
     r = toku_cachetable_openf(&f, t, fname, O_RDWR|O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO);
     assert(r == 0);
 
@@ -641,7 +641,7 @@ static void test_size_flush() {
     assert(r == 0);
 
     char *fname = __FILE__ "test.dat";
-    unlink(fname);
+    unlink_file_and_bit(fname);
     r = toku_cachetable_openf(&f, t, fname, O_RDWR|O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO);
     assert(r == 0);
 

newbrt/tests/cachetable-test2.c

@@ -150,7 +150,7 @@ static void test_chaining (void) {
     for (i=0; i<N_FILES; i++) {
 	r = snprintf(fname[i], FILENAME_LEN, __FILE__ ".%ld.dat", i);
 	assert(r>0 && r<FILENAME_LEN);
-	unlink(fname[i]);
+	unlink_file_and_bit(fname[i]);
 	r = toku_cachetable_openf(&f[i], ct, fname[i], O_RDWR|O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO);   assert(r==0);
 	}
     for (i=0; i<N_PRESENT_LIMIT; i++) {

newbrt/tests/cachetable-unpin-and-remove-test.c

@@ -38,7 +38,7 @@ cachetable_unpin_and_remove_test (int n) {
     CACHETABLE ct;
     r = toku_create_cachetable(&ct, table_limit, ZERO_LSN, NULL_LOGGER); assert(r == 0);
     char fname1[] = __FILE__ "test1.dat";
-    unlink(fname1);
+    unlink_file_and_bit(fname1);
     CACHEFILE f1;
     r = toku_cachetable_openf(&f1, ct, fname1, O_RDWR|O_CREAT, 0777); assert(r == 0);
 
@@ -101,7 +101,7 @@ cachetable_put_evict_remove_test (int n) {
     CACHETABLE ct;
     r = toku_create_cachetable(&ct, table_limit, ZERO_LSN, NULL_LOGGER); assert(r == 0);
     char fname1[] = __FILE__ "test1.dat";
-    unlink(fname1);
+    unlink_file_and_bit(fname1);
     CACHEFILE f1;
     r = toku_cachetable_openf(&f1, ct, fname1, O_RDWR|O_CREAT, 0777); assert(r == 0);
 

newbrt/tests/cachetable-unpin-test.c

@@ -34,7 +34,7 @@ cachetable_unpin_test (int n) {
     CACHETABLE ct;
     r = toku_create_cachetable(&ct, test_limit, ZERO_LSN, NULL_LOGGER); assert(r == 0);
     char fname1[] = __FILE__ "test1.dat";
-    unlink(fname1);
+    unlink_file_and_bit(fname1);
     CACHEFILE f1;
     r = toku_cachetable_openf(&f1, ct, fname1, O_RDWR|O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO); assert(r == 0);
 

newbrt/tests/dup-delete-all.c

@@ -18,7 +18,7 @@ static DB * const null_db = 0;
 static void test_delete_all (void) {
     char fname[]= __FILE__ ".brt";
     u_int32_t limit =200;
-    unlink(fname);
+    unlink_file_and_bit(fname);
     CACHETABLE ct;
     int r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);                                assert(r==0);
     BRT t;

newbrt/tests/keyrange-dupsort-unflat.c

@@ -13,7 +13,7 @@ static void test_flat (void) {
     char fname[]= __FILE__ ".brt";
     u_int64_t limit=100;
     u_int64_t ilimit=100;
-    unlink(fname);
+    unlink_file_and_bit(fname);
     CACHETABLE ct;
     // set the cachetable to size 1 so that things won't fit.
     int r = toku_brt_create_cachetable(&ct, 1, ZERO_LSN, NULL_LOGGER);                                assert(r==0);

newbrt/tests/keyrange-dupsort.c

@@ -13,7 +13,7 @@ static void test_flat (void) {
     char fname[]= __FILE__ ".brt";
     u_int64_t limit=100;
     u_int64_t ilimit=100;
-    unlink(fname);
+    unlink_file_and_bit(fname);
     CACHETABLE ct;
     int r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);                                assert(r==0);
     BRT t;

newbrt/tests/keyrange-unflat.c

@@ -13,7 +13,7 @@ static void test_flat (void) {
     char fname[]= __FILE__ ".brt";
     const u_int64_t limit=10000;
     u_int64_t permute[limit];
-    unlink(fname);
+    unlink_file_and_bit(fname);
     CACHETABLE ct;
     // set the cachetable to size 1 so that things won't fit.
     int r = toku_brt_create_cachetable(&ct, 1, ZERO_LSN, NULL_LOGGER);                                assert(r==0);

newbrt/tests/keyrange.c

@@ -12,7 +12,7 @@ static DB * const null_db = 0;
 static void test_flat (void) {
     char fname[]= __FILE__ ".brt";
     u_int64_t limit=30000;
-    unlink(fname);
+    unlink_file_and_bit(fname);
     CACHETABLE ct;
     int r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);                                assert(r==0);
     BRT t;

newbrt/tests/shortcut.c

@@ -19,7 +19,7 @@ test_main (int argc __attribute__((__unused__)), const char *argv[]  __attribute
     DB *db = &a_db;
     DBT key,val;
 
-    unlink(fname);
+    unlink_file_and_bit(fname);
 
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);                               assert(r==0);
     r = toku_open_brt(fname, 0, 1, &brt, 1<<12, ct, null_txn, test_brt_cursor_keycompare, db);   assert(r==0);

newbrt/tests/test-brt-delete-both.c

@@ -19,7 +19,7 @@ doit (void) {
     DBT k,v;
     if (verbose) printf("%s\n", __FUNCTION__);
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER); assert(r==0);
-    unlink(fname);
+    unlink_file_and_bit(fname);
     r = toku_brt_create(&t); assert(r==0);
     r = toku_brt_open(t, fname, fname, 0, 1, 1, ct, null_txn, (DB*)0); assert(r==0);
 

newbrt/tests/test-brt-overflow.c

@@ -17,7 +17,7 @@ test_overflow (void) {
     CACHETABLE ct;
     u_int32_t nodesize = 1<<20; 
     int r;
-    unlink(fname);
+    unlink_file_and_bit(fname);
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);         assert(r==0);
     r = toku_open_brt(fname, 0, 1, &t, nodesize, ct, null_txn, toku_default_compare_fun, null_db); assert(r==0);
 

newbrt/tests/test-del-inorder.c

@@ -27,7 +27,7 @@ doit (void) {
 
     snprintf(fname, fnamelen, "%s.brt", __FILE__);
     r = toku_brt_create_cachetable(&ct, 16*1024, ZERO_LSN, NULL_LOGGER); assert(r==0);
-    unlink(fname);
+    unlink_file_and_bit(fname);
     r = toku_open_brt(fname, 0, 1, &t, NODESIZE, ct, null_txn, toku_default_compare_fun, null_db);
     assert(r==0);
     toku_free(fname);

newbrt/tests/test-dump-brt.c

@@ -13,7 +13,7 @@ test_main (int argc __attribute__((__unused__)), char *argv[] __attribute__((__u
     BRT t;
     CACHETABLE ct;
     FILE *f = fopen("test-dump-brt.out", "w");
-    unlink(n);
+    unlink_file_and_bit(n);
     assert(f);
     r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);   assert(r==0);
     r = toku_open_brt(n, 0, 1, &t, 1<<12, ct, null_txn, toku_default_compare_fun, null_db); assert(r==0);

newbrt/tests/test-inc-split.c

@@ -56,7 +56,7 @@ doit (int ksize __attribute__((__unused__))) {
 
     snprintf(fname, fnamelen, "%s.brt", __FILE__);
     r = toku_brt_create_cachetable(&ct, 16*1024, ZERO_LSN, NULL_LOGGER); assert(r==0);
-    unlink(fname);
+    unlink_file_and_bit(fname);
     r = toku_open_brt(fname, 0, 1, &t, NODESIZE, ct, null_txn, toku_default_compare_fun, null_db);
     assert(r==0);
     toku_free(fname);

newbrt/tests/test.h

@@ -7,6 +7,17 @@
 
 int verbose=0;
 
+void
+unlink_file_and_bit(const char *name) {
+    char dirty[strlen(name) + sizeof(".dirty")];
+    char clean[strlen(name) + sizeof(".clean")];
+    sprintf(dirty, "%s.dirty", name);
+    sprintf(clean, "%s.clean", name);
+    unlink(name);
+    unlink(dirty);
+    unlink(clean);
+}
+
 static inline void
 default_parse_args (int argc, const char *argv[]) {
     const char *progname=argv[0];

newbrt/tests/test1305.c

@@ -25,7 +25,7 @@
 
 static void
 test (u_int64_t fsize) {
-    unlink(FNAME);
+    unlink_file_and_bit(FNAME);
     // Create a file of size fsize.  Fill it with 8-byte values which are integers, in order)
     assert(fsize%(N_BIGINTS*sizeof(u_int64_t)) == 0); // Make sure the fsize is a multiple of the buffer size.
     u_int64_t i = 0;
@@ -95,7 +95,7 @@ assert(sizeof(buf) == N_BIGINTS * BIGINT_SIZE);
     }
     //printf("Did %" PRIu64 "\n", fsize);
     //system("ls -l " FNAME);
-    unlink(FNAME);
+    unlink_file_and_bit(FNAME);
 }
 
 int

newbrt/tests/test1308a.c

@@ -15,7 +15,7 @@
 int
 test_main (int argc __attribute__((__unused__)), const char *argv[] __attribute__((__unused__)))
 {
-    unlink(FNAME);
+    unlink_file_and_bit(FNAME);
     
     int fd;
     {
@@ -45,6 +45,6 @@ test_main (int argc __attribute__((__unused__)), const char *argv[] __attribute_
     assert(file_size==file_size2);
     close(fd);
 
-    unlink(FNAME);
+    unlink_file_and_bit(FNAME);
     return 0;
 }

newbrt/tests/test_oexcl.c

@@ -9,7 +9,7 @@
 
 int
 test_main (int argc __attribute__((__unused__)), const char *argv[] __attribute__((__unused__))) {
-    unlink(fname);
+    unlink_file_and_bit(fname);
     int fd0 = open (fname, O_RDWR|O_CREAT|O_EXCL, S_IRWXU|S_IRWXG|S_IRWXO);
     assert(fd0>=0);
     int fd1 = open (fname, O_RDWR|O_CREAT|O_EXCL, S_IRWXU|S_IRWXG|S_IRWXO);

src/tests/test_abort1.c

@@ -21,6 +21,7 @@ test_db_open_aborts (void) {
     DB *db;
 
     int r;
+    struct stat buf;
     system("rm -rf " ENVDIR);
     r=toku_os_mkdir(ENVDIR, S_IRWXU+S_IRWXG+S_IRWXO);       assert(r==0);
     r=db_env_create(&env, 0); assert(r==0);
@@ -58,13 +59,24 @@ test_db_open_aborts (void) {
 	r=tid->abort(tid);        assert(r==0);
     }
     {
-	struct stat buf;
 	r=stat(ENVDIR "/foo.db", &buf);
 	assert(r!=0);
 	assert(errno==ENOENT);
+	r=stat(ENVDIR "/foo.db.clean", &buf);
+	assert(r!=0);
+	assert(errno==ENOENT);
+	r=stat(ENVDIR "/foo.db.dirty", &buf);
+	assert(r!=0);
+	assert(errno==ENOENT);
     }
 
     r=db->close(db, 0);       assert(r==0);
+    r=stat(ENVDIR "/foo.db.clean", &buf);
+    assert(r!=0);
+    assert(errno==ENOENT);
+    r=stat(ENVDIR "/foo.db.dirty", &buf);
+    assert(r!=0);
+    assert(errno==ENOENT);
     r=env->close(env, 0);     assert(r==0);
 }
 

src/ydb.c

@@ -3204,6 +3204,7 @@ static int toku_db_remove(DB * db, const char *fname, const char *dbname, u_int3
 	need_close = FALSE;
 	if (r!=0) { goto cleanup; }
         if (unlink(full_name) != 0) { r = errno; goto cleanup; }
+        if (toku_graceful_delete(full_name) !=0) { r = errno; goto cleanup; }
     } else {
 	r = toku_db_close(db, 0);
 	need_close = FALSE;

toku_include/Makefile.include

@@ -283,7 +283,7 @@ TAGS: $(TOKUROOT)*/*.[ch] $(TOKUROOT)*/*/*.[ch] $(TOKUROOT)*/*/*/*.[ch]
 	etags $(TOKUROOT)*/*.[ch] $(TOKUROOT)*/*/*.[ch] $(TOKUROOT)*/*/*/*.[ch]
 
 cscope.files: $(TOKUROOT)*/*.[ch] $(TOKUROOT)*/*/*.[ch] $(TOKUROOT)*/*/*/*.[ch]
-	(echo $(TOKUROOT)*/*.[ch] $(TOKUROOT)*/*/*.[ch] $(TOKUROOT)*/*/*/*.[ch] | tr " " "\n") > $@ # Very long command line quieted.
+	(echo $(TOKUROOT)*/*.[ch] $(TOKUROOT)*/*/*.[ch] $(TOKUROOT)*/*/*/*.[ch] | tr " " "\n") > $@
 
 cscope.out: cscope.files $(TOKUROOT)*/*.[ch] $(TOKUROOT)*/*/*.[ch] $(TOKUROOT)*/*/*/*.[ch]
 	$(CSCOPE) -qb