refs #61, several code simplifications:

 - break up cachetable_flush_cachefile into more digestable functions,
 - decouple hash_id from filenum
 - break up close_userdata into close_userdata and free_userdata

ft/cachetable-internal.h

@@ -184,7 +184,7 @@ struct cachefile {
     // they are managed whenever we add or remove a pair from
     // the cachetable. As of Riddler, this linked list is only used to
     // make cachetable_flush_cachefile more efficient
-    PAIR cf_head;
+    PAIR cf_head; // doubly linked list that is NOT circular
     uint32_t num_pairs; // count on number of pairs in the cachetable belong to this cachefile
 
     bool for_checkpoint; //True if part of the in-progress checkpoint
@@ -209,6 +209,7 @@ struct cachefile {
     void *userdata;
     void (*log_fassociate_during_checkpoint)(CACHEFILE cf, void *userdata); // When starting a checkpoint we must log all open files.
     void (*close_userdata)(CACHEFILE cf, int fd, void *userdata, bool lsnvalid, LSN); // when closing the last reference to a cachefile, first call this function. 
+    void (*free_userdata)(CACHEFILE cf, void *userdata); // when closing the last reference to a cachefile, first call this function. 
     void (*begin_checkpoint_userdata)(LSN lsn_of_checkpoint, void *userdata); // before checkpointing cachefiles call this function.
     void (*checkpoint_userdata)(CACHEFILE cf, int fd, void *userdata); // when checkpointing a cachefile, call this function.
     void (*end_checkpoint_userdata)(CACHEFILE cf, int fd, void *userdata); // after checkpointing cachefiles call this function.
@@ -427,11 +428,13 @@ public:
     void add_cf_unlocked(CACHEFILE newcf);
     void remove_cf(CACHEFILE cf);
     FILENUM reserve_filenum();
+    uint32_t get_new_hash_id_unlocked();
     CACHEFILE find_cachefile_unlocked(struct fileid* fileid);
     void verify_unused_filenum(FILENUM filenum);
     // access to these fields are protected by the lock
-    CACHEFILE m_head;
+    CACHEFILE m_active_head; // head of CACHEFILEs that are active
     FILENUM m_next_filenum_to_use;
+    uint32_t m_next_hash_id_to_use;
     toku_pthread_rwlock_t m_lock; // this field is publoc so we are still POD
 };
 

ft/cachetable.h

@@ -275,6 +275,7 @@ typedef void (*CACHETABLE_REMOVE_KEY)(CACHEKEY* cachekey, bool for_checkpoint, v
 void toku_cachefile_set_userdata(CACHEFILE cf, void *userdata,
     void (*log_fassociate_during_checkpoint)(CACHEFILE, void*),
     void (*close_userdata)(CACHEFILE, int, void*, bool, LSN),
+    void (*free_userdata)(CACHEFILE, void*),
     void (*checkpoint_userdata)(CACHEFILE, int, void*),
     void (*begin_checkpoint_userdata)(LSN, void*),
     void (*end_checkpoint_userdata)(CACHEFILE, int, void*),

ft/ft.cc

@@ -317,6 +317,11 @@ static void ft_close(CACHEFILE cachefile, int fd, void *header_v, bool oplsn_val
         ft_end_checkpoint(cachefile, fd, header_v);
         assert(!ft->h->dirty); // dirty bit should be cleared by begin_checkpoint and never set again (because we're closing the dictionary)
     }
+}
+
+// maps to cf->free_userdata
+static void ft_free(CACHEFILE cachefile UU(), void *header_v) {
+    FT ft = (FT) header_v;
     toku_ft_free(ft);
 }
 
@@ -392,6 +397,7 @@ static void ft_init(FT ft, FT_OPTIONS options, CACHEFILE cf) {
                                 ft,
                                 ft_log_fassociate_during_checkpoint,
                                 ft_close,
+                                ft_free,
                                 ft_checkpoint,
                                 ft_begin_checkpoint,
                                 ft_end_checkpoint,
@@ -494,6 +500,7 @@ int toku_read_ft_and_store_in_cachefile (FT_HANDLE brt, CACHEFILE cf, LSN max_ac
                                 (void*)h,
                                 ft_log_fassociate_during_checkpoint,
                                 ft_close,
+                                ft_free,
                                 ft_checkpoint,
                                 ft_begin_checkpoint,
                                 ft_end_checkpoint,

ft/tests/cachetable-checkpointer-class.cc

@@ -136,7 +136,7 @@ void checkpointer_test::test_begin_checkpoint() {
     struct cachefile cf;
     cf.next = NULL;
     cf.for_checkpoint = false;
-    m_cp.m_cf_list->m_head = &cf;
+    m_cp.m_cf_list->m_active_head = &cf;
     create_dummy_functions(&cf);
 
     m_cp.begin_checkpoint();
@@ -146,7 +146,7 @@ void checkpointer_test::test_begin_checkpoint() {
     // 3. Call checkpoint with MANY cachefiles.
     const uint32_t count = 3;
     struct cachefile cfs[count];
-    m_cp.m_cf_list->m_head = &cfs[0];
+    m_cp.m_cf_list->m_active_head = &cfs[0];
     for (uint32_t i = 0; i < count; ++i) {
         cfs[i].for_checkpoint = false;
         create_dummy_functions(&cfs[i]);
@@ -196,7 +196,7 @@ void checkpointer_test::test_pending_bits() {
     memset(&cf, 0, sizeof(cf));
     cf.next = NULL;
     cf.for_checkpoint = true;
-    m_cp.m_cf_list->m_head = &cf;
+    m_cp.m_cf_list->m_active_head = &cf;
     create_dummy_functions(&cf);
 
     CACHEKEY k;
@@ -341,7 +341,7 @@ void checkpointer_test::test_end_checkpoint() {
 
     ZERO_STRUCT(m_cp);
     m_cp.init(&ctbl.list, NULL, &ctbl.ev, &cfl);
-    m_cp.m_cf_list->m_head = &cf;
+    m_cp.m_cf_list->m_active_head = &cf;
 
     // 2. Add data before running checkpoint.
     const uint32_t count = 6;

ft/tests/cachetable-pin-checkpoint.cc

@@ -422,6 +422,7 @@ cachetable_test (void) {
         NULL,
         &dummy_log_fassociate,
         &dummy_close_usr,
+        &dummy_free_usr,
         &dummy_chckpnt_usr,
         &test_begin_checkpoint,
         &dummy_end,

ft/tests/cachetable-put-checkpoint.cc

@@ -554,6 +554,7 @@ cachetable_test (void) {
         NULL,
         &dummy_log_fassociate,
         &dummy_close_usr,
+        &dummy_free_usr,
         &dummy_chckpnt_usr,
         test_begin_checkpoint, // called in begin_checkpoint
         &dummy_end,

ft/tests/cachetable-test.h

@@ -95,6 +95,7 @@ PATENT RIGHTS GRANT:
 //
 static void dummy_log_fassociate(CACHEFILE UU(cf), void* UU(p)) { }
 static void dummy_close_usr(CACHEFILE UU(cf), int UU(i), void* UU(p), bool UU(b), LSN UU(lsn))  { }
+static void dummy_free_usr(CACHEFILE UU(cf), void* UU(p))  { }
 static void dummy_chckpnt_usr(CACHEFILE UU(cf), int UU(i), void* UU(p)) { }
 static void dummy_begin(LSN UU(lsn), void* UU(p)) { }
 static void dummy_end(CACHEFILE UU(cf), int UU(i), void* UU(p)) { }
@@ -112,6 +113,7 @@ create_dummy_functions(CACHEFILE cf)
                                ud,
                                &dummy_log_fassociate,
                                &dummy_close_usr,
+                               &dummy_free_usr,
                                &dummy_chckpnt_usr,
                                &dummy_begin,
                                &dummy_end,