FT-242 Begin breaking up fttypes.h by moving many things to their

appropriate headers

ft/block_table.h

@@ -92,7 +92,7 @@ PATENT RIGHTS GRANT:
 #ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
 
 #include "fttypes.h"
-
+#include "ft/ft-internal.h"
 
 typedef struct block_table *BLOCK_TABLE;
 

ft/cachetable.h

@@ -111,6 +111,42 @@ PATENT RIGHTS GRANT:
 
 typedef BLOCKNUM CACHEKEY;
 
+class checkpointer;
+typedef class checkpointer *CHECKPOINTER;
+typedef struct cachetable *CACHETABLE;
+typedef struct cachefile *CACHEFILE;
+typedef struct ctpair *PAIR;
+
+// This struct hold information about values stored in the cachetable.
+// As one can tell from the names, we are probably violating an
+// abstraction layer by placing names.
+//
+// The purpose of having this struct is to have a way for the 
+// cachetable to accumulate the some totals we are interested in.
+// Breaking this abstraction layer by having these names was the 
+// easiest way.
+//
+typedef struct pair_attr_s {
+    long size; // size PAIR's value takes in memory
+    long nonleaf_size; // size if PAIR is a nonleaf node, 0 otherwise, used only for engine status
+    long leaf_size; // size if PAIR is a leaf node, 0 otherwise, used only for engine status
+    long rollback_size; // size of PAIR is a rollback node, 0 otherwise, used only for engine status
+    long cache_pressure_size; // amount PAIR contributes to cache pressure, is sum of buffer sizes and workdone counts
+    bool is_valid;
+} PAIR_ATTR;
+
+static inline PAIR_ATTR make_pair_attr(long size) { 
+    PAIR_ATTR result={
+        .size = size, 
+        .nonleaf_size = 0, 
+        .leaf_size = 0, 
+        .rollback_size = 0, 
+        .cache_pressure_size = 0,
+        .is_valid = true
+    }; 
+    return result; 
+}
+
 void toku_set_cleaner_period (CACHETABLE ct, uint32_t new_period);
 uint32_t toku_get_cleaner_period_unlocked (CACHETABLE ct);
 void toku_set_cleaner_iterations (CACHETABLE ct, uint32_t new_iterations);
@@ -394,8 +430,9 @@ struct unlockers {
     bool       locked;
     void (*f)(void* extra);
     void      *extra;
-    UNLOCKERS  next;
+    struct unlockers *next;
 };
+typedef struct unlockers *UNLOCKERS;
 
 // Effect:  If the block is in the cachetable, then return it.
 //   Otherwise call the functions in unlockers, fetch the data (but don't pin it, since we'll just end up pinning it again later), and return TOKUDB_TRY_AGAIN.

ft/cursor.h

+/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
+
+/*
+COPYING CONDITIONS NOTICE:
+
+  This program is free software; you can redistribute it and/or modify
+  it under the terms of version 2 of the GNU General Public License as
+  published by the Free Software Foundation, and provided that the
+  following conditions are met:
+
+      * Redistributions of source code must retain this COPYING
+        CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
+        DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
+        PATENT MARKING NOTICE (below), and the PATENT RIGHTS
+        GRANT (below).
+
+      * Redistributions in binary form must reproduce this COPYING
+        CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
+        DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
+        PATENT MARKING NOTICE (below), and the PATENT RIGHTS
+        GRANT (below) in the documentation and/or other materials
+        provided with the distribution.
+
+  You should have received a copy of the GNU General Public License
+  along with this program; if not, write to the Free Software
+  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+  02110-1301, USA.
+
+COPYRIGHT NOTICE:
+
+  TokuDB, Tokutek Fractal Tree Indexing Library.
+  Copyright (C) 2014 Tokutek, Inc.
+
+DISCLAIMER:
+
+  This program is distributed in the hope that it will be useful, but
+  WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+  General Public License for more details.
+
+UNIVERSITY PATENT NOTICE:
+
+  The technology is licensed by the Massachusetts Institute of
+  Technology, Rutgers State University of New Jersey, and the Research
+  Foundation of State University of New York at Stony Brook under
+  United States of America Serial No. 11/760379 and to the patents
+  and/or patent applications resulting from it.
+
+PATENT MARKING NOTICE:
+
+  This software is covered by US Patent No. 8,185,551.
+  This software is covered by US Patent No. 8,489,638.
+
+PATENT RIGHTS GRANT:
+
+  "THIS IMPLEMENTATION" means the copyrightable works distributed by
+  Tokutek as part of the Fractal Tree project.
+
+  "PATENT CLAIMS" means the claims of patents that are owned or
+  licensable by Tokutek, both currently or in the future; and that in
+  the absence of this license would be infringed by THIS
+  IMPLEMENTATION or by using or running THIS IMPLEMENTATION.
+
+  "PATENT CHALLENGE" shall mean a challenge to the validity,
+  patentability, enforceability and/or non-infringement of any of the
+  PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS.
+
+  Tokutek hereby grants to you, for the term and geographical scope of
+  the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free,
+  irrevocable (except as stated in this section) patent license to
+  make, have made, use, offer to sell, sell, import, transfer, and
+  otherwise run, modify, and propagate the contents of THIS
+  IMPLEMENTATION, where such license applies only to the PATENT
+  CLAIMS.  This grant does not include claims that would be infringed
+  only as a consequence of further modifications of THIS
+  IMPLEMENTATION.  If you or your agent or licensee institute or order
+  or agree to the institution of patent litigation against any entity
+  (including a cross-claim or counterclaim in a lawsuit) alleging that
+  THIS IMPLEMENTATION constitutes direct or contributory patent
+  infringement, or inducement of patent infringement, then any rights
+  granted to you under this License shall terminate as of the date
+  such litigation is filed.  If you or your agent or exclusive
+  licensee institute or order or agree to the institution of a PATENT
+  CHALLENGE, then Tokutek may terminate any rights granted to you
+  under this License.
+*/
+
+#pragma once
+
+#include <db.h>
+
+typedef bool(*FT_CHECK_INTERRUPT_CALLBACK)(void* extra);
+
+/* an ft cursor is represented as a kv pair in a tree */
+struct ft_cursor {
+    FT_HANDLE ft_handle;
+    DBT key, val;             // The key-value pair that the cursor currently points to
+    DBT range_lock_left_key, range_lock_right_key;
+    bool prefetching;
+    bool left_is_neg_infty, right_is_pos_infty;
+    bool is_snapshot_read; // true if query is read_committed, false otherwise
+    bool is_leaf_mode;
+    bool disable_prefetching;
+    bool is_temporary;
+    int out_of_range_error;
+    int direction;
+    TOKUTXN ttxn;
+    FT_CHECK_INTERRUPT_CALLBACK interrupt_cb;
+    void *interrupt_cb_extra;
+};
+typedef struct ft_cursor *FT_CURSOR;

ft/fifo.cc

@@ -172,7 +172,7 @@ int toku_fifo_enq(FIFO fifo, const void *key, unsigned int keylen, const void *d
         toku_fifo_resize(fifo, next_2);
     }
     struct fifo_entry *entry = (struct fifo_entry *)(fifo->memory + fifo->memory_used);
-    fifo_entry_set_msg_type(entry, type);
+    entry->type = (unsigned char) type;
     entry->msn = msn;
     xids_cpy(&entry->xids_s, xids);
     entry->is_fresh = is_fresh;

ft/fifo.h

@@ -91,10 +91,10 @@ PATENT RIGHTS GRANT:
 #ident "Copyright (c) 2007-2013 Tokutek Inc.  All rights reserved."
 #ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
 
-#include "fttypes.h"
-#include "xids-internal.h"
-#include "xids.h"
-
+#include "ft/fttypes.h"
+#include "ft/xids-internal.h"
+#include "ft/xids.h"
+#include "ft/ft_msg.h"
 
 // If the fifo_entry is unpacked, the compiler aligns the xids array and we waste a lot of space
 struct __attribute__((__packed__)) fifo_entry {
@@ -106,24 +106,6 @@ struct __attribute__((__packed__)) fifo_entry {
     XIDS_S        xids_s;
 };
 
-// get and set the ft message type for a fifo entry.
-// it is internally stored as a single unsigned char.
-static inline enum ft_msg_type 
-fifo_entry_get_msg_type(const struct fifo_entry * entry)
-{
-    enum ft_msg_type msg_type;
-    msg_type = (enum ft_msg_type) entry->type;
-    return msg_type;
-}
-
-static inline void
-fifo_entry_set_msg_type(struct fifo_entry * entry,
-        enum ft_msg_type msg_type)
-{
-    unsigned char type = (unsigned char) msg_type;
-    entry->type = type;
-}
-
 typedef struct fifo *FIFO;
 
 int toku_fifo_create(FIFO *);
@@ -150,7 +132,7 @@ void toku_fifo_iterate(FIFO, void(*f)(bytevec key,ITEMLEN keylen,bytevec data,IT
       struct fifo_entry *e = toku_fifo_iterate_internal_get_entry(fifo, fifo_iterate_off);                \
       ITEMLEN keylenvar = e->keylen;                                                                      \
       ITEMLEN datalenvar = e->vallen;                                                                     \
-      enum ft_msg_type typevar = fifo_entry_get_msg_type(e);                                              \
+      enum ft_msg_type typevar = (enum ft_msg_type) e->type;                                              \
       MSN     msnvar  = e->msn;                                                                           \
       XIDS    xidsvar = &e->xids_s;                                                                       \
       bytevec keyvar  = xids_get_end_of_array(xidsvar);                                                   \

ft/ft-cachetable-wrappers.h

@@ -92,8 +92,8 @@ PATENT RIGHTS GRANT:
 #ident "Copyright (c) 2007-2013 Tokutek Inc.  All rights reserved."
 #ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
 
-#include <fttypes.h>
-#include "cachetable.h"
+#include "ft/ft-internal.h"
+#include "ft/cachetable.h"
 
 /**
  * Put an empty node (that is, no fields filled) into the cachetable. 

ft/ft-flusher.h

@@ -91,8 +91,7 @@ PATENT RIGHTS GRANT:
 #ident "Copyright (c) 2007-2013 Tokutek Inc.  All rights reserved."
 #ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
 
-// This must be first to make the 64-bit file mode work right in Linux
-#include "fttypes.h"
+#include "ft/ft-internal.h"
 
 typedef enum {
     FT_FLUSHER_CLEANER_TOTAL_NODES = 0,     // total number of nodes whose buffers are potentially flushed by cleaner thread
@@ -158,6 +157,23 @@ toku_ft_flush_node_on_background_thread(
     FTNODE parent
     );
 
+// Given pinned node and pinned child, split child into two
+// and update node with information about its new child.
+void toku_ft_split_child(
+    FT h,
+    FTNODE node,
+    int childnum,
+    FTNODE child,
+    enum split_mode split_mode
+    );
+// Given pinned node, merge childnum with a neighbor and update node with
+// information about the change
+void toku_ft_merge_child(
+    FT ft,
+    FTNODE node,
+    int childnum
+    );
+
 /**
  * Effect: Split a leaf node.
  * Argument "node" is node to be split.
@@ -166,6 +182,7 @@ toku_ft_flush_node_on_background_thread(
  *   nodea is the left node that results from the split
  *   splitk is the right-most key of nodea
  */
+// TODO: Rename toku_ft_leaf_split
 void
 ftleaf_split(
     FT h,
@@ -189,6 +206,7 @@ ftleaf_split(
  *    but it does not guarantee that the resulting nodes are smaller than nodesize.
  */
 void
+// TODO: Rename toku_ft_nonleaf_split
 ft_nonleaf_split(
     FT h,
     FTNODE node,

ft/ft-internal.h

@@ -115,7 +115,9 @@ PATENT RIGHTS GRANT:
 #include "compress.h"
 #include <util/mempool.h>
 #include <util/omt.h>
-#include "bndata.h"
+#include "ft/bndata.h"
+#include "ft/rollback.h"
+#include "ft/ft-search.h"
 
 enum { KEY_VALUE_OVERHEAD = 8 }; /* Must store the two lengths. */
 enum { FT_MSG_OVERHEAD = (2 + sizeof(MSN)) };   // the type plus freshness plus MSN
@@ -140,6 +142,18 @@ enum ftnode_fetch_type {
     ftnode_fetch_keymatch, // one child is needed if it holds both keys
 };
 
+enum split_mode {
+    SPLIT_EVENLY,
+    SPLIT_LEFT_HEAVY,
+    SPLIT_RIGHT_HEAVY
+};
+
+enum reactivity {
+    RE_STABLE,
+    RE_FUSIBLE,
+    RE_FISSIBLE
+};
+
 static bool is_valid_ftnode_fetch_type(enum ftnode_fetch_type type) UU();
 static bool is_valid_ftnode_fetch_type(enum ftnode_fetch_type type) {
     switch (type) {
@@ -191,6 +205,7 @@ struct ftnode_fetch_extra {
     tokutime_t decompress_time;
     tokutime_t deserialize_time;
 };
+typedef struct ftnode_fetch_extra *FTNODE_FETCH_EXTRA;
 
 struct toku_fifo_entry_key_msn_heaviside_extra {
     DESCRIPTOR desc;
@@ -593,6 +608,7 @@ struct ft {
     // - if our attempt fails because the key was not in range of the rightmost leaf, we reset the score back to 0
     uint32_t seqinsert_score;
 };
+typedef struct ft *FT;
 
 // Allocate a DB struct off the stack and only set its comparison
 // descriptor. We don't bother setting any other fields because
@@ -758,22 +774,6 @@ int toku_ftnode_cleaner_callback( void *ftnode_pv, BLOCKNUM blocknum, uint32_t f
 void toku_evict_bn_from_memory(FTNODE node, int childnum, FT h);
 BASEMENTNODE toku_detach_bn(FTNODE node, int childnum);
 
-// Given pinned node and pinned child, split child into two
-// and update node with information about its new child.
-void toku_ft_split_child(
-    FT h,
-    FTNODE node,
-    int childnum,
-    FTNODE child,
-    enum split_mode split_mode
-    );
-// Given pinned node, merge childnum with a neighbor and update node with
-// information about the change
-void toku_ft_merge_child(
-    FT ft,
-    FTNODE node,
-    int childnum
-    );
 static inline CACHETABLE_WRITE_CALLBACK get_write_callbacks_for_node(FT h) {
     CACHETABLE_WRITE_CALLBACK wc;
     wc.flush_callback = toku_ftnode_flush_callback;
@@ -786,27 +786,6 @@ static inline CACHETABLE_WRITE_CALLBACK get_write_callbacks_for_node(FT h) {
     return wc;
 }
 
-static const FTNODE null_ftnode=0;
-
-/* an ft cursor is represented as a kv pair in a tree */
-struct ft_cursor {
-    struct toku_list cursors_link;
-    FT_HANDLE ft_handle;
-    DBT key, val;             // The key-value pair that the cursor currently points to
-    DBT range_lock_left_key, range_lock_right_key;
-    bool prefetching;
-    bool left_is_neg_infty, right_is_pos_infty;
-    bool is_snapshot_read; // true if query is read_committed, false otherwise
-    bool is_leaf_mode;
-    bool disable_prefetching;
-    bool is_temporary;
-    int out_of_range_error;
-    int direction;
-    TOKUTXN ttxn;
-    FT_CHECK_INTERRUPT_CALLBACK interrupt_cb;
-    void *interrupt_cb_extra;
-};
-
 //
 // Helper function to fill a ftnode_fetch_extra with data
 // that will tell the fetch callback that the entire node is
@@ -942,43 +921,22 @@ static inline void destroy_bfe_for_prefetch(struct ftnode_fetch_extra *bfe) {
 }
 
 // this is in a strange place because it needs the cursor struct to be defined
-static inline void fill_bfe_for_prefetch(struct ftnode_fetch_extra *bfe,
+void fill_bfe_for_prefetch(struct ftnode_fetch_extra *bfe,
                            FT h,
-                                         FT_CURSOR c) {
-    paranoid_invariant(h->h->type == FT_CURRENT);
-    bfe->type = ftnode_fetch_prefetch;
-    bfe->h = h;
-    bfe->search = NULL;
-    toku_init_dbt(&bfe->range_lock_left_key);
-    toku_init_dbt(&bfe->range_lock_right_key);
-    const DBT *left = &c->range_lock_left_key;
-    if (left->data) {
-        toku_clone_dbt(&bfe->range_lock_left_key, *left);
-    }
-    const DBT *right = &c->range_lock_right_key;
-    if (right->data) {
-        toku_clone_dbt(&bfe->range_lock_right_key, *right);
-    }
-    bfe->left_is_neg_infty = c->left_is_neg_infty;
-    bfe->right_is_pos_infty = c->right_is_pos_infty;
-    bfe->child_to_read = -1;
-    bfe->disable_prefetching = c->disable_prefetching;
-    bfe->read_all_partitions = false;
-    bfe->bytes_read = 0;
-    bfe->io_time = 0;
-    bfe->deserialize_time = 0;
-    bfe->decompress_time = 0;
-}
+                           struct ft_cursor *c);
 
 struct ancestors {
     FTNODE   node;     // This is the root node if next is NULL.
     int       childnum; // which buffer holds messages destined to the node whose ancestors this list represents.
-    ANCESTORS next;     // Parent of this node (so next->node.(next->childnum) refers to this node).
+    struct ancestors *next;     // Parent of this node (so next->node.(next->childnum) refers to this node).
 };
+typedef struct ancestors *ANCESTORS;
+
 struct pivot_bounds {
     const DBT * const lower_bound_exclusive;
     const DBT * const upper_bound_inclusive; // NULL to indicate negative or positive infinity (which are in practice exclusive since there are now transfinite keys in messages).
 };
+typedef struct pivot_bounds const * const PIVOT_BOUNDS;
 
 __attribute__((nonnull))
 void toku_move_ftnode_messages_to_stale(FT ft, FTNODE node);

ft/ft-ops.cc

@@ -422,6 +422,7 @@ toku_ft_get_status(FT_STATUS s) {
         }                                                                           \
     } while (0)
 
+
 void toku_note_deserialized_basement_node(bool fixed_key_size) {
     if (fixed_key_size) {
         STATUS_INC(FT_BASEMENT_DESERIALIZE_FIXED_KEYSIZE, 1);
@@ -4703,7 +4704,7 @@ do_bn_apply_msg(FT_HANDLE t, BASEMENTNODE bn, struct fifo_entry *entry, txn_gc_i
     if (entry->msn.msn > bn->max_msn_applied.msn) {
         ITEMLEN keylen = entry->keylen;
         ITEMLEN vallen = entry->vallen;
-        enum ft_msg_type type = fifo_entry_get_msg_type(entry);
+        enum ft_msg_type type = (enum ft_msg_type) entry->type;
         MSN msn = entry->msn;
         const XIDS xids = (XIDS) &entry->xids_s;
         bytevec key = xids_get_end_of_array(xids);
@@ -5439,6 +5440,34 @@ ftnode_pf_callback_and_free_bfe(void *ftnode_pv, void* disk_data, void *read_ext
     return r;
 }
 
+void fill_bfe_for_prefetch(struct ftnode_fetch_extra *bfe,
+                           FT h,
+                           struct ft_cursor *c) {
+    paranoid_invariant(h->h->type == FT_CURRENT);
+    bfe->type = ftnode_fetch_prefetch;
+    bfe->h = h;
+    bfe->search = NULL;
+    toku_init_dbt(&bfe->range_lock_left_key);
+    toku_init_dbt(&bfe->range_lock_right_key);
+    const DBT *left = &c->range_lock_left_key;
+    if (left->data) {
+        toku_clone_dbt(&bfe->range_lock_left_key, *left);
+    }
+    const DBT *right = &c->range_lock_right_key;
+    if (right->data) {
+        toku_clone_dbt(&bfe->range_lock_right_key, *right);
+    }
+    bfe->left_is_neg_infty = c->left_is_neg_infty;
+    bfe->right_is_pos_infty = c->right_is_pos_infty;
+    bfe->child_to_read = -1;
+    bfe->disable_prefetching = c->disable_prefetching;
+    bfe->read_all_partitions = false;
+    bfe->bytes_read = 0;
+    bfe->io_time = 0;
+    bfe->deserialize_time = 0;
+    bfe->decompress_time = 0;
+}
+
 static void
 ft_node_maybe_prefetch(FT_HANDLE ft_handle, FTNODE node, int childnum, FT_CURSOR ftcursor, bool *doprefetch) {
     // the number of nodes to prefetch

ft/ft-ops.h

@@ -100,6 +100,8 @@ PATENT RIGHTS GRANT:
 #include "log.h"
 #include "ft-search.h"
 #include "compress.h"
+#include "ft_msg.h"
+#include "ft/cursor.h"
 
 // A callback function is invoked with the key, and the data.
 // The pointers (to the bytevecs) must not be modified.  The data must be copied out before the callback function returns.
@@ -114,8 +116,6 @@ PATENT RIGHTS GRANT:
 // When lock_only is true, the callback only does optional lock tree locking.
 typedef int(*FT_GET_CALLBACK_FUNCTION)(ITEMLEN keylen, bytevec key, ITEMLEN vallen, bytevec val, void *extra, bool lock_only);
 
-typedef bool(*FT_CHECK_INTERRUPT_CALLBACK)(void* extra);
-
 int toku_open_ft_handle (const char *fname, int is_create, FT_HANDLE *, int nodesize, int basementnodesize, enum toku_compression_method compression_method, CACHETABLE, TOKUTXN, int(*)(DB *,const DBT*,const DBT*)) __attribute__ ((warn_unused_result));
 
 // effect: changes the descriptor for the ft of the given handle.
@@ -249,6 +249,7 @@ void toku_ft_maybe_delete (FT_HANDLE ft_h, DBT *k, TOKUTXN txn, bool oplsn_valid
 TXNID toku_ft_get_oldest_referenced_xid_estimate(FT_HANDLE ft_h);
 TXN_MANAGER toku_ft_get_txn_manager(FT_HANDLE ft_h);
 
+class txn_gc_info;
 void toku_ft_send_insert(FT_HANDLE ft_h, DBT *key, DBT *val, XIDS xids, enum ft_msg_type type, txn_gc_info *gc_info);
 void toku_ft_send_delete(FT_HANDLE ft_h, DBT *key, XIDS xids, txn_gc_info *gc_info);
 void toku_ft_send_commit_any(FT_HANDLE ft_h, DBT *key, XIDS xids, txn_gc_info *gc_info);
@@ -261,7 +262,6 @@ extern int toku_ft_debug_mode;
 int toku_verify_ft (FT_HANDLE ft_h)  __attribute__ ((warn_unused_result));
 int toku_verify_ft_with_progress (FT_HANDLE ft_h, int (*progress_callback)(void *extra, float progress), void *extra, int verbose, int keep_going)  __attribute__ ((warn_unused_result));
 
-typedef struct ft_cursor *FT_CURSOR;
 int toku_ft_cursor (FT_HANDLE, FT_CURSOR*, TOKUTXN, bool, bool)  __attribute__ ((warn_unused_result));
 void toku_ft_cursor_set_leaf_mode(FT_CURSOR);
 // Sets a boolean on the ft cursor that prevents uncessary copying of

ft/ft-search.h

@@ -92,6 +92,7 @@ PATENT RIGHTS GRANT:
 #ifndef FT_SEARCH_H
 #define FT_SEARCH_H
 
+#include "ft/ybt.h"
 
 enum ft_search_direction_e {
     FT_SEARCH_LEFT = 1,  /* search left -> right, finds min xy as defined by the compare function */

ft/ft_msg.h

@@ -97,6 +97,107 @@ PATENT RIGHTS GRANT:
 #ident "Copyright (c) 2007-2013 Tokutek Inc.  All rights reserved."
 #ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
 
+/* tree command types */
+enum ft_msg_type {
+    FT_NONE = 0,
+    FT_INSERT = 1,
+    FT_DELETE_ANY = 2,  // Delete any matching key.  This used to be called FT_DELETE.
+    //FT_DELETE_BOTH = 3,
+    FT_ABORT_ANY = 4,   // Abort any commands on any matching key.
+    //FT_ABORT_BOTH  = 5, // Abort commands that match both the key and the value
+    FT_COMMIT_ANY  = 6,
+    //FT_COMMIT_BOTH = 7,
+    FT_COMMIT_BROADCAST_ALL = 8, // Broadcast to all leafentries, (commit all transactions).
+    FT_COMMIT_BROADCAST_TXN = 9, // Broadcast to all leafentries, (commit specific transaction).
+    FT_ABORT_BROADCAST_TXN  = 10, // Broadcast to all leafentries, (commit specific transaction).
+    FT_INSERT_NO_OVERWRITE = 11,
+    FT_OPTIMIZE = 12,             // Broadcast
+    FT_OPTIMIZE_FOR_UPGRADE = 13, // same as FT_OPTIMIZE, but record version number in leafnode
+    FT_UPDATE = 14,
+    FT_UPDATE_BROADCAST_ALL = 15
+};
+
+static inline bool
+ft_msg_type_applies_once(enum ft_msg_type type)
+{
+    bool ret_val;
+    switch (type) {
+    case FT_INSERT_NO_OVERWRITE:
+    case FT_INSERT:
+    case FT_DELETE_ANY:
+    case FT_ABORT_ANY:
+    case FT_COMMIT_ANY:
+    case FT_UPDATE:
+        ret_val = true;
+        break;
+    case FT_COMMIT_BROADCAST_ALL:
+    case FT_COMMIT_BROADCAST_TXN:
+    case FT_ABORT_BROADCAST_TXN:
+    case FT_OPTIMIZE:
+    case FT_OPTIMIZE_FOR_UPGRADE:
+    case FT_UPDATE_BROADCAST_ALL:
+    case FT_NONE:
+        ret_val = false;
+        break;
+    default:
+        assert(false);
+    }
+    return ret_val;
+}
+
+static inline bool
+ft_msg_type_applies_all(enum ft_msg_type type)
+{
+    bool ret_val;
+    switch (type) {
+    case FT_NONE:
+    case FT_INSERT_NO_OVERWRITE:
+    case FT_INSERT:
+    case FT_DELETE_ANY:
+    case FT_ABORT_ANY:
+    case FT_COMMIT_ANY:
+    case FT_UPDATE:
+        ret_val = false;
+        break;
+    case FT_COMMIT_BROADCAST_ALL:
+    case FT_COMMIT_BROADCAST_TXN:
+    case FT_ABORT_BROADCAST_TXN:
+    case FT_OPTIMIZE:
+    case FT_OPTIMIZE_FOR_UPGRADE:
+    case FT_UPDATE_BROADCAST_ALL:
+        ret_val = true;
+        break;
+    default:
+        assert(false);
+    }
+    return ret_val;
+}
+
+static inline bool
+ft_msg_type_does_nothing(enum ft_msg_type type)
+{
+    return (type == FT_NONE);
+}
+
+typedef struct xids_t *XIDS;
+
+/* tree commands */
+struct ft_msg {
+    enum ft_msg_type type;
+    MSN          msn;          // message sequence number
+    XIDS         xids;
+    union {
+        /* insert or delete */
+        struct ft_msg_insert_delete {
+            const DBT *key;   // for insert, delete, upsertdel
+            const DBT *val;   // for insert, delete, (and it is the "extra" for upsertdel, upsertdel_broadcast_all)
+        } id;
+    } u;
+};
+
+// Message sent into the ft to implement insert, delete, update, etc
+typedef struct ft_msg FT_MSG_S;
+typedef struct ft_msg *FT_MSG;
 
 uint32_t ft_msg_get_keylen(FT_MSG ft_msg);
 
@@ -104,21 +205,10 @@ uint32_t ft_msg_get_vallen(FT_MSG ft_msg);
 
 XIDS ft_msg_get_xids(FT_MSG ft_msg);
 
-void * ft_msg_get_key(FT_MSG ft_msg);
+void *ft_msg_get_key(FT_MSG ft_msg);
 
-void * ft_msg_get_val(FT_MSG ft_msg);
+void *ft_msg_get_val(FT_MSG ft_msg);
 
 enum ft_msg_type ft_msg_get_type(FT_MSG ft_msg);
 
-void ft_msg_from_fifo_msg(FT_MSG ft_msg, FIFO_MSG fifo_msg);
-
-#if 0
-
-void ft_msg_from_dbts(FT_MSG ft_msg, DBT *key, DBT *val, XIDS xids, enum ft_msg_type type);
-
 #endif
-
-
-
-#endif  // FT_MSG_H
-

ft/ft_node-serialize.cc

@@ -98,6 +98,8 @@ PATENT RIGHTS GRANT:
 #include "ft.h"
 #include <util/status.h>
 #include <util/scoped_malloc.h>
+#include "ft/cachetable.h"
+#include "ft/rollback.h"
 
 static FT_UPGRADE_STATUS_S ft_upgrade_status;
 

ft/ftloader.h

@@ -92,6 +92,8 @@ PATENT RIGHTS GRANT:
 #ident "Copyright (c) 2007-2013 Tokutek Inc.  All rights reserved."
 #ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
 
+#include "ft/cachetable.h"
+
 // The loader callbacks are C functions and need to be defined as such
 
 typedef void (*ft_loader_error_func)(DB *, int which_db, int err, DBT *key, DBT *val, void *extra);

ft/fttypes.h

@@ -115,9 +115,6 @@ typedef struct ft *FT;
 typedef struct ft_header *FT_HEADER;
 typedef struct ft_options *FT_OPTIONS;
 
-struct wbuf;
-struct dbuf;
-
 typedef unsigned int ITEMLEN;
 typedef const void *bytevec;
 
@@ -143,36 +140,6 @@ typedef TOKU_XA_XID *XIDP; // this is the type that's passed to the logger code
 
 static inline BLOCKNUM make_blocknum(int64_t b) { BLOCKNUM result={b}; return result; }
 
-// This struct hold information about values stored in the cachetable.
-// As one can tell from the names, we are probably violating an
-// abstraction layer by placing names.
-//
-// The purpose of having this struct is to have a way for the 
-// cachetable to accumulate the some totals we are interested in.
-// Breaking this abstraction layer by having these names was the 
-// easiest way.
-//
-typedef struct pair_attr_s {
-    long size; // size PAIR's value takes in memory
-    long nonleaf_size; // size if PAIR is a nonleaf node, 0 otherwise, used only for engine status
-    long leaf_size; // size if PAIR is a leaf node, 0 otherwise, used only for engine status
-    long rollback_size; // size of PAIR is a rollback node, 0 otherwise, used only for engine status
-    long cache_pressure_size; // amount PAIR contributes to cache pressure, is sum of buffer sizes and workdone counts
-    bool is_valid;
-} PAIR_ATTR;
-
-static inline PAIR_ATTR make_pair_attr(long size) { 
-    PAIR_ATTR result={
-        .size = size, 
-        .nonleaf_size = 0, 
-        .leaf_size = 0, 
-        .rollback_size = 0, 
-        .cache_pressure_size = 0,
-        .is_valid = true
-    }; 
-    return result; 
-}
-
 typedef struct {
     uint32_t len;
     char *data;
@@ -218,165 +185,16 @@ typedef struct tokulogger *TOKULOGGER;
 typedef struct txn_manager *TXN_MANAGER;
 #define NULL_LOGGER ((TOKULOGGER)0)
 typedef struct tokutxn    *TOKUTXN;
-typedef struct txninfo    *TXNINFO;
 #define NULL_TXN ((TOKUTXN)0)
 
-struct logged_btt_pair {
-    DISKOFF off;
-    int32_t size;
-};
-
-typedef struct cachetable *CACHETABLE;
-typedef struct cachefile *CACHEFILE;
-typedef struct ctpair *PAIR;
-typedef class checkpointer *CHECKPOINTER;
-class bn_data;
-
-/* tree command types */
-enum ft_msg_type {
-    FT_NONE = 0,
-    FT_INSERT = 1,
-    FT_DELETE_ANY = 2,  // Delete any matching key.  This used to be called FT_DELETE.
-    //FT_DELETE_BOTH = 3,
-    FT_ABORT_ANY = 4,   // Abort any commands on any matching key.
-    //FT_ABORT_BOTH  = 5, // Abort commands that match both the key and the value
-    FT_COMMIT_ANY  = 6,
-    //FT_COMMIT_BOTH = 7,
-    FT_COMMIT_BROADCAST_ALL = 8, // Broadcast to all leafentries, (commit all transactions).
-    FT_COMMIT_BROADCAST_TXN = 9, // Broadcast to all leafentries, (commit specific transaction).
-    FT_ABORT_BROADCAST_TXN  = 10, // Broadcast to all leafentries, (commit specific transaction).
-    FT_INSERT_NO_OVERWRITE = 11,
-    FT_OPTIMIZE = 12,             // Broadcast
-    FT_OPTIMIZE_FOR_UPGRADE = 13, // same as FT_OPTIMIZE, but record version number in leafnode
-    FT_UPDATE = 14,
-    FT_UPDATE_BROADCAST_ALL = 15
-};
-
-static inline bool
-ft_msg_type_applies_once(enum ft_msg_type type)
-{
-    bool ret_val;
-    switch (type) {
-    case FT_INSERT_NO_OVERWRITE:
-    case FT_INSERT:
-    case FT_DELETE_ANY:
-    case FT_ABORT_ANY:
-    case FT_COMMIT_ANY:
-    case FT_UPDATE:
-        ret_val = true;
-        break;
-    case FT_COMMIT_BROADCAST_ALL:
-    case FT_COMMIT_BROADCAST_TXN:
-    case FT_ABORT_BROADCAST_TXN:
-    case FT_OPTIMIZE:
-    case FT_OPTIMIZE_FOR_UPGRADE:
-    case FT_UPDATE_BROADCAST_ALL:
-    case FT_NONE:
-        ret_val = false;
-        break;
-    default:
-        assert(false);
-    }
-    return ret_val;
-}
-
-static inline bool
-ft_msg_type_applies_all(enum ft_msg_type type)
-{
-    bool ret_val;
-    switch (type) {
-    case FT_NONE:
-    case FT_INSERT_NO_OVERWRITE:
-    case FT_INSERT:
-    case FT_DELETE_ANY:
-    case FT_ABORT_ANY:
-    case FT_COMMIT_ANY:
-    case FT_UPDATE:
-        ret_val = false;
-        break;
-    case FT_COMMIT_BROADCAST_ALL:
-    case FT_COMMIT_BROADCAST_TXN:
-    case FT_ABORT_BROADCAST_TXN:
-    case FT_OPTIMIZE:
-    case FT_OPTIMIZE_FOR_UPGRADE:
-    case FT_UPDATE_BROADCAST_ALL:
-        ret_val = true;
-        break;
-    default:
-        assert(false);
-    }
-    return ret_val;
-}
-
-static inline bool
-ft_msg_type_does_nothing(enum ft_msg_type type)
-{
-    return (type == FT_NONE);
-}
-
 typedef struct xids_t *XIDS;
-typedef struct fifo_msg_t *FIFO_MSG;
-/* tree commands */
-struct ft_msg {
-    enum ft_msg_type type;
-    MSN          msn;          // message sequence number
-    XIDS         xids;
-    union {
-        /* insert or delete */
-        struct ft_msg_insert_delete {
-            const DBT *key;   // for insert, delete, upsertdel
-            const DBT *val;   // for insert, delete, (and it is the "extra" for upsertdel, upsertdel_broadcast_all)
-        } id;
-    } u;
-};
-
-// Message sent into the ft to implement insert, delete, update, etc
-typedef struct ft_msg FT_MSG_S;
-typedef struct ft_msg *FT_MSG;
 
 typedef int (*ft_compare_func)(DB *, const DBT *, const DBT *);
 typedef void (*setval_func)(const DBT *, void *);
 typedef int (*ft_update_func)(DB *, const DBT *, const DBT *, const DBT *, setval_func, void *);
-typedef void (*on_redirect_callback)(FT_HANDLE, void*);
 typedef void (*remove_ft_ref_callback)(FT, void*);
+typedef void (*on_redirect_callback)(FT_HANDLE, void*);
 
 #define UU(x) x __attribute__((__unused__))
 
-typedef struct memarena *MEMARENA;
-typedef struct rollback_log_node *ROLLBACK_LOG_NODE;
-typedef struct serialized_rollback_log_node *SERIALIZED_ROLLBACK_LOG_NODE;
-
-//
-// Types of snapshots that can be taken by a tokutxn
-//  - TXN_SNAPSHOT_NONE: means that there is no snapshot. Reads do not use snapshot reads.
-//                       used for SERIALIZABLE and READ UNCOMMITTED
-//  - TXN_SNAPSHOT_ROOT: means that all tokutxns use their root transaction's snapshot
-//                       used for REPEATABLE READ
-//  - TXN_SNAPSHOT_CHILD: means that each child tokutxn creates its own snapshot
-//                        used for READ COMMITTED
-//
-
-typedef enum __TXN_SNAPSHOT_TYPE { 
-    TXN_SNAPSHOT_NONE=0,
-    TXN_SNAPSHOT_ROOT=1,
-    TXN_SNAPSHOT_CHILD=2
-} TXN_SNAPSHOT_TYPE;
-
-typedef struct ancestors *ANCESTORS;
-typedef struct pivot_bounds const * const PIVOT_BOUNDS;
-typedef struct ftnode_fetch_extra *FTNODE_FETCH_EXTRA;
-typedef struct unlockers *UNLOCKERS;
-
-enum reactivity {
-    RE_STABLE,
-    RE_FUSIBLE,
-    RE_FISSIBLE
-};
-
-enum split_mode {
-    SPLIT_EVENLY,
-    SPLIT_LEFT_HEAVY,
-    SPLIT_RIGHT_HEAVY
-};
-
 #endif

ft/le-cursor.cc

@@ -89,9 +89,10 @@ PATENT RIGHTS GRANT:
 #ident "Copyright (c) 2010-2013 Tokutek Inc.  All rights reserved."
 #ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
 
-#include "ft.h"
-#include "ft-internal.h"
-#include "le-cursor.h"
+#include "ft/ft.h"
+#include "ft/ft-internal.h"
+#include "ft/le-cursor.h"
+#include "ft/cursor.h"
 
 // A LE_CURSOR is a special purpose FT_CURSOR that:
 //  - enables prefetching

ft/leafentry.h

@@ -98,8 +98,9 @@ PATENT RIGHTS GRANT:
 #include <util/mempool.h>
 #include <util/omt.h>
 
-#include "txn_manager.h"
-#include "rbuf.h"
+#include "ft/txn_manager.h"
+#include "ft/rbuf.h"
+#include "ft/ft_msg.h"
 
 /*
     Memory format of packed leaf entry
@@ -241,6 +242,8 @@ toku_le_upgrade_13_14(LEAFENTRY_13 old_leafentry, // NULL if there was no stored
                       size_t *new_leafentry_memorysize,
                       LEAFENTRY *new_leafentry_p);
 
+class bn_data;
+
 void
 toku_le_apply_msg(FT_MSG   msg,
                   LEAFENTRY old_leafentry, // NULL if there was no stored data.

ft/log-internal.h

@@ -119,6 +119,7 @@ using namespace toku;
 
 #define LOGGER_MIN_BUF_SIZE (1<<24)
 
+// TODO: Remove mylock, it has no value
 struct mylock {
     toku_mutex_t lock;
 };
@@ -283,6 +284,7 @@ struct tokutxn {
                       // txn to not transition to commit or abort
     uint64_t client_id;
 };
+typedef struct tokutxn    *TOKUTXN;
 
 static inline int
 txn_has_current_rollback_log(TOKUTXN txn) {
@@ -306,6 +308,9 @@ struct txninfo {
     BLOCKNUM   current_rollback;
 };
 
+// TODO: Remove null txn
+#define NULL_TXN ((TOKUTXN)0)
+
 static inline int toku_logsizeof_uint8_t (uint32_t v __attribute__((__unused__))) {
     return 1;
 }

ft/logformat.cc

@@ -195,7 +195,7 @@ const struct logtype logtypes[] = {
                             {"BYTESTRING", "iname", 0},   // pathname of file
                             {"uint8_t", "unlink_on_close", 0},
                             NULLFIELD}, IGNORE_LOG_BEGIN},
-    //We do not use a TXNINFO struct since recovery log has
+    //We do not use a txninfo struct since recovery log has
     //FILENUMS and TOKUTXN has FTs (for open_fts)
     {"xstillopen", 's', FA{{"TXNID_PAIR", "xid", 0}, 
                            {"TXNID_PAIR", "parentxid", 0}, 

ft/logger.h

@@ -92,8 +92,9 @@ PATENT RIGHTS GRANT:
 #ident "Copyright (c) 2007-2013 Tokutek Inc.  All rights reserved."
 #ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
 
-#include "fttypes.h"
-#include "ft_layout_version.h"
+#include "ft/fttypes.h"
+#include "ft/ft-internal.h"
+#include "ft/ft_layout_version.h"
 
 enum {
     TOKU_LOG_VERSION_1 = 1,

ft/rollback.h

@@ -92,8 +92,13 @@ PATENT RIGHTS GRANT:
 #ident "Copyright (c) 2007-2013 Tokutek Inc.  All rights reserved."
 #ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
 
-#include <util/omt.h>
-#include "sub_block.h"
+#include "ft/sub_block.h"
+#include "ft/cachetable.h"
+
+#include "util/memarena.h"
+
+typedef struct rollback_log_node *ROLLBACK_LOG_NODE;
+typedef struct serialized_rollback_log_node *SERIALIZED_ROLLBACK_LOG_NODE;
 
 void toku_poll_txn_progress_function(TOKUTXN txn, uint8_t is_commit, uint8_t stall_for_checkpoint);
 
@@ -172,6 +177,7 @@ struct serialized_rollback_log_node {
     BLOCKNUM blocknum;
     struct sub_block sub_block[max_sub_blocks];
 };
+typedef struct serialized_rollback_log_node *SERIALIZED_ROLLBACK_LOG_NODE;
 
 static inline void
 toku_static_serialized_rollback_log_destroy(SERIALIZED_ROLLBACK_LOG_NODE log) {

ft/sub_block.h

@@ -112,6 +112,7 @@ struct sub_block {
 
     uint32_t xsum;                    // sub block checksum
 };
+typedef struct sub_block *SUB_BLOCK;
 
 struct stored_sub_block {
     uint32_t uncompressed_size;

ft/txn.cc

@@ -388,7 +388,7 @@ toku_txn_update_xids_in_txn(TOKUTXN txn, TXNID xid)
 
 //Used on recovery to recover a transaction.
 int
-toku_txn_load_txninfo (TOKUTXN txn, TXNINFO info) {
+toku_txn_load_txninfo (TOKUTXN txn, struct txninfo *info) {
     txn->roll_info.rollentry_raw_count = info->rollentry_raw_count;
     uint32_t i;
     for (i = 0; i < info->num_fts; i++) {

ft/txn.h

@@ -142,7 +142,7 @@ int toku_txn_begin_with_xid (
 
 void toku_txn_update_xids_in_txn(TOKUTXN txn, TXNID xid);
 
-int toku_txn_load_txninfo (TOKUTXN txn, TXNINFO info);
+int toku_txn_load_txninfo (TOKUTXN txn, struct txninfo *info);
 
 int toku_txn_commit_txn (TOKUTXN txn, int nosync,
                          TXN_PROGRESS_POLL_FUNCTION poll, void *poll_extra);

ft/txn_manager.h

@@ -123,6 +123,7 @@ struct txn_manager {
     TXNID last_xid_seen_for_recover;
     TXNID last_calculated_oldest_referenced_xid;
 };
+typedef struct txn_manager *TXN_MANAGER;
 
 struct txn_manager_state { 
     txn_manager_state(TXN_MANAGER mgr) :
@@ -189,6 +190,22 @@ TXNID toku_txn_manager_get_oldest_living_xid(TXN_MANAGER txn_manager);
 
 TXNID toku_txn_manager_get_oldest_referenced_xid_estimate(TXN_MANAGER txn_manager);
 
+//
+// Types of snapshots that can be taken by a tokutxn
+//  - TXN_SNAPSHOT_NONE: means that there is no snapshot. Reads do not use snapshot reads.
+//                       used for SERIALIZABLE and READ UNCOMMITTED
+//  - TXN_SNAPSHOT_ROOT: means that all tokutxns use their root transaction's snapshot
+//                       used for REPEATABLE READ
+//  - TXN_SNAPSHOT_CHILD: means that each child tokutxn creates its own snapshot
+//                        used for READ COMMITTED
+//
+
+typedef enum __TXN_SNAPSHOT_TYPE { 
+    TXN_SNAPSHOT_NONE=0,
+    TXN_SNAPSHOT_ROOT=1,
+    TXN_SNAPSHOT_CHILD=2
+} TXN_SNAPSHOT_TYPE;
+
 void toku_txn_manager_handle_snapshot_create_for_child_txn(
     TOKUTXN txn,
     TXN_MANAGER txn_manager,