[t:4570] Merging verification tool and partial removal of deserialization custom errors.

git-svn-id: file:///svn/toku/tokudb@44286 c7de825b-a66e-492c-adef-691d508d4ae1

buildheader/make_tdb.c

@@ -119,6 +119,7 @@ enum {
         TOKUDB_TRY_AGAIN               = -100012,
         TOKUDB_NEEDS_REPAIR            = -100013,
         TOKUDB_CURSOR_CONTINUE         = -100014,
+        TOKUDB_BAD_CHECKSUM            = -100015,
         DONTUSE_I_JUST_PUT_THIS_HERE_SO_I_COULD_HAVE_A_COMMA_AFTER_EACH_ITEM
 };
 
@@ -269,6 +270,7 @@ static void print_defines (void) {
     dodefine(TOKUDB_TRY_AGAIN);
     dodefine(TOKUDB_NEEDS_REPAIR);
     dodefine(TOKUDB_CURSOR_CONTINUE);
+    dodefine(TOKUDB_BAD_CHECKSUM);
 
     /* LOADER flags */
     printf("/* LOADER flags */\n");

ft/CMakeLists.txt

@@ -37,6 +37,7 @@ set(FT_SOURCES
   ftloader-callback.c
   ft_msg.c
   ft_node-serialize.c
+  ft-node-deserialize.c
   ft-ops.c
   ft-pwrite.c
   ft-serialize.c
@@ -111,6 +112,7 @@ set(bins
   ftdump
   tdb_logprint
   tdb-recover
+  ftverify
   )
 foreach(bin ${bins})
   add_executable(${bin} ${bin}.c)
@@ -122,6 +124,10 @@ foreach(bin ${bins})
   target_link_libraries(${bin}_static ft_static ${LIBTOKUPORTABILITY_STATIC})
 endforeach(bin)
 
+# link in math.h library just for this tool.
+target_link_libraries(ftverify m)
+target_link_libraries(ftverify_static m)
+
 install(
   TARGETS ftdump_static
   DESTINATION bin

ft/ft-internal.h

@@ -518,9 +518,68 @@ int toku_serialize_rollback_log_to (int fd, BLOCKNUM blocknum, ROLLBACK_LOG_NODE
                                     FT h, int n_workitems, int n_threads,
                                     BOOL for_checkpoint);
 int toku_deserialize_rollback_log_from (int fd, BLOCKNUM blocknum, u_int32_t fullhash, ROLLBACK_LOG_NODE *logp, FT h);
-enum deserialize_error_code toku_deserialize_bp_from_disk(FTNODE node, FTNODE_DISK_DATA ndd, int childnum, int fd, struct ftnode_fetch_extra* bfe);
-enum deserialize_error_code toku_deserialize_bp_from_compressed(FTNODE node, int childnum, DESCRIPTOR desc, ft_compare_func cmp);
-enum deserialize_error_code toku_deserialize_ftnode_from (int fd, BLOCKNUM off, u_int32_t /*fullhash*/, FTNODE *ftnode, FTNODE_DISK_DATA* ndd, struct ftnode_fetch_extra* bfe);
+int toku_deserialize_bp_from_disk(FTNODE node, FTNODE_DISK_DATA ndd, int childnum, int fd, struct ftnode_fetch_extra* bfe);
+int toku_deserialize_bp_from_compressed(FTNODE node, int childnum, DESCRIPTOR desc, ft_compare_func cmp);
+int toku_deserialize_ftnode_from (int fd, BLOCKNUM off, u_int32_t /*fullhash*/, FTNODE *ftnode, FTNODE_DISK_DATA* ndd, struct ftnode_fetch_extra* bfe);
+
+//////////////// <CER> TODO: Move these function declarations
+int
+deserialize_ft_from_fd_into_rbuf(int fd,
+                                 toku_off_t offset_of_header,
+                                 struct rbuf *rb,
+                                 u_int64_t *checkpoint_count,
+                                 LSN *checkpoint_lsn,
+                                 u_int32_t * version_p,
+                                 enum deserialize_error_code *e);
+
+enum deserialize_error_code
+deserialize_ft_versioned(int fd, struct rbuf *rb, FT *ft, uint32_t version);
+
+int
+read_block_from_fd_into_rbuf(
+    int fd, 
+    BLOCKNUM blocknum,
+    FT h,
+    struct rbuf *rb
+    );
+
+int
+read_compressed_sub_block(struct rbuf *rb, struct sub_block *sb);
+
+int
+verify_ftnode_sub_block (struct sub_block *sb);
+
+void
+just_decompress_sub_block(struct sub_block *sb);
+
+/* Beginning of ft-node-deserialize.c helper functions. */
+
+//
+inline void
+initialize_ftnode(FTNODE node, BLOCKNUM blocknum);
+
+//
+inline int
+read_and_check_magic(struct rbuf *rb);
+
+//
+inline int
+read_and_check_version(FTNODE node, struct rbuf *rb);
+
+//
+inline void
+read_node_info(FTNODE node, struct rbuf *rb, int version);
+
+//
+inline void
+allocate_and_read_partition_offsets(FTNODE node, struct rbuf *rb, FTNODE_DISK_DATA *ndd);
+
+//
+inline int
+check_node_info_checksum(struct rbuf *rb);
+
+//////////////// <CER>
+
 unsigned int toku_serialize_ftnode_size(FTNODE node); /* How much space will it take? */
 int toku_keycompare (bytevec key1, ITEMLEN key1len, bytevec key2, ITEMLEN key2len);
 
@@ -983,6 +1042,6 @@ toku_ft_node_put_cmd (
 
 void toku_flusher_thread_set_callback(void (*callback_f)(int, void*), void* extra);
 
-enum deserialize_error_code toku_upgrade_subtree_estimates_to_stat64info(int fd, FT h);
+int toku_upgrade_subtree_estimates_to_stat64info(int fd, FT h);
 
 #endif

ft/ft-node-deserialize.c

+/* -*- mode: C; c-basic-offset: 4 -*- */
+// vim: expandtab:ts=8:sw=4:softtabstop=4:
+#ident "$Id: ft-serialize.c 43686 2012-05-18 23:21:00Z leifwalsh $"
+#ident "Copyright (c) 2007-2010 Tokutek Inc.  All rights reserved."
+#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
+
+#include <ft-internal.h>
+#include <db.h>
+
+////////////////////////////////////////
+// 1. Sets initial values
+//
+inline void
+initialize_ftnode(FTNODE node, BLOCKNUM blocknum)
+{
+    node->fullhash = 0xDEADBEEF; // <CER> Is this 'spoof' ok?
+    node->thisnodename = blocknum;
+    node->dirty = 0;
+    node->bp = NULL;
+    // <CER> Can we use this initialization as a correctness assert in
+    // a later function?
+    node->layout_version_read_from_disk = 0;
+}
+
+// TODO:
+
+/************************
+ * In other deserialization code, we check the rb size member.  We
+ * verify that it is greater than or equal to 24.  Ignoring this magic
+ * number for a moment, should we put this check in its own function? *
+*************************/
+
+/////////////////////////////////////////////////////////////////////
+// 2. Read and check the 'magic' bytes on disk.  Returns an error if
+// the magic does not match.
+//
+inline int
+read_and_check_magic(struct rbuf *rb)
+{
+    int r = 0;
+    bytevec magic;
+    rbuf_literal_bytes(rb, &magic, 8);
+    if (memcmp(magic, "tokuleaf", 8)!=0 &&
+        memcmp(magic, "tokunode", 8)!=0) {
+	r = DB_BADFORMAT; // TODO: Return more meaningful error.
+    }
+
+    return r;
+}
+
+////////////////////
+// 3. 
+inline int
+read_and_check_version(FTNODE node, struct rbuf *rb)
+{
+    int r = 0;
+    int version = rbuf_int(rb);
+    node->layout_version_read_from_disk = version;
+    if (version < FT_LAYOUT_MIN_SUPPORTED_VERSION) {
+	r = 1; // TODO: Better error reporting.
+    }
+
+    return r;
+}
+
+////////////////////
+// 4. 
+inline void
+read_node_info(FTNODE node, struct rbuf *rb, int version)
+{
+    node->layout_version = version;
+    node->layout_version_original = rbuf_int(rb);
+    node->build_id = rbuf_int(rb);
+    node->n_children = rbuf_int(rb);
+}
+
+////////////////////
+// 5.
+// <CER> Should these be two seperate functions?
+inline void
+allocate_and_read_partition_offsets(FTNODE node, struct rbuf *rb, FTNODE_DISK_DATA *ndd)
+{
+    XMALLOC_N(node->n_children, node->bp);
+    // TODO: Fix this to use xmalloc_n
+    *ndd = toku_xmalloc(node->n_children * sizeof(**ndd));
+    // Read the partition locations.
+    for (int i = 0; i < node->n_children; i++) {
+        BP_START(*ndd, i) = rbuf_int(rb);
+        BP_SIZE (*ndd, i) = rbuf_int(rb);
+    }
+}
+
+////////////////////
+// 6. Check the node info checksum.
+inline int
+check_node_info_checksum(struct rbuf *rb)
+{
+    int r = 0;
+    // Verify checksum of header stored.
+    u_int32_t checksum = x1764_memory(rb->buf, rb->ndone);
+    u_int32_t stored_checksum = rbuf_int(rb);
+
+    if (stored_checksum != checksum) {
+        // TODO: dump_bad_block(rb->buf, rb->size);
+	r = TOKUDB_BAD_CHECKSUM;
+    }
+
+    return r;
+}
+
+
+// Two functions, at this point, SHOULD be split into sperate read and
+// checksum check calls: 
+
+// 1. read_and_decompress_sub_block - this is generic, used elsewhere.
+
+// So....
+// a. call read_compressed_sub_block() directly
+// then....
+
+
+//
+
+
+// 2. deserialize_ftnode_info() - this actually reads in the node
+// 'info' fields, such as height, nodesize, etc.
+
+
+/////////////////////////////////
+// ?. ----
+// setup_ftnode_paritions() - 
+// calls :
+// a. update_bfe_using_ftnode
+// b. setup_partitions_using_bfe()
+
+/////////////////////////////////
+// ?. partition from sub-block deserialization.
+// A.decompress_and_deserialize_worker()
+// calls:
+// a. read_and_decompress_sub_block (SEE ABOVE) - 
+// -calls:
+// -ii. read_compressed_sub_block() - Returns checksum error AND reads out buffer. 
+// b. deserialize_ftnode_partition()
+// -calls:
+// -ii. verify_ftnode_sub_block() - JUST verifies checksum.
+//
+// OR
+//
+// B. check_and_copy_compressed_sub_block_worker
+// calls:
+// a. read_compressed_sub_block() - Returns checksum AND reads out of buffer.
+
+/////////////////////////////////
+// 1. first calls verify_ftnode_sub_block() which must be refactored
+// into two seperate calls.

ft/ft-serialize.c

@@ -138,7 +138,7 @@ deserialize_descriptor_from(int fd, BLOCK_TABLE bt, DESCRIPTOR desc, int layout_
 }
 
 // We only deserialize brt header once and then share everything with all the brts.
-static enum deserialize_error_code
+enum deserialize_error_code
 deserialize_ft_versioned(int fd, struct rbuf *rb, FT *ftp, uint32_t version)
 {
     enum deserialize_error_code e = DS_OK;
@@ -423,7 +423,7 @@ serialize_ft_min_size (u_int32_t version) {
 //
 // TOKUDB_DICTIONARY_NO_HEADER means we can overwrite everything in the
 // file AND the header is useless
-static int
+int
 deserialize_ft_from_fd_into_rbuf(int fd,
                                         toku_off_t offset_of_header,
                                         struct rbuf *rb,