refs #5025 Updated verification tool to read non-upgraded nodes and check the...

refs #5025 Updated verification tool to read non-upgraded nodes and check the end to end checksum for version 14 nodes.

git-svn-id: file:///svn/toku/tokudb@45558 c7de825b-a66e-492c-adef-691d508d4ae1

ft/ft-internal.h

@@ -556,32 +556,15 @@ void
 just_decompress_sub_block(struct sub_block *sb);
 
 /* Beginning of ft-node-deserialize.c helper functions. */
-
-//
-void
-initialize_ftnode(FTNODE node, BLOCKNUM blocknum);
-
-//
-int
-read_and_check_magic(struct rbuf *rb);
-
-//
-int
-read_and_check_version(FTNODE node, struct rbuf *rb);
-
-//
-void
-read_node_info(FTNODE node, struct rbuf *rb, int version);
-
-//
-void
-allocate_and_read_partition_offsets(FTNODE node, struct rbuf *rb, FTNODE_DISK_DATA *ndd);
-
-//
-int
-check_node_info_checksum(struct rbuf *rb);
-
-//////////////// <CER>
+void initialize_ftnode(FTNODE node, BLOCKNUM blocknum);
+int read_and_check_magic(struct rbuf *rb);
+int read_and_check_version(FTNODE node, struct rbuf *rb);
+void read_node_info(FTNODE node, struct rbuf *rb, int version);
+void allocate_and_read_partition_offsets(FTNODE node, struct rbuf *rb, FTNODE_DISK_DATA *ndd);
+int check_node_info_checksum(struct rbuf *rb);
+void read_legacy_node_info(FTNODE node, struct rbuf *rb, int version);
+int check_legacy_end_checksum(struct rbuf *rb);
+/* End of ft-node-deserialization.c helper functions. */
 
 unsigned int toku_serialize_ftnode_size(FTNODE node); /* How much space will it take? */
 int toku_keycompare (bytevec key1, ITEMLEN key1len, bytevec key2, ITEMLEN key2len);

ft/ft-node-deserialize.c

@@ -92,6 +92,7 @@ allocate_and_read_partition_offsets(FTNODE node, struct rbuf *rb, FTNODE_DISK_DA
 {
     XMALLOC_N(node->n_children, node->bp);
     // TODO: Fix this to use xmalloc_n
+    // XMALLOC_N(node->n_children, *ndd);
     *ndd = toku_xmalloc(node->n_children * sizeof(**ndd));
     // Read the partition locations.
     for (int i = 0; i < node->n_children; i++) {
@@ -118,3 +119,36 @@ check_node_info_checksum(struct rbuf *rb)
 
     return r;
 }
+
+// Reads node info from older (13 and 14) fractal tree nodes
+// out of the given buffer.
+void
+read_legacy_node_info(FTNODE node, struct rbuf *rb, int version)
+{
+    node->nodesize = rbuf_int(rb); // 1. nodesize
+    node->flags = rbuf_int(rb);    // 2. flags
+    node->height = rbuf_int(rb);   // 3. height
+    
+    // If the version is less than 14, there are two extra ints here.
+    // we would need to ignore them if they are there.
+    if (version == FT_LAYOUT_VERSION_13) {
+        (void) rbuf_int(rb);       // 4. rand4
+        (void) rbuf_int(rb);       // 5. local
+    }
+}
+
+// Assuming the given buffer is in the correct position,
+// this checks to see if the stored checksum matches the
+// checksum of the entire buffer.
+int
+check_legacy_end_checksum(struct rbuf *rb)
+{
+    int r = 0;
+    u_int32_t expected_xsum = rbuf_int(rb);
+    u_int32_t actual_xsum = x1764_memory(rb->buf, rb->size - 4);
+    if (expected_xsum != actual_xsum) {
+        r = TOKUDB_BAD_CHECKSUM;
+    }
+    
+    return r;
+}
\ No newline at end of file

ft/ftverify.c

 /* -*- mode: C; c-basic-offset: 4 -*- */
 // vim: expandtab:ts=8:sw=4:softtabstop=4:
-#ident "$Id: ft-serialize.c 43686 2012-05-18 23:21:00Z leifwalsh $"
+#ident "$Id: ftverify.c 43686 2012-05-18 23:21:00Z leifwalsh $"
 #ident "Copyright (c) 2007-2010 Tokutek Inc.  All rights reserved."
 #ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
 
@@ -166,6 +166,26 @@ struct check_block_table_extra {
     struct ft *h;
 };
 
+// Check non-upgraded (legacy) node.
+// NOTE: These nodes have less checksumming than more 
+// recent nodes.  This effectively means that we are 
+// skipping over these nodes.
+static int
+check_old_node(FTNODE node, struct rbuf *rb, int version)
+{
+    int r = 0;
+    read_legacy_node_info(node, rb, version);
+    // For version 14 nodes, advance the buffer to the end
+    // and verify the checksum.
+    if (version == FT_FIRST_LAYOUT_VERSION_WITH_END_TO_END_CHECKSUM) {
+        // Advance the buffer to the end.
+        rb->ndone = rb->size - 4;
+        r = check_legacy_end_checksum(rb);
+    }
+    
+    return r;
+}
+
 // Read, decompress, and check the given block.
 static int
 check_block(BLOCKNUM blocknum, int64_t UU(blocksize), int64_t UU(address), void *extra)
@@ -213,6 +233,19 @@ check_block(BLOCKNUM blocknum, int64_t UU(blocksize), int64_t UU(address), void
      // UPGRADE FORK GOES HERE //
     ////////////////////////////
     
+    // Check nodes before major layout changes in version 15.
+    // All newer versions should follow the same layout, for now.
+    // This predicate would need to be changed if the layout
+    // of the nodes on disk does indeed change in the future.
+    if (version < FT_FIRST_LAYOUT_VERSION_WITH_BASEMENT_NODES)
+    {
+        r = check_old_node(node, &rb, version);
+        if (r != 0) {
+            failure++;
+        }
+        goto cleanup;
+    }
+
     read_node_info(node, &rb, version);
 
     FTNODE_DISK_DATA ndd;
@@ -266,6 +299,7 @@ check_block(BLOCKNUM blocknum, int64_t UU(blocksize), int64_t UU(address), void
 	// point.
     }
 
+cleanup:
     // Cleanup and error incrementing.
     if (failure) {
        	cbte->blocks_failed++;