#4079 add last verification time to the brt header closes[t:4079]

git-svn-id: file:///svn/toku/tokudb@36487 c7de825b-a66e-492c-adef-691d508d4ae1

buildheader/db.h_4_1

@@ -45,6 +45,7 @@ typedef struct __toku_db_btree_stat64 {
   u_int64_t bt_fsize; /* how big is the underlying file                                                         */
   u_int64_t bt_create_time_sec; /* Creation time, in seconds */
   u_int64_t bt_modify_time_sec; /* Time of last serialization, in seconds */
+  u_int64_t bt_verify_time_sec; /* Time of last verification, in seconds */
 } DB_BTREE_STAT64;
 typedef struct __toku_loader DB_LOADER;
 struct __toku_loader_internal;

buildheader/db.h_4_3

@@ -45,6 +45,7 @@ typedef struct __toku_db_btree_stat64 {
   u_int64_t bt_fsize; /* how big is the underlying file                                                         */
   u_int64_t bt_create_time_sec; /* Creation time, in seconds */
   u_int64_t bt_modify_time_sec; /* Time of last serialization, in seconds */
+  u_int64_t bt_verify_time_sec; /* Time of last verification, in seconds */
 } DB_BTREE_STAT64;
 typedef struct __toku_loader DB_LOADER;
 struct __toku_loader_internal;

buildheader/db.h_4_4

@@ -45,6 +45,7 @@ typedef struct __toku_db_btree_stat64 {
   u_int64_t bt_fsize; /* how big is the underlying file                                                         */
   u_int64_t bt_create_time_sec; /* Creation time, in seconds */
   u_int64_t bt_modify_time_sec; /* Time of last serialization, in seconds */
+  u_int64_t bt_verify_time_sec; /* Time of last verification, in seconds */
 } DB_BTREE_STAT64;
 typedef struct __toku_loader DB_LOADER;
 struct __toku_loader_internal;

buildheader/db.h_4_5

@@ -45,6 +45,7 @@ typedef struct __toku_db_btree_stat64 {
   u_int64_t bt_fsize; /* how big is the underlying file                                                         */
   u_int64_t bt_create_time_sec; /* Creation time, in seconds */
   u_int64_t bt_modify_time_sec; /* Time of last serialization, in seconds */
+  u_int64_t bt_verify_time_sec; /* Time of last verification, in seconds */
 } DB_BTREE_STAT64;
 typedef struct __toku_loader DB_LOADER;
 struct __toku_loader_internal;

buildheader/db.h_4_6

@@ -45,6 +45,7 @@ typedef struct __toku_db_btree_stat64 {
   u_int64_t bt_fsize; /* how big is the underlying file                                                         */
   u_int64_t bt_create_time_sec; /* Creation time, in seconds */
   u_int64_t bt_modify_time_sec; /* Time of last serialization, in seconds */
+  u_int64_t bt_verify_time_sec; /* Time of last verification, in seconds */
 } DB_BTREE_STAT64;
 typedef struct __toku_loader DB_LOADER;
 struct __toku_loader_internal;

buildheader/make_db_h.c

@@ -433,6 +433,7 @@ int main (int argc __attribute__((__unused__)), char *const argv[] __attribute__
     // 4018
     printf("  u_int64_t bt_create_time_sec; /* Creation time, in seconds */\n");
     printf("  u_int64_t bt_modify_time_sec; /* Time of last serialization, in seconds */\n");
+    printf("  u_int64_t bt_verify_time_sec; /* Time of last verification, in seconds */\n");
     printf("} DB_BTREE_STAT64;\n");
 
     //bulk loader

buildheader/tdb.h

@@ -45,6 +45,7 @@ typedef struct __toku_db_btree_stat64 {
   u_int64_t bt_fsize; /* how big is the underlying file                                                         */
   u_int64_t bt_create_time_sec; /* Creation time, in seconds */
   u_int64_t bt_modify_time_sec; /* Time of last serialization, in seconds */
+  u_int64_t bt_verify_time_sec; /* Time of last verification, in seconds */
 } DB_BTREE_STAT64;
 typedef struct __toku_loader DB_LOADER;
 struct __toku_loader_internal;

include/db.h

@@ -45,6 +45,7 @@ typedef struct __toku_db_btree_stat64 {
   u_int64_t bt_fsize; /* how big is the underlying file                                                         */
   u_int64_t bt_create_time_sec; /* Creation time, in seconds */
   u_int64_t bt_modify_time_sec; /* Time of last serialization, in seconds */
+  u_int64_t bt_verify_time_sec; /* Time of last verification, in seconds */
 } DB_BTREE_STAT64;
 typedef struct __toku_loader DB_LOADER;
 struct __toku_loader_internal;

newbrt/brt-internal.h

@@ -457,6 +457,7 @@ struct brt_header {
     uint32_t build_id_original;         // build_id of software that created this tree (read from disk, overwritten when written to disk)
     uint64_t time_of_creation;          // time this tree was created
     uint64_t time_of_last_modification; // last time this header was serialized to disk (read from disk, overwritten when written to disk)
+    uint64_t time_of_last_verification; // last time that this tree was verified
     BOOL upgrade_brt_performed;         // initially FALSE, set TRUE when brt has been fully updated (even though nodes may not have been)
     int64_t num_blocks_to_upgrade_13;   // Number of v13 blocks still not newest version. 
     int64_t num_blocks_to_upgrade_14;   // Number of v14 blocks still not newest version. 

newbrt/brt-serialize.c

@@ -1776,6 +1776,7 @@ serialize_brt_header_min_size (u_int32_t version) {
         case BRT_LAYOUT_VERSION_15:
             size += 4;  // basement node size
             size += 8;  // num_blocks_to_upgrade_14 (previously num_blocks_to_upgrade, now one int each for upgrade from 13, 14
+            size += 8;  // time of last verification
         case BRT_LAYOUT_VERSION_14:
             size += 8;  //TXNID that created
         case BRT_LAYOUT_VERSION_13:
@@ -1841,6 +1842,7 @@ int toku_serialize_brt_header_to_wbuf (struct wbuf *wbuf, struct brt_header *h,
     wbuf_ulonglong(wbuf, h->num_blocks_to_upgrade_14);
     wbuf_TXNID(wbuf, h->root_xid_that_created);
     wbuf_int(wbuf, h->basementnodesize);
+    wbuf_ulonglong(wbuf, h->time_of_last_verification);
     u_int32_t checksum = x1764_finish(&wbuf->checksum);
     wbuf_int(wbuf, checksum);
     lazy_assert(wbuf->ndone == wbuf->size);
@@ -2100,6 +2102,7 @@ deserialize_brtheader (int fd, struct rbuf *rb, struct brt_header **brth) {
     h->build_id_original = rbuf_int(&rc);
     h->time_of_creation  = rbuf_ulonglong(&rc);
     h->time_of_last_modification = rbuf_ulonglong(&rc);
+    h->time_of_last_verification = 0;
     h->num_blocks_to_upgrade_13  = rbuf_ulonglong(&rc);
     h->num_blocks_to_upgrade_14  = rbuf_ulonglong(&rc);
 
@@ -2109,6 +2112,7 @@ deserialize_brtheader (int fd, struct rbuf *rb, struct brt_header **brth) {
     }
     if (h->layout_version >= BRT_LAYOUT_VERSION_15) {
         h->basementnodesize = rbuf_int(&rc);
+        h->time_of_last_verification = rbuf_ulonglong(&rc);
     }
     (void)rbuf_int(&rc); //Read in checksum and ignore (already verified).
     if (rc.ndone!=rc.size) {ret = EINVAL; goto died1;}

newbrt/brt-verify.c

@@ -348,6 +348,12 @@ toku_verify_brt_with_progress (BRT brt, int (*progress_callback)(void *extra, fl
     u_int32_t root_hash;
     CACHEKEY *rootp = toku_calculate_root_offset_pointer(brt, &root_hash);
     int r = toku_verify_brtnode(brt, ZERO_MSN, ZERO_MSN, *rootp, -1, NULL, NULL, progress_callback, progress_extra, 1, verbose, keep_on_going);
+    if (r == 0) {
+        toku_brtheader_lock(brt->h);
+        brt->h->time_of_last_verification = time(NULL);
+        brt->h->dirty = 1;
+        toku_brtheader_unlock(brt->h);
+    }
     return r;
 }
 

newbrt/brt.c

@@ -4225,6 +4225,7 @@ brt_alloc_init_header(BRT t, TOKUTXN txn) {
     uint64_t now = (uint64_t) time(NULL);
     t->h->time_of_creation = now;
     t->h->time_of_last_modification = now;
+    t->h->time_of_last_verification = 0;
 
     memset(&t->h->descriptor, 0, sizeof(t->h->descriptor));
 
@@ -6894,6 +6895,7 @@ int toku_brt_stat64 (BRT brt, TOKUTXN UU(txn), struct brtstat64_s *s) {
     // 4018
     s->create_time_sec = brt->h->time_of_creation;
     s->modify_time_sec = brt->h->time_of_last_modification;
+    s->verify_time_sec = brt->h->time_of_last_verification;
     
     int r = toku_cachetable_unpin(brt->cf, root, fullhash, CACHETABLE_CLEAN, 0);
     if (r!=0) return r;

newbrt/brt.h

@@ -244,8 +244,9 @@ struct brtstat64_s {
     u_int64_t fsize;  /* the size of the underlying file                                                */
     u_int64_t ffree; /* Number of free bytes in the underlying file                                    */
     // 4018
-    u_int64_t create_time_sec;  /* creation time in seconds. */
+    u_int64_t create_time_sec; /* creation time in seconds. */
     u_int64_t modify_time_sec; /* time of last serialization, in seconds. */ 
+    u_int64_t verify_time_sec; /* time of last verification, in seconds */
 };
 int toku_brt_stat64 (BRT, TOKUTXN,
 		     struct brtstat64_s *stat

newbrt/brt_layout_version.h

@@ -18,7 +18,7 @@ enum brt_layout_version_e {
     BRT_LAYOUT_VERSION_12 = 12, // Diff from 11 to 12: Added BRT_CMD 'BRT_INSERT_NO_OVERWRITE', compressed block format, num old blocks
     BRT_LAYOUT_VERSION_13 = 13, // Diff from 12 to 13: Fixed loader pivot bug, added build_id to every node, timestamps to brtheader 
     BRT_LAYOUT_VERSION_14 = 14, // Diff from 13 to 14: Added MVCC; deprecated TOKU_DB_VALCMP_BUILTIN(_13); Remove fingerprints; Support QUICKLZ; add end-to-end checksum on uncompressed data.
-    BRT_LAYOUT_VERSION_15 = 15, // Diff from 14 to 15: basement nodes, TODO
+    BRT_LAYOUT_VERSION_15 = 15, // Diff from 14 to 15: basement nodes, last verification time
     BRT_NEXT_VERSION,           // the version after the current version
     BRT_LAYOUT_VERSION   = BRT_NEXT_VERSION-1, // A hack so I don't have to change this line.
     BRT_LAYOUT_MIN_SUPPORTED_VERSION = BRT_LAYOUT_VERSION_13, // Minimum version supported

newbrt/brtloader.c

@@ -2866,6 +2866,7 @@ write_header (struct dbout *out, long long translation_location_on_disk, long lo
     uint64_t now = (uint64_t) time(NULL);
     h.time_of_creation = now;
     h.time_of_last_modification = now;
+    h.time_of_last_verification = 0;
     h.checkpoint_count = 1;
     h.checkpoint_lsn   = load_lsn;
     h.nodesize         = target_nodesize;

src/tests/Makefile

@@ -142,6 +142,7 @@ BDB_DONTRUN_TESTS = \
         keyrange-dupsort \
         keyrange-dupsort-unflat \
         keyrange-unflat \
+	last-verify-time \
         loader-cleanup-test \
 	loader-create-abort \
 	loader-create-close \

src/tests/last-verify-time.c

+#include "test.h"
+
+// test for the last verify time
+
+static void
+test_verify_time_after_create(DB_ENV *env) {
+    int r;
+
+    DB *db = NULL;
+    r = db_create(&db, env, 0); assert_zero(r);
+
+    r = db->open(db, NULL, "test.tdb", NULL, DB_BTREE, DB_AUTO_COMMIT+DB_CREATE, S_IRWXU+S_IRWXG+S_IRWXO); assert_zero(r);
+
+    DB_BTREE_STAT64 stats;
+    r = db->stat64(db, NULL, &stats); assert_zero(r);
+    assert(stats.bt_verify_time_sec == 0);
+
+    r = db->close(db, 0); assert_zero(r);
+}
+
+static void
+test_verify_time_after_open(DB_ENV *env) {
+    int r;
+
+    DB *db = NULL;
+    r = db_create(&db, env, 0); assert_zero(r);
+
+    r = db->open(db, NULL, "test.tdb", NULL, DB_BTREE, DB_AUTO_COMMIT+DB_CREATE, S_IRWXU+S_IRWXG+S_IRWXO); assert_zero(r);
+
+    DB_BTREE_STAT64 stats;
+    r = db->stat64(db, NULL, &stats); assert_zero(r);
+    assert(stats.bt_verify_time_sec == 0);
+
+    r = db->close(db, 0); assert_zero(r);
+}
+
+static void
+test_verify_time_after_check(DB_ENV *env) {
+    int r;
+
+    DB *db = NULL;
+    r = db_create(&db, env, 0); assert_zero(r);
+
+    r = db->open(db, NULL, "test.tdb", NULL, DB_BTREE, DB_AUTO_COMMIT+DB_CREATE, S_IRWXU+S_IRWXG+S_IRWXO); assert_zero(r);
+
+    DB_BTREE_STAT64 stats;
+    r = db->stat64(db, NULL, &stats); assert_zero(r);
+    assert(stats.bt_verify_time_sec == 0);
+
+    r = db->verify_with_progress(db, NULL, NULL, 0, 0); assert_zero(r);
+
+    r = db->stat64(db, NULL, &stats); assert_zero(r);
+    assert(stats.bt_verify_time_sec != 0);
+
+    r = db->close(db, 0); assert_zero(r);
+}
+
+static void
+test_verify_time_after_reopen(DB_ENV *env) {
+    int r;
+
+    DB *db = NULL;
+    r = db_create(&db, env, 0); assert_zero(r);
+
+    r = db->open(db, NULL, "test.tdb", NULL, DB_BTREE, DB_AUTO_COMMIT+DB_CREATE, S_IRWXU+S_IRWXG+S_IRWXO); assert_zero(r);
+
+    DB_BTREE_STAT64 stats;
+    r = db->stat64(db, NULL, &stats); assert_zero(r);
+    assert(stats.bt_verify_time_sec != 0);
+
+    r = db->close(db, 0); assert_zero(r);
+}
+
+int
+test_main(int argc, char * const argv[]) {
+    int r;
+
+    // parse_args(argc, argv);
+    for (int i = 1; i < argc; i++) {
+        char * const arg = argv[i];
+        if (strcmp(arg, "-v") == 0) {
+            verbose++;
+            continue;
+        }
+        if (strcmp(arg, "-q") == 0) {
+            verbose = 0;
+            continue;
+        }
+        assert(0);
+    }
+
+    r = system("rm -rf " ENVDIR); assert_zero(r);
+    r = toku_os_mkdir(ENVDIR, S_IRWXU+S_IRWXG+S_IRWXO); assert_zero(r);
+
+    DB_ENV *env = NULL;
+    r = db_env_create(&env, 0); assert_zero(r);
+
+    r = env->open(env, ENVDIR, DB_INIT_MPOOL|DB_CREATE|DB_THREAD |DB_INIT_LOCK|DB_INIT_LOG|DB_INIT_TXN|DB_PRIVATE, S_IRWXU+S_IRWXG+S_IRWXO); assert_zero(r);
+
+    test_verify_time_after_create(env);
+
+    test_verify_time_after_open(env);
+
+    test_verify_time_after_check(env);
+
+    test_verify_time_after_reopen(env);
+
+    r = env->close(env, 0); assert_zero(r);
+
+    return 0;
+}
+

src/ydb.c

@@ -5550,6 +5550,7 @@ toku_db_stat64(DB * db, DB_TXN *txn, DB_BTREE_STAT64 *s) {
         // 4018
         s->bt_create_time_sec = brtstat.create_time_sec;
         s->bt_modify_time_sec = brtstat.modify_time_sec;
+        s->bt_verify_time_sec = brtstat.verify_time_sec;
     }
     return r;
 }