[t:4018] 4018 changes to main

git-svn-id: file:///svn/toku/tokudb@35662 c7de825b-a66e-492c-adef-691d508d4ae1

buildheader/db.h_4_1

@@ -43,6 +43,8 @@ typedef struct __toku_db_btree_stat64 {
   u_int64_t bt_ndata; /* how many key-value pairs (an estimate, but exact when flattened)                       */
   u_int64_t bt_dsize; /* how big are the keys+values (not counting the lengths) (an estimate, unless flattened) */
   u_int64_t bt_fsize; /* how big is the underlying file                                                         */
+  u_int64_t bt_create_time_sec; /* Creation time, in seconds */
+  u_int64_t bt_modify_time_sec; /* Time of last serialization, in seconds */
 } DB_BTREE_STAT64;
 typedef struct __toku_loader DB_LOADER;
 struct __toku_loader_internal;

buildheader/db.h_4_3

@@ -43,6 +43,8 @@ typedef struct __toku_db_btree_stat64 {
   u_int64_t bt_ndata; /* how many key-value pairs (an estimate, but exact when flattened)                       */
   u_int64_t bt_dsize; /* how big are the keys+values (not counting the lengths) (an estimate, unless flattened) */
   u_int64_t bt_fsize; /* how big is the underlying file                                                         */
+  u_int64_t bt_create_time_sec; /* Creation time, in seconds */
+  u_int64_t bt_modify_time_sec; /* Time of last serialization, in seconds */
 } DB_BTREE_STAT64;
 typedef struct __toku_loader DB_LOADER;
 struct __toku_loader_internal;

buildheader/db.h_4_4

@@ -43,6 +43,8 @@ typedef struct __toku_db_btree_stat64 {
   u_int64_t bt_ndata; /* how many key-value pairs (an estimate, but exact when flattened)                       */
   u_int64_t bt_dsize; /* how big are the keys+values (not counting the lengths) (an estimate, unless flattened) */
   u_int64_t bt_fsize; /* how big is the underlying file                                                         */
+  u_int64_t bt_create_time_sec; /* Creation time, in seconds */
+  u_int64_t bt_modify_time_sec; /* Time of last serialization, in seconds */
 } DB_BTREE_STAT64;
 typedef struct __toku_loader DB_LOADER;
 struct __toku_loader_internal;

buildheader/db.h_4_5

@@ -43,6 +43,8 @@ typedef struct __toku_db_btree_stat64 {
   u_int64_t bt_ndata; /* how many key-value pairs (an estimate, but exact when flattened)                       */
   u_int64_t bt_dsize; /* how big are the keys+values (not counting the lengths) (an estimate, unless flattened) */
   u_int64_t bt_fsize; /* how big is the underlying file                                                         */
+  u_int64_t bt_create_time_sec; /* Creation time, in seconds */
+  u_int64_t bt_modify_time_sec; /* Time of last serialization, in seconds */
 } DB_BTREE_STAT64;
 typedef struct __toku_loader DB_LOADER;
 struct __toku_loader_internal;

buildheader/db.h_4_6

@@ -43,6 +43,8 @@ typedef struct __toku_db_btree_stat64 {
   u_int64_t bt_ndata; /* how many key-value pairs (an estimate, but exact when flattened)                       */
   u_int64_t bt_dsize; /* how big are the keys+values (not counting the lengths) (an estimate, unless flattened) */
   u_int64_t bt_fsize; /* how big is the underlying file                                                         */
+  u_int64_t bt_create_time_sec; /* Creation time, in seconds */
+  u_int64_t bt_modify_time_sec; /* Time of last serialization, in seconds */
 } DB_BTREE_STAT64;
 typedef struct __toku_loader DB_LOADER;
 struct __toku_loader_internal;

buildheader/make_db_h.c

@@ -430,6 +430,9 @@ int main (int argc __attribute__((__unused__)), char *const argv[] __attribute__
     printf("  u_int64_t bt_ndata; /* how many key-value pairs (an estimate, but exact when flattened)                       */\n");
     printf("  u_int64_t bt_dsize; /* how big are the keys+values (not counting the lengths) (an estimate, unless flattened) */\n");
     printf("  u_int64_t bt_fsize; /* how big is the underlying file                                                         */\n");
+    // 4018
+    printf("  u_int64_t bt_create_time_sec; /* Creation time, in seconds */\n");
+    printf("  u_int64_t bt_modify_time_sec; /* Time of last serialization, in seconds */\n");
     printf("} DB_BTREE_STAT64;\n");
 
     //bulk loader

buildheader/tdb.h

@@ -43,6 +43,8 @@ typedef struct __toku_db_btree_stat64 {
   u_int64_t bt_ndata; /* how many key-value pairs (an estimate, but exact when flattened)                       */
   u_int64_t bt_dsize; /* how big are the keys+values (not counting the lengths) (an estimate, unless flattened) */
   u_int64_t bt_fsize; /* how big is the underlying file                                                         */
+  u_int64_t bt_create_time_sec; /* Creation time, in seconds */
+  u_int64_t bt_modify_time_sec; /* Time of last serialization, in seconds */
 } DB_BTREE_STAT64;
 typedef struct __toku_loader DB_LOADER;
 struct __toku_loader_internal;

include/db.h

@@ -43,6 +43,8 @@ typedef struct __toku_db_btree_stat64 {
   u_int64_t bt_ndata; /* how many key-value pairs (an estimate, but exact when flattened)                       */
   u_int64_t bt_dsize; /* how big are the keys+values (not counting the lengths) (an estimate, unless flattened) */
   u_int64_t bt_fsize; /* how big is the underlying file                                                         */
+  u_int64_t bt_create_time_sec; /* Creation time, in seconds */
+  u_int64_t bt_modify_time_sec; /* Time of last serialization, in seconds */
 } DB_BTREE_STAT64;
 typedef struct __toku_loader DB_LOADER;
 struct __toku_loader_internal;

newbrt/brt-serialize.c

@@ -1667,10 +1667,7 @@ int toku_serialize_brt_header_to_wbuf (struct wbuf *wbuf, struct brt_header *h,
     wbuf_int(wbuf, h->layout_version_original);
     wbuf_int(wbuf, h->build_id_original);
     wbuf_ulonglong(wbuf, h->time_of_creation);
-    {   // time_of_last_modification
-	uint64_t now = (uint64_t) time(NULL);
-	wbuf_ulonglong(wbuf, now);
-    }
+    wbuf_ulonglong(wbuf, h->time_of_last_modification);
     wbuf_ulonglong(wbuf, h->num_blocks_to_upgrade);
     wbuf_TXNID(wbuf, h->root_xid_that_created);
     wbuf_int(wbuf, h->basementnodesize);

newbrt/brt.c

@@ -776,9 +776,7 @@ void toku_brtnode_pe_est_callback(
     // we estimate the compressed size of data to be how large
     // the compressed data is on disk
     for (int i = 0; i < node->n_children; i++) {
-        if (BP_STATE(node,i) == PT_AVAIL && 
-            BP_SHOULD_EVICT(node,i) && 
-            toku_bnc_nbytesinbuf(BNC(node, i)) > 0) {
+        if (BP_STATE(node,i) == PT_AVAIL && BP_SHOULD_EVICT(node,i)) {
             // calculate how much data would be freed if
             // we compress this node and add it to
             // bytes_to_free
@@ -838,9 +836,7 @@ int toku_brtnode_pe_callback (void *brtnode_pv, long UU(bytes_to_free), long* by
         for (int i = 0; i < node->n_children; i++) {
             if (BP_STATE(node,i) == PT_AVAIL) {
                 if (BP_SHOULD_EVICT(node,i)) {
-                    if (toku_bnc_nbytesinbuf(BNC(node, i)) > 0) {
-                        cilk_spawn compress_internal_node_partition(node, i);
-                    }
+                    cilk_spawn compress_internal_node_partition(node, i);
                 }
                 else {
                     BP_SWEEP_CLOCK(node,i);
@@ -2834,7 +2830,7 @@ maybe_merge_pinned_nodes (BRTNODE parent, int childnum_of_parent, struct kv_pair
 // As output, two of node's children are merged or rebalanced, and node is unlocked
 //
 static void
-brt_merge_child (BRT t, BRTNODE node, int childnum_to_merge)
+brt_merge_child (BRT t, BRTNODE node, int childnum_to_merge, BOOL *did_react)
 {
     if (node->n_children < 2) {
         toku_unpin_brtnode(t, node);
@@ -2908,6 +2904,7 @@ brt_merge_child (BRT t, BRTNODE node, int childnum_to_merge)
 	if (childa->height>0) { int i; for (i=0; i+1<childa->n_children; i++) assert(childa->childkeys[i]); }
 	//toku_verify_estimates(t,childa);
 	// the tree did react if a merge (did_merge) or rebalance (new spkit key) occurred
+	*did_react = (BOOL)(did_merge || did_rebalance);
 	if (did_merge) assert(!splitk_kvpair); else assert(splitk_kvpair);
 
 	node->totalchildkeylens -= deleted_size; // The key was free()'d inside the maybe_merge_pinned_nodes.
@@ -3222,6 +3219,7 @@ flush_some_child (BRT t, BRTNODE parent)
         brt_split_child(t, parent, childnum, child);
     }
     else if (child_re == RE_FUSIBLE) {
+        BOOL did_react;
         //
         // There is probably a way to pass BRTNODE child
         // into brt_merge_child, but for simplicity for now,
@@ -3233,7 +3231,7 @@ flush_some_child (BRT t, BRTNODE parent)
         //
         // it is responsibility of brt_merge_child to unlock parent
         //
-        brt_merge_child(t, parent, childnum);
+        brt_merge_child(t, parent, childnum, &did_react);
     }
     else {
         assert(FALSE);
@@ -5008,15 +5006,14 @@ toku_brtheader_checkpoint (CACHEFILE cf, int fd, void *header_v)
 	    r = toku_logger_fsync_if_lsn_not_fsynced(logger, ch->checkpoint_lsn);
 	    if (r!=0) goto handle_error;
 	}
-	{
-	    ch->checkpoint_count++;
-	    // write translation and header to disk (or at least to OS internal buffer)
-	    r = toku_serialize_brt_header_to(fd, ch);
-	    if (r!=0) goto handle_error;
-	}
+        h->time_of_last_modification = ch->time_of_last_modification = time(NULL); // 4018
+        ch->checkpoint_count++;
+        // write translation and header to disk (or at least to OS internal buffer)
+        r = toku_serialize_brt_header_to(fd, ch);
+        if (r!=0) goto handle_error;
 	ch->dirty = 0;		      // this is only place this bit is cleared (in checkpoint_header)
-    }
-    else toku_block_translation_note_skipped_checkpoint(ch->blocktable);
+    } else 
+        toku_block_translation_note_skipped_checkpoint(ch->blocktable);
     if (0) {
 handle_error:
 	if (h->panic) r = h->panic;
@@ -6796,10 +6793,13 @@ int toku_brt_stat64 (BRT brt, TOKUTXN UU(txn), struct brtstat64_s *s) {
     int i;
     for (i=0; i<node->n_children; i++) {
         SUBTREE_EST se = &BP_SUBTREE_EST(node,i);
-	s->nkeys += se->nkeys;
-	s->ndata += se->ndata;
-	s->dsize += se->dsize;
+        s->nkeys += se->nkeys;
+        s->ndata += se->ndata;
+        s->dsize += se->dsize;
     }
+    // 4018
+    s->create_time_sec = brt->h->time_of_creation;
+    s->modify_time_sec = brt->h->time_of_last_modification;
     
     int r = toku_cachetable_unpin(brt->cf, root, fullhash, CACHETABLE_CLEAN, 0);
     if (r!=0) return r;

newbrt/brt.h

@@ -243,6 +243,9 @@ struct brtstat64_s {
     u_int64_t dsize; /* estimate the sum of the sizes of the pairs (exact when flattened and committed) */
     u_int64_t fsize;  /* the size of the underlying file                                                */
     u_int64_t ffree; /* Number of free bytes in the underlying file                                    */
+    // 4018
+    u_int64_t create_time_sec;  /* creation time in seconds. */
+    u_int64_t modify_time_sec; /* time of last serialization, in seconds. */ 
 };
 int toku_brt_stat64 (BRT, TOKUTXN,
 		     struct brtstat64_s *stat

src/tests/Makefile

@@ -227,7 +227,7 @@ BDB_DONTRUN_TESTS = \
         root_fifo_32 \
         root_fifo_41 \
 	shutdown-3344 \
-        stat64 stat64_flatten stat64-null-txn \
+        stat64 stat64-create-modify-times stat64_flatten stat64-null-txn \
 	stress-gc \
         test1324 \
         test1426 \

src/tests/stat64-create-modify-times.c

+/* -*- mode: C; c-basic-offset: 4 -*- */
+#ident "Copyright (c) 2011 Tokutek Inc.  All rights reserved."
+
+// verify that the tree create time is retuned in db->stat64
+
+#include "test.h"
+
+#include <db.h>
+#include <sys/stat.h>
+
+static void
+test_stat64_create_time (uint64_t n) {
+    if (verbose) printf("%s:%u\n", __FUNCTION__, __LINE__);
+
+    int r;
+    r = system("rm -rf " ENVDIR);
+    CKERR(r);
+    toku_os_mkdir(ENVDIR, S_IRWXU+S_IRWXG+S_IRWXO);
+    
+    DB_ENV *env = NULL;
+    r = db_env_create(&env, 0);                                           CKERR(r);
+
+    r = env->set_cachesize(env, 0, 20*1000000, 1);
+    r = env->open(env, ENVDIR, DB_INIT_LOCK|DB_INIT_LOG|DB_INIT_MPOOL|DB_INIT_TXN|DB_CREATE|DB_PRIVATE, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
+
+    DB *db = NULL;
+    r = db_create(&db, env, 0);                                           CKERR(r);
+
+    DB_TXN *txn = NULL;
+    r = env->txn_begin(env, 0, &txn, 0); assert(r == 0);
+
+    r = db->set_pagesize(db, 4096); assert(r == 0);
+    r = db->open(db, txn, "foo.db", 0, DB_BTREE, DB_CREATE, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
+
+    // initial stats
+    DB_BTREE_STAT64 s0;
+    r = db->stat64(db, txn, &s0); assert(r == 0);
+    printf("initial stat create %lu\n", s0.bt_create_time_sec);
+    assert(s0.bt_create_time_sec != 0);
+    printf("initial stat modify %lu\n", s0.bt_modify_time_sec);
+    assert(s0.bt_modify_time_sec == 0);
+    r = txn->commit(txn, 0);    assert(r == 0);
+
+    // stats after create is committed
+    r = env->txn_begin(env, 0, &txn, 0);  CKERR(r);
+    DB_BTREE_STAT64 s1;
+    r = db->stat64(db, txn, &s1); assert(r == 0);
+    assert(s1.bt_create_time_sec == s0.bt_create_time_sec);
+    assert(s1.bt_modify_time_sec == s0.bt_modify_time_sec);
+    r = txn->commit(txn, 0);    assert(r == 0);
+
+    // stats after checkpoint
+    sleep(10);
+    r = env->txn_checkpoint(env, 0, 0, 0); assert(r == 0);
+    r = env->txn_begin(env, 0, &txn, 0);  CKERR(r);
+    DB_BTREE_STAT64 s2;
+    r = db->stat64(db, txn, &s2); assert(r == 0);
+    assert(s2.bt_create_time_sec == s1.bt_create_time_sec);
+    assert(s2.bt_modify_time_sec > s1.bt_modify_time_sec);
+    r = txn->commit(txn, 0);    assert(r == 0);
+
+    // stats after insertion
+    DB_BTREE_STAT64 s3;
+    assert(n > 0);
+    for (uint64_t i = 0; i < n; i++) {
+        r = env->txn_begin(env, 0, &txn, 0);  CKERR(r);
+        DBT key, val;
+        r = db->put(db, txn, dbt_init(&key, &i, sizeof i), dbt_init(&val, &i, sizeof i), 0);
+        r = db->stat64(db, txn, &s3); assert(r == 0);
+        assert(s3.bt_create_time_sec == s2.bt_create_time_sec);
+        assert(s3.bt_modify_time_sec == s2.bt_modify_time_sec);
+        r = txn->commit(txn, 0);    assert(r == 0);
+    }
+
+    // stats after checkpoint
+    sleep(10);
+    r = env->txn_checkpoint(env, 0, 0, 0); assert(r == 0);
+    r = env->txn_begin(env, 0, &txn, 0);  CKERR(r);
+    DB_BTREE_STAT64 s4;
+    r = db->stat64(db, txn, &s4); assert(r == 0);
+    assert(s4.bt_create_time_sec == s3.bt_create_time_sec);
+    assert(s4.bt_modify_time_sec > s3.bt_modify_time_sec);
+    r = txn->commit(txn, 0);    assert(r == 0);
+
+    r = db->close(db, 0); CKERR(r);
+
+    r = env->close(env, 0); CKERR(r);
+}
+
+int
+test_main (int argc, char *const argv[]) {
+    parse_args(argc, argv);
+    test_stat64_create_time(1);
+    test_stat64_create_time(1000);
+    return 0;
+}

src/ydb.c

@@ -5527,10 +5527,13 @@ toku_db_stat64(DB * db, DB_TXN *txn, DB_BTREE_STAT64 *s) {
     }
     int r = toku_brt_stat64(db->i->brt, tokutxn, &brtstat);
     if (r==0) {
-	s->bt_nkeys = brtstat.nkeys;
-	s->bt_ndata = brtstat.ndata;
-	s->bt_dsize = brtstat.dsize;
-	s->bt_fsize = brtstat.fsize;
+        s->bt_nkeys = brtstat.nkeys;
+        s->bt_ndata = brtstat.ndata;
+        s->bt_dsize = brtstat.dsize;
+        s->bt_fsize = brtstat.fsize;
+        // 4018
+        s->bt_create_time_sec = brtstat.create_time_sec;
+        s->bt_modify_time_sec = brtstat.modify_time_sec;
     }
     return r;
 }