[t:3988] #3988 Collect some statistics. Still need to add to header and deliver to engine status.

git-svn-id: file:///svn/toku/tokudb@36845 c7de825b-a66e-492c-adef-691d508d4ae1

newbrt/brt-internal.h

@@ -748,6 +748,12 @@ typedef struct brt_status {
     uint64_t  search_root_retries;         // number of searches that required the root node to be fetched more than once
     uint64_t  search_tries_gt_height;      // number of searches that required more tries than the height of the tree
     uint64_t  search_tries_gt_heightplus3; // number of searches that required more tries than the height of the tree plus three
+    uint64_t  disk_flush_leaf;             // number of leaf nodes flushed to disk, not for checkpoint
+    uint64_t  disk_flush_nonleaf;          // number of nonleaf nodes flushed to disk, not for checkpoint
+    uint64_t  disk_flush_leaf_for_checkpoint;             // number of leaf nodes flushed to disk for checkpoint
+    uint64_t  disk_flush_nonleaf_for_checkpoint;          // number of nonleaf nodes flushed to disk for checkpoint
+    uint64_t  destroy_leaf;                // number of leaf nodes destroyed
+    uint64_t  destroy_nonleaf;             // number of nonleaf nodes destroyed
     uint64_t  cleaner_total_nodes;         // total number of nodes whose buffers are potentially flushed by cleaner thread
     uint64_t  cleaner_h1_nodes;            // number of nodes of height one whose message buffers are flushed by cleaner thread
     uint64_t  cleaner_hgt1_nodes;          // number of nodes of height > 1 whose message buffers are flushed by cleaner thread

newbrt/brt.c

@@ -671,6 +671,7 @@ void toku_brtnode_flush_callback (CACHEFILE cachefile, int fd, BLOCKNUM nodename
     struct brt_header *h = extraargs;
     BRTNODE brtnode = brtnode_v;
     assert(brtnode->thisnodename.b==nodename.b);
+    int height = brtnode->height;
     //printf("%s:%d %p->mdict[0]=%p\n", __FILE__, __LINE__, brtnode, brtnode->mdicts[0]);
     if (write_me) {
 	if (!h->panic) { // if the brt panicked, stop writing, otherwise try to write it.
@@ -689,6 +690,18 @@ void toku_brtnode_flush_callback (CACHEFILE cachefile, int fd, BLOCKNUM nodename
 		}
 	    }
 	}
+	if (height == 0) {  // statistics incremented only when disk I/O is done, so worth the threadsafe count
+	    if (for_checkpoint)
+		(void) toku_sync_fetch_and_increment_uint64(&brt_status.disk_flush_leaf_for_checkpoint);
+	    else
+		(void) toku_sync_fetch_and_increment_uint64(&brt_status.disk_flush_leaf);
+	}
+	else {
+	    if (for_checkpoint)
+		(void) toku_sync_fetch_and_increment_uint64(&brt_status.disk_flush_nonleaf_for_checkpoint);
+	    else
+		(void) toku_sync_fetch_and_increment_uint64(&brt_status.disk_flush_nonleaf);
+	}
     }
     //printf("%s:%d %p->mdict[0]=%p\n", __FILE__, __LINE__, brtnode, brtnode->mdicts[0]);
     *new_size = make_brtnode_pair_attr(brtnode);
@@ -1152,7 +1165,10 @@ void toku_brtnode_free (BRTNODE *nodep) {
 		toku_mempool_destroy(mp);
             }
 	}
+        toku_sync_fetch_and_increment_uint64(&brt_status.destroy_leaf);
     }
+    else
+        toku_sync_fetch_and_increment_uint64(&brt_status.destroy_nonleaf);
     toku_destroy_brtnode_internals(node);
     toku_free(node);
     *nodep=0;

newbrt/txn.c

@@ -15,10 +15,7 @@ BOOL garbage_collection_debug = FALSE;
 static void verify_snapshot_system(TOKULOGGER logger);
 
 // accountability
-static TXN_STATUS_S status = {.begin  = 0, 
-                              .commit = 0,
-                              .abort  = 0,
-                              .close  = 0};
+static TXN_STATUS_S status;
 
 
 void 
@@ -278,6 +275,9 @@ int toku_txn_begin_with_xid (
 
     *tokutxn = result;
     status.begin++;
+    status.num_open++;
+    if (status.num_open > status.max_open)
+	status.max_open = status.num_open;
     if (garbage_collection_debug) {
         verify_snapshot_system(logger);
     }
@@ -470,6 +470,7 @@ void toku_txn_close_txn(TOKUTXN txn) {
         verify_snapshot_system(logger);
 
     status.close++;
+    status.num_open--;
     return;
 }
 

newbrt/txn.h

@@ -64,6 +64,8 @@ typedef struct txn_status {
     u_int64_t   commit;      // successful commits
     u_int64_t   abort;
     u_int64_t   close;       // should be sum of aborts and commits
+    u_int64_t   num_open;    // should be begin - close
+    u_int64_t   max_open;    // max value of num_open
 } TXN_STATUS_S, *TXN_STATUS;
 
 void toku_txn_get_status(TXN_STATUS s);

src/lock_tree/locktree.c

@@ -1455,7 +1455,13 @@ toku_ltm_get_lt(toku_ltm* mgr, toku_lock_tree** ptree,
     *ptree = tree;
     r = 0;
 cleanup:
-    if (r != 0) {
+    if (r == 0) {
+	mgr->status.lt_create++;
+	mgr->status.lt_num++;
+	if (mgr->status.lt_num > mgr->status.lt_num_max)
+	    mgr->status.lt_num_max = mgr->status.lt_num;
+    }
+    else {
         if (tree != NULL) {
             if (added_to_ltm)
                 toku_ltm_remove_lt(mgr, tree);
@@ -1465,6 +1471,7 @@ cleanup:
                 lt_remove_db(tree, db);
             toku_lt_close(tree); 
         }
+	mgr->status.lt_create_fail++;
     }
     return r;
 }
@@ -1476,6 +1483,8 @@ toku_lt_close(toku_lock_tree* tree) {
     if (!tree) { 
         r = EINVAL; goto cleanup; 
     }
+    tree->mgr->status.lt_destroy++;
+    tree->mgr->status.lt_num--;
     toku_lock_request_tree_destroy(tree);
     r = toku_rt_close(tree->borderwrite);
     if (!first_error && r != 0)

src/lock_tree/locktree.h

@@ -128,6 +128,11 @@ typedef struct ltm_status {
     uint64_t   write_lock;                 // number of times write lock taken successfully
     uint64_t   write_lock_fail;            // number of times write lock denied
     uint64_t   out_of_write_locks;         // number of times write lock denied for out_of_locks
+    uint64_t   lt_create;                  // number of locktrees created
+    uint64_t   lt_create_fail;             // number of locktrees unable to be created
+    uint64_t   lt_destroy;                 // number of locktrees destroyed
+    uint64_t   lt_num;                     // number of locktrees (should be created - destroyed)
+    uint64_t   lt_num_max;                 // max number of locktrees that have existed simultaneously
 } LTM_STATUS_S, *LTM_STATUS;
 
 

src/ydb.c

@@ -79,6 +79,10 @@ static u_int64_t num_multi_updates;
 static u_int64_t num_multi_updates_fail;
 static u_int64_t num_point_queries;
 static u_int64_t num_sequential_queries;
+static u_int64_t num_db_open;
+static u_int64_t num_db_close;
+static u_int64_t max_db_open;               
+static u_int64_t num_open_dbs;
 
 static u_int64_t directory_read_locks;        /* total directory read locks taken */ 
 static u_int64_t directory_read_locks_fail;   /* total directory read locks unable to be taken */ 
@@ -2992,6 +2996,10 @@ env_note_db_opened(DB_ENV *env, DB *db) {
     OMTVALUE dbv;
     uint32_t idx;
     env->i->num_open_dbs++;
+    num_open_dbs = env->i->num_open_dbs;
+    num_db_open++;
+    if (num_open_dbs > max_db_open)
+	max_db_open = num_open_dbs;
     r = toku_omt_find_zero(env->i->open_dbs, find_db_by_db, db, &dbv, &idx);
     assert(r==DB_NOTFOUND); //Must not already be there.
     r = toku_omt_insert_at(env->i->open_dbs, db, idx);
@@ -3007,6 +3015,8 @@ env_note_db_closed(DB_ENV *env, DB *db) {
     OMTVALUE dbv;
     uint32_t idx;
     env->i->num_open_dbs--;
+    num_open_dbs = env->i->num_open_dbs;
+    num_db_close++;
     r = toku_omt_find_zero(env->i->open_dbs, find_db_by_db, db, &dbv, &idx);
     assert(r==0); //Must already be there.
     assert((DB*)dbv == db);