[t:3825] #3825 Merged changesets 33440, 33527, 33536 from 5.1.4 branch to main.

git-svn-id: file:///svn/toku/tokudb@33975 c7de825b-a66e-492c-adef-691d508d4ae1

buildheader/db.h_4_1

@@ -107,6 +107,9 @@ typedef struct __toku_engine_status {
   u_int64_t        maybe_get_and_pin_if_in_memorys;  /* how many times has get_and_pin_if_in_memory been called */ 
   int64_t          cachetable_size_current; /* sum of the sizes of the nodes represented in the cachetable */ 
   int64_t          cachetable_size_limit;   /* the limit to the sum of the node sizes */ 
+  int64_t          cachetable_size_max;     /* the max value (high water mark) of cachetable_size_current */ 
+  uint64_t         cachetable_size_leaf;    /* the number of bytes of leaf nodes */ 
+  uint64_t         cachetable_size_nonleaf; /* the number of bytes of nonleaf nodes */ 
   int64_t          cachetable_size_writing; /* the sum of the sizes of the nodes being written */ 
   int64_t          get_and_pin_footprint;   /* state of get_and_pin procedure */ 
   int64_t          local_checkpoint;        /* number of times a local checkpoint is taken for commit */ 

buildheader/db.h_4_3

@@ -107,6 +107,9 @@ typedef struct __toku_engine_status {
   u_int64_t        maybe_get_and_pin_if_in_memorys;  /* how many times has get_and_pin_if_in_memory been called */ 
   int64_t          cachetable_size_current; /* sum of the sizes of the nodes represented in the cachetable */ 
   int64_t          cachetable_size_limit;   /* the limit to the sum of the node sizes */ 
+  int64_t          cachetable_size_max;     /* the max value (high water mark) of cachetable_size_current */ 
+  uint64_t         cachetable_size_leaf;    /* the number of bytes of leaf nodes */ 
+  uint64_t         cachetable_size_nonleaf; /* the number of bytes of nonleaf nodes */ 
   int64_t          cachetable_size_writing; /* the sum of the sizes of the nodes being written */ 
   int64_t          get_and_pin_footprint;   /* state of get_and_pin procedure */ 
   int64_t          local_checkpoint;        /* number of times a local checkpoint is taken for commit */ 

buildheader/db.h_4_4

@@ -107,6 +107,9 @@ typedef struct __toku_engine_status {
   u_int64_t        maybe_get_and_pin_if_in_memorys;  /* how many times has get_and_pin_if_in_memory been called */ 
   int64_t          cachetable_size_current; /* sum of the sizes of the nodes represented in the cachetable */ 
   int64_t          cachetable_size_limit;   /* the limit to the sum of the node sizes */ 
+  int64_t          cachetable_size_max;     /* the max value (high water mark) of cachetable_size_current */ 
+  uint64_t         cachetable_size_leaf;    /* the number of bytes of leaf nodes */ 
+  uint64_t         cachetable_size_nonleaf; /* the number of bytes of nonleaf nodes */ 
   int64_t          cachetable_size_writing; /* the sum of the sizes of the nodes being written */ 
   int64_t          get_and_pin_footprint;   /* state of get_and_pin procedure */ 
   int64_t          local_checkpoint;        /* number of times a local checkpoint is taken for commit */ 

buildheader/db.h_4_5

@@ -107,6 +107,9 @@ typedef struct __toku_engine_status {
   u_int64_t        maybe_get_and_pin_if_in_memorys;  /* how many times has get_and_pin_if_in_memory been called */ 
   int64_t          cachetable_size_current; /* sum of the sizes of the nodes represented in the cachetable */ 
   int64_t          cachetable_size_limit;   /* the limit to the sum of the node sizes */ 
+  int64_t          cachetable_size_max;     /* the max value (high water mark) of cachetable_size_current */ 
+  uint64_t         cachetable_size_leaf;    /* the number of bytes of leaf nodes */ 
+  uint64_t         cachetable_size_nonleaf; /* the number of bytes of nonleaf nodes */ 
   int64_t          cachetable_size_writing; /* the sum of the sizes of the nodes being written */ 
   int64_t          get_and_pin_footprint;   /* state of get_and_pin procedure */ 
   int64_t          local_checkpoint;        /* number of times a local checkpoint is taken for commit */ 

buildheader/db.h_4_6

@@ -107,6 +107,9 @@ typedef struct __toku_engine_status {
   u_int64_t        maybe_get_and_pin_if_in_memorys;  /* how many times has get_and_pin_if_in_memory been called */ 
   int64_t          cachetable_size_current; /* sum of the sizes of the nodes represented in the cachetable */ 
   int64_t          cachetable_size_limit;   /* the limit to the sum of the node sizes */ 
+  int64_t          cachetable_size_max;     /* the max value (high water mark) of cachetable_size_current */ 
+  uint64_t         cachetable_size_leaf;    /* the number of bytes of leaf nodes */ 
+  uint64_t         cachetable_size_nonleaf; /* the number of bytes of nonleaf nodes */ 
   int64_t          cachetable_size_writing; /* the sum of the sizes of the nodes being written */ 
   int64_t          get_and_pin_footprint;   /* state of get_and_pin procedure */ 
   int64_t          local_checkpoint;        /* number of times a local checkpoint is taken for commit */ 

buildheader/make_db_h.c

@@ -501,6 +501,9 @@ int main (int argc __attribute__((__unused__)), char *const argv[] __attribute__
     printf("  u_int64_t        maybe_get_and_pin_if_in_memorys;  /* how many times has get_and_pin_if_in_memory been called */ \n");
     printf("  int64_t          cachetable_size_current; /* sum of the sizes of the nodes represented in the cachetable */ \n");
     printf("  int64_t          cachetable_size_limit;   /* the limit to the sum of the node sizes */ \n");
+    printf("  int64_t          cachetable_size_max;     /* the max value (high water mark) of cachetable_size_current */ \n");
+    printf("  uint64_t         cachetable_size_leaf;    /* the number of bytes of leaf nodes */ \n");
+    printf("  uint64_t         cachetable_size_nonleaf; /* the number of bytes of nonleaf nodes */ \n");
     printf("  int64_t          cachetable_size_writing; /* the sum of the sizes of the nodes being written */ \n");
     printf("  int64_t          get_and_pin_footprint;   /* state of get_and_pin procedure */ \n");
     printf("  int64_t          local_checkpoint;        /* number of times a local checkpoint is taken for commit */ \n");

buildheader/tdb.h

@@ -107,6 +107,9 @@ typedef struct __toku_engine_status {
   u_int64_t        maybe_get_and_pin_if_in_memorys;  /* how many times has get_and_pin_if_in_memory been called */ 
   int64_t          cachetable_size_current; /* sum of the sizes of the nodes represented in the cachetable */ 
   int64_t          cachetable_size_limit;   /* the limit to the sum of the node sizes */ 
+  int64_t          cachetable_size_max;     /* the max value (high water mark) of cachetable_size_current */ 
+  uint64_t         cachetable_size_leaf;    /* the number of bytes of leaf nodes */ 
+  uint64_t         cachetable_size_nonleaf; /* the number of bytes of nonleaf nodes */ 
   int64_t          cachetable_size_writing; /* the sum of the sizes of the nodes being written */ 
   int64_t          get_and_pin_footprint;   /* state of get_and_pin procedure */ 
   int64_t          local_checkpoint;        /* number of times a local checkpoint is taken for commit */ 

include/db.h

@@ -107,6 +107,9 @@ typedef struct __toku_engine_status {
   u_int64_t        maybe_get_and_pin_if_in_memorys;  /* how many times has get_and_pin_if_in_memory been called */ 
   int64_t          cachetable_size_current; /* sum of the sizes of the nodes represented in the cachetable */ 
   int64_t          cachetable_size_limit;   /* the limit to the sum of the node sizes */ 
+  int64_t          cachetable_size_max;     /* the max value (high water mark) of cachetable_size_current */ 
+  uint64_t         cachetable_size_leaf;    /* the number of bytes of leaf nodes */ 
+  uint64_t         cachetable_size_nonleaf; /* the number of bytes of nonleaf nodes */ 
   int64_t          cachetable_size_writing; /* the sum of the sizes of the nodes being written */ 
   int64_t          get_and_pin_footprint;   /* state of get_and_pin procedure */ 
   int64_t          local_checkpoint;        /* number of times a local checkpoint is taken for commit */ 

newbrt/brt-internal.h

@@ -717,6 +717,8 @@ typedef struct brt_status {
     u_int64_t msn_discards;             // how many messages were ignored by leaf because of msn
     u_int64_t max_workdone;             // max workdone value of any buffer
     u_int64_t dsn_gap;                  // dsn has detected a gap in continuity of root-to-leaf path (internal node was evicted and re-read)
+    u_int64_t bytes_leaf;               // memory used by leaf nodes
+    u_int64_t bytes_nonleaf;            // memory used by nonleaf nodes
 } BRT_STATUS_S, *BRT_STATUS;
 
 void toku_brt_get_status(BRT_STATUS);

newbrt/brt.c

@@ -639,6 +639,11 @@ void toku_brtnode_flush_callback (CACHEFILE cachefile, int fd, BLOCKNUM nodename
     }
     //printf("%s:%d %p->mdict[0]=%p\n", __FILE__, __LINE__, brtnode, brtnode->mdicts[0]);
     if (!keep_me) {
+	uint64_t bytes_freed = brtnode_memory_size(brtnode);
+	if (brtnode->height == 0)
+	    brt_status.bytes_leaf -= bytes_freed;
+	else
+	    brt_status.bytes_nonleaf -= bytes_freed;
 	toku_brtnode_free(&brtnode);
     }
     //printf("%s:%d n_items_malloced=%lld\n", __FILE__, __LINE__, n_items_malloced);
@@ -660,6 +665,10 @@ int toku_brtnode_fetch_callback (CACHEFILE UU(cachefile), int fd, BLOCKNUM noden
 	*dirtyp = (*result)->dirty;
     }
     //    printf("fetch node %"PRIu64"\n", nodename.b);
+    if ((*result)->height == 0)
+	brt_status.bytes_leaf += *sizep;
+    else
+	brt_status.bytes_nonleaf += *sizep;
     return r;
 }
 
@@ -713,6 +722,11 @@ int toku_brtnode_pe_callback (void *brtnode_pv, long bytes_to_free, long* bytes_
         }
     }
     *bytes_freed = orig_size - brtnode_memory_size(node);
+    invariant(*bytes_freed >= 0);
+    if (node->height == 0)
+	brt_status.bytes_leaf -= *bytes_freed;
+    else
+	brt_status.bytes_nonleaf -= *bytes_freed;
     return 0;
 }
 
@@ -811,6 +825,7 @@ BOOL toku_brtnode_pf_req_callback(void* brtnode_pv, void* read_extraargs) {
 // could have just used toku_brtnode_fetch_callback, but wanted to separate the two cases to separate functions
 int toku_brtnode_pf_callback(void* brtnode_pv, void* read_extraargs, int fd, long* sizep) {
     BRTNODE node = brtnode_pv;
+    long orig_size = brtnode_memory_size(node);
     struct brtnode_fetch_extra *bfe = read_extraargs;
     // there must be a reason this is being called. If we get a garbage type or the type is brtnode_fetch_none,
     // then something went wrong
@@ -844,6 +859,11 @@ int toku_brtnode_pf_callback(void* brtnode_pv, void* read_extraargs, int fd, lon
     }
     cilk_sync;
     *sizep = brtnode_memory_size(node);
+    uint64_t bytes_added = *sizep - orig_size;
+    if (node->height == 0)
+	brt_status.bytes_leaf += bytes_added;
+    else
+	brt_status.bytes_nonleaf += bytes_added;
     return 0;
 }
 

newbrt/cachetable.c

@@ -138,6 +138,7 @@ struct cachetable {
     int64_t size_reserved;           // How much memory is reserved (e.g., by the loader)
     int64_t size_current;            // the sum of the sizes of the pairs in the cachetable
     int64_t size_limit;              // the limit to the sum of the pair sizes
+    int64_t size_max;                // high water mark of size_current (max value size_current ever had)
     TOKULOGGER logger;
     toku_pthread_mutex_t *mutex;  // coarse lock that protects the cachetable, the cachefiles, and the pairs
     toku_pthread_mutex_t cachefiles_mutex;  // lock that protects the cachefiles list
@@ -292,6 +293,8 @@ u_int64_t toku_cachetable_reserve_memory(CACHETABLE ct, double fraction) {
 	}
     }
     ct->size_current += reserved_memory;
+    if (ct->size_current > ct->size_max)
+	ct->size_max = ct->size_current;
     cachetable_unlock(ct);
     return reserved_memory;
 }
@@ -1133,6 +1136,8 @@ static int cachetable_fetch_pair(
         p->value = toku_value;
         p->size = size;
         ct->size_current += size;
+	if (ct->size_current > ct->size_max)
+	    ct->size_max = ct->size_current;
         if (p->cq) {
             workitem_init(&p->asyncwork, NULL, p);
             workqueue_enq(p->cq, &p->asyncwork, 1);
@@ -1356,6 +1361,8 @@ static PAIR cachetable_insert_at(CACHETABLE ct,
     ct->table[h] = p;
     ct->n_in_table++;
     ct->size_current += size;
+    if (ct->size_current > ct->size_max)
+	ct->size_max = ct->size_current;
     if (ct->n_in_table > ct->table_size) {
         cachetable_rehash(ct, ct->table_size*2);
     }
@@ -1498,6 +1505,8 @@ do_partial_fetch(CACHETABLE ct, CACHEFILE cachefile, PAIR p, CACHETABLE_PARTIAL_
     p->size = size;
     ct->size_current += size;
     ct->size_current -= old_size;
+    if (ct->size_current > ct->size_max)
+	ct->size_max = ct->size_current;
     p->state = CTPAIR_IDLE;
     if (p->cq) {
         workitem_init(&p->asyncwork, NULL, p);
@@ -1811,6 +1820,8 @@ toku_cachetable_unpin_internal(CACHEFILE cachefile, CACHEKEY key, u_int32_t full
                 ct->size_current -= p->size; 
                 p->size = size;
                 ct->size_current += p->size; 
+		if (ct->size_current > ct->size_max)
+		    ct->size_max = ct->size_current;
             }
 	    WHEN_TRACE_CT(printf("[count=%lld]\n", p->pinned));
 	    {
@@ -2804,7 +2815,7 @@ void toku_cachetable_print_state (CACHETABLE ct) {
     cachetable_unlock(ct);
 }
 
-void toku_cachetable_get_state (CACHETABLE ct, int *num_entries_ptr, int *hash_size_ptr, long *size_current_ptr, long *size_limit_ptr) {
+void toku_cachetable_get_state (CACHETABLE ct, int *num_entries_ptr, int *hash_size_ptr, long *size_current_ptr, long *size_limit_ptr, int64_t *size_max_ptr) {
     cachetable_lock(ct);
     if (num_entries_ptr) 
         *num_entries_ptr = ct->n_in_table;
@@ -2814,6 +2825,8 @@ void toku_cachetable_get_state (CACHETABLE ct, int *num_entries_ptr, int *hash_s
         *size_current_ptr = ct->size_current;
     if (size_limit_ptr)
         *size_limit_ptr = ct->size_limit;
+    if (size_max_ptr)
+        *size_max_ptr = ct->size_max;
     cachetable_unlock(ct);
 }
 
@@ -2945,6 +2958,7 @@ void toku_cachetable_get_status(CACHETABLE ct, CACHETABLE_STATUS s) {
     s->get_and_pin_if_in_memorys = cachetable_get_and_pin_if_in_memorys;
     s->size_current = ct->size_current;          
     s->size_limit   = ct->size_limit;            
+    s->size_max     = ct->size_max;
     s->size_writing = 0;          
     s->get_and_pin_footprint = get_and_pin_footprint;
     s->local_checkpoint      = local_checkpoint;

newbrt/cachetable.h

@@ -364,7 +364,7 @@ u_int32_t toku_cachefile_fullhash_of_header (CACHEFILE cachefile);
 void toku_cachetable_print_state (CACHETABLE ct);
 
 // Get the state of the cachetable. This is used to verify the cachetable
-void toku_cachetable_get_state(CACHETABLE ct, int *num_entries_ptr, int *hash_size_ptr, long *size_current_ptr, long *size_limit_ptr);
+void toku_cachetable_get_state(CACHETABLE ct, int *num_entries_ptr, int *hash_size_ptr, long *size_current_ptr, long *size_limit_ptr, int64_t *size_max_ptr);
 
 // Get the state of a cachetable entry by key. This is used to verify the cachetable
 int toku_cachetable_get_key_state(CACHETABLE ct, CACHEKEY key, CACHEFILE cf,
@@ -406,6 +406,7 @@ typedef struct cachetable_status {
     u_int64_t get_and_pin_if_in_memorys; // how many times has get_and_pin_if_in_memory been called?
     int64_t   size_current;            // the sum of the sizes of the nodes represented in the cachetable
     int64_t   size_limit;              // the limit to the sum of the node sizes
+    int64_t   size_max;                // high water mark of size_current (max value size_current ever had)
     int64_t   size_writing;            // the sum of the sizes of the nodes being written
     u_int64_t get_and_pin_footprint;   
     uint64_t  local_checkpoint;        // number of times a local checkpoint was taken for a commit (2440)

newbrt/tests/cachetable-debug-test.c

@@ -40,8 +40,8 @@ cachetable_debug_test (int n) {
     CACHEFILE f1;
     r = toku_cachetable_openf(&f1, ct, fname1, O_RDWR|O_CREAT, S_IRWXU|S_IRWXG|S_IRWXO); assert(r == 0);
 
-    int num_entries, hash_size; long size_current, size_limit;
-    toku_cachetable_get_state(ct, &num_entries, &hash_size, &size_current, &size_limit);
+    int num_entries, hash_size; long size_current, size_limit, size_max;
+    toku_cachetable_get_state(ct, &num_entries, &hash_size, &size_current, &size_limit, &size_max);
     assert(num_entries == 0);
     assert(size_current == 0);
     assert(size_limit == n);
@@ -66,7 +66,7 @@ cachetable_debug_test (int n) {
         r = toku_cachetable_unpin(f1, make_blocknum(i), hi, CACHETABLE_CLEAN, 1);
         assert(r == 0);
 
-        toku_cachetable_get_state(ct, &num_entries, &hash_size, &size_current, &size_limit);
+        toku_cachetable_get_state(ct, &num_entries, &hash_size, &size_current, &size_limit, &size_max);
         assert(num_entries == i);
         assert(size_current == i);
         assert(size_limit == n);

newbrt/tests/cachetable-test.c

@@ -689,11 +689,11 @@ static void test_size_flush(void) {
         r = toku_cachetable_put(f, key, hkey, value, size, test_size_flush_callback, pe_callback, 0);
         assert(r == 0);
 
-        int n_entries, hash_size; long size_current, size_limit;
-        toku_cachetable_get_state(t, &n_entries, &hash_size, &size_current, &size_limit);
+        int n_entries, hash_size; long size_current, size_limit, size_max;
+        toku_cachetable_get_state(t, &n_entries, &hash_size, &size_current, &size_limit, &size_max);
         while (n_entries != min2(i+1, n)) {
             toku_pthread_yield(); maybe_flush(t);
-            toku_cachetable_get_state(t, &n_entries, 0, 0, 0);
+            toku_cachetable_get_state(t, &n_entries, 0, 0, 0, 0);
         }
         assert(n_entries == min2(i+1, n));
 

newbrt/tests/cachetable-unpin-and-remove-test.c

@@ -108,7 +108,7 @@ cachetable_unpin_and_remove_test (int n) {
 
     // verify that the cachtable is empty
     int nentries;
-    toku_cachetable_get_state(ct, &nentries, NULL, NULL, NULL);
+    toku_cachetable_get_state(ct, &nentries, NULL, NULL, NULL, NULL);
     assert(nentries == 0);
 
     char *error_string;

src/ydb.c

@@ -1879,6 +1879,7 @@ env_get_engine_status(DB_ENV * env, ENGINE_STATUS * engstat, char * env_panic_st
 	    engstat->maybe_get_and_pin_hits   = ctstat.maybe_get_and_pin_hits;
 	    engstat->cachetable_size_current  = ctstat.size_current;
 	    engstat->cachetable_size_limit    = ctstat.size_limit;
+	    engstat->cachetable_size_max      = ctstat.size_max;
 	    engstat->cachetable_size_writing  = ctstat.size_writing;
 	    engstat->get_and_pin_footprint    = ctstat.get_and_pin_footprint;
 	    engstat->local_checkpoint         = ctstat.local_checkpoint;
@@ -1941,6 +1942,8 @@ env_get_engine_status(DB_ENV * env, ENGINE_STATUS * engstat, char * env_panic_st
 	    engstat->msn_discards = brt_stat.msn_discards;
 	    engstat->max_workdone = brt_stat.max_workdone;
 	    engstat->dsn_gap = brt_stat.dsn_gap;	    
+	    engstat->cachetable_size_leaf = brt_stat.bytes_leaf;
+	    engstat->cachetable_size_nonleaf = brt_stat.bytes_nonleaf;
 	}
 	{
 	    u_int64_t fsync_count, fsync_time;
@@ -2113,6 +2116,9 @@ env_get_engine_status_text(DB_ENV * env, char * buff, int bufsiz) {
 	n += snprintf(buff + n, bufsiz - n, "maybe_get_and_pin_hits           %"PRIu64"\n", engstat.maybe_get_and_pin_hits);
 	n += snprintf(buff + n, bufsiz - n, "cachetable_size_current          %"PRId64"\n", engstat.cachetable_size_current);
 	n += snprintf(buff + n, bufsiz - n, "cachetable_size_limit            %"PRId64"\n", engstat.cachetable_size_limit);
+	n += snprintf(buff + n, bufsiz - n, "cachetable_size_max              %"PRId64"\n", engstat.cachetable_size_max);
+	n += snprintf(buff + n, bufsiz - n, "cachetable_size_leaf             %"PRId64"\n", engstat.cachetable_size_leaf);
+	n += snprintf(buff + n, bufsiz - n, "cachetable_size_nonleaf          %"PRId64"\n", engstat.cachetable_size_nonleaf);
 	n += snprintf(buff + n, bufsiz - n, "cachetable_size_writing          %"PRId64"\n", engstat.cachetable_size_writing);
 	n += snprintf(buff + n, bufsiz - n, "get_and_pin_footprint            %"PRId64"\n", engstat.get_and_pin_footprint);
 	n += snprintf(buff + n, bufsiz - n, "local_checkpoint                 %"PRId64"\n", engstat.local_checkpoint);