Make the leaf nodes have accurate memory footprint info.

git-svn-id: file:///svn/tokudb@4178 c7de825b-a66e-492c-adef-691d508d4ae1

db-benchmark-test/Makefile

@@ -32,7 +32,7 @@ TDB_LDFLAGS = -L../lib -ltokudb -Wl,-rpath,$(PWD)/../lib
 
 TARGET_BDB = db-benchmark-test-bdb
 TARGET_TDB = db-benchmark-test-tokudb
-TARGETS = $(TARGET_BDB) $(TARGET_TDB)
+TARGETS = $(TARGET_BDB) $(TARGET_TDB) scanscan-tokudb
 
 default: build
 build: $(TARGETS) 

db-benchmark-test/db-benchmark-test.c

@@ -329,6 +329,12 @@ int main (int argc, const char *argv[]) {
 	printf("Total time %9.6fs for %lld insertions = %8.0f/s\n", tdiff(&t3, &t1), 
 	       (!noserial+!norandom)*total_n_items, (!noserial+!norandom)*total_n_items/tdiff(&t3, &t1));
     }
+#ifdef TOKUDB
+    if (verbose) {
+	extern unsigned long toku_get_maxrss(void);
+	printf("maxrss=%.2fMB\n", toku_get_maxrss()/256.0);
+    }
+#endif
     return 0;
 }
 

db-benchmark-test/scanscan.c

@@ -4,6 +4,8 @@
 #include <assert.h>
 #include <string.h>
 #include <sys/time.h>
+#include <sys/resource.h>
+#include <unistd.h>
 
 DB_ENV *env;
 DB *db;
@@ -19,6 +21,7 @@ char *dbfilename = "bench.db";
 void setup (void) {
     int r;
     r = db_env_create(&env, 0);                                 assert(r==0);
+    r = env->set_cachesize(env, 0, 127*1024*1024, 1);           assert(r==0);
     r = env->open(env, dbdir, env_open_flags, 0644);            assert(r==0);
     r = db_create(&db, env, 0);                                 assert(r==0);
     r = env->txn_begin(env, 0, &tid, 0);                        assert(r==0);
@@ -32,6 +35,10 @@ void shutdown (void) {
     r = db->close(db, 0);                                       assert(r==0);
     r = tid->commit(tid, 0);                                    assert(r==0);
     r = env->close(env, 0);                                     assert(r==0);
+    {
+	extern unsigned long toku_get_maxrss(void);
+	printf("maxrss=%.2fMB\n", toku_get_maxrss()/256.0);
+    }
 }
 
 double gettime (void) {

newbrt/brt.c

@@ -84,8 +84,25 @@ void toku_brtnode_free (BRTNODE *nodep) {
     *nodep=0;
 }
 
-static long brtnode_size(BRTNODE node) {
+static long brtnode_memory_size(BRTNODE node) {
+    if (node->height>0) {
 	return toku_serialize_brtnode_size(node);
+#if 0
+	int n_children = node->u.n.n_children;
+	int fifo_sum=0;
+	int i;
+	for (i=0; i<n_children; i++) {
+	    fifo_sum+=toku_fifo_memory_size(node->u.n.childinfos[i].buffer);
+	}
+	return sizeof(*node)
+	    +(1+n_children)*(sizeof(node->u.n.childinfos[0]))
+	    +(n_children)+(sizeof(node->u.n.childkeys[0]))
+	    +node->u.n.totalchildkeylens
+	    +fifo_sum;
+#endif
+    } else {
+	return sizeof(*node)+toku_omt_memory_size(node->u.l.buffer)+toku_mempool_memory_size(&node->u.l.buffer_mempool);
+    }
 }
 
 
@@ -163,7 +180,7 @@ int toku_brtnode_fetch_callback (CACHEFILE cachefile, DISKOFF nodename, void **b
     BRTNODE *result=(BRTNODE*)brtnode_pv;
     int r = toku_deserialize_brtnode_from(toku_cachefile_fd(cachefile), nodename, result);
     if (r == 0) {
-        *sizep = brtnode_size(*result);
+        *sizep = brtnode_memory_size(*result);
 	*written_lsn = (*result)->disk_lsn;
     }
     //(*result)->parent_brtnode = 0; /* Don't know it right now. */
@@ -233,7 +250,7 @@ int toku_unpin_brtnode (BRT brt, BRTNODE node) {
 //	//if (node->log_lsn.lsn>33320) printf("%s:%d node%lld lsn=%lld\n", __FILE__, __LINE__, node->thisnodename, node->log_lsn.lsn);
 //    }
     VERIFY_NODE(node);
-    return toku_cachetable_unpin(brt->cf, node->thisnodename, node->dirty, brtnode_size(node));
+    return toku_cachetable_unpin(brt->cf, node->thisnodename, node->dirty, brtnode_memory_size(node));
 }
 
 typedef struct kvpair {
@@ -345,7 +362,7 @@ int toku_create_new_brtnode (BRT t, BRTNODE *result, int height, TOKULOGGER logg
     assert(n->nodesize>0);
     //    n->brt            = t;
     //printf("%s:%d putting %p (%lld)\n", __FILE__, __LINE__, n, n->thisnodename);
-    r=toku_cachetable_put(t->cf, n->thisnodename, n, brtnode_size(n),
+    r=toku_cachetable_put(t->cf, n->thisnodename, n, brtnode_memory_size(n),
 			  toku_brtnode_flush_callback, toku_brtnode_fetch_callback, t);
     assert(r==0);
     return 0;
@@ -1852,7 +1869,7 @@ static int setup_initial_brt_root_node (BRT t, DISKOFF offset, TOKULOGGER logger
 	printf("%s:%d put root at %lld\n", __FILE__, __LINE__, offset);
     }
     //printf("%s:%d putting %p (%lld)\n", __FILE__, __LINE__, node, node->thisnodename);
-    r=toku_cachetable_put(t->cf, offset, node, brtnode_size(node),
+    r=toku_cachetable_put(t->cf, offset, node, brtnode_memory_size(node),
 			  toku_brtnode_flush_callback, toku_brtnode_fetch_callback, t);
     if (r!=0) {
 	toku_free(node);
@@ -2287,7 +2304,7 @@ static int brt_init_new_root(BRT brt, BRTNODE nodea, BRTNODE nodeb, DBT splitk,
     r = toku_unpin_brtnode(brt, nodeb);
     if (r!=0) return r;
     //printf("%s:%d put %lld\n", __FILE__, __LINE__, newroot_diskoff);
-    toku_cachetable_put(brt->cf, newroot_diskoff, newroot, brtnode_size(newroot),
+    toku_cachetable_put(brt->cf, newroot_diskoff, newroot, brtnode_memory_size(newroot),
                         toku_brtnode_flush_callback, toku_brtnode_fetch_callback, brt);
     *newrootp = newroot;
     return 0;
@@ -2587,7 +2604,7 @@ static int brt_search_child(BRT brt, BRTNODE node, int childnum, brt_search_t *s
         } else {
             if (r == EAGAIN) 
                 continue;
-            rr = toku_cachetable_unpin(brt->cf, childnode->thisnodename, childnode->dirty, brtnode_size(childnode)); 
+            rr = toku_cachetable_unpin(brt->cf, childnode->thisnodename, childnode->dirty, brtnode_memory_size(childnode)); 
             assert(rr == 0);
             break;
         }

newbrt/cachetable.c

@@ -345,11 +345,35 @@ static void flush_and_remove (CACHETABLE t, PAIR remove_me, int write_me) {
     toku_free(remove_me);
 }
 
+static unsigned long toku_maxrss=0;
+unsigned long toku_get_maxrss(void) __attribute__((__visibility__("default")));
+unsigned long toku_get_maxrss(void) {
+    return toku_maxrss;
+}
+
+static unsigned long check_maxrss (void) __attribute__((__unused__));
+static unsigned long check_maxrss (void) {
+    pid_t pid = getpid();
+    char fname[100];
+    snprintf(fname, sizeof(fname), "/proc/%u/statm", pid);
+    FILE *f = fopen(fname, "r");
+    unsigned long ignore, rss;
+    fscanf(f, "%lu %lu", &ignore, &rss);
+    fclose(f);
+    if (toku_maxrss<rss) toku_maxrss=rss;
+    return rss;
+}
+
+
 static int maybe_flush_some (CACHETABLE t, long size __attribute__((unused))) {
     int r = 0;
 again:
 //    if (t->n_in_table >= t->table_size) {
     if (size + t->size_current > t->size_limit) {
+	{
+	    unsigned long rss __attribute__((__unused__)) = check_maxrss();
+	    //printf("this-size=%.6fMB projected size = %.2fMB  limit=%2.fMB  rss=%2.fMB\n", size/(1024.0*1024.0), (size+t->size_current)/(1024.0*1024.0), t->size_limit/(1024.0*1024.0), rss/256.0);
+	}
         /* Try to remove one. */
 	PAIR remove_me;
 	for (remove_me = t->tail; remove_me; remove_me = remove_me->prev) {
@@ -439,7 +463,6 @@ int toku_cachetable_get_and_pin(CACHEFILE cachefile, CACHEKEY key, void**value,
 	}
     }
     int r;
-    if ((r=maybe_flush_some(t, 1))) return r;
     // Note.  hashit(t,key) may have changed as a result of flushing.
     {
 	void *toku_value; 
@@ -455,6 +478,7 @@ int toku_cachetable_get_and_pin(CACHEFILE cachefile, CACHEKEY key, void**value,
             *sizep = size;
         // maybe_flush_some(t, size);
     }
+    if ((r=maybe_flush_some(t, 0))) return r;
     WHEN_TRACE_CT(printf("%s:%d did fetch: cachtable_get_and_pin(%lld)--> %p\n", __FILE__, __LINE__, key, *value));
     return 0;
 }
@@ -491,6 +515,10 @@ int toku_cachetable_unpin(CACHEFILE cachefile, CACHEKEY key, int dirty, long siz
                 t->size_current += p->size;
             }
 	    WHEN_TRACE_CT(printf("[count=%lld]\n", p->pinned));
+	    {
+		int r;
+		if ((r=maybe_flush_some(t, 0))) return r;
+	    }
 	    return 0;
 	}
     }

newbrt/mempool.c

@@ -66,3 +66,7 @@ void toku_mempool_mfree(struct mempool *mp, void *vp, int size) {
     mp->frag_size += size;
     assert(mp->frag_size <= mp->size);
 }
+
+unsigned long toku_mempool_memory_size(struct mempool *mp) {
+    return mp->size; 
+}

newbrt/mempool.h

@@ -55,4 +55,7 @@ static inline int toku_mempool_inrange(struct mempool *mp, void *vp, int size) {
     return mp->base <= vp && vp + size <= mp->base + mp->size;
 }
 
+unsigned long toku_mempool_memory_size(struct mempool *mp);
+// Effect: Return the number of bytes that the mempool is using in main memory.  Include fragmented space.  Don't include the mp itself.
+
 #endif

newbrt/omt.c

@@ -524,3 +524,6 @@ void toku_omt_clear(OMT omt) {
     omt->root     = NODE_NULL;
 }
 
+unsigned long toku_omt_memory_size (OMT omt) {
+    return sizeof(*omt)+omt->node_capacity*sizeof(omt->nodes[0]) + omt->tmparray_size*sizeof(omt->tmparray[0]);
+}

newbrt/omt.h

@@ -282,5 +282,8 @@ void toku_omt_clear(OMT omt);
 //  Note: Will not resize the array, since void precludes allowing a malloc.
 // Performance: time=O(1)
 
+unsigned long toku_omt_memory_size (OMT omt);
+// Effect: Return the size (in bytes) of the omt, as it resides in main memory.  Don't include any of the OMTVALUES.
+
 #endif  /* #ifndef OMT_H */
 

src/export.map

@@ -7,6 +7,8 @@
    log_compare;
    db_env_set_func_fsync;   
 
+   toku_get_maxrss;
+
    toku_ydb_error_all_cases;
    toku_set_trace_file;
    toku_close_trace_file;