[t:3825] #3825 Add memory usage statistics to engine status.

git-svn-id: file:///svn/toku/tokudb@33928 c7de825b-a66e-492c-adef-691d508d4ae1

buildheader/db.h_4_1

@@ -199,6 +199,12 @@ typedef struct __toku_engine_status {
   u_int64_t        env_panic;               /* non-zero if environment is panicked */ 
   u_int64_t        logger_panic;            /* non-zero if logger is panicked */ 
   u_int64_t        logger_panic_errno;      /* non-zero if environment is panicked */ 
+  uint64_t         malloc_count;            /* number of malloc operations */ 
+  uint64_t         free_count;              /* number of free operations */ 
+  uint64_t         realloc_count;           /* number of realloc operations */ 
+  uint64_t         mem_requested;           /* number of bytes requested via malloc/realloc */ 
+  uint64_t         mem_used;                /* number of bytes used (obtained from malloc_usable_size()) */ 
+  uint64_t         mem_freed;               /* number of bytes freed */ 
 } ENGINE_STATUS;
 typedef enum {
  DB_BTREE=1,

buildheader/db.h_4_3

@@ -199,6 +199,12 @@ typedef struct __toku_engine_status {
   u_int64_t        env_panic;               /* non-zero if environment is panicked */ 
   u_int64_t        logger_panic;            /* non-zero if logger is panicked */ 
   u_int64_t        logger_panic_errno;      /* non-zero if environment is panicked */ 
+  uint64_t         malloc_count;            /* number of malloc operations */ 
+  uint64_t         free_count;              /* number of free operations */ 
+  uint64_t         realloc_count;           /* number of realloc operations */ 
+  uint64_t         mem_requested;           /* number of bytes requested via malloc/realloc */ 
+  uint64_t         mem_used;                /* number of bytes used (obtained from malloc_usable_size()) */ 
+  uint64_t         mem_freed;               /* number of bytes freed */ 
 } ENGINE_STATUS;
 typedef enum {
  DB_BTREE=1,

buildheader/db.h_4_4

@@ -199,6 +199,12 @@ typedef struct __toku_engine_status {
   u_int64_t        env_panic;               /* non-zero if environment is panicked */ 
   u_int64_t        logger_panic;            /* non-zero if logger is panicked */ 
   u_int64_t        logger_panic_errno;      /* non-zero if environment is panicked */ 
+  uint64_t         malloc_count;            /* number of malloc operations */ 
+  uint64_t         free_count;              /* number of free operations */ 
+  uint64_t         realloc_count;           /* number of realloc operations */ 
+  uint64_t         mem_requested;           /* number of bytes requested via malloc/realloc */ 
+  uint64_t         mem_used;                /* number of bytes used (obtained from malloc_usable_size()) */ 
+  uint64_t         mem_freed;               /* number of bytes freed */ 
 } ENGINE_STATUS;
 typedef enum {
  DB_BTREE=1,

buildheader/db.h_4_5

@@ -199,6 +199,12 @@ typedef struct __toku_engine_status {
   u_int64_t        env_panic;               /* non-zero if environment is panicked */ 
   u_int64_t        logger_panic;            /* non-zero if logger is panicked */ 
   u_int64_t        logger_panic_errno;      /* non-zero if environment is panicked */ 
+  uint64_t         malloc_count;            /* number of malloc operations */ 
+  uint64_t         free_count;              /* number of free operations */ 
+  uint64_t         realloc_count;           /* number of realloc operations */ 
+  uint64_t         mem_requested;           /* number of bytes requested via malloc/realloc */ 
+  uint64_t         mem_used;                /* number of bytes used (obtained from malloc_usable_size()) */ 
+  uint64_t         mem_freed;               /* number of bytes freed */ 
 } ENGINE_STATUS;
 typedef enum {
  DB_BTREE=1,

buildheader/db.h_4_6

@@ -199,6 +199,12 @@ typedef struct __toku_engine_status {
   u_int64_t        env_panic;               /* non-zero if environment is panicked */ 
   u_int64_t        logger_panic;            /* non-zero if logger is panicked */ 
   u_int64_t        logger_panic_errno;      /* non-zero if environment is panicked */ 
+  uint64_t         malloc_count;            /* number of malloc operations */ 
+  uint64_t         free_count;              /* number of free operations */ 
+  uint64_t         realloc_count;           /* number of realloc operations */ 
+  uint64_t         mem_requested;           /* number of bytes requested via malloc/realloc */ 
+  uint64_t         mem_used;                /* number of bytes used (obtained from malloc_usable_size()) */ 
+  uint64_t         mem_freed;               /* number of bytes freed */ 
 } ENGINE_STATUS;
 typedef enum {
  DB_BTREE=1,

buildheader/make_db_h.c

@@ -593,6 +593,12 @@ int main (int argc __attribute__((__unused__)), char *const argv[] __attribute__
     printf("  u_int64_t        env_panic;               /* non-zero if environment is panicked */ \n");
     printf("  u_int64_t        logger_panic;            /* non-zero if logger is panicked */ \n");
     printf("  u_int64_t        logger_panic_errno;      /* non-zero if environment is panicked */ \n");
+    printf("  uint64_t         malloc_count;            /* number of malloc operations */ \n");
+    printf("  uint64_t         free_count;              /* number of free operations */ \n");
+    printf("  uint64_t         realloc_count;           /* number of realloc operations */ \n");
+    printf("  uint64_t         mem_requested;           /* number of bytes requested via malloc/realloc */ \n");
+    printf("  uint64_t         mem_used;                /* number of bytes used (obtained from malloc_usable_size()) */ \n");
+    printf("  uint64_t         mem_freed;               /* number of bytes freed */ \n");
     printf("} ENGINE_STATUS;\n");
 
     print_dbtype();

buildheader/tdb.h

@@ -199,6 +199,12 @@ typedef struct __toku_engine_status {
   u_int64_t        env_panic;               /* non-zero if environment is panicked */ 
   u_int64_t        logger_panic;            /* non-zero if logger is panicked */ 
   u_int64_t        logger_panic_errno;      /* non-zero if environment is panicked */ 
+  uint64_t         malloc_count;            /* number of malloc operations */ 
+  uint64_t         free_count;              /* number of free operations */ 
+  uint64_t         realloc_count;           /* number of realloc operations */ 
+  uint64_t         mem_requested;           /* number of bytes requested via malloc/realloc */ 
+  uint64_t         mem_used;                /* number of bytes used (obtained from malloc_usable_size()) */ 
+  uint64_t         mem_freed;               /* number of bytes freed */ 
 } ENGINE_STATUS;
 typedef enum {
  DB_BTREE=1,

include/db.h

@@ -199,6 +199,12 @@ typedef struct __toku_engine_status {
   u_int64_t        env_panic;               /* non-zero if environment is panicked */ 
   u_int64_t        logger_panic;            /* non-zero if logger is panicked */ 
   u_int64_t        logger_panic_errno;      /* non-zero if environment is panicked */ 
+  uint64_t         malloc_count;            /* number of malloc operations */ 
+  uint64_t         free_count;              /* number of free operations */ 
+  uint64_t         realloc_count;           /* number of realloc operations */ 
+  uint64_t         mem_requested;           /* number of bytes requested via malloc/realloc */ 
+  uint64_t         mem_used;                /* number of bytes used (obtained from malloc_usable_size()) */ 
+  uint64_t         mem_freed;               /* number of bytes freed */ 
 } ENGINE_STATUS;
 typedef enum {
  DB_BTREE=1,

src/ydb.c

@@ -2015,6 +2015,16 @@ env_get_engine_status(DB_ENV * env, ENGINE_STATUS * engstat, char * env_panic_st
 	    engstat->last_lsn_v13       = persistent_last_lsn_of_v13;
 	    format_time(&persistent_upgrade_v14_time, engstat->upgrade_v14_time);
 	}
+	{
+	    MEMORY_STATUS_S memory_status;
+	    toku_memory_get_status(&memory_status);
+	    engstat->malloc_count  = memory_status.malloc_count;
+	    engstat->free_count    = memory_status.free_count;
+	    engstat->realloc_count = memory_status.realloc_count;
+	    engstat->mem_requested = memory_status.requested;
+	    engstat->mem_used      = memory_status.used;
+	    engstat->mem_freed     = memory_status.freed;
+	}
     }
     return r;
 }
@@ -2184,6 +2194,12 @@ env_get_engine_status_text(DB_ENV * env, char * buff, int bufsiz) {
 	n += snprintf(buff + n, bufsiz - n, "ver_at_startup                   %"PRIu64"\n", engstat.ver_at_startup);
 	n += snprintf(buff + n, bufsiz - n, "last_lsn_v13                     %"PRIu64"\n", engstat.last_lsn_v13);
 	n += snprintf(buff + n, bufsiz - n, "upgrade_v14_time                 %s \n", engstat.upgrade_v14_time);
+	n += snprintf(buff + n, bufsiz - n, "malloc_count                     %"PRIu64"\n", engstat.malloc_count);
+	n += snprintf(buff + n, bufsiz - n, "free_count                       %"PRIu64"\n", engstat.free_count);
+	n += snprintf(buff + n, bufsiz - n, "realloc_count                    %"PRIu64"\n", engstat.realloc_count);
+	n += snprintf(buff + n, bufsiz - n, "mem_requested                    %"PRIu64"\n", engstat.mem_requested);
+	n += snprintf(buff + n, bufsiz - n, "mem_used                         %"PRIu64"\n", engstat.mem_used);
+	n += snprintf(buff + n, bufsiz - n, "mem_freed                        %"PRIu64"\n", engstat.mem_freed);
     }
     if (n > bufsiz) {
 	char * errmsg = "BUFFER TOO SMALL\n";

toku_include/memory.h

@@ -107,6 +107,18 @@ void toku_set_func_xrealloc_only(realloc_fun_t f);
 void toku_set_func_realloc_only(realloc_fun_t f);
 void toku_set_func_free(free_fun_t f);
 
+typedef struct memory_status {
+  uint64_t malloc_count;    // number of malloc operations
+  uint64_t free_count;      // number of free operations
+  uint64_t realloc_count;   // number of realloc operations
+  uint64_t requested;       // number of bytes requested
+  uint64_t used;            // number of bytes used (requested + overhead), obtained from malloc_usable_size()
+  uint64_t freed;           // number of bytes freed;
+} MEMORY_STATUS_S, *MEMORY_STATUS;
+
+void toku_memory_get_status(MEMORY_STATUS s);
+
+
 #if defined(__cplusplus) || defined(__cilkplusplus)
 }
 #endif

windows/memory.c

@@ -5,6 +5,7 @@
 #include <string.h>
 #include <stdio.h>
 #include <stdlib.h>
+#include <malloc.h>
 #include "toku_assert.h"
 
 int toku_memory_check=0;
@@ -15,8 +16,21 @@ static free_fun_t    t_free    = 0;
 static realloc_fun_t t_realloc = 0;
 static realloc_fun_t t_xrealloc = 0;
 
+static MEMORY_STATUS_S status;
+
+void 
+toku_memory_get_status(MEMORY_STATUS s) {
+    *s = status;
+}
+
+
+
 void *toku_malloc(size_t size) {
     void *p = t_malloc ? t_malloc(size) : os_malloc(size);
+    size_t used = malloc_usable_size(p);
+    __sync_add_and_fetch(&status.malloc_count, 1L);
+    __sync_add_and_fetch(&status.requested, size);
+    __sync_add_and_fetch(&status.used, used);
     return p;
 }
 
@@ -30,7 +44,13 @@ toku_calloc(size_t nmemb, size_t size) {
 
 void *
 toku_realloc(void *p, size_t size) {
+    size_t used_orig = malloc_usable_size(p);
     void *q = t_realloc ? t_realloc(p, size) : os_realloc(p, size);
+    size_t used = malloc_usable_size(q);
+    __sync_add_and_fetch(&status.realloc_count, 1L);
+    __sync_add_and_fetch(&status.requested, size);
+    __sync_add_and_fetch(&status.used, used);
+    __sync_add_and_fetch(&status.freed, used_orig);
     return q;
 }
 
@@ -48,6 +68,9 @@ toku_strdup(const char *s) {
 
 void
 toku_free(void *p) {
+    size_t used = malloc_usable_size(p);
+    __sync_add_and_fetch(&status.free_count, 1L);
+    __sync_add_and_fetch(&status.freed, used);
     if (t_free)
 	t_free(p);
     else
@@ -64,6 +87,10 @@ toku_xmalloc(size_t size) {
     void *p = t_xmalloc ? t_xmalloc(size) : os_malloc(size);
     if (p == NULL)  // avoid function call in common case
         resource_assert(p);
+    size_t used = malloc_usable_size(p);
+    __sync_add_and_fetch(&status.malloc_count, 1L);
+    __sync_add_and_fetch(&status.requested, size);
+    __sync_add_and_fetch(&status.used, used);
     return p;
 }
 
@@ -77,9 +104,15 @@ toku_xcalloc(size_t nmemb, size_t size) {
 
 void *
 toku_xrealloc(void *v, size_t size) {
+    size_t used_orig = malloc_usable_size(v);
     void *p = t_xrealloc ? t_xrealloc(v, size) : os_realloc(v, size);
+    size_t used = malloc_usable_size(p);
     if (p == 0)  // avoid function call in common case
         resource_assert(p);
+    __sync_add_and_fetch(&status.realloc_count, 1L);
+    __sync_add_and_fetch(&status.requested, size);
+    __sync_add_and_fetch(&status.used, used);
+    __sync_add_and_fetch(&status.freed, used_orig);
     return p;
 }