[t:4182] #4182 Merge tokudb.4182 to main. Add mallocator version string to engine status.

git-svn-id: file:///svn/toku/tokudb@37423 c7de825b-a66e-492c-adef-691d508d4ae1

buildheader/db.h_4_1

@@ -269,6 +269,7 @@ typedef struct __toku_engine_status {
   uint64_t         mem_used;                /* number of bytes used (obtained from malloc_usable_size()) */ 
   uint64_t         mem_freed;               /* number of bytes freed */ 
   uint64_t         max_mem_in_use;          /* estimated max value of (used - freed) */ 
+  const char *     mallocator_version;      /* version string from malloc lib */ 
 } ENGINE_STATUS;
 typedef enum {
  DB_BTREE=1,

buildheader/db.h_4_3

@@ -269,6 +269,7 @@ typedef struct __toku_engine_status {
   uint64_t         mem_used;                /* number of bytes used (obtained from malloc_usable_size()) */ 
   uint64_t         mem_freed;               /* number of bytes freed */ 
   uint64_t         max_mem_in_use;          /* estimated max value of (used - freed) */ 
+  const char *     mallocator_version;      /* version string from malloc lib */ 
 } ENGINE_STATUS;
 typedef enum {
  DB_BTREE=1,

buildheader/db.h_4_4

@@ -269,6 +269,7 @@ typedef struct __toku_engine_status {
   uint64_t         mem_used;                /* number of bytes used (obtained from malloc_usable_size()) */ 
   uint64_t         mem_freed;               /* number of bytes freed */ 
   uint64_t         max_mem_in_use;          /* estimated max value of (used - freed) */ 
+  const char *     mallocator_version;      /* version string from malloc lib */ 
 } ENGINE_STATUS;
 typedef enum {
  DB_BTREE=1,

buildheader/db.h_4_5

@@ -269,6 +269,7 @@ typedef struct __toku_engine_status {
   uint64_t         mem_used;                /* number of bytes used (obtained from malloc_usable_size()) */ 
   uint64_t         mem_freed;               /* number of bytes freed */ 
   uint64_t         max_mem_in_use;          /* estimated max value of (used - freed) */ 
+  const char *     mallocator_version;      /* version string from malloc lib */ 
 } ENGINE_STATUS;
 typedef enum {
  DB_BTREE=1,

buildheader/db.h_4_6

@@ -269,6 +269,7 @@ typedef struct __toku_engine_status {
   uint64_t         mem_used;                /* number of bytes used (obtained from malloc_usable_size()) */ 
   uint64_t         mem_freed;               /* number of bytes freed */ 
   uint64_t         max_mem_in_use;          /* estimated max value of (used - freed) */ 
+  const char *     mallocator_version;      /* version string from malloc lib */ 
 } ENGINE_STATUS;
 typedef enum {
  DB_BTREE=1,

buildheader/make_db_h.c

@@ -664,6 +664,7 @@ int main (int argc __attribute__((__unused__)), char *const argv[] __attribute__
     printf("  uint64_t         mem_used;                /* number of bytes used (obtained from malloc_usable_size()) */ \n");
     printf("  uint64_t         mem_freed;               /* number of bytes freed */ \n");
     printf("  uint64_t         max_mem_in_use;          /* estimated max value of (used - freed) */ \n");
+    printf("  const char *     mallocator_version;      /* version string from malloc lib */ \n");
     printf("} ENGINE_STATUS;\n");
 
     print_dbtype();

buildheader/tdb.h

@@ -269,6 +269,7 @@ typedef struct __toku_engine_status {
   uint64_t         mem_used;                /* number of bytes used (obtained from malloc_usable_size()) */ 
   uint64_t         mem_freed;               /* number of bytes freed */ 
   uint64_t         max_mem_in_use;          /* estimated max value of (used - freed) */ 
+  const char *     mallocator_version;      /* version string from malloc lib */ 
 } ENGINE_STATUS;
 typedef enum {
  DB_BTREE=1,

include/db.h

@@ -269,6 +269,7 @@ typedef struct __toku_engine_status {
   uint64_t         mem_used;                /* number of bytes used (obtained from malloc_usable_size()) */ 
   uint64_t         mem_freed;               /* number of bytes freed */ 
   uint64_t         max_mem_in_use;          /* estimated max value of (used - freed) */ 
+  const char *     mallocator_version;      /* version string from malloc lib */ 
 } ENGINE_STATUS;
 typedef enum {
  DB_BTREE=1,

linux/memory.c

@@ -2,13 +2,13 @@
 #ident "Copyright (c) 2007-2011 Tokutek Inc.  All rights reserved."
 
 #include <toku_portability.h>
-#include "memory.h"
 #include <string.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <malloc.h>
+#include <dlfcn.h>
+#include "memory.h"
 #include "toku_assert.h"
-#include "toku_pthread.h"
 
 static malloc_fun_t  t_malloc  = 0;
 static malloc_fun_t  t_xmalloc = 0;
@@ -18,16 +18,28 @@ static realloc_fun_t t_xrealloc = 0;
 
 static MEMORY_STATUS_S status;
 
-static size_t
-my_malloc_usable_size(void *p) {
-    return p == NULL ? 0 : malloc_usable_size(p);
-}
-
 void 
 toku_memory_get_status(MEMORY_STATUS s) {
+    if (status.mallocator_version == NULL) {
+        // mallctl in jemalloc can be used to get the version string
+        int (*mallctl)(const char *, void *, size_t *, void *, size_t);
+        mallctl = dlsym(NULL, "mallctl");
+        if (mallctl) {
+            size_t version_length = sizeof status.mallocator_version;
+            int r = mallctl("version", &status.mallocator_version, &version_length, NULL, 0);
+            assert(r == 0);
+        } else
+            status.mallocator_version = "libc";
+    }
     *s = status;
 }
 
+// jemalloc's malloc_usable_size does not work with a NULL pointer, so we implement a version that works
+static size_t
+my_malloc_usable_size(void *p) {
+    return p == NULL ? 0 : malloc_usable_size(p);
+}
+
 // max_in_use may be slightly off because use of max_in_use is not thread-safe.
 // It is not worth the overhead to make it completely accurate.
 static inline void 

linux/tests/Makefile

@@ -16,7 +16,7 @@ ifneq ($(GCOV),)
 	CFLAGS += -fprofile-arcs -ftest-coverage -DGCOV
 endif
 LDFLAGS = -L../../lib -Wl,-rpath,../../lib
-LDLIBS = -ltokuportability -lpthread
+LDLIBS = -ltokuportability -lpthread -ldl
 SRCS=$(sort $(filter-out dir.%.c,$(wildcard *.c)))
 TARGETS = $(patsubst %.c,%,$(SRCS))
 RUNTARGETS = $(patsubst %,%.tdbrun,$(TARGETS))
@@ -53,7 +53,7 @@ test-pwrite4g.tdbrun: TEST_EXTRA_ARGS=.
 ifeq ($(VGRIND),)
 	./$< $(TEST_EXTRA_ARGS) $(SUMMARIZE_CMD)
 else
-	$(VGRIND) --error-exitcode=1 --quiet --leak-check=full --show-reachable=yes --log-file=$<.check.valgrind ./$< $(TEST_EXTRA_ARGS) >$<.check.output 2>&1; \
+	$(VGRIND) --error-exitcode=1 --suppressions=../../newbrt/valgrind.suppressions --quiet --leak-check=full --show-reachable=yes --log-file=$<.check.valgrind ./$< $(TEST_EXTRA_ARGS) >$<.check.output 2>&1; \
 	if [ $$? = 0 ] ; then \
 		lines=`cat $<.check.valgrind | wc -l`; \
 		if [ $$lines -ne 0 ] ; then cat $<.check.valgrind; test 0 = 1; fi \
@@ -68,7 +68,7 @@ try-leak-lost.tdbrun: try-leak-lost
 ifeq ($(VGRIND),)
 	./$< $(SUMMARIZE_SHOULD_FAIL)
 else
-	$(VGRIND) --error-exitcode=1 --quiet --leak-check=full --show-reachable=yes --log-file=$<.check.valgrind ./$< >$<.check.output 2>&1; \
+	$(VGRIND) --error-exitcode=1 --suppressions=../../newbrt/valgrind.suppressions --quiet --leak-check=full --show-reachable=yes --log-file=$<.check.valgrind ./$< >$<.check.output 2>&1; \
 	if [ $$? = 0 ] ; then \
 		lines=`cat $<.check.valgrind | wc -l`; \
 		if [ $$lines -ne 0 ] ; then cat $<.check.valgrind; test 0 = 1; fi \
@@ -83,7 +83,7 @@ try-leak-reachable.tdbrun: try-leak-reachable
 ifeq ($(VGRIND),)
 	./$< $(SUMMARIZE_SHOULD_FAIL)
 else
-	$(VGRIND) --error-exitcode=1 --quiet --leak-check=full --show-reachable=yes --log-file=$<.check.valgrind ./$< >$<.check.output 2>&1; \
+	$(VGRIND) --error-exitcode=1 --suppressions=../../newbrt/valgrind.suppressions --quiet --leak-check=full --show-reachable=yes --log-file=$<.check.valgrind ./$< >$<.check.output 2>&1; \
 	if [ $$? = 0 ] ; then \
 		lines=`cat $<.check.valgrind | wc -l`; \
 		if [ $$lines -ne 0 ] ; then cat $<.check.valgrind; test 0 = 1; fi \
@@ -98,7 +98,7 @@ try-uninit.tdbrun: try-uninit
 ifeq ($(VGRIND),)
 	./$< $(SUMMARIZE_SHOULD_FAIL)
 else
-	$(VGRIND) --error-exitcode=1 --quiet --leak-check=full --show-reachable=yes --log-file=$<.check.valgrind ./$< >$<.check.output 2>&1; \
+	$(VGRIND) --error-exitcode=1 --suppressions=../../newbrt/valgrind.suppressions --quiet --leak-check=full --show-reachable=yes --log-file=$<.check.valgrind ./$< >$<.check.output 2>&1; \
 	if [ $$? = 0 ] ; then \
 		lines=`cat $<.check.valgrind | wc -l`; \
 		if [ $$lines -ne 0 ] ; then cat $<.check.valgrind; test 0 = 1; fi \
@@ -120,6 +120,9 @@ clean:
 
 try-assert0.tdbrun:
 
+check_memory_status: test-memory-status
+	./test-memory-status
+	LD_PRELOAD=../../../../mysql/jemalloc-2.2.5/lib/libjemalloc.so ./test-memory-status
 
 foo:
 	echo $(TARGETS)

linux/tests/test-memory-status.c

+#include <stdio.h>
+#include "memory.h"
+
+int main(void) {
+    MEMORY_STATUS_S s;
+    toku_memory_get_status(&s);
+    printf("tokudb mallocator: %s\n", s.mallocator_version);
+    return 0;
+}

newbrt/valgrind.suppressions

@@ -115,3 +115,23 @@
    Memcheck:Value8
    fun:qlz_compress
 }
+{
+   dlsym_on_centos
+   Memcheck:Leak
+   fun:calloc
+   fun:_dlerror_run
+   fun:dlsym
+}
+{
+   dlsym_on_centos
+   Memcheck:Leak
+   fun:malloc
+   fun:_dl_signal_error
+   fun:_dl_signal_cerror
+   fun:_dl_lookup_symbol_x
+   fun:do_sym
+   fun:dlsym_doit
+   fun:_dl_catch_error
+   fun:_dlerror_run
+   fun:dlsym
+}

src/ydb.c

@@ -2181,6 +2181,7 @@ env_get_engine_status(DB_ENV * env, ENGINE_STATUS * engstat, char * env_panic_st
 	    engstat->mem_used       = memory_status.used;
 	    engstat->mem_freed      = memory_status.freed;
 	    engstat->max_mem_in_use = memory_status.max_in_use;
+	    engstat->mallocator_version = memory_status.mallocator_version;
 	}
     }
     return r;
@@ -2418,6 +2419,7 @@ env_get_engine_status_text(DB_ENV * env, char * buff, int bufsiz) {
 	n += snprintf(buff + n, bufsiz - n, "mem_used                         %"PRIu64"\n", engstat.mem_used);
 	n += snprintf(buff + n, bufsiz - n, "mem_freed                        %"PRIu64"\n", engstat.mem_freed);
 	n += snprintf(buff + n, bufsiz - n, "max_mem_in_use                   %"PRIu64"\n", engstat.max_mem_in_use);
+	n += snprintf(buff + n, bufsiz - n, "mallocator_version               %s\n",        engstat.mallocator_version);
     }
     if (n > bufsiz) {
 	char * errmsg = "BUFFER TOO SMALL\n";

toku_include/Makefile.include

@@ -167,7 +167,7 @@ LIBPORTABILITY_SO=$(TOKUROOT)lib/libtokuportability.$(SOEXT)
 LIBPORTABILITY_A=$(TOKUROOT)lib/libtokuportability.$(AEXT)
 
 PORTABILITY_HEADERS= $(TOKUROOT)$(SYSTEM)
-ALWAYS_LINK= -lz -lpthread 
+ALWAYS_LINK= -lz -lpthread -ldl
 ifeq ($(CC),icc)
 # needed for things like intel_fast_memset(), intel_sse2_strlen()
 ifeq ($(DEBUG),0)

toku_include/memory.h

@@ -95,15 +95,16 @@ void toku_set_func_realloc_only(realloc_fun_t f);
 void toku_set_func_free(free_fun_t f);
 
 typedef struct memory_status {
-  uint64_t malloc_count;    // number of malloc operations
-  uint64_t free_count;      // number of free operations
-  uint64_t realloc_count;   // number of realloc operations
-  uint64_t malloc_fail;     // number of malloc operations that failed 
-  uint64_t realloc_fail;    // number of realloc operations that failed 
-  uint64_t requested;       // number of bytes requested
-  uint64_t used;            // number of bytes used (requested + overhead), obtained from malloc_usable_size()
-  uint64_t freed;           // number of bytes freed;
-  uint64_t max_in_use;      // maximum memory footprint (used - freed), approximate (not worth threadsafety overhead for exact)
+    uint64_t malloc_count;    // number of malloc operations
+    uint64_t free_count;      // number of free operations
+    uint64_t realloc_count;   // number of realloc operations
+    uint64_t malloc_fail;     // number of malloc operations that failed 
+    uint64_t realloc_fail;    // number of realloc operations that failed 
+    uint64_t requested;       // number of bytes requested
+    uint64_t used;            // number of bytes used (requested + overhead), obtained from malloc_usable_size()
+    uint64_t freed;           // number of bytes freed;
+    uint64_t max_in_use;      // maximum memory footprint (used - freed), approximate (not worth threadsafety overhead for exact)
+    const char *mallocator_version;
 } MEMORY_STATUS_S, *MEMORY_STATUS;
 
 void toku_memory_get_status(MEMORY_STATUS s);