Closes #1295

Merged in 1.08.1 fixes as well as some minor changes required to get them to work/Makefiles

git-svn-id: file:///svn/toku/tokudb.1032b@8148 c7de825b-a66e-492c-adef-691d508d4ae1

include/Makefile.include

@@ -141,11 +141,15 @@ endif
 
 DEPEND_COMPILE += $(wildcard $(TOKUROOT)$(OS_CHOICE)/*.h)
 
+SUPPRESSIONS=no
+
 #Tools
 VGRIND=valgrind --quiet --error-exitcode=1 --leak-check=full --show-reachable=yes \
        --suppressions=$(TOKUROOT)newbrt/valgrind.suppressions \
-       --suppressions=$(TOKUROOT)src/tests/bdb.suppressions
-HGRIND=valgrind --quiet --tool=helgrind
+       --suppressions=$(TOKUROOT)src/tests/bdb.suppressions \
+       --gen-suppressions=$(SUPPRESSIONS)
+
+HGRIND=valgrind --quiet --tool=helgrind --error-exitcode=1
 
 # When debugging, try: valgrind --show-reachable=yes --leak-check=full ./brt-test
 # When debugging, try: valgrind --show-reachable=yes --leak-check=full ./brt-test

newbrt/brt.c

@@ -2186,11 +2186,12 @@ brt_merge_child (BRT t, BRTNODE node, int childnum_to_merge, BOOL *did_io, TOKUL
 	    node->u.n.childkeys[childnuma] = splitk_kvpair;
 	    node->u.n.totalchildkeylens += toku_brt_pivot_key_len(t, node->u.n.childkeys[childnuma]);
 	    verify_local_fingerprint_nonleaf(node);
+	    node->dirty = 1;
 	}
     }
+    assert(node->dirty);
  return_r:
     // Unpin both, and return the first nonzero error code that is found
-    assert(node->dirty);
     {
 	int rrb1 = 0;
 	int rra = toku_unpin_brtnode(t, childa);

newbrt/valgrind.suppressions

@@ -40,3 +40,11 @@
    fun:deflate
    fun:compress2
 }
+{
+   compress_is_not_valgrind_clean_ubuntu804
+   Memcheck:Cond
+   obj:/usr/lib/libz.so.1.2.3.3
+   fun:deflate
+   fun:compress2
+}  
+

src/tests/Makefile

@@ -18,6 +18,7 @@ TLIBTDB=../libtokudbtrace.$(SOEXT)
 ifeq ($(CC),icc)
 SKIP_WARNING += $(ICC_NOWARN)1418 #Non static functions do not need prototypes.
 endif
+BDBVGRIND=
 
 $(LIBTDB) $(TLIBTDB):
 	$(MAYBEATSIGN)cd $(@D) && $(MAKE) $(@F)
@@ -40,6 +41,8 @@ BDB_DONTRUN_TESTS = \
 	keyrange-dupsort-unflat \
 	manyfiles \
 	test938c \
+	helgrind1 \
+	helgrind 2\
 #\ ends prev line
 
 BDB_TESTS = $(patsubst %.c,%.bdb$(BINSUF),$(filter-out $(patsubst %,%.c,$(BDB_DONTRUN)),$(SRCS)))
@@ -55,6 +58,9 @@ TDB_TESTS_THAT_SHOULD_FAIL= \
 	test_truncate_subdb \
 	test_txn_nested_abort3 \
 	test_txn_nested_abort4 \
+	helgrind1 \
+	helgrind2 \
+	helgrind3 \
 #\ ends prev line
 
 TDB_TESTS_THAT_SHOULD_FAIL_LIT= \
@@ -118,11 +124,12 @@ SHOULD_FAIL = $(TDB_TESTS_THAT_SHOULD_FAIL_LIT) $(patsubst %,%.tdbrun,$(TDB_TEST
 $(SHOULD_FAIL): MAYBEINVERTER=$(INVERTER)
 $(SHOULD_FAIL): SUMMARIZE_CMD=$(SUMMARIZE_SHOULD_FAIL)
 
+# Use -s on the command line to make things quiet.
 # Use -s on the command line to make things quiet.
 %.bdbrun: %.bdb$(BINSUF) $(DEPEND_COMPILE) $(DEPEND_LINK)
-	$(MAYBEATSIGN)          ./$< $(VERBVERBOSE) $(SUMMARIZE_CMD)
+	$(MAYBEATSIGN)$(BDBVGRIND) ./$< $(VERBVERBOSE) $(SUMMARIZE_CMD)
 %.tdbrun: %.tdb$(BINSUF) $(DEPEND_COMPILE) $(DEPEND_LINK) $(notdir $(LIBTDB))
-	$(MAYBEATSIGN)$(VGRIND) ./$< $(VERBVERBOSE) $(MAYBEINVERTER) $(SUMMARIZE_CMD)
+	$(MAYBEATSIGN)$(VGRIND)    ./$< $(VERBVERBOSE) $(MAYBEINVERTER) $(SUMMARIZE_CMD)
 
 %.recover: %.tdb$(BINSUF)
 	$(MAYBEATSIGN)$(VGRIND) ./$< && \
@@ -154,9 +161,9 @@ endif
 	$(MAYBEATSIGN)$(CC) $< $(BIN_FROM_C_FLAGS)
 
 %.tdb$(BINSUF):  DLINK_FILES+=$(LIBTDB)
-%.tdb$(BINSUF):  RPATH_DIRS+=$(dir LIBTDB)
+%.tdb$(BINSUF):  RPATH_DIRS+=$(dir $(LIBTDB))
 %.tdbt$(BINSUF): DLINK_FILES+=$(TLIBTDB)
-%.tdbt$(BINSUF): RPATH_DIRS+=$(dir TLIBTDB)
+%.tdbt$(BINSUF): RPATH_DIRS+=$(dir $(TLIBTDB))
 %.tdb$(BINSUF) %.tdbt$(BINSUF): CFLAGS+= -DENVDIR=\"dir.$<.tdb\" -DUSE_TDB -DIS_TDB=1
 
 %.tdb$(BINSUF) %.tdbt$(BINSUF): %.c $(DEPEND_COMPILE) $(DEPEND_LINK)
@@ -266,6 +273,14 @@ tgbr_%_b.tdbrun: test_get_both_range.tdb$(BINSUF)
 tgbr_%_c.tdbrun: test_get_both_range.tdb$(BINSUF)
 	$(MAYBEATSIGN)$(VGRIND) ./$< $(VERBVERBOSE) -i $* -c $(MAYBEINVERTER) $(SUMMARIZE_CMD)
 
+# helgrind1 is supposed to fail.
+helgrind1.tdbrun: VGRIND=$(HGRIND) --log-file=helgrind1.tdb.deleteme
+helgrind1.tdbrun: MAYBEINVERTER=$(INVERTER)
+helgrind2.tdbrun: VGRIND=$(HGRIND)
+helgrind2.bdbrun: BDBVGRIND=$(HGRIND)
+helgrind3.tdbrun: VGRIND=$(HGRIND)
+helgrind3.bdbrun: BDBVGRIND=$(HGRIND)
+
 dumpit:
 	../../newbrt/brtdump  dir.test_log5.c.tdb.recover/foo.db > dump.r && ../../newbrt/brtdump  dir.test_log5.c.tdb/foo.db > dump.o && diff dump.o dump.r
 

src/tests/helgrind1.c

+// The helgrind1.tdbrun test should fail.  This is merely a check to verify that helgrind actually notices a race.
+#include <assert.h>
+#include <pthread.h>
+#include "test.h"
+int x;
+
+void *starta(void* ignore __attribute__((__unused__))) {
+    x++;
+    return 0;
+}
+void *startb(void* ignore __attribute__((__unused__))) {
+    x++;
+    return 0;
+}
+
+int main (int argc, const char *argv[]) {
+    parse_args(argc, argv);
+    pthread_t a,b;
+    { int x_l = pthread_create(&a, NULL, starta, NULL); assert(x_l==0); }
+    { int x_l = pthread_create(&b, NULL, startb, NULL); assert(x_l==0); }
+    { int x_l = pthread_join(a, NULL);           assert(x_l==0); }
+    { int x_l = pthread_join(b, NULL);           assert(x_l==0); }
+    return 0;
+}

src/tests/helgrind2.c

+// The helgrind2 test performs a DB->get() in two different concurrent threads.
+#include <arpa/inet.h>
+#include <assert.h>
+#include <db.h>
+#include <pthread.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include "test.h"
+
+DB_ENV *env;
+DB *db;
+
+void initialize (void) {
+    int r;
+    system("rm -rf " ENVDIR);
+    mkdir(ENVDIR, 0777);
+
+    // setup environment
+    {
+        r = db_env_create(&env, 0); assert(r == 0);
+        r = env->set_data_dir(env, ENVDIR);
+        r = env->set_lg_dir(env, ENVDIR);
+        env->set_errfile(env, stdout);
+        r = env->open(env, 0, DB_INIT_MPOOL + DB_PRIVATE + DB_CREATE, 0777); 
+        assert(r == 0);
+    }
+
+    // setup DB
+    {
+        DB_TXN *txn = 0;
+        r = db_create(&db, env, 0); assert(r == 0);
+        r = db->open(db, txn, "test.db", 0, DB_BTREE, DB_CREATE, 0777); assert(r == 0);
+    }
+
+    // Put some stuff in
+    {    
+        char v[10];
+        DB_TXN *txn = 0;
+        int i;
+	const int n = 10;
+	memset(v, 0, sizeof(v));
+        for (i=0; i<n; i++) {
+            int k = htonl(i);
+            DBT key, val;
+            r = db->put(db, txn, dbt_init(&key, &k, sizeof k), dbt_init(&val, v, sizeof(v)), 0);
+            assert(r == 0);
+        }
+    }
+}
+
+void finish (void) {
+    int r;
+    r = db->close(db, 0); assert(r==0);
+    r = env->close(env, 0); assert(r==0);
+}
+
+void *starta(void* ignore __attribute__((__unused__))) {
+    DB_TXN *txn = 0;
+    DBT key, val;
+    memset(&key, 0, sizeof(key));
+    memset(&val, 0, sizeof(val));
+    val.flags |= DB_DBT_MALLOC;
+    int k = htonl(0);
+    int r = db->get(db, txn, dbt_init(&key, &k, sizeof k), &val, 0);
+    assert(r==0);
+    //printf("val.data=%p\n", val.data);
+    int i; for (i=0; i<10; i++) assert(((char*)val.data)[i]==0);
+    free(val.data);
+    return 0;
+}
+void *startb(void* ignore __attribute__((__unused__))) {
+    DB_TXN *txn = 0;
+    DBT key, val;
+    memset(&key, 0, sizeof(key));
+    memset(&val, 0, sizeof(val));
+    int k = htonl(0);
+    val.flags |= DB_DBT_MALLOC;
+    int r = db->get(db, txn, dbt_init(&key, &k, sizeof k), &val, 0);
+    assert(r==0);
+    //printf("val.data=%p\n", val.data);
+    int i; for (i=0; i<10; i++) assert(((char*)val.data)[i]==0);
+    free(val.data);
+    return 0;
+}
+
+int main (int argc, const char *argv[]) {
+    parse_args(argc, argv);
+    pthread_t a,b;
+    initialize();
+    { int x = pthread_create(&a, NULL, starta, NULL); assert(x==0); }
+    { int x = pthread_create(&b, NULL, startb, NULL); assert(x==0); }
+    { int x = pthread_join(a, NULL);           assert(x==0); }
+    { int x = pthread_join(b, NULL);           assert(x==0); }
+    finish();
+    return 0;
+}

src/tests/helgrind3.c

+// The helgrind2 test performs a DB->get() in two different concurrent threads.
+#include <arpa/inet.h>
+#include <assert.h>
+#include <db.h>
+#include <pthread.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include "test.h"
+
+DB_ENV *env;
+DB *db;
+
+void initialize (void) {
+    int r;
+    system("rm -rf " ENVDIR);
+    mkdir(ENVDIR, 0777);
+
+    // setup environment
+    {
+        r = db_env_create(&env, 0); assert(r == 0);
+        r = env->set_data_dir(env, ENVDIR);
+        r = env->set_lg_dir(env, ENVDIR);
+        env->set_errfile(env, stdout);
+        r = env->open(env, 0, DB_INIT_MPOOL + DB_PRIVATE + DB_CREATE, 0777); 
+        assert(r == 0);
+    }
+
+    // setup DB
+    {
+        DB_TXN *txn = 0;
+        r = db_create(&db, env, 0); assert(r == 0);
+        r = db->open(db, txn, "test.db", 0, DB_BTREE, DB_CREATE, 0777); assert(r == 0);
+    }
+
+    // Put some stuff in
+    {    
+        char v[10];
+        DB_TXN *txn = 0;
+        int i;
+	const int n = 10;
+	memset(v, 0, sizeof(v));
+        for (i=0; i<n; i++) {
+            int k = htonl(i);
+            DBT key, val;
+            r = db->put(db, txn, dbt_init(&key, &k, sizeof k), dbt_init(&val, v, sizeof(v)), 0);
+            assert(r == 0);
+        }
+    }
+}
+
+void finish (void) {
+    int r;
+    r = db->close(db, 0); assert(r==0);
+    r = env->close(env, 0); assert(r==0);
+}
+
+void *starta(void* ignore __attribute__((__unused__))) {
+    DB_TXN *txn = 0;
+    DBT key, val;
+    char data[10];
+    val.data = data;
+    memset(&key, 0, sizeof(key));
+    memset(&val, 0, sizeof(val));
+    val.flags |= DB_DBT_MALLOC;
+    int k = htonl(99);
+    int r = db->put(db, txn, dbt_init(&key, &k, sizeof k), &val, 0);
+    assert(r==0);
+    //printf("val.data=%p\n", val.data);
+    return 0;
+}
+void *startb(void* ignore __attribute__((__unused__))) {
+    DB_TXN *txn = 0;
+    DBT key, val;
+    memset(&key, 0, sizeof(key));
+    memset(&val, 0, sizeof(val));
+    int k = htonl(0);
+    val.flags |= DB_DBT_MALLOC;
+    int r = db->get(db, txn, dbt_init(&key, &k, sizeof k), &val, 0);
+    assert(r==0);
+    //printf("val.data=%p\n", val.data);
+    int i; for (i=0; i<10; i++) assert(((char*)val.data)[i]==0);
+    free(val.data);
+    return 0;
+}
+
+int main (int argc, const char *argv[]) {
+    parse_args(argc, argv);
+    pthread_t a,b;
+    initialize();
+    { int x = pthread_create(&a, NULL, starta, NULL); assert(x==0); }
+    { int x = pthread_create(&b, NULL, startb, NULL); assert(x==0); }
+    { int x = pthread_join(a, NULL);           assert(x==0); }
+    { int x = pthread_join(b, NULL);           assert(x==0); }
+    finish();
+    return 0;
+}