Merge all stuff from source:tokudb/toku/tokudb.1032 since the creation of this...

Merge all stuff from source:tokudb/toku/tokudb.1032 since the creation of this branch at [8452] to the present [8556], as
{{{
svn merge -r 8452:8556 https://svn.tokutek.com/tokudb/toku/tokudb.1032b/
}}}
And resolve conflicts.

It compiles, and we'll see if it runs.

Addresses #1032, #1343.


git-svn-id: file:///svn/toku/tokudb.1032b+1343@8557 c7de825b-a66e-492c-adef-691d508d4ae1

cxx/Makefile

+# -*- Mode: Makefile -*-
+# standard build: make
+#ident "Copyright (c) 2007 Tokutek Inc.  All rights reserved."
+
+.DEFAULT_GOAL= default
+TOKUROOT=../
+INCLUDEDIRS=-I.
+DEPEND_COMPILE += \
+	./*.h \
+#end
+
+HERE = cxx
+include $(TOKUROOT)include/Makefile.include
+#
+#
 # OPTFLAGS = -O2
 # GCOV_FLAGS = -fprofile-arcs -ftest-coverage
 CPPFLAGS = -I../include
@@ -28,7 +43,7 @@ test1: test1.o dbt.o db.o dbenv.o ../lib/libdb.a
 $(LIBNAME).a: $(OBJS)
 	$(AR) cr $@ $(OBJS)
 clean:
-	rm -f $(OBJS) $(LIBNAME).a $(LIBNAME).so *.gcno *.gcda *.gcov
-	cd tests; $(MAKE) clean
+	$(MAYBEATSIGN)rm -f $(OBJS) $(LIBNAME).a $(LIBNAME).so *.gcno *.gcda *.gcov
+	$(MAYBEATSIGN)cd tests && $(MAKE) clean
 
 

cxx/tests/Makefile

+# -*- Mode: Makefile -*-
+# standard build: make
+#ident "Copyright (c) 2007 Tokutek Inc.  All rights reserved."
+
+.DEFAULT_GOAL= default
+TOKUROOT=../../
+INCLUDEDIRS=-I.
+DEPEND_COMPILE += \
+	../*.h \
+	./*.h \
+#end
+
+HERE = cxx/tests
+include $(TOKUROOT)include/Makefile.include
+
 SHOULD_FAIL = 
 $(SHOULD_FAIL): MAYBEINVERTER=;test $$? -ne 0
 
@@ -38,7 +53,7 @@ $(DBCXX):
 	cd ..;make
 
 clean:
-	rm -rf $(TARGETS) *.gcno *.gcda *.gcov *.db dir.test.db.assoc3 test_reverse_compare_fun.cpp.dir
+	$(MAYBEATSIGN)rm -rf $(TARGETS) *.gcno *.gcda *.gcov *.db dir.test.db.assoc3 test_reverse_compare_fun.cpp.dir
 
 check_test1: test1
 	$(VGRIND) ./$< $(SUMMARIZE_CMD)

db-benchmark-test-cxx/Makefile

@@ -6,6 +6,7 @@
 # build with TokuDB: make BDBDIR=~/svn/tokudb
 # build with g++: make CC=g++
 
+
 .DEFAULT_GOAL= build
 TOKUROOT=../
 INCLUDEDIRS=-I. -I../ -I$(TOKUROOT)newbrt -I../range_tree -I../lock_tree

db-benchmark-test/runbenchmarks.sh

@@ -6,7 +6,7 @@ alias db='./db-benchmark-test-tokudb$BINSUF' # standard db-benchmark test
 alias db='rm -rf bench.db && ./db-benchmark-test-tokudb$BINSUF' # standard db-benchmark test
 alias dbtxn='db -x --singlex' # db-benchmark test with single transaction
 alias dbabort='dbtxn --abort' # db-benchmark test with single transaction (AND ABORT) at end
-alias scan='./scanscan-tokudb$BINSUF --lwc --prelock --prelockflag --cachesize 536870912'     # scanscan default, cache large enough to scan properly.
+alias scan='./scanscan-tokudb$BINSUF --lwc --prelock --prelockflag --cachesize 268435456'     # scanscan default, cache of windows (256MB)
 alias flatteneddb='db && scan'
 alias flattenedtxndb='dbtxn && scan'
 alias flatteningscan='(db 2>&1 >/dev/null) && scan'

include/Makefile.include

@@ -41,6 +41,10 @@
 # To make warnings be warnngs instead of errors do
 #   make WERROR=
 
+ifeq ($(filter --no-print-directory,$(MAKE)),)
+    MAKE+= --no-print-directory #Do not print 'entering directory' infomration.
+endif
+
 ifeq ($(DEBUG),)
     DEBUG = 0
 endif
@@ -135,10 +139,13 @@ DEPEND_LINK += \
 
 ifeq ($(CYGWIN),)
     OS_CHOICE=linux
+    OS_NOTCHOICE=windows
 else ifneq ($(CC),icc)
     OS_CHOICE=linux
+    OS_NOTCHOICE=windows
 else
     OS_CHOICE=windows
+    OS_NOTCHOICE=linux
 endif
 
 DEPEND_COMPILE += $(wildcard $(TOKUROOT)$(OS_CHOICE)/*.h)
@@ -186,7 +193,8 @@ ifneq ($(CYGWIN),)
     BINSUF=.exe
     WRONGBINSUF=#empty
     ALWAYS_LINK=$(LIBPORTABILITY) /usr/lib/libz.a
-    VGRIND   =#No Valgrind in cygwin
+    SRCPATH_DIRS = `find $(TOKUROOT) -type d | grep -v '/\.[^.]' | grep -F -v "$(TOKUROOT)$(OS_NOTCHOICE)" | tr '\n' ';'` 
+    VGRIND   = windbg -Q -c "ddddddddd;g" -y "$(SRCPATH_DIRS)" -srcpath "$(SRCPATH_DIRS)" #No Valgrind in cygwin #-G is quit at exit, but does not return ERROR on error!!! and quits on error to quickly
     HGRIND   =#No Hgrind in cygwin
     FPICFLAGS=#FPIC is default and not allowed as an option.
     VISIBILITY=#Not supported
@@ -281,9 +289,9 @@ cscope.files: $(TOKUROOT)*/*.[ch] $(TOKUROOT)*/*/*.[ch] $(TOKUROOT)*/*/*/*.[ch]
 	$(MAYBEATSIGN)(echo $(TOKUROOT)*/*.[ch] $(TOKUROOT)*/*/*.[ch] $(TOKUROOT)*/*/*/*.[ch] | tr " " "\n") > $@ # Very long command line quieted.
 
 cscope.out: cscope.files $(TOKUROOT)*/*.[ch] $(TOKUROOT)*/*/*.[ch] $(TOKUROOT)*/*/*/*.[ch]
-	$(MAYBEATSIGN)$(CSCOPE) -b
+	$(MAYBEATSIGN)$(CSCOPE) -qb
 
-.PHONY: clean clean-default
+.PHONY: clean clean-default %.dir.clean
 clean: clean-default
 
 %.dir.clean:
@@ -407,3 +415,18 @@ BINS = $(patsubst %,%$(BINSUF),$(BINS_RAW))
 OBJS = $(patsubst %,%.$(OEXT),$(OBJS_RAW))
 REGRESSION_TESTS = $(patsubst %,%$(BINSUF),$(REGRESSION_TESTS_RAW))
 
+APPVERIFIER_TESTS=exceptions handles heaps locks memory threadpool tls dangerousapis dirtystacks filepaths inputoutput
+APPVERIFIER_TESTS_NONDEFAULT=heaps.backward:TRUE heaps.protect:TRUE
+ENABLE_APPVERIFIER_FOR=appverif -enable $(APPVERIFIER_TESTS) -with $(APPVERIFIER_TESTS_NONDEFAULT) -for
+DISABLE_APPVERIFIER_FOR=appverif -disable \* -for
+%.appverifyenable:
+	$(MAYBEATSIGN)$(ENABLE_APPVERIFIER_FOR) $*.exe
+%.appverifydisable:
+	$(MAYBEATSIGN)$(DISABLE_APPVERIFIER_FOR) $*.exe
+
+.PHONY:%.appverifyenable %.appverifydisable %.appverifymaybeenable %.appverifymaybedisable
+
+EXE_FILES = $(wildcard *.exe)
+appverifyenable: $(patsubst %.exe,%.appverifyenable,$(EXE_FILES));
+appverifydisable: $(patsubst %.exe,%.appverifydisable,$(EXE_FILES));
+

newbrt/brt.c

@@ -628,7 +628,7 @@ free_diskblocknumber (BLOCKNUM *b, struct brt_header *h, TOKULOGGER logger __att
 //  Watch out for the case where the disk block was never yet written to disk and is beyond the translated_blocknum_limit.
 {
     extend_block_translation(*b, h);
-    assert((u_int64_t)b->b <= h->translated_blocknum_limit);
+    assert((u_int64_t)b->b < h->translated_blocknum_limit); // as a "limit" it should be <
     assert(h->block_translation[b->b].size != size_is_free);
     h->block_translation[b->b].size = size_is_free;
     h->block_translation[b->b].diskoff = h->free_blocks.b;
@@ -2199,6 +2199,14 @@ brt_merge_child (BRT t, BRTNODE node, int childnum_to_merge, BOOL *did_io, TOKUL
 	if (did_merge) {
 	    BLOCKNUM bn = childb->thisnodename;
 	    rrb = toku_cachetable_unpin_and_remove(t->cf, bn);
+	    // If the block_translation indicates that the size is <=0 then there is no block allocated.
+	    // The block translation might not be big enough, and that also indicates no block allocated.
+	    assert(0 <= bn.b); // the blocknumber better be good
+            if ((unsigned)bn.b < t->h->translated_blocknum_limit) {
+                if (t->h->block_translation[bn.b].size > 0) {
+                    block_allocator_free_block(t->h->block_allocator, t->h->block_translation[bn.b].diskoff);
+                }
+            }
 	    rrb1 = free_diskblocknumber(&bn, t->h, logger);
 	} else {
 	    rrb = toku_unpin_brtnode(t, childb);

newbrt/cachetable.c

@@ -4,11 +4,8 @@
 #include "includes.h"
 
 // execute the cachetable callbacks using a writer thread 0->no 1->yes
-#if defined(_WIN32)
-#define DO_WRITER_THREAD 0
-#else
 #define DO_WRITER_THREAD 1
-#endif
+
 #if DO_WRITER_THREAD
 static void *cachetable_writer(void *);
 #endif

src/tests/test.h

@@ -108,11 +108,11 @@ int test_main (int argc, const char *argv[]);
 int
 main(int argc, const char *argv[]) {
 #if defined(_WIN32) || defined(_WIN64)
-    toku_ydb_init();
+    //toku_ydb_init(); //Windows also uses dll (FOR TESTS)
 #endif
     int r = test_main(argc, argv);
 #if defined(_WIN32) || defined(_WIN64)
-    toku_ydb_destroy();
+    //toku_ydb_destroy(); //Windows also uses dll (FOR TESTS)
 #endif
     return r;
 }

utils/Makefile

+# -*- Mode: Makefile -*-
+# standard build: make
+#ident "Copyright (c) 2007 Tokutek Inc.  All rights reserved."
+
+.DEFAULT_GOAL= build
+TOKUROOT=../
+INCLUDEDIRS=-I.
+DEPEND_COMPILE += \
+	./*.h \
+#end
+
+HERE = utils
+include $(TOKUROOT)include/Makefile.include
 ifndef BDBDIR
 BDBDIR=/usr/local/BerkeleyDB.4.4
 endif
@@ -168,5 +181,5 @@ test_dupsort:
 
 #if $(DIFF) -q <(echo "foo") <(echo "foo") > /dev/null; then echo yes; else echo no; fi
 clean:
-	rm -rf *.so *.o $(UTILS) $(BDB_UTILS) $(STATIC_UTILS) *.temp *.gcno *.gcda *.gcov
+	$(MAYBEATSIGN)rm -rf *.so *.o $(UTILS) $(BDB_UTILS) $(STATIC_UTILS) *.temp *.gcno *.gcda *.gcov
 

windows/files.c

@@ -22,6 +22,10 @@ pread(int fildes, void *buf, size_t nbyte, int64_t offset) {
     if (!r) {
         r = GetLastError();
         if (r==ERROR_HANDLE_EOF) r = bytes_read;
+        else {
+            errno = r;
+            r = -1;
+        }
     }
     else    r = bytes_read;
 
@@ -45,7 +49,10 @@ pwrite(int fildes, const void *buf, size_t nbyte, int64_t offset) {
 
     DWORD bytes_written;
     r = WriteFile(filehandle, buf, nbyte, &bytes_written, &win_offset);
-    if (!r) r = GetLastError();
+    if (!r) {
+        errno = GetLastError();
+        r = -1;
+    }
     else    r = bytes_written;
 
     // printf("%s: %d %p %u %I64d %I64d\n", __FUNCTION__, fildes, buf, nbyte, offset, r); fflush(stdout);

windows/memory.c

@@ -18,18 +18,39 @@ int toku_realloc_counter = 0;
 int toku_free_counter = 0;
 
 static inline size_t resize(size_t n) {
-#if defined(_WIN32) || defined(_WIN64)
-    if (64*1024 < n && n < 1024*1024)
-	n = 1024*1024;
-    return n;
-#else
-    return n;
+    if (n >= 1*1024*1024) 
+        n = (n+7) & ~7; // round up to make windbg !heap happy
+#define DO_PAD_64K 0
+#if DO_PAD_64K
+    else if (64*1024 <= n && n < 1*1024*1024)
+	n = 1*1024*1024; // map anything >= 64K to 1M
 #endif
+#define DO_ROUND_POW2 1
+#if DO_ROUND_POW2
+    else {
+        // make all buffers a power of 2 in size including the windows overhead
+        size_t r = 0;
+        size_t newn = 1<<r;
+        size_t overhead = 0x24;
+        n += overhead;
+        while (n > newn) {
+            r++;
+            newn = 1<<r;
+        }
+        n = newn - overhead;
+    }
+#endif
+    return n;
 }
 
 void *toku_calloc(size_t nmemb, size_t size) {
+    void *vp;
+    size_t newsize = resize(nmemb * size);
     toku_calloc_counter++;
-    return calloc(nmemb, resize(size));
+    vp = malloc(newsize);
+    if (vp) 
+        memset(vp, 0, newsize);
+    return vp;
 }
 
 void *toku_malloc(size_t size) {