Merge the 558 branch back to the main with

{{{
svn merge -r3061:3225 https://svn.tokutek.com/tokudb/tokudb.558
}}}

Fixes #630.
Addresses #558.


git-svn-id: file:///svn/tokudb@3226 c7de825b-a66e-492c-adef-691d508d4ae1

Makefile

+default: build
+
+
 TAGS: */*.c */*.h
 	etags */*.c */*.h src/lock_tree/*.c src/lock_tree/*.h src/range_tree/*.c src/range_tree/*.h
 
-SRCDIRS = newbrt src src/tests src/range_tree src/range_tree/tests src/lock_tree src/lock_tree/tests cxx cxx/tests \
-		utils db-benchmark-test db-benchmark-test-cxx
+SRCDIRS = newbrt src cxx utils db-benchmark-test db-benchmark-test-cxx
 BUILDDIRS = $(SRCDIRS) man/texi
 
-build:
-	for d in $(BUILDDIRS); do (cd $$d; $(MAKE) -k); done
+src.dir: newbrt.dir
+cxx.dir: src.dir
+db-benchmark-test.dir: src.dir
+db-benchmark-test-cxx.dir: cxx.dir
+utils.dir: src.dir
+
+%.dir:
+	cd $(patsubst %.dir, %, $@);$(MAKE) build
+
+build: $(patsubst %,%.dir, $(SRCDIRS))
 
 CHECKS = $(patsubst %,checkdir_%,$(SRCDIRS))
 
@@ -16,11 +26,7 @@ CHECKS = $(patsubst %,checkdir_%,$(SRCDIRS))
 #check:
 #	for d in $(SRCDIRS); do (cd $$d; $(MAKE) -k check); done
 
-checkdir_%: build
-	cd $(patsubst checkdir_%,%,$@) ; $(MAKE) -k check
-checkdir_src/%: build
-	cd $(patsubst checkdir_%,%,$@) ; $(MAKE) -k check
-checkdir_cxx/%: build
+checkdir_%:
 	cd $(patsubst checkdir_%,%,$@) ; $(MAKE) -k check
 
 check: $(CHECKS)
@@ -45,7 +51,7 @@ check-coverage: check-coverage-newbrt check-coverage-src-tests check-coverage-ut
 		check-coverage-range-tree-tests check-coverage-lock-tree-tests
 
 check-coverage-newbrt:
-	(cd newbrt; $(MAKE) -k check DTOOL="")
+	(cd newbrt; $(MAKE) -k check VGRIND="")
 check-coverage-src-tests:
 	(cd src/tests; $(MAKE) -k check.tdb VGRIND="")
 check-coverage-utils:

cxx/Makefile

@@ -9,7 +9,11 @@ OBJS = $(patsubst %.cpp, %.o, $(SRCS))
 
 LIBNAME = libtokudb_cxx
 
-default: install
+default: install build
+build: $(LIBNAME).a
+	cd tests; $(MAKE) build
+check:
+	cd tests; $(MAKE) check
 install: $(LIBNAME).a
 	cp $< ../lib/
 $(OBJS): ../include/db_cxx.h
@@ -19,5 +23,6 @@ $(LIBNAME).a: $(OBJS)
 	$(AR) rv $@ $(OBJS)
 clean:
 	rm -f $(OBJS) $(LIBNAME).a $(LIBNAME).so *.gcno *.gcda *.gcov
+	cd tests; $(MAKE) clean
 
 

cxx/tests/Makefile

@@ -13,7 +13,8 @@ else
  VGRIND=valgrind --quiet --error-exitcode=1 --leak-check=yes
 endif
 
-all: $(TARGETS)
+default: build
+build all: $(TARGETS)
 $(TARGETS): $(DBCXX)
 
 $(DBCXX):
@@ -61,4 +62,4 @@ check_permissions:
 	rm -f test.db
 	./db_create test.db 1 1
 	chmod -w test.db
-	./db_create test.db 2 2; let exitcode=$$?; if [ $$exitcode -ne 0 ] ; then exit 0; else exit 1; fi
+	./db_create test.db 2 2; let exitcode=$$?; if [ $$exitcode -ne 0 ] ; then exit 0; else exit 1; fi
\ No newline at end of file

db-benchmark-test-cxx/Makefile

@@ -22,7 +22,8 @@ TARGET_BDB = db-benchmark-test-bdb
 TARGET_TDB = db-benchmark-test-tokudb
 TARGETS = $(TARGET_BDB) $(TARGET_TDB)
 
-default: $(TARGETS)
+default: build
+build: $(TARGETS)
 
 check: check-default
 

db-benchmark-test/Makefile

@@ -25,7 +25,8 @@ TARGET_BDB = db-benchmark-test-bdb
 TARGET_TDB = db-benchmark-test-tokudb
 TARGETS = $(TARGET_BDB) $(TARGET_TDB)
 
-default: $(TARGETS)
+default: build
+build: $(TARGETS)
 
 check: check-default check-xfast
 

newbrt/benchmark-test.c

@@ -9,6 +9,7 @@
 #include <errno.h>
 #include <stdio.h>
 #include <stdlib.h>
+#include <string.h>
 #include <sys/time.h>
 #include <unistd.h>
 

newbrt/brt-internal.h

@@ -3,12 +3,16 @@
 
 #ident "Copyright (c) 2007, 2008 Tokutek Inc.  All rights reserved."
 
+#include "toku_assert.h"
 #include "cachetable.h"
 #include "fifo.h"
-#include "pma.h"
+#include "yerror.h"
+#include "gpma.h"
 #include "brt.h"
 #include "crc.h"
 #include "list.h"
+#include "mempool.h"
+#include "kv-pair.h"
 
 #ifndef BRT_FANOUT
 #define BRT_FANOUT 16
@@ -77,8 +81,9 @@ struct brtnode {
 						         However, in the absense of duplicate keys, child 1's keys *are* > childkeys[0]. */
         } n;
 	struct leaf {
-	    PMA buffer;
+	    GPMA buffer;
 	    unsigned int n_bytes_in_buffer; /* How many bytes to represent the PMA (including the per-key overheads, but not including the overheads for the node. */
+	    struct mempool buffer_mempool;
 	} l;
     } u;
 };
@@ -101,11 +106,6 @@ struct brt_header {
     unsigned int flags;
 };
 
-enum brt_header_flags {
-    TOKU_DB_DUP = 1,
-    TOKU_DB_DUPSORT = 2,
-};
-
 struct brt {
     CACHEFILE cf;
     char *database_name;
@@ -125,7 +125,7 @@ struct brt {
 
 /* serialization code */
 void toku_serialize_brtnode_to(int fd, DISKOFF off, DISKOFF size, BRTNODE node);
-int toku_deserialize_brtnode_from (int fd, DISKOFF off, BRTNODE *brtnode, int flags, int nodesize, int (*bt_compare)(DB *, const DBT*, const DBT*), int (*dup_compare)(DB *, const DBT *, const DBT *), DB *db, FILENUM filenum);
+int toku_deserialize_brtnode_from (int fd, DISKOFF off, BRTNODE *brtnode, unsigned int flags, int nodesize);
 unsigned int toku_serialize_brtnode_size(BRTNODE node); /* How much space will it take? */
 int toku_keycompare (bytevec key1, ITEMLEN key1len, bytevec key2, ITEMLEN key2len);
 
@@ -160,6 +160,7 @@ extern CACHEKEY* toku_calculate_root_offset_pointer (BRT brt);
 static const BRTNODE null_brtnode=0;
 
 extern u_int32_t toku_calccrc32_kvpair (const void *key, int keylen, const void *val, int vallen);
+extern u_int32_t toku_calccrc32_kvpair_struct (const struct kv_pair *kvp);
 extern u_int32_t toku_calccrc32_cmd (int type, TXNID xid, const void *key, int keylen, const void *val, int vallen);
 extern u_int32_t toku_calccrc32_cmdstruct (BRT_CMD cmd);
 
@@ -191,6 +192,34 @@ int toku_testsetup_insert_to_nonleaf (BRT brt, DISKOFF diskoff, enum brt_cmd_typ
 
 int toku_set_func_fsync (int (*fsync_function)(int));
 
-
+/* allocate a kv pair from a kv memory pool */
+//static inline struct kv_pair *kv_pair_malloc_mempool(const void *key, int keylen, const void *val, int vallen, struct mempool *mp) {
+//    struct kv_pair *kv = toku_mempool_malloc(mp, sizeof (struct kv_pair) + keylen + vallen, 4);
+//    if (kv)
+//        kv_pair_init(kv, key, keylen, val, vallen);
+//    return kv;
+//}
+
+int toku_brtnode_compress_kvspace (GPMA pma, struct mempool *mp);
+
+static inline struct kv_pair *brtnode_malloc_kv_pair (GPMA pma, struct mempool *mp, const void *key, unsigned int keylen, const void *val, unsigned int vallen) {
+    struct kv_pair *kv = toku_mempool_malloc(mp, sizeof (struct kv_pair) + keylen + vallen, 4);
+    if (kv == 0) {
+	if (0 == toku_brtnode_compress_kvspace (pma, mp)) {
+	    kv = toku_mempool_malloc(mp, sizeof (struct kv_pair) + keylen + vallen, 4);
+	    toku_verify_gpma(pma);
+	    assert(kv);
+	}
+    }
+    kv_pair_init(kv, key, keylen, val, vallen);
+    return kv;
+}
+
+// used for the leaf compare fun
+struct lc_pair {
+    BRT t;
+    int compare_both; // compare_both is set if it is a DUPSORT database and both keys are needed (e.g, for DB_DELETE_ANY)
+};
+int toku_brtleaf_compare_fun (u_int32_t alen __attribute__((__unused__)), void *aval, u_int32_t blen __attribute__((__unused__)), void *bval, void *lc /*this is (struct lc_pair *) cast to (void*). */) ;
 
 #endif

newbrt/brt-search.h

 #ifndef BRT_SEARCH_H
 #define BRT_SEARCH_H
 
-enum {
+enum brt_search_direction_e {
     BRT_SEARCH_LEFT = 1,  /* search left -> right, finds min xy as defined by the compare function */
     BRT_SEARCH_RIGHT = 2, /* search right -> left, finds max xy as defined by the compare function */
-    BRT_SEARCH_ONE = 4,   /* look into only one subtree, used for point queries */
 };
 
 struct brt_search;
@@ -20,7 +19,7 @@ typedef int (*brt_search_compare_func_t)(struct brt_search */*so*/, DBT */*x*/,
 
 typedef struct brt_search {
     brt_search_compare_func_t compare;
-    int direction;
+    enum brt_search_direction_e direction;
     DBT *k;
     DBT *v;
     void *context;

newbrt/brt-serialize-test.c

 /* -*- mode: C; c-basic-offset: 4 -*- */
 #ident "Copyright (c) 2007, 2008 Tokutek Inc.  All rights reserved."
 
-#include "assert.h"
+#include "toku_assert.h"
 #include "brt-internal.h"
+#include "kv-pair.h"
 
 #include <fcntl.h>
 #include <string.h>
@@ -26,7 +27,7 @@ static void test_serialize(void) {
     sn.thisnodename = sn.nodesize*20;
     sn.disk_lsn.lsn = 789;
     sn.log_lsn.lsn  = 123456;
-    sn.layout_version = 2;
+    sn.layout_version = 3;
     sn.height = 1;
     sn.rand4fingerprint = randval;
     sn.local_fingerprint = 0;
@@ -49,14 +50,14 @@ static void test_serialize(void) {
     BNC_NBYTESINBUF(&sn, 1) = 1*(BRT_CMD_OVERHEAD+KEY_VALUE_OVERHEAD+2+5);
     sn.u.n.n_bytes_in_buffers = 3*(BRT_CMD_OVERHEAD+KEY_VALUE_OVERHEAD+2+5);
 
-    toku_serialize_brtnode_to(fd, sn.nodesize*20, sn.nodesize, &sn);  assert(r==0);
+    toku_serialize_brtnode_to(fd, sn.nodesize*(DISKOFF)20, (DISKOFF)sn.nodesize, &sn);  assert(r==0);
 
-    r = toku_deserialize_brtnode_from(fd, nodesize*20, &dn, sn.flags, nodesize, 0, 0, 0, (FILENUM){0});
+    r = toku_deserialize_brtnode_from(fd, nodesize*(DISKOFF)20, &dn, sn.flags, nodesize);
     assert(r==0);
 
     assert(dn->thisnodename==nodesize*20);
     assert(dn->disk_lsn.lsn==123456);
-    assert(dn->layout_version ==2);
+    assert(dn->layout_version ==3);
     assert(dn->height == 1);
     assert(dn->rand4fingerprint==randval);
     assert(dn->u.n.n_children==2);
@@ -100,6 +101,8 @@ static void test_serialize(void) {
     toku_free(hello_string);
     toku_fifo_free(&BNC_BUFFER(&sn,0));
     toku_fifo_free(&BNC_BUFFER(&sn,1));
+    toku_free(sn.u.n.childinfos);
+    toku_free(sn.u.n.childkeys);
 }
 
 int main (int argc __attribute__((__unused__)), char *argv[] __attribute__((__unused__))) {

newbrt/brt-serialize.c

@@ -9,6 +9,8 @@
 #include "key.h"
 #include "rbuf.h"
 #include "wbuf.h"
+#include "kv-pair.h"
+#include "mempool.h"
 
 #include <unistd.h>
 #include <stdio.h>
@@ -57,10 +59,13 @@ static unsigned int toku_serialize_brtnode_size_slow(BRTNODE node) {
 	return size+hsize+csize;
     } else {
 	unsigned int hsize=0;
-	PMA_ITERATE(node->u.l.buffer,
-		    key __attribute__((__unused__)), keylen,
-		    data __attribute__((__unused__)), datalen,
-		    (hsize+=PMA_ITEM_OVERHEAD+KEY_VALUE_OVERHEAD+keylen+datalen));
+	GPMA_ITERATE(node->u.l.buffer,
+		     idx, vlen, vdata,
+		     ({
+			 struct kv_pair *p=vdata;
+			 assert(vlen==sizeof(*p)+kv_pair_keylen(p)+kv_pair_vallen(p));
+			 hsize+=PMA_ITEM_OVERHEAD+KEY_VALUE_OVERHEAD+kv_pair_keylen(p)+kv_pair_vallen(p);
+		     }));
 	assert(hsize==node->u.l.n_bytes_in_buffer);
 	hsize+=4; /* the PMA size */
 	hsize+=4; /* add n entries in buffer table. */
@@ -110,12 +115,13 @@ void toku_serialize_brtnode_to(int fd, DISKOFF off, DISKOFF size, BRTNODE node)
     wbuf_int(&w, node->layout_version);
     wbuf_ulonglong(&w, node->log_lsn.lsn);
     //printf("%s:%d %lld.calculated_size=%d\n", __FILE__, __LINE__, off, calculated_size);
-    wbuf_int(&w, calculated_size);
-    wbuf_int(&w, node->flags);
-    wbuf_int(&w, node->height);
+    wbuf_uint(&w, calculated_size);
+    wbuf_uint(&w, node->flags);
+    wbuf_uint(&w, node->height);
     //printf("%s:%d %lld rand=%08x sum=%08x height=%d\n", __FILE__, __LINE__, node->thisnodename, node->rand4fingerprint, node->subtree_fingerprint, node->height);
     wbuf_int(&w, node->rand4fingerprint);
     wbuf_int(&w, node->local_fingerprint);
+//    printf("%s:%d wrote %08x for node %lld\n", __FILE__, __LINE__, node->local_fingerprint, (long long)node->thisnodename);
     //printf("%s:%d local_fingerprint=%8x\n", __FILE__, __LINE__, node->local_fingerprint);
     //printf("%s:%d w.ndone=%d n_children=%d\n", __FILE__, __LINE__, w.ndone, node->n_children);
     if (node->height>0) {
@@ -169,15 +175,19 @@ void toku_serialize_brtnode_to(int fd, DISKOFF off, DISKOFF size, BRTNODE node)
 	}
     } else {
 	//printf(" n_entries=%d\n", toku_pma_n_entries(node->u.l.buffer));
-	wbuf_int(&w, toku_pma_n_entries(node->u.l.buffer));
-	wbuf_int(&w, toku_pma_index_limit(node->u.l.buffer));
-	PMA_ITERATE_IDX(node->u.l.buffer, idx,
-			key, keylen, data, datalen,
-			({
-			    wbuf_int(&w, idx);
-			    wbuf_bytes(&w, key, keylen);
-			    wbuf_bytes(&w, data, datalen);
-			}));
+	wbuf_int(&w, toku_gpma_n_entries(node->u.l.buffer));
+	wbuf_int(&w, toku_gpma_index_limit(node->u.l.buffer));
+	GPMA_ITERATE(node->u.l.buffer, idx, vlen, vdata,
+		     ({
+			 struct kv_pair *p=vdata;
+			 assert((char*)node->u.l.buffer_mempool.base<= (char*)p && (char*)p < (char*)node->u.l.buffer_mempool.base+node->u.l.buffer_mempool.size );
+			 int keylen=kv_pair_keylen(p);
+			 int datalen=kv_pair_vallen(p);
+			 assert(vlen==sizeof(*p)+keylen+datalen);
+			 wbuf_int(&w, idx);
+			 wbuf_bytes(&w, kv_pair_key(p), keylen);
+			 wbuf_bytes(&w, kv_pair_val(p), datalen);
+		     }));
     }
     assert(w.ndone<=w.size);
 #ifdef CRC_ATEND
@@ -205,10 +215,7 @@ void toku_serialize_brtnode_to(int fd, DISKOFF off, DISKOFF size, BRTNODE node)
     toku_free(buf);
 }
 
-int toku_deserialize_brtnode_from (int fd, DISKOFF off, BRTNODE *brtnode, int flags, int nodesize,
-				   int (*bt_compare)(DB *, const DBT *, const DBT *),
-				   int (*dup_compare)(DB *, const DBT *, const DBT *),
-				   DB *db, FILENUM filenum) {
+int toku_deserialize_brtnode_from (int fd, DISKOFF off, BRTNODE *brtnode, unsigned int flags, int nodesize) {
     TAGMALLOC(BRTNODE, result);
     struct rbuf rc;
     int i;
@@ -262,7 +269,7 @@ int toku_deserialize_brtnode_from (int fd, DISKOFF off, BRTNODE *brtnode, int fl
 	}
     }
     result->layout_version    = rbuf_int(&rc);
-    if (result->layout_version!=2) {
+    if (result->layout_version!=3) {
 	r=DB_BADFORMAT;
 	goto died1;
     }
@@ -278,6 +285,7 @@ int toku_deserialize_brtnode_from (int fd, DISKOFF off, BRTNODE *brtnode, int fl
     result->height = rbuf_int(&rc);
     result->rand4fingerprint = rbuf_int(&rc);
     result->local_fingerprint = rbuf_int(&rc);
+//    printf("%s:%d read %08x\n", __FILE__, __LINE__, result->local_fingerprint);
     result->dirty = 0;
     //printf("height==%d\n", result->height);
     if (result->height>0) {
@@ -365,27 +373,32 @@ int toku_deserialize_brtnode_from (int fd, DISKOFF off, BRTNODE *brtnode, int fl
 	int n_in_buf = rbuf_int(&rc);
         int index_limit = rbuf_int(&rc);
 	result->u.l.n_bytes_in_buffer = 0;
-	r=toku_pma_create(&result->u.l.buffer, bt_compare, db, filenum, nodesize, index_limit);
+	r=toku_gpma_create(&result->u.l.buffer, index_limit);
 	if (r!=0) {
-	    if (0) { died_21: toku_pma_free(&result->u.l.buffer); }
+	    if (0) { died_21: toku_gpma_free(&result->u.l.buffer, 0, 0); }
 	    goto died1;
 	}
-        toku_pma_set_dup_mode(result->u.l.buffer, flags);
-        toku_pma_set_dup_compare(result->u.l.buffer, dup_compare);
 	//printf("%s:%d r PMA= %p\n", __FILE__, __LINE__, result->u.l.buffer); 
-	toku_verify_counts(result);
+	{
+	    void *mp = toku_malloc(nodesize);
+	    if (mp==0) return ENOMEM;
+	    toku_mempool_init(&result->u.l.buffer_mempool, mp, nodesize);
+	}
 
 	u_int32_t actual_sum = 0;
 	for (i=0; i<n_in_buf; i++) {
 	    bytevec key; ITEMLEN keylen; 
 	    bytevec val; ITEMLEN vallen;
 	    int idx = rbuf_int(&rc);
-	    DBT keydbt, datadbt;
 	    rbuf_bytes(&rc, &key, &keylen); /* Returns a pointer into the rbuf. */
 	    rbuf_bytes(&rc, &val, &vallen);
 	    result->u.l.n_bytes_in_buffer += keylen + vallen + KEY_VALUE_OVERHEAD + PMA_ITEM_OVERHEAD;
-	    r = toku_pma_set_at_index(result->u.l.buffer, idx, toku_fill_dbt(&keydbt, key, keylen), toku_fill_dbt(&datadbt, val, vallen));
-	    actual_sum += result->rand4fingerprint*toku_calccrc32_kvpair(key, keylen, val, vallen);
+	    struct kv_pair *pair = brtnode_malloc_kv_pair(result->u.l.buffer, &result->u.l.buffer_mempool, key, keylen, val, vallen);
+	    assert(pair);
+	    int pairlen = kv_pair_size(pair);
+	    toku_gpma_set_at_index(result->u.l.buffer, idx, pairlen, pair);
+	    actual_sum += result->rand4fingerprint*toku_calccrc32_kvpair_struct(pair);
+//	    printf("%s:%d rand4=%08x actual=%08x this=%08x expect=%08x\n", __FILE__, __LINE__, result->rand4fingerprint, actual_sum, toku_calccrc32_kvpair_struct(pair), result->local_fingerprint);
 	}
             
 	if (r!=0) goto died_21;
@@ -425,8 +438,13 @@ void toku_verify_counts (BRTNODE node) {
     if (node->height==0) {
 	assert(node->u.l.buffer);
 	unsigned int sum=0;
-	PMA_ITERATE(node->u.l.buffer, key __attribute__((__unused__)), keylen, data  __attribute__((__unused__)), datalen,
-		    sum+=(PMA_ITEM_OVERHEAD + KEY_VALUE_OVERHEAD + keylen + datalen));
+	unsigned int count=0;
+	GPMA_ITERATE(node->u.l.buffer, idx, dlen, dvadata,
+		     ({
+			 count++;
+			 sum+=(PMA_ITEM_OVERHEAD + dlen);
+		     }));
+	assert(count==toku_gpma_n_entries(node->u.l.buffer));
 	assert(sum==node->u.l.n_bytes_in_buffer);
     } else {
 	unsigned int sum = 0;

newbrt/brt-test-cursor-2.c

@@ -3,7 +3,6 @@
 
 #include "brt.h"
 #include "key.h"
-#include "pma.h"
 #include "brt-internal.h"
 #include "memory.h"
 #include "toku_assert.h"

newbrt/brt-test-cursor.c

@@ -3,7 +3,6 @@
 
 #include "brt.h"
 #include "key.h"
-#include "pma.h"
 #include "brt-internal.h"
 #include "memory.h"
 #include "toku_assert.h"

newbrt/brt-test-helpers.c

@@ -71,24 +71,29 @@ int toku_testsetup_insert_to_leaf (BRT brt, DISKOFF diskoff, char *key, int keyl
     if (r!=0) return r;
     BRTNODE node=node_v;
     assert(node->height==0);
-    DBT k,v;
-    int replaced_v_size;
-    enum pma_errors pma_status =
-	toku_pma_insert_or_replace(node->u.l.buffer,
-				   toku_fill_dbt(&k, key, keylen),
-				   toku_fill_dbt(&v, val, vallen),
-				   &replaced_v_size,
-				   (TOKULOGGER)0, (TXNID)0,
-				   toku_cachefile_filenum(brt->cf),
-				   node->thisnodename, node->rand4fingerprint,
-				   &node->local_fingerprint,
-				   &node->log_lsn);
-    assert(pma_status==BRT_OK);
-    if (replaced_v_size>=0) {
-	node->u.l.n_bytes_in_buffer += v.size - replaced_v_size;
+
+    struct kv_pair *kv = kv_pair_malloc(key, keylen, val, vallen);
+    struct lc_pair  lc = {brt, node->flags & TOKU_DB_DUPSORT};
+    u_int32_t storedlen;
+    void *storeddata;
+    u_int32_t idx;
+    r = toku_gpma_lookup_item(node->u.l.buffer, kv_pair_size(kv), kv, toku_brtleaf_compare_fun, &lc, &storedlen, &storeddata, &idx);
+
+    if (r==0) {
+	// It's already there.  So now we have to remove it and put the new one back in.
+	node->u.l.n_bytes_in_buffer -= PMA_ITEM_OVERHEAD + storedlen;
+	node->local_fingerprint     -= toku_crc32(toku_null_crc, storeddata, storedlen);
+	toku_mempool_mfree(&node->u.l.buffer_mempool, storeddata, storedlen);
+	// Now put the new kv in.
+	toku_gpma_set_at_index(node->u.l.buffer, idx, kv_pair_size(kv), kv);
     } else {
-	node->u.l.n_bytes_in_buffer += k.size + v.size + KEY_VALUE_OVERHEAD + PMA_ITEM_OVERHEAD;
+	r = toku_gpma_insert(node->u.l.buffer, kv_pair_size(kv), kv, toku_brtleaf_compare_fun, &lc, 0, 0, 0);
+	assert(r==0);
     }
+
+    node->u.l.n_bytes_in_buffer += PMA_ITEM_OVERHEAD + kv_pair_size(kv);
+    node->local_fingerprint += toku_crc32(toku_null_crc, kv, kv_pair_size(kv));
+
     node->dirty=1;
     *subtree_fingerprint = node->local_fingerprint;
     r = toku_unpin_brtnode(brt, node_v);

newbrt/brt-test-named-db.c

+/* -*- mode: C; c-basic-offset: 4 -*- */
+#ident "Copyright (c) 2007, 2008 Tokutek Inc.  All rights reserved."
+
+#include "brt.h"
+#include "key.h"
+#include "test.h"
+#include "toku_assert.h"
+#include <unistd.h>
+
+static TOKUTXN const null_txn = 0;
+static DB * const null_db = 0;
+
+static void test_named_db (void) {
+    const char *n0 = "brt-test-named-db-0.brt";
+    CACHETABLE ct;
+    BRT t0;
+    int r;
+    DBT k,v;
+
+    if (verbose) printf("test_named_db\n");
+    unlink(n0);
+    toku_memory_check_all_free();
+    r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);           assert(r==0);
+    r = toku_open_brt(n0, "db1", 1, &t0, 1<<12, ct, null_txn, toku_default_compare_fun, null_db); assert(r==0);
+
+    toku_brt_insert(t0, toku_fill_dbt(&k, "good", 5), toku_fill_dbt(&v, "day", 4), null_txn); assert(r==0);
+
+    r = toku_close_brt(t0); assert(r==0);
+    r = toku_cachetable_close(&ct); assert(r==0);
+    toku_memory_check_all_free();
+
+    toku_memory_check_all_free();
+    r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);           assert(r==0);
+    r = toku_open_brt(n0, "db1", 0, &t0, 1<<12, ct, null_txn, toku_default_compare_fun, null_db); assert(r==0);
+
+    {
+	r = toku_brt_lookup(t0, toku_fill_dbt(&k, "good", 5), toku_init_dbt(&v));
+	assert(r==0);
+	assert(v.size==4);
+	assert(strcmp(v.data,"day")==0);
+    }
+
+    r = toku_close_brt(t0); assert(r==0);
+    r = toku_cachetable_close(&ct); assert(r==0);
+    toku_memory_check_all_free();
+}
+
+int main (int argc , const char *argv[]) {
+    default_parse_args(argc, argv);
+    test_named_db();
+    toku_malloc_cleanup();
+    if (verbose) printf("test_named_db ok\n");
+    return 0;
+}

newbrt/brt-test.c

@@ -3,7 +3,7 @@
 
 #include "brt.h"
 #include "key.h"
-#include "pma.h"
+#include "gpma.h"
 #include "brt-internal.h"
 #include "memory.h"
 #include "toku_assert.h"
@@ -21,93 +21,6 @@
 static TOKUTXN const null_txn = 0;
 static DB * const null_db = 0;
 
-static void test0 (void) {
-    BRT t;
-    int r;
-    CACHETABLE ct;
-    char fname[]="testbrt.brt";
-    if (verbose) printf("%s:%d test0\n", __FILE__, __LINE__);
-    toku_memory_check=1;
-    toku_memory_check_all_free();
-    r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
-    assert(r==0);
-    if (verbose) printf("%s:%d test0\n", __FILE__, __LINE__);
-    unlink(fname);
-    r = toku_open_brt(fname, 0, 1, &t, 1024, ct, null_txn, toku_default_compare_fun, null_db);
-    assert(r==0);
-    //printf("%s:%d test0\n", __FILE__, __LINE__);
-    //printf("%s:%d n_items_malloced=%lld\n", __FILE__, __LINE__, n_items_malloced);
-    r = toku_close_brt(t);     assert(r==0);
-    //printf("%s:%d n_items_malloced=%lld\n", __FILE__, __LINE__, n_items_malloced);
-    r = toku_cachetable_close(&ct);
-    assert(r==0);
-    toku_memory_check_all_free();
-}
-
-static void test1 (void) {
-    BRT t;
-    int r;
-    CACHETABLE ct;
-    char fname[]="testbrt.brt";
-    DBT k,v;
-    toku_memory_check=1;
-    toku_memory_check_all_free();
-    r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
-    assert(r==0);
-    unlink(fname);
-    r = toku_open_brt(fname, 0, 1, &t, 1024, ct, null_txn, toku_default_compare_fun, null_db);
-    assert(r==0);
-    toku_brt_insert(t, toku_fill_dbt(&k, "hello", 6), toku_fill_dbt(&v, "there", 6), null_txn);
-    {
-	r = toku_brt_lookup(t, toku_fill_dbt(&k, "hello", 6), toku_init_dbt(&v));
-	assert(r==0);
-	assert(strcmp(v.data, "there")==0);
-	assert(v.size==6);
-    }
-    r = toku_close_brt(t);              assert(r==0);
-    r = toku_cachetable_close(&ct);     assert(r==0);
-    toku_memory_check_all_free();
-    if (verbose) printf("test1 ok\n");
-}
-
-static void test2 (int memcheck) {
-    BRT t;
-    int r;
-    int i;
-    CACHETABLE ct;
-    char fname[]="testbrt.brt";
-    toku_memory_check=memcheck;
-    if (verbose) printf("%s:%d checking\n", __FILE__, __LINE__);
-    toku_memory_check_all_free();
-    r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER); assert(r==0);
-    unlink(fname);
-    r = toku_open_brt(fname, 0, 1, &t, 1024, ct, null_txn, toku_default_compare_fun, null_db);
-    if (verbose) printf("%s:%d did setup\n", __FILE__, __LINE__);
-    assert(r==0);
-    for (i=0; i<2048; i++) {
-	DBT k,v;
-	char key[100],val[100];
-	snprintf(key,100,"hello%d",i);
-	snprintf(val,100,"there%d",i);
-	toku_brt_insert(t, toku_fill_dbt(&k, key, 1+strlen(key)), toku_fill_dbt(&v, val, 1+strlen(val)), null_txn);
-	//printf("%s:%d did insert %d\n", __FILE__, __LINE__, i);
-	if (0) {
-	    brt_flush(t);
-	    {
-		int n = toku_get_n_items_malloced();
-		if (verbose) printf("%s:%d i=%d n_items_malloced=%d\n", __FILE__, __LINE__, i, n);
-		if (n!=3) toku_print_malloced_items();
-		assert(n==3);
-	    }
-	}
-    }
-    if (verbose) printf("%s:%d inserted\n", __FILE__, __LINE__);
-    r = toku_close_brt(t);              assert(r==0);
-    r = toku_cachetable_close(&ct);     assert(r==0);
-    toku_memory_check_all_free();
-    if (verbose) printf("test2 ok\n");
-}
-
 static void test5 (void) {
     int r;
     BRT t;
@@ -240,44 +153,6 @@ static void test_multiple_files (void) {
     test_multiple_files_of_size (1<<20);
 }
 
-static void test_named_db (void) {
-    const char *n0 = "test0.brt";
-    const char *n1 = "test1.brt";
-    CACHETABLE ct;
-    BRT t0;
-    int r;
-    DBT k,v;
-
-    if (verbose) printf("test_named_db\n");
-    unlink(n0);
-    unlink(n1);
-    toku_memory_check_all_free();
-    r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);           assert(r==0);
-    r = toku_open_brt(n0, "db1", 1, &t0, 1<<12, ct, null_txn, toku_default_compare_fun, null_db); assert(r==0);
-
-
-    toku_brt_insert(t0, toku_fill_dbt(&k, "good", 5), toku_fill_dbt(&v, "day", 4), null_txn); assert(r==0);
-
-    r = toku_close_brt(t0); assert(r==0);
-    r = toku_cachetable_close(&ct); assert(r==0);
-    toku_memory_check_all_free();
-
-    toku_memory_check_all_free();
-    r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);           assert(r==0);
-    r = toku_open_brt(n0, "db1", 0, &t0, 1<<12, ct, null_txn, toku_default_compare_fun, null_db); assert(r==0);
-
-    {
-	r = toku_brt_lookup(t0, toku_fill_dbt(&k, "good", 5), toku_init_dbt(&v));
-	assert(r==0);
-	assert(v.size==4);
-	assert(strcmp(v.data,"day")==0);
-    }
-
-    r = toku_close_brt(t0); assert(r==0);
-    r = toku_cachetable_close(&ct); assert(r==0);
-    toku_memory_check_all_free();
-}
-
 static void test_multiple_dbs (void) {
     const char *n0 = "test0.brt";
     const char *n1 = "test1.brt";
@@ -1640,20 +1515,9 @@ static void brt_blackbox_test (void) {
     test_cursor_last_empty();             toku_memory_check_all_free();
     test_multiple_brts_one_db_one_file(); toku_memory_check_all_free();
     test_dump_empty_db();                 toku_memory_check_all_free();
-    test_named_db();
     toku_memory_check_all_free();
     test_multiple_dbs();
     toku_memory_check_all_free();
-    if (verbose) printf("test0 A\n");
-    test0();
-    if (verbose) printf("test0 B\n");
-    test0(); /* Make sure it works twice. */
-    if (verbose) printf("test1\n");
-    test1();
-    if (verbose) printf("test2 checking memory\n");
-    test2(1);
-    if (verbose) printf("test2 faster\n");
-    test2(0);
     if (verbose) printf("test5\n");
     test5();
     if (verbose) printf("test_multiple_files\n");
@@ -1672,14 +1536,6 @@ static void brt_blackbox_test (void) {
 
     toku_brt_do_push_cmd = old_brt_do_push_cmd;
 
-//    test3(1<<19, 1<<20, 0);
-
-//    test3(1<<20, 1<<20, 0);
-
-//    test3(1<<20, 1<<21, 0);
-
-//    test3(1<<20, 1<<22, 0);
-
 }
 
 int main (int argc , const char *argv[]) {

newbrt/brt-test0.c

+/* -*- mode: C; c-basic-offset: 4 -*- */
+#ident "Copyright (c) 2007, 2008 Tokutek Inc.  All rights reserved."
+
+#include "brt.h"
+#include "key.h"
+#include "test.h"
+#include "toku_assert.h"
+#include <unistd.h>
+
+static TOKUTXN const null_txn = 0;
+static DB * const null_db = 0;
+
+static void test0 (void) {
+    BRT t;
+    int r;
+    CACHETABLE ct;
+    char fname[]="brt-test0.brt";
+    if (verbose) printf("%s:%d test0\n", __FILE__, __LINE__);
+    toku_memory_check=1;
+    toku_memory_check_all_free();
+    r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
+    assert(r==0);
+    if (verbose) printf("%s:%d test0\n", __FILE__, __LINE__);
+    unlink(fname);
+    r = toku_open_brt(fname, 0, 1, &t, 1024, ct, null_txn, toku_default_compare_fun, null_db);
+    assert(r==0);
+    //printf("%s:%d test0\n", __FILE__, __LINE__);
+    //printf("%s:%d n_items_malloced=%lld\n", __FILE__, __LINE__, n_items_malloced);
+    r = toku_close_brt(t);     assert(r==0);
+    //printf("%s:%d n_items_malloced=%lld\n", __FILE__, __LINE__, n_items_malloced);
+    r = toku_cachetable_close(&ct);
+    assert(r==0);
+    toku_memory_check_all_free();
+}
+
+int main (int argc , const char *argv[]) {
+    default_parse_args(argc, argv);
+    if (verbose) printf("test0 A\n");
+    test0();
+    if (verbose) printf("test0 B\n");
+    test0(); /* Make sure it works twice. */
+    toku_malloc_cleanup();
+    if (verbose) printf("test0 ok\n");
+    return 0;
+}

newbrt/brt-test1.c

+/* -*- mode: C; c-basic-offset: 4 -*- */
+#ident "Copyright (c) 2007, 2008 Tokutek Inc.  All rights reserved."
+
+#include "brt.h"
+#include "key.h"
+#include "test.h"
+#include "toku_assert.h"
+#include <unistd.h>
+
+static TOKUTXN const null_txn = 0;
+static DB * const null_db = 0;
+
+static void test1 (void) {
+    BRT t;
+    int r;
+    CACHETABLE ct;
+    char fname[]="brt-test1.brt";
+    DBT k,v;
+    toku_memory_check=1;
+    toku_memory_check_all_free();
+    r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
+    assert(r==0);
+    unlink(fname);
+    r = toku_open_brt(fname, 0, 1, &t, 1024, ct, null_txn, toku_default_compare_fun, null_db);
+    assert(r==0);
+    toku_brt_insert(t, toku_fill_dbt(&k, "hello", 6), toku_fill_dbt(&v, "there", 6), null_txn);
+    {
+	r = toku_brt_lookup(t, toku_fill_dbt(&k, "hello", 6), toku_init_dbt(&v));
+	assert(r==0);
+	assert(strcmp(v.data, "there")==0);
+	assert(v.size==6);
+    }
+    r = toku_close_brt(t);              assert(r==0);
+    r = toku_cachetable_close(&ct);     assert(r==0);
+    toku_memory_check_all_free();
+    if (verbose) printf("test1 ok\n");
+}
+int main (int argc , const char *argv[]) {
+    default_parse_args(argc, argv);
+     if (verbose) printf("test1\n");
+    test1();
+    toku_malloc_cleanup();
+    if (verbose) printf("test1 ok\n");
+    return 0;
+}

newbrt/brt-test2.c

+/* -*- mode: C; c-basic-offset: 4 -*- */
+#ident "Copyright (c) 2007, 2008 Tokutek Inc.  All rights reserved."
+
+#include "brt.h"
+#include "key.h"
+#include "test.h"
+#include "toku_assert.h"
+#include <unistd.h>
+
+static TOKUTXN const null_txn = 0;
+static DB * const null_db = 0;
+
+static void test2 (int memcheck) {
+    BRT t;
+    int r;
+    int i;
+    CACHETABLE ct;
+    char fname[]="brt-test2.brt";
+    toku_memory_check=memcheck;
+    if (verbose) printf("%s:%d checking\n", __FILE__, __LINE__);
+    toku_memory_check_all_free();
+    r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER); assert(r==0);
+    unlink(fname);
+    r = toku_open_brt(fname, 0, 1, &t, 1024, ct, null_txn, toku_default_compare_fun, null_db);
+    if (verbose) printf("%s:%d did setup\n", __FILE__, __LINE__);
+    assert(r==0);
+    for (i=0; i<4096; i++) {
+	DBT k,v;
+	char key[100],val[100];
+	snprintf(key,100,"hello%d",i);
+	snprintf(val,100,"there%d",i);
+	toku_brt_insert(t, toku_fill_dbt(&k, key, 1+strlen(key)), toku_fill_dbt(&v, val, 1+strlen(val)), null_txn);
+	//printf("%s:%d did insert %d\n", __FILE__, __LINE__, i);
+	if (0) {
+	    brt_flush(t);
+	    {
+		int n = toku_get_n_items_malloced();
+		if (verbose) printf("%s:%d i=%d n_items_malloced=%d\n", __FILE__, __LINE__, i, n);
+		if (n!=3) toku_print_malloced_items();
+		assert(n==3);
+	    }
+	}
+    }
+    if (verbose) printf("%s:%d inserted\n", __FILE__, __LINE__);
+    r = toku_close_brt(t);              assert(r==0);
+    r = toku_cachetable_close(&ct);     assert(r==0);
+    toku_memory_check_all_free();
+    if (verbose) printf("test2 ok\n");
+}
+
+int main (int argc , const char *argv[]) {
+    default_parse_args(argc, argv);
+    if (verbose) printf("test2 checking memory\n");
+//    test2(1);
+    if (verbose) printf("test2 faster\n");
+    test2(0);
+    toku_malloc_cleanup();
+    if (verbose) printf("test1 ok\n");
+    return 0;
+}

newbrt/brt-test3.c

@@ -3,7 +3,6 @@
 
 #include "brt.h"
 #include "key.h"
-#include "pma.h"
 #include "brt-internal.h"
 #include "memory.h"
 #include "toku_assert.h"
@@ -59,7 +58,7 @@ static void brt_blackbox_test (void) {
     test3(2048, 1<<15, 1);
     if (verbose) printf("test3 fast\n");
 
-    if (verbose) toku_pma_show_stats();
+    //if (verbose) toku_pma_show_stats();
 
     test3(1<<15, 1024, 1);
     if (verbose) printf("test3 fast\n");

newbrt/brt-test4.c

@@ -3,7 +3,6 @@
 
 #include "brt.h"
 #include "key.h"
-#include "pma.h"
 #include "brt-internal.h"
 #include "memory.h"
 #include "toku_assert.h"
@@ -57,7 +56,7 @@ static void brt_blackbox_test (void) {
     if (verbose) printf("test4 slow\n");
     test4(2048, 1<<15, 1);
 
-    if (verbose) toku_pma_show_stats();
+    //if (verbose) toku_pma_show_stats();
 
     test4(1<<15, 1024, 1);
 

newbrt/brt-verify.c

@@ -17,6 +17,16 @@
 
 #include "brt-internal.h"
 #include "toku_assert.h"
+#include "kv-pair.h"
+
+static void gpma_verify_fingerprint (GPMA pma, u_int32_t rand4fingerprint, u_int32_t fingerprint) {
+    u_int32_t actual_fingerprint=0;
+    GPMA_ITERATE(pma, idx, len, val,
+		 actual_fingerprint+=rand4fingerprint*toku_calccrc32_kvpair_struct(val)
+		 );
+    assert(actual_fingerprint==fingerprint);
+}
+
 
 static void verify_local_fingerprint (BRTNODE node) {
     u_int32_t fp=0;
@@ -29,7 +39,7 @@ static void verify_local_fingerprint (BRTNODE node) {
 			      }));
 	assert(fp==node->local_fingerprint);
     } else {
-	toku_pma_verify_fingerprint(node->u.l.buffer, node->rand4fingerprint, node->local_fingerprint);
+	gpma_verify_fingerprint(node->u.l.buffer, node->rand4fingerprint, node->local_fingerprint);
     }
 }
 

newbrt/brt.h

@@ -69,4 +69,9 @@ int toku_brt_height_of_root(BRT, int *height); // for an open brt, return the cu
 // Special hack for recovery
 int toku_brt_nonleaf_expunge_xaction(BRT brt, DISKOFF diskoff, TXNID xid);
 
+enum brt_header_flags {
+    TOKU_DB_DUP = 1,
+    TOKU_DB_DUPSORT = 2,
+};
+
 #endif

newbrt/brt2.c

@@ -219,7 +219,7 @@ static void initialize_brtnode (BRT t, BRTNODE n, DISKOFF nodename, int height)
     n->thisnodename = nodename;
     n->disk_lsn.lsn = 0; // a new one can always be 0.
     n->log_lsn = n->disk_lsn;
-    n->layout_version = 2;
+    n->layout_version = 3;
     n->height       = height;
     n->rand4fingerprint = random();
     n->local_fingerprint = 0;

newbrt/brtdump.c

@@ -44,9 +44,7 @@ void print_item (bytevec val, ITEMLEN len) {
 
 void dump_node (int f, DISKOFF off, struct brt_header *h) {
     BRTNODE n;
-    int r = toku_deserialize_brtnode_from (f, off, &n, h->flags, h->nodesize,
-					   toku_default_compare_fun, toku_default_compare_fun,
-					   (DB*)0, (FILENUM){0});
+    int r = toku_deserialize_brtnode_from (f, off, &n, h->flags, h->nodesize);
     assert(r==0);
     assert(n!=0);
     printf("brtnode\n");
@@ -105,13 +103,13 @@ void dump_node (int f, DISKOFF off, struct brt_header *h) {
 	}
     } else {
 	printf(" n_bytes_in_buffer=%d\n", n->u.l.n_bytes_in_buffer);
-	printf(" items_in_buffer  =%d\n", toku_pma_n_entries(n->u.l.buffer));
-	PMA_ITERATE_IDX(n->u.l.buffer, idx, key, keylen, data, datalen,
+	printf(" items_in_buffer  =%d\n", toku_gpma_n_entries(n->u.l.buffer));
+	GPMA_ITERATE(n->u.l.buffer, idx, len, data,
 			({
 			  printf("%d: ", idx);
-			  print_item(key, keylen);
+			  print_item(kv_pair_key(data), kv_pair_keylen(data));
 			  printf(" ");
-			  print_item(data, datalen);
+			  print_item(kv_pair_val(data), kv_pair_vallen(data));
 			  printf("\n");
 			}));
     }

newbrt/brttypes.h

@@ -23,7 +23,7 @@ typedef long long DISKOFF;  /* Offset in a disk. -1 is the NULL pointer. */
 typedef u_int64_t TXNID;
 
 typedef struct {
-    int len;
+    u_int32_t len;
     char *data;
 } BYTESTRING;
 

newbrt/cachetable-test2.c

@@ -170,7 +170,7 @@ static void test_chaining (void) {
     r = toku_cachetable_close(&ct); assert(r==0);
 }
 
-void usage (const char *progname) {
+void __attribute__((__noreturn__)) usage (const char *progname) {
     fprintf(stderr, "Usage:\n %s [-v] [-q]\n", progname);
     exit(1);
 }

newbrt/crc.h

@@ -8,7 +8,7 @@
 // zlib crc32 has a bug:  If len==0 then it should return oldcrc32, but crc32 returns 0.
 static inline u_int32_t toku_crc32 (u_int32_t oldcrc32, const void *data, u_int32_t len) {
     if (len==0) return oldcrc32;
-    else return crc32((unsigned long)oldcrc32, data, len);
+    else return crc32((unsigned long)oldcrc32, data, (uInt)len);
 }
 
 static const u_int32_t toku_null_crc = 0;

newbrt/fingerprint.c

@@ -21,6 +21,11 @@ u_int32_t toku_calccrc32_kvpair (const void *key, int keylen, const void *val, i
     return toku_calc_more_crc32_kvpair(toku_null_crc, key, keylen, val, vallen);
 }
 
+u_int32_t toku_calccrc32_kvpair_struct (const struct kv_pair *kvp) {
+    return toku_calccrc32_kvpair(kv_pair_key_const(kvp), kv_pair_keylen(kvp),
+				 kv_pair_val_const(kvp), kv_pair_vallen(kvp));
+}
+
 u_int32_t toku_calccrc32_cmd (int type, TXNID xid, const void *key, int keylen, const void *val, int vallen) {
     unsigned char type_c = type;
     unsigned int a = htonl(xid>>32);

newbrt/gpma-internal.h

+#include "memory.h"
+
+struct gpma {
+    enum typ_tag tag;
+    unsigned int N;      /* How long is the array? Always a power of two >= 4. */
+    u_int32_t n_items_present; /* How many array elements are non-null.         */
+    struct gitem *items; /* A malloced array.  If any item's DATA is null, then it's not in use. */
+    
+
+    double udt_step;     /* upper density threshold step */     
+                         /* Each doubling decreases the density by density step.
+			  * For example if array_len=256 and uplgN=8 then there are 5 doublings.
+			  * Regions of size 8 are full.  Regions of size 16 are 90% full.
+			  *  Regions of size 32 are 80% full.  Regions of size 64 are 70% full.
+			  *  Regions of size 128 are 60% full.  Regions of size 256 are 50% full.
+			  *  The density step is 0.10. */
+    double ldt_step;     /* lower density threshold step */
+};
+
+#define GPMA_MIN_ARRAY_SIZE 4
+
+/* density thresholds */
+#define GPMA_LDT_HIGH  0.25
+#define GPMA_LDT_LOW  0.40
+#define GPMA_UDT_HIGH  1.00
+#define GPMA_UDT_LOW  0.50
+
+/* Expose these for testing purposes */
+u_int32_t toku_gpma_find_index_bes (GPMA pma, gpma_besselfun_t besf, int direction, void *extra, int *found);
+u_int32_t toku_gpma_find_index (GPMA pma, u_int32_t len, void *data, gpma_compare_fun_t compare, void *extra, int *found);
+int toku_lg (unsigned int n);
+u_int32_t toku_hyperceil (u_int32_t v);
+int toku_max_int (int, int);
+int toku_gpma_smooth_region (GPMA pma,
+			     u_int32_t lo, u_int32_t hi,
+			     u_int32_t count, // The number of nonnull values
+			     u_int32_t idx, u_int32_t *newidxp, gpma_renumber_callback_t rcall, void *extra,
+			     u_int32_t old_N);
+int toku_make_space_at (GPMA pma, u_int32_t idx, u_int32_t *newidx, gpma_renumber_callback_t rcall, void *extra);
+
+void toku_gpma_distribute (GPMA pma,
+			   u_int32_t lo, u_int32_t hi,
+			   u_int32_t count,
+			   struct gitem *items, // some of these may be NULL data, be we leave space for them anyway.
+			   /*out*/ u_int32_t *tos   // the indices where the values end up (we fill this in)
+			   );
+int toku_smooth_deleted_region (GPMA pma, u_int32_t minidx, u_int32_t maxidx, gpma_renumber_callback_t renumberf, void *extra_for_renumberf);

newbrt/gpma.h

+#ifndef GPMA_H
+#define GPMA_H
+
+#ident "Copyright (c) 2007 Tokutek Inc.  All rights reserved."
+
+// Need this to get the u_int32_t types and so forth
+#include <sys/types.h>
+
+typedef struct gpma *GPMA;
+struct gitem {
+    u_int32_t len;
+    void *data;
+};
+
+typedef int (*gpma_compare_fun_t)(u_int32_t alen, void *aval, u_int32_t blen, void *bval, void*extra);
+typedef int (*gpma_besselfun_t)(u_int32_t dlen, void *dval, void *extra);  // return a number, not an error code.
+typedef int (*gpma_delete_callback_t)(u_int32_t slotnum, u_int32_t deletelen, void*deletedata, void*extra); // return 0 if OK.
+// If the pma moves things around and/or changes the size of the pma, it calls this function to indicate what happened.
+typedef int (*gpma_renumber_callback_t)(u_int32_t nitems,  // How many things moved
+					u_int32_t *froms,  // An array of indices indicating where things moved from
+					u_int32_t *tos,    // An array of indices indicating where thigns moved to
+					struct gitem *items, // The actual items that were moved
+					u_int32_t old_N,     // The old size of the array
+					u_int32_t new_N,     // The new size of teh array
+					void *extra);        // Context
+typedef void (*gpma_free_callback_t)(u_int32_t len, void*freeme, void*extra);
+
+// initial_index_limit must be zero or a power of two.
+int toku_gpma_create (GPMA*, int initial_index_limit);
+/* Return 0 if OK, and sets the referenced GPMA to NULL. */
+void toku_gpma_free (GPMA*, gpma_free_callback_t, void*);
+// How many items are present
+u_int32_t toku_gpma_n_entries (GPMA);
+// What is the maximum index limit
+u_int32_t toku_gpma_index_limit (GPMA);
+
+// Require that the item not be already present, according ot the compare function
+// The data in the DBT is passed in.
+int toku_gpma_insert (GPMA,
+		      u_int32_t len, void*data,
+		      gpma_compare_fun_t comparef, void*extra_for_comparef,
+		      gpma_renumber_callback_t renumberf, void*extra_for_renumberf,   // if anything gets renumbered, let the caller know
+		      u_int32_t *indexp // Where did the item get stored?
+		      );
+
+// Delete anything for which the besselfun is zero.  The besselfun must be monotonically increasing compared to the comparison function.
+// That is, if two othings compare to be < then their besselfun's must yield <=, and if the compare to be = their besselfuns must be =, and if they are > then their besselfuns must be >=
+// Note the delete_callback would be responsible for calling free on the object.
+int toku_gpma_delete_bessel (GPMA,
+			     gpma_besselfun_t,
+			     void*extra_for_besself,
+			     gpma_delete_callback_t,
+			     void*extra_for_deletef,
+			     gpma_renumber_callback_t, // if anything gets renumbered, let the caller know
+			     void*extra_for_renumberf);
+
+// Delete any items for which the compare function says things are zero.
+// For each item deleted, invoke deletef.
+// For any items moved around, invoke renumberf.
+int toku_gpma_delete_item (GPMA,
+			   u_int32_t len, void *data,
+			   gpma_compare_fun_t comparef,        void *extra_for_comparef,
+			   gpma_delete_callback_t deletef,     void *extra_for_deletef,
+			   gpma_renumber_callback_t renumberf, void *extra_for_renumberf);
+
+// Look up a particular item, using the compare function.  Find some X such that compf(len,data, X.len, X.data)==0
+//  (Note that the len and data passed here are always passed as the first pair of arguments to compf. )
+//  The item being looked up is the second pair of arguments.
+int toku_gpma_lookup_item (GPMA, u_int32_t len, void *data, gpma_compare_fun_t compf, void*extra, u_int32_t *resultlen, void **resultdata, u_int32_t *idx);
+
+// Lookup something according to the besselfun.
+// If direction==0 then return something for which the besselfun is zero (or return DB_NOTFOUND).
+// If direction>0  then return the first thing for which the besselfun is positive (or return DB_NOTFOUND).
+// If direction<0  then return the last thing for which the besselfun is negative (or return DB_NOTFOUND).
+int toku_gpma_lookup_bessel (GPMA, gpma_besselfun_t, int direction, void*extra, u_int32_t *len, void **data, u_int32_t *idx);
+void toku_gpma_iterate (GPMA, void(*)(u_int32_t len, void*data, void*extra), void*extra);
+#define GPMA_ITERATE(table,idx,vallen,val,body) ({                  \
+  u_int32_t idx;                                                    \
+  for (idx=0; idx<toku_gpma_index_limit(table); idx++) {            \
+      u_int32_t vallen; void*val;                                   \
+      if (0==toku_gpma_get_from_index(table, idx, &vallen, &val)) { \
+          body;                                                     \
+      } } })
+
+int toku_gpma_valididx (GPMA, u_int32_t idx);
+int toku_gpma_get_from_index (GPMA, u_int32_t idx, u_int32_t *len, void **data);
+
+// Whatever is in the slot gets overwritten.  Watch out that you free the thing before overwriting it.
+void toku_gpma_set_at_index (GPMA, u_int32_t idx, u_int32_t len, void*data);
+void toku_gpma_clear_at_index (GPMA, u_int32_t idx);
+
+int toku_gpma_move_inside_pma_by_renumbering (GPMA,
+					      u_int32_t nitems,
+					      u_int32_t *froms, u_int32_t *tos);
+
+int toku_gpma_split (GPMA pma, GPMA newpma, u_int32_t overhead,
+		     int (*realloc_data)(u_int32_t len, void *odata, void **ndata, void *extra),
+		     gpma_renumber_callback_t rcall,
+		     gpma_renumber_callback_t rcall_across_pmas, // This one is called for everything that moved
+		     void *extra);
+
+void toku_verify_gpma (GPMA pma);
+
+// Change the size of the PMA.  Anything beyond the oldsize is discarded (if the newsize is smaller) or zerod (if the newsize is larger)
+int toku_resize_gpma_exactly (GPMA pma, u_int32_t newsize);
+
+#endif

newbrt/kv-pair.h

@@ -57,7 +57,7 @@ static inline const void *kv_pair_key_const(const struct kv_pair *pair) {
     return pair->key;
 }
 
-static inline unsigned int kv_pair_keylen(struct kv_pair *pair) {
+static inline unsigned int kv_pair_keylen(const struct kv_pair *pair) {
     return pair->keylen;
 }
 
@@ -69,7 +69,7 @@ static inline const void *kv_pair_val_const(const struct kv_pair *pair) {
     return pair->key + pair->keylen;
 }
 
-static inline unsigned int kv_pair_vallen(struct kv_pair *pair) {
+static inline unsigned int kv_pair_vallen(const struct kv_pair *pair) {
     return pair->vallen;
 }
 

newbrt/log-internal.h

@@ -7,6 +7,7 @@
 #include <stdio.h>
 #include <pthread.h>
 #include <sys/types.h>
+#include <string.h>
 
 // Locking for the logger
 //  For most purposes we use the big ydb lock.

newbrt/log.c

@@ -494,7 +494,7 @@ int toku_fread_BYTESTRING (FILE *f, BYTESTRING *bs, u_int32_t *crc, u_int32_t *l
     int r=toku_fread_u_int32_t(f, (u_int32_t*)&bs->len, crc, len);
     if (r!=0) return r;
     bs->data = toku_malloc(bs->len);
-    int i;
+    u_int32_t i;
     for (i=0; i<bs->len; i++) {
 	r=toku_fread_u_int8_t(f, (u_int8_t*)&bs->data[i], crc, len);
 	if (r!=0) {
@@ -574,7 +574,7 @@ int toku_logprint_BYTESTRING (FILE *outf, FILE *inf, const char *fieldname, u_in
     int r = toku_fread_BYTESTRING(inf, &bs, crc, len);
     if (r!=0) return r;
     fprintf(outf, " %s={len=%d data=\"", fieldname, bs.len);
-    int i;
+    u_int32_t i;
     for (i=0; i<bs.len; i++) {
 	switch (bs.data[i]) {
 	case '"':  fprintf(outf, "\\\""); break;

newbrt/log.h

@@ -7,7 +7,7 @@
 
 #include "../include/db.h"
 #include "brttypes.h"
-#include "kv-pair.h"
+#include "memory.h"
 
 struct logbytes;
 struct logbytes {
@@ -33,8 +33,6 @@ LSN toku_logger_last_lsn(TOKULOGGER);
 int toku_logger_set_lg_max (TOKULOGGER logger, u_int32_t);
 int toku_logger_get_lg_max (TOKULOGGER logger, u_int32_t *);
 
-int toku_logger_log_phys_add_or_delete_in_leaf    (DB *db, TOKUTXN txn, DISKOFF diskoff, int is_add, const struct kv_pair *pair);
-
 int toku_logger_commit (TOKUTXN txn, int no_sync);
 
 int toku_logger_txn_begin (TOKUTXN /*parent*/,TOKUTXN *, TOKULOGGER /*logger*/);

newbrt/logformat.c

@@ -41,9 +41,9 @@ int logformat_version_number = 0;
 const struct logtype rollbacks[] = {
     {"fcreate", 'F', FA{{"BYTESTRING", "fname", 0},
 			NULLFIELD}},
-    {"fclose", 'c', FA{{"FILENUM", "filenum", 0},
-		       {"BYTESTRING", "fname", 0},
-		       NULLFIELD}},
+//    {"fclose", 'c', FA{{"FILENUM", "filenum", 0},
+//		       {"BYTESTRING", "fname", 0},
+//		       NULLFIELD}},
     {"deleteatleaf", 'd', FA{{"FILENUM", "filenum", 0}, // Note a delete for rollback.   The delete takes place in a leaf.
 			     {"BYTESTRING", "key", 0},
 			     {"BYTESTRING", "data", 0},
@@ -175,6 +175,8 @@ const struct logtype logtypes[] = {
 			      {"DISKOFF",    "old_diskoff", 0},
 			      {"DISKOFF",    "new_diskoff", 0},
 			      {"INTPAIRARRAY",   "fromto",    0},
+			      {"u_int32_t",      "old_N", 0},
+			      {"u_int32_t",      "new_N", 0},
 			      NULLFIELD}},
     {0,0,FA{NULLFIELD}}
 };
@@ -199,7 +201,7 @@ const struct logtype logtypes[] = {
     } })
 
 
-void fprintf2 (FILE *f1, FILE *f2, const char *format, ...) {
+static void __attribute__((format (printf, 3, 4))) fprintf2 (FILE *f1, FILE *f2, const char *format, ...) {
     va_list ap;
     int r;
     va_start(ap, format);

newbrt/memory-debug.c

+/* An implementation of memory that can be made to return NULL and ENOMEM on certain mallocs. */
+#include "memory.h"
+#include "toku_assert.h"
+
+#include <errno.h>
+#include <string.h>
+
+int *toku_dead_mallocs=0;
+int toku_malloc_counter=0;
+
+void toku_malloc_cleanup(void) {}
+void toku_free(void*x) {
+    free(x);
+}
+
+// if it fails, return 1, and set errno
+static int does_malloc_fail (void) {
+    if (toku_dead_mallocs) {
+	int mnum = *toku_dead_mallocs;
+	if (mnum==toku_malloc_counter) {
+	    toku_malloc_counter++;
+	    toku_dead_mallocs++;
+	    errno=ENOMEM;
+	    return 1;
+	}
+    }
+    toku_malloc_counter++;
+    return 0;
+}
+
+void* toku_malloc(size_t n) {
+    if (does_malloc_fail()) return 0;
+    return malloc(n);
+}
+void *toku_tagmalloc(size_t size, enum typ_tag typtag) {
+    //printf("%s:%d tagmalloc\n", __FILE__, __LINE__);
+    void *r = toku_malloc(size);
+    if (!r) return 0;
+    assert(size>sizeof(int));
+    ((int*)r)[0] = typtag;
+    return r;
+}
+
+void *toku_memdup (const void *v, size_t len) {
+    void *r=toku_malloc(len);
+    memcpy(r,v,len);
+    return r;
+}
+
+char *toku_strdup (const char *s) {
+    return toku_memdup(s, strlen(s)+1);
+}
+
+void *toku_realloc(void *p, size_t size) {
+    if (p==0) return toku_malloc(size);
+    if (does_malloc_fail()) return 0;
+    return realloc(p, size);
+}

newbrt/memory.c

@@ -21,7 +21,7 @@ static int overflowed=0;
 static void *items[items_limit];
 static long sizes[items_limit];
 
-static void note_did_malloc (void *p, long size) {
+static void note_did_malloc (void *p, size_t size) {
     static long long count=0;
     WHEN_MEM_DEBUG(
 		   if (n_items_malloced<items_limit) { items[n_items_malloced]=p; sizes[n_items_malloced]=size; }
@@ -168,9 +168,10 @@ void *toku_malloc(size_t size) {
     //if ((long)r==0x80523f8) { printf("%s:%d %p size=%ld\n", __FILE__, __LINE__, r, size);   }
     return r;
 }
-void *toku_tagmalloc(size_t size, int typtag) {
+void *toku_tagmalloc(size_t size, enum typ_tag typtag) {
     //printf("%s:%d tagmalloc\n", __FILE__, __LINE__);
     void *r = toku_malloc(size);
+    if (!r) return 0;
     assert(size>sizeof(int));
     ((int*)r)[0] = typtag;
     return r;
@@ -216,10 +217,14 @@ char *toku_strdup (const char *s) {
 
 void toku_memory_check_all_free (void) {
     if (n_items_malloced>0) {
-	printf("n_items_malloced=%lld\n", n_items_malloced);
-	if (toku_memory_check)
-	    printf(" one item is %p size=%ld\n", items[0], sizes[0]);
-	exit(1);
+	fprintf(stderr, "n_items_malloced=%lld\n", n_items_malloced);
+	if (toku_memory_check) {
+	    int i;
+	    for (i=0; i<n_items_malloced && i< items_limit && i<10; i++) {
+		fprintf(stderr, " one item is %p size=%ld\n", items[i], sizes[i]);
+	    }
+	}
+	return;
     }
     assert(n_items_malloced==0);
 }

newbrt/memory.h

@@ -10,6 +10,14 @@
 
 /* Generally: errno is set to 0 or a value to indicate problems. */
 
+enum typ_tag { TYP_BRTNODE = 0xdead0001,
+	       TYP_CACHETABLE, TYP_PAIR, /* for cachetables */
+	       TYP_PMA,
+	       TYP_GPMA,
+               TYP_TOKULOGGER,
+	       TYP_TOKUTXN
+};
+
 /* Everything should call toku_malloc() instead of malloc(), and toku_calloc() instead of calloc() */
 void *toku_calloc(size_t nmemb, size_t size);
 void *toku_malloc(size_t size);
@@ -17,7 +25,7 @@ void *toku_malloc(size_t size);
  * This "tag" is useful if you are debugging and run across a void* that is
  * really a (struct foo *), and you want to figure out what it is.
  */
-void *toku_tagmalloc(size_t size, int typ);
+void *toku_tagmalloc(size_t size, enum typ_tag typ);
 void toku_free(void*);
 /* toku_free_n() should be used if the caller knows the size of the malloc'd object. */
 void toku_free_n(void*, size_t size);
@@ -73,4 +81,9 @@ int toku_get_n_items_malloced(void); /* How many items are malloc'd but not free
 void toku_print_malloced_items(void); /* Try to print some malloced-but-not-freed items.  May be a noop. */
 void toku_malloc_report (void); /* report on statistics about number of mallocs.  Maybe a no-op. */ 
 
+// For memory-debug.c  Set this to an array of integers that say which mallocs should return NULL and ENOMEM.
+// The array is terminated by a -1.
+extern int *toku_dead_mallocs;
+extern int toku_malloc_counter; // so you can reset it
+
 #endif

newbrt/mempool.c

@@ -41,10 +41,10 @@ int toku_mempool_get_frag_size(struct mempool *mp) {
     return mp->frag_size;
 }
 
-void *toku_mempool_malloc(struct mempool *mp, int size, int alignment) {
+void *toku_mempool_malloc(struct mempool *mp, size_t size, int alignment) {
     assert(mp->free_offset <= mp->size);
     void *vp;
-    int offset = (mp->free_offset + (alignment-1)) & ~(alignment-1);
+    size_t offset = (mp->free_offset + (alignment-1)) & ~(alignment-1);
     if (offset + size > mp->size) {
         vp = 0;
     } else {

newbrt/mempool.h

@@ -8,15 +8,17 @@
    when the memory pool no longer has free space, the allocated chunks 
    must be relocated by the application to a new memory pool. */ 
 
+#include <sys/types.h>
+
 struct mempool;
 
 typedef int (*mempool_compress_func)(struct mempool *mp, void *arg);
 
 struct mempool {
     void *base;           /* the base address of the memory */
-    int free_offset;      /* the offset of the memory pool free space */
-    int size;             /* the size of the memory */
-    int frag_size;        /* the size of the fragmented memory */
+    size_t free_offset;      /* the offset of the memory pool free space */
+    size_t size;             /* the size of the memory */
+    size_t frag_size;        /* the size of the fragmented memory */
     mempool_compress_func compress_func;
     void *compress_arg;
 };
@@ -42,7 +44,7 @@ int toku_mempool_get_size(struct mempool *mp);
 int toku_mempool_get_frag_size(struct mempool *mp);
 
 /* allocate a chunk of memory from the memory pool suitably aligned */
-void *toku_mempool_malloc(struct mempool *mp, int size, int alignment);
+void *toku_mempool_malloc(struct mempool *mp, size_t size, int alignment);
 
 /* free a previously allocated chunk of memory.  the free only updates
    a count of the amount of free space in the memory pool.  the memory

newbrt/pma-internal.h

@@ -8,7 +8,7 @@ struct pma {
     int dup_mode;
     unsigned int N;      /* How long is the array? Always a power of two >= 4. */
     int n_pairs_present; /* How many array elements are non-null.         */
-    struct kv_pair **pairs;
+    LEAFENTRY *pairs;
     int uplgN;           /* The smallest power of two >= lg(N)            */
     double udt_step;     /* upper density threshold step */     
                          /* Each doubling decreases the density by density step.
@@ -26,10 +26,10 @@ struct pma {
     struct mempool kvspace;
 };
 
-int toku_pmainternal_count_region (struct kv_pair *pairs[], int lo, int hi);
+int toku_pmainternal_count_region (LEAFENTRY pairs[], int lo, int hi);
 void toku_pmainternal_calculate_parameters (PMA pma);
-int toku_pmainternal_smooth_region (TOKULOGGER, FILENUM, DISKOFF, struct kv_pair */*pairs*/[], int /*n*/, int /*idx*/, int /*base*/, PMA /*pma*/, int */*new_idx*/, LSN */*node_lsn*/);
-int toku_pmainternal_printpairs (struct kv_pair *pairs[], int N);
+int toku_pmainternal_smooth_region (TOKULOGGER, FILENUM, DISKOFF, LEAFENTRY/*pairs*/[], int /*n*/, int /*idx*/, int /*base*/, PMA /*pma*/, int */*new_idx*/, LSN */*node_lsn*/);
+int toku_pmainternal_printpairs (LEAFENTRY pairs[], int N);
 int toku_pmainternal_make_space_at (TOKULOGGER, FILENUM, DISKOFF, PMA pma, int idx, unsigned int *new_index, LSN *node_lsn);
 int toku_pmainternal_find (PMA pma, DBT *); // The DB is so the comparison fuction can be called.
 void toku_print_pma (PMA pma); /* useful for debugging, so keep the name short. I.e., not pmainternal_print_pma() */

newbrt/pma-test.c

@@ -1149,98 +1149,6 @@ static void test_pma_split(void) {
     test_pma_split_varkey(); local_memory_check_all_free();
 }
 
-/*
- * test the toku_pma_bulk_insert function by creating n kv pairs and bulk 
- * inserting them into an empty pma.  verify that the pma contains all
- * of the kv pairs.
- */
-static void test_pma_bulk_insert_n(int n) {
-    PMA pma;
-    int r;
-    int i;
-    DBT *keys, *vals;
-
-    u_int32_t rand4fingerprint = random();
-    u_int32_t sum = 0;
-    u_int32_t expect_fingerprint = 0;
-
-    if (verbose) printf("test_pma_bulk_insert_n: %d\n", n);
-
-    r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0, 0);
-    assert(r == 0);
-
-    /* init n kv pairs */
-    keys = toku_malloc(n * sizeof (DBT));
-    assert(keys);
-    vals = toku_malloc(n * sizeof (DBT));
-    assert(vals);
-
-    /* init n kv pairs */
-    for (i=0; i<n; i++) {
-        char kstring[11];
-        char *k; int klen;
-        int *v; int vlen;
-
-        snprintf(kstring, sizeof kstring, "%.10d", i);
-        klen = strlen(kstring) + 1;
-        k = toku_malloc(klen);
-        assert(k);
-        strcpy(k, kstring);
-        toku_fill_dbt(&keys[i], k, klen);
-
-        vlen = sizeof (int);
-        v = toku_malloc(vlen);
-        assert(v);
-        *v = i;
-        toku_fill_dbt(&vals[i], v, vlen);
-
-	expect_fingerprint += rand4fingerprint*toku_calccrc32_kvpair (k, klen, v, vlen);
-    }
-
-    /* bulk insert n kv pairs */
-    r = toku_pma_bulk_insert(null_logger, null_filenum, (DISKOFF)0, pma, keys, vals, n, rand4fingerprint, &sum, 0);
-    assert(r == 0);
-    assert(sum==expect_fingerprint);
-    toku_pma_verify(pma);
-    toku_pma_verify_fingerprint(pma, rand4fingerprint, sum);
-
-    /* verify */
-    if (0) toku_print_pma(pma);
-    assert(n == toku_pma_n_entries(pma));
-    for (i=0; i<n; i++) {
-        DBT val;
-        toku_init_dbt(&val); val.flags = DB_DBT_MALLOC;
-        r = toku_pma_lookup(pma, &keys[i], &val);
-        assert(r == 0);
-        assert(vals[i].size == val.size);
-        assert(memcmp(vals[i].data, val.data, val.size) == 0);
-        toku_free(val.data);
-    }
-
-    /* cleanup */
-    for (i=0; i<n; i++) {
-        toku_free(keys[i].data);
-        toku_free(vals[i].data);
-    }
-
-    r = toku_pma_free(&pma);
-    assert(r == 0);
-    
-    toku_free(keys);
-    toku_free(vals);
-}
-
-static void test_pma_bulk_insert(void) {
-    test_pma_bulk_insert_n(0); local_memory_check_all_free();
-    test_pma_bulk_insert_n(1); local_memory_check_all_free();
-    test_pma_bulk_insert_n(2); local_memory_check_all_free();
-    test_pma_bulk_insert_n(3); local_memory_check_all_free();
-    test_pma_bulk_insert_n(4); local_memory_check_all_free();
-    test_pma_bulk_insert_n(5); local_memory_check_all_free();
-    test_pma_bulk_insert_n(8); local_memory_check_all_free();
-    test_pma_bulk_insert_n(32); local_memory_check_all_free();
-}
-
 static void test_pma_insert_or_replace(void) {
     PMA pma;
     int r;
@@ -2486,7 +2394,6 @@ static void pma_tests (void) {
     test_pma_cursor();            local_memory_check_all_free();
 
     test_pma_split();             local_memory_check_all_free();
-    test_pma_bulk_insert();       local_memory_check_all_free();
     test_pma_insert_or_replace(); local_memory_check_all_free();
     test_pma_delete();
     test_pma_already_there();     local_memory_check_all_free();

newbrt/pma.h

-#ifndef PMA_H
-#define PMA_H
-
-#ident "Copyright (c) 2007 Tokutek Inc.  All rights reserved."
-
-#include "brttypes.h"
-#include "ybt.h"
-#include "yerror.h"
-#include "../include/db.h"
-#include "log.h"
-#include "brt-search.h"
-
-/* An in-memory Packed Memory Array dictionary. */
-/* There is a built-in-cursor. */
-
-/* All functions return 0 on success. */
-
-typedef struct pma *PMA;
-// typedef struct pma_cursor *PMA_CURSOR;
-
-/* compare 2 DBT's. return a value < 0, = 0, > 0 if a < b, a == b, a > b respectively */
-typedef int (*pma_compare_fun_t)(DB *, const DBT *a, const DBT *b);
-
-int toku_pma_create(PMA *, pma_compare_fun_t compare_fun, DB *, FILENUM filenum, int maxsize, int initial_n_pairs);
-
-int toku_pma_set_compare(PMA pma, pma_compare_fun_t compare_fun);
-
-/* set the duplicate mode 
-   0 -> no duplications, TOKU_DB_DUP, TOKU_DB_DUPSORT */
-int toku_pma_set_dup_mode(PMA pma, int mode);
-
-/* set the duplicate compare function */
-int toku_pma_set_dup_compare(PMA pma, pma_compare_fun_t dup_compare_fun);
-
-/* verify the integrity of a pma */
-void toku_pma_verify(PMA pma);
-
-/* returns 0 if OK.
- * You must have freed all the cursors, otherwise returns nonzero and does nothing. */
-int toku_pma_free (PMA *);
-
-int  toku_pma_n_entries (PMA);
-
-/* Returns an error if the key is already present. */
-/* The values returned should not be modified.by the caller. */
-/* Any cursors should be updated. */
-/* Duplicates the key and keylen. */
-//enum pma_errors toku_pma_insert (PMA, bytevec key, ITEMLEN keylen, bytevec data, ITEMLEN datalen);
-
-// The DB pointer is there so that the comparison function can be called.
-enum pma_errors toku_pma_insert (PMA, DBT*, DBT*, TOKULOGGER, TXNID, FILENUM, DISKOFF, u_int32_t /*random for fingerprint */, u_int32_t */*fingerprint*/, LSN *node_lsn);
-
-/* This returns an error if the key is NOT present. */
-int pma_replace (PMA, bytevec key, ITEMLEN keylen, bytevec data, ITEMLEN datalen);
-
-/* Delete pairs from the pma.
- * If val is 0 then delete all pairs from the pma that match the key.
- * If val is not 0 then only delete the pair that matches both the key and the val.
- *   (This even works if there is no such pair (in which case DB_NOTFOUND is returned, and
- *    no changes are made.)
- *   The case where val!=0 should work for both DUP and NODUP dictionaries.
- *    For NODUP dictionaries, the value is deleted only if both the key and the value match.
- */
-int toku_pma_delete (PMA, DBT */*key*/, DBT */*val*/,
-		     TOKULOGGER, TXNID, DISKOFF,
-		     u_int32_t /*random for fingerprint*/, u_int32_t */*fingerprint*/, u_int32_t *deleted_size, LSN*);
-
-int toku_pma_delete_fixupsize (PMA, DBT */*key*/, DBT */*val*/,
-			       TOKULOGGER, TXNID, DISKOFF,
-			       u_int32_t /*random for fingerprint*/, u_int32_t */*fingerprint*/, LSN*, u_int32_t *n_bytes_in_buffer_including_overheads);
-
-int toku_pma_strong_insert_or_replace (PMA pma, DBT *k, DBT *v,
-				     TOKULOGGER, TXNID, FILENUM, DISKOFF,
-				     u_int32_t rand4fingerprint, u_int32_t *fingerprint,
-				     LSN *node_lsn,
-				     u_int32_t *n_bytes_in_buffer_including_overheads);
-int toku_pma_insert_or_replace_ws (PMA pma, DBT *k, DBT *v,
-				   TOKULOGGER, TXNID, FILENUM, DISKOFF,
-				   u_int32_t rand4fingerprint, u_int32_t *fingerprint,
-				   LSN *node_lsn,
-				   u_int32_t *n_bytes_in_buffer_including_overheads,
-				   int weak_p);
-
-int toku_pma_insert_or_replace (PMA /*pma*/, DBT */*k*/, DBT */*v*/,
-				int */*replaced_v_size*/, /* If it is a replacement, set to the size of the old value, otherwise set to -1. */
-				TOKULOGGER, TXNID, FILENUM, DISKOFF,
-				u_int32_t /*random for fingerprint*/, u_int32_t */*fingerprint*/,
-				LSN */*node_lsn*/);
-//??  __attribute__((deprecated));
-
-
-/* Exposes internals of the PMA by returning a pointer to the guts.
- * Don't modify the returned data.  Don't free it. */
-enum pma_errors toku_pma_lookup (PMA, DBT*, DBT*);
-
-int toku_pma_search(PMA, brt_search_t *, DBT *, DBT *);
-
-/*
- * The kv pairs in PMA are split into two (nearly) equal sized sets.
- * THe ones in the left half are left in PMA, the ones in the right half are put into NEWPMA.
- * The size is determined by the sum of the sizes of the keys and values. 
- * The NEWPMA must be empty.
- *
- * DISKOFF  - the disk offset of the node containing the PMA to be split.  (Needed for logging)
- * PMA      - the pma to be split.
- * PMA_SIZE - a variable containing the size of the disk image of the PMA.
- * RAND4SUM - the random number for fingerprinting
- * FINGERPRINT - the current fingerprint of the PMA.
- *
- * NEWDISKOFF, NEWPMA, NEWPMASIZE, NEWRAND4SUM, NEWFINGERPRINT -  The same information fo the pma to hold the stuff to be moved out of PMA.
- *
- * SPLITK  filled in with the resulting pivot key.
- *   The original PMA gets keys <= pivot key
- *   The NEWPMA gets keys > pivot key
- */
-int toku_pma_split(TOKULOGGER, FILENUM,
-		   DISKOFF /*diskoff*/,    PMA /*pma*/,     unsigned int */*pma_size*/,     u_int32_t /*rand4sum*/,  u_int32_t */*fingerprint*/,       LSN* /*lsn*/,
-		   DBT */*splitk*/,
-		   DISKOFF /*newdiskoff*/, PMA /*newpma*/,  unsigned int */*newpma_size*/,  u_int32_t /*newrand4sum*/,  u_int32_t */*newfingerprint*/, LSN* /*newlsn*/);
-
-/*
- * Insert several key value pairs into an empty pma.  
- * Doesn't delete any existing keys (even if they are duplicates)
- * Requires: The keys are sorted
- *
- * pma - the pma that the key value pairs will be inserted into.
- *      must be empty with no cursors.
- * keys - an array of keys
- * vals - an array of values
- * n_newpairs - the number of key value pairs
- */
-int toku_pma_bulk_insert(TOKULOGGER, FILENUM, DISKOFF, PMA pma, DBT *keys, DBT *vals, int n_newpairs, u_int32_t rand4sem, u_int32_t *fingerprint, LSN */*node_lsn*/);
-
-int toku_pma_random_pick(PMA, bytevec *key, ITEMLEN *keylen, bytevec *data, ITEMLEN *datalen);
-
-unsigned int toku_pma_index_limit(PMA); // How many slots are in the PMA right now?
-int toku_pmanode_valid(PMA, unsigned int);
-bytevec toku_pmanode_key(PMA, unsigned int);
-ITEMLEN toku_pmanode_keylen(PMA, unsigned int);
-bytevec toku_pmanode_val(PMA, unsigned int);
-ITEMLEN toku_pmanode_vallen(PMA, unsigned int);
-
-void toku_pma_iterate (PMA, void(*)(bytevec,ITEMLEN,bytevec,ITEMLEN, void*), void*);
-
-#define PMA_ITERATE_IDX(table,idx,keyvar,keylenvar,datavar,datalenvar,body) ({ \
-  unsigned int idx;							\
-  for (idx=0; idx<toku_pma_index_limit(table); idx++) {		       \
-    if (toku_pmanode_valid(table,idx)) {                                    \
-      bytevec keyvar = toku_pmanode_key(table,idx);                         \
-      ITEMLEN keylenvar = toku_pmanode_keylen(table,idx);                   \
-      bytevec datavar = toku_pmanode_val(table, idx);                       \
-      ITEMLEN datalenvar = toku_pmanode_vallen(table, idx);                 \
-      body;                                                            \
-    } } })
-
-#define PMA_ITERATE(table,keyvar,keylenvar,datavar,datalenvar,body) PMA_ITERATE_IDX(table, __i, keyvar, keylenvar, datavar, datalenvar, body)
-
-void toku_pma_verify_fingerprint (PMA pma, u_int32_t rand4fingerprint, u_int32_t fingerprint);
-
-// Set the PMA's size, without moving anything.
-int toku_resize_pma_exactly (PMA pma, int oldsize, int newsize);
-
-int toku_pma_set_at_index (PMA, unsigned int /*index*/, DBT */*key*/, DBT */*value*/); // If the index is wrong or there is a value already, return nonzero
-int toku_pma_clear_at_index (PMA, unsigned int /*index*/); // If the index is wrong or there is a value already, return nonzero
-
-
-// Requires:  No open cursors on the pma.
-// Return nonzero if the indices are somehow wrong.
-int toku_pma_move_indices (PMA pma_from, PMA pma_to, INTPAIRARRAY fromto,
-			   u_int32_t rand_from, u_int32_t *fingerprint_from,
-			   u_int32_t rand_to,   u_int32_t *fingerprint_to,
-			   u_int32_t *n_in_buf_from, u_int32_t *n_in_buf_to);
-// Move things backwards according to fromto.
-int toku_pma_move_indices_back (PMA pma_backto, PMA pma_backfrom, INTPAIRARRAY fromto,
-				u_int32_t rand_backto,   u_int32_t *fingerprint_backto,
-				u_int32_t rand_backfrom, u_int32_t *fingerprint_backfrom,
-				u_int32_t *n_in_buf_backto, u_int32_t *n_in_buf_backfrom
-				);
-
-void toku_pma_show_stats (void);
-
-
-#endif

newbrt/recover.c

@@ -12,6 +12,8 @@
 #include "log-internal.h"
 #include "log_header.h"
 #include "toku_assert.h"
+#include "kv-pair.h"
+#include "gpma-internal.h"
 
 #include <fcntl.h>
 #include <stdlib.h>
@@ -146,16 +148,22 @@ void toku_recover_newbrtnode (LSN lsn, FILENUM filenum,DISKOFF diskoff,u_int32_t
     n->thisnodename = diskoff;
     n->log_lsn = n->disk_lsn  = lsn;
     //printf("%s:%d %p->disk_lsn=%"PRId64"\n", __FILE__, __LINE__, n, n->disk_lsn.lsn);
-    n->layout_version = 2;
+    n->layout_version = 3;
     n->height         = height;
     n->rand4fingerprint = rand4fingerprint;
     n->flags = is_dup_sort ? TOKU_DB_DUPSORT : 0; // Don't have TOKU_DB_DUP ???
     n->local_fingerprint = 0; // nothing there yet
     n->dirty = 1;
     if (height==0) {
-	r=toku_pma_create(&n->u.l.buffer, toku_dont_call_this_compare_fun, null_db, filenum, nodesize, 0);
+	r=toku_gpma_create(&n->u.l.buffer, 0);
 	assert(r==0);
 	n->u.l.n_bytes_in_buffer=0;
+	{
+	    void *mp = toku_malloc(n->nodesize);
+	    assert(mp);
+	    toku_mempool_init(&n->u.l.buffer_mempool, mp, n->nodesize);
+	}
+
     } else {
 	n->u.n.n_children = 0;
 	n->u.n.totalchildkeylens = 0;
@@ -392,10 +400,11 @@ void toku_recover_insertinleaf (LSN lsn, TXNID UU(txnid), FILENUM filenum, DISKO
     BRTNODE node = node_v;
     assert(node->height==0);
     VERIFY_COUNTS(node);
-    DBT key,data;
-    r = toku_pma_set_at_index(node->u.l.buffer, pmaidx, toku_fill_dbt(&key, keybs.data, keybs.len), toku_fill_dbt(&data, databs.data, databs.len));
-    assert(r==0);
+    struct kv_pair *kvp = brtnode_malloc_kv_pair(node->u.l.buffer, &node->u.l.buffer_mempool, keybs.data, keybs.len, databs.data, databs.len);
+    assert(pair);
+    toku_gpma_set_at_index(node->u.l.buffer, pmaidx, kv_pair_size(kvp), kvp);
     node->local_fingerprint += node->rand4fingerprint*toku_calccrc32_kvpair(keybs.data, keybs.len, databs.data, databs.len);
+//    printf("%s:%d local_fingerprint=%08x (this=%08x)\n", __FILE__, __LINE__, node->local_fingerprint, toku_calccrc32_kvpair(keybs.data, keybs.len, databs.data, databs.len));
     node->u.l.n_bytes_in_buffer += PMA_ITEM_OVERHEAD + KEY_VALUE_OVERHEAD + keybs.len + databs.len; 
 
 //    PMA_ITERATE_IDX(node->u.l.buffer, idx, skey, keylen __attribute__((__unused__)), sdata, datalen __attribute__((__unused__)),
@@ -421,8 +430,15 @@ void toku_recover_deleteinleaf (LSN lsn, TXNID UU(txnid), FILENUM filenum, DISKO
     BRTNODE node = node_v;
     assert(node->height==0);
     VERIFY_COUNTS(node);
-    r = toku_pma_clear_at_index(node->u.l.buffer, pmaidx);
-    assert (r==0);
+    {
+	u_int32_t len;
+	void *data;
+	r = toku_gpma_get_from_index(node->u.l.buffer, pmaidx, &len, &data);
+	if (r==0) {
+	    toku_mempool_mfree(&node->u.l.buffer_mempool, data, len);
+	}
+    }
+    toku_gpma_clear_at_index(node->u.l.buffer, pmaidx);
     node->local_fingerprint -= node->rand4fingerprint*toku_calccrc32_kvpair(keybs.data, keybs.len, databs.data, databs.len);
     node->u.l.n_bytes_in_buffer -= PMA_ITEM_OVERHEAD + KEY_VALUE_OVERHEAD + keybs.len + databs.len; 
     VERIFY_COUNTS(node);
@@ -434,7 +450,7 @@ void toku_recover_deleteinleaf (LSN lsn, TXNID UU(txnid), FILENUM filenum, DISKO
 }
 
 // a newbrtnode should have been done before this
-void toku_recover_resizepma (LSN lsn, FILENUM filenum, DISKOFF diskoff, u_int32_t oldsize, u_int32_t newsize) {
+void toku_recover_resizepma (LSN lsn, FILENUM filenum, DISKOFF diskoff, u_int32_t oldsize __attribute__((__unused__)), u_int32_t newsize) {
     struct cf_pair *pair = NULL;
     int r = find_cachefile(filenum, &pair);
     assert(r==0);
@@ -444,7 +460,7 @@ void toku_recover_resizepma (LSN lsn, FILENUM filenum, DISKOFF diskoff, u_int32_
     assert(r==0);
     BRTNODE node = node_v;
     assert(node->height==0);
-    r = toku_resize_pma_exactly (node->u.l.buffer, oldsize, newsize);
+    r = toku_resize_gpma_exactly (node->u.l.buffer, newsize);
     assert(r==0);
     
     VERIFY_COUNTS(node);
@@ -454,7 +470,35 @@ void toku_recover_resizepma (LSN lsn, FILENUM filenum, DISKOFF diskoff, u_int32_
     assert(r==0);
 }
 
-void toku_recover_pmadistribute (LSN lsn, FILENUM filenum, DISKOFF old_diskoff, DISKOFF new_diskoff, INTPAIRARRAY fromto) {
+int move_indices (GPMA from, GPMA to, INTPAIRARRAY fromto,
+		  u_int32_t a_rand, u_int32_t *a_fp,
+		  u_int32_t b_rand, u_int32_t *b_fp,
+		  u_int32_t *a_nbytes, u_int32_t *b_nbytes) {
+    struct gitem *MALLOC_N(fromto.size, items);
+    if (items==0) return errno;
+    u_int32_t i;
+    u_int32_t fp=0;
+    u_int32_t sizediff=0;
+    for (i=0; i<fromto.size; i++) {
+	int idx = fromto.array[i].a;
+	struct gitem item = from->items[idx];
+	items[i]=item;
+	from->items[idx].data = 0;
+	fp += toku_crc32(toku_null_crc, item.data, item.len);
+	sizediff += PMA_ITEM_OVERHEAD + item.len;
+    }
+    for (i=0; i<fromto.size; i++) {
+	to->items[fromto.array[i].b] = items[i];
+    }
+    *a_fp -= a_rand * fp;
+    *b_fp += b_rand * fp;
+    *a_nbytes -= sizediff;
+    *b_nbytes += sizediff;
+    toku_free(items);
+    return 0;
+}
+
+void toku_recover_pmadistribute (LSN lsn, FILENUM filenum, DISKOFF old_diskoff, DISKOFF new_diskoff, INTPAIRARRAY fromto, u_int32_t old_N __attribute__((__unused__)), u_int32_t new_N) {
     struct cf_pair *pair = NULL;
     int r = find_cachefile(filenum, &pair);
     assert(r==0);
@@ -466,21 +510,25 @@ void toku_recover_pmadistribute (LSN lsn, FILENUM filenum, DISKOFF old_diskoff,
     assert(r==0);
     BRTNODE nodea = node_va;      assert(nodea->height==0);
     BRTNODE nodeb = node_vb;      assert(nodeb->height==0);
+    if (new_N!=toku_gpma_index_limit(nodeb->u.l.buffer)) {
+	r = toku_resize_gpma_exactly(nodeb->u.l.buffer, new_N);
+	assert(r==0);
+    }
     {    
 	unsigned int i;
 	//printf("{");
 	for (i=0; i<fromto.size; i++) {
 	    //printf(" {%d %d}", fromto.array[i].a, fromto.array[i].b);
-	    assert(fromto.array[i].a < toku_pma_index_limit(nodea->u.l.buffer));
-	    assert(fromto.array[i].b < toku_pma_index_limit(nodeb->u.l.buffer));
+	    assert(fromto.array[i].a < toku_gpma_index_limit(nodea->u.l.buffer));
+	    assert(fromto.array[i].b < toku_gpma_index_limit(nodeb->u.l.buffer));
 	}
 	//printf("}\n");
     }
-    r = toku_pma_move_indices (nodea->u.l.buffer, nodeb->u.l.buffer, fromto,
-			       nodea->rand4fingerprint, &nodea->local_fingerprint,
-			       nodeb->rand4fingerprint, &nodeb->local_fingerprint,
-			       &nodea->u.l.n_bytes_in_buffer, &nodeb->u.l.n_bytes_in_buffer
-			       );
+    r = move_indices (nodea->u.l.buffer, nodeb->u.l.buffer, fromto,
+		      nodea->rand4fingerprint, &nodea->local_fingerprint,
+		      nodeb->rand4fingerprint, &nodeb->local_fingerprint,
+		      &nodea->u.l.n_bytes_in_buffer, &nodeb->u.l.n_bytes_in_buffer
+		      );
     // The bytes in buffer and fingerprint shouldn't change
 
 //    PMA_ITERATE_IDX(nodeb->u.l.buffer, idx, key, keylen __attribute__((__unused__)), data, datalen __attribute__((__unused__)),

newbrt/roll.c

@@ -26,6 +26,7 @@ int toku_rollback_fcreate (BYTESTRING bs_fname,
     return 0;
 }
 
+#if 0
 int toku_rollback_fclose (FILENUM filenum, BYTESTRING bs_fname, TOKUTXN txn) {
     abort();
     filenum=filenum;
@@ -55,7 +56,7 @@ int toku_rollback_fclose (FILENUM filenum, BYTESTRING bs_fname, TOKUTXN txn) {
     return 0;
 #endif
 }
-			  
+#endif	
 
 //int toku_rollback_newbrtnode (struct logtype_newbrtnode *le, TOKUTXN txn) {
 //    // All that must be done is to put the node on the freelist.

newbrt/test-assert.c

 /* -*- mode: C; c-basic-offset: 4 -*- */
 #include "toku_assert.h"
+#include "brttypes.h"
 #include <signal.h>
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
 
-static void catch_abort (int sig __attribute__((__unused__))) {
+static __attribute__((__noreturn__)) void catch_abort (int sig __attribute__((__unused__))) {
     exit(1);
 }
 
+static BOOL foo (void) {
+    return TRUE;
+}
+    
+
 int main (int argc, const char *argv[]) {
     signal (SIGABRT, catch_abort);
     if (argc!=2) { printf("argcount should be 2.\n");  exit(1); }
     const char *str=argv[1];
     assert(strcmp(str,"ok")==0);
+    assert(foo());
+    assert(0x8000000000000000UL);
+    assert(0x4000000000000000UL);
+    assert(argv[1]);
     return 0;
 }

newbrt/toku_assert.c

@@ -2,7 +2,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 
-void toku_do_assert(long expr,const char* expr_as_string,const char *function,const char*file,int line) {
+void toku_do_assert(int expr,const char* expr_as_string,const char *function,const char*file,int line) {
     if (expr==0) {
 	fprintf(stderr, "%s:%d %s: Assertion `%s' failed\n", file,line,function,expr_as_string);
 	abort();

newbrt/toku_assert.h

@@ -8,11 +8,11 @@
 #error NDEBUG should not be set
 #endif
 
-void toku_do_assert(long,const char*/*expr_as_string*/,const char */*fun*/,const char*/*file*/,int/*line*/);
+void toku_do_assert(int,const char*/*expr_as_string*/,const char */*fun*/,const char*/*file*/,int/*line*/);
 
 #ifndef FAST_ASSERT
-#define assert(expr) toku_do_assert((long)(expr), #expr, __FUNCTION__, __FILE__, __LINE__)
+#define assert(expr) toku_do_assert((expr) != 0, #expr, __FUNCTION__, __FILE__, __LINE__)
 #else
-#define assert(expr) ({ long __assert_expr = (int)(expr); if (__assert_expr==0) toku_do_assert(0, #expr, __FUNCTION__, __FILE__, __LINE__); })
+#define assert(expr) ({ if ((expr)==0) toku_do_assert(0, #expr, __FUNCTION__, __FILE__, __LINE__); })
 #endif
 #endif

newbrt/wbuf.h

@@ -34,7 +34,7 @@ static inline void wbuf_init (struct wbuf *w, void *buf, DISKOFF size) {
     w->size=size;
     w->ndone=0;
 #ifdef CRC_INCR
-    w->crc32 = toku_crc32(0L, Z_NULL, 0);
+    w->crc32 = toku_crc32(toku_null_crc, Z_NULL, 0);
 #endif
 }
 
@@ -47,7 +47,7 @@ static inline void wbuf_char (struct wbuf *w, int ch) {
 #endif
 }
 
-static void wbuf_int (struct wbuf *w, unsigned int i) {
+static void wbuf_int (struct wbuf *w, int32_t i) {
 #if 0
     wbuf_char(w, i>>24);
     wbuf_char(w, i>>16);
@@ -65,14 +65,17 @@ static void wbuf_int (struct wbuf *w, unsigned int i) {
     w->ndone += 4;
 #endif
 }
+static void wbuf_uint (struct wbuf *w, u_int32_t i) {
+    wbuf_int(w, (int32_t)i);
+}
 
-static inline void wbuf_literal_bytes(struct wbuf *w, bytevec bytes_bv, int nbytes) {
+static inline void wbuf_literal_bytes(struct wbuf *w, bytevec bytes_bv, u_int32_t nbytes) {
     const unsigned char *bytes=bytes_bv; 
 #if 0
     { int i; for (i=0; i<nbytes; i++) wbuf_char(w, bytes[i]); }
 #else
     assert(w->ndone + nbytes <= w->size);
-    memcpy(w->buf + w->ndone, bytes, nbytes);
+    memcpy(w->buf + w->ndone, bytes, (size_t)nbytes);
  #ifdef CRC_INCR
     w->crc32 = toku_crc32(w->crc32, &w->buf[w->ndone], nbytes);
  #endif
@@ -81,14 +84,14 @@ static inline void wbuf_literal_bytes(struct wbuf *w, bytevec bytes_bv, int nbyt
     
 }
 
-static void wbuf_bytes (struct wbuf *w, bytevec bytes_bv, int nbytes) {
-    wbuf_int(w, nbytes);
+static void wbuf_bytes (struct wbuf *w, bytevec bytes_bv, u_int32_t nbytes) {
+    wbuf_uint(w, nbytes);
     wbuf_literal_bytes(w, bytes_bv, nbytes);
 }
 
-static void wbuf_ulonglong (struct wbuf *w, unsigned long long ull) {
-    wbuf_int(w, ull>>32);
-    wbuf_int(w, ull&0xFFFFFFFF);
+static void wbuf_ulonglong (struct wbuf *w, u_int64_t ull) {
+    wbuf_uint(w, (u_int32_t)(ull>>32));
+    wbuf_uint(w, (u_int32_t)(ull&0xFFFFFFFF));
 }
 
 static inline void wbuf_BYTESTRING (struct wbuf *w, BYTESTRING v) {
@@ -100,11 +103,11 @@ static inline void wbuf_u_int8_t (struct wbuf *w, u_int8_t v) {
 }
 
 static inline void wbuf_u_int32_t (struct wbuf *w, u_int32_t v) {
-    wbuf_int(w, v);
+    wbuf_uint(w, v);
 }
 
 static inline void wbuf_DISKOFF (struct wbuf *w, DISKOFF off) {
-    wbuf_ulonglong(w, off);
+    wbuf_ulonglong(w, (u_int64_t)off);
 }
 
 static inline void wbuf_TXNID (struct wbuf *w, TXNID tid) {
@@ -116,13 +119,13 @@ static inline void wbuf_LSN (struct wbuf *w, LSN lsn) {
 }
 
 static inline void wbuf_FILENUM (struct wbuf *w, FILENUM fileid) {
-    wbuf_int(w, fileid.fileid);
+    wbuf_uint(w, fileid.fileid);
 }
 
 static inline void wbuf_LOGGEDBRTHEADER (struct wbuf *w, LOGGEDBRTHEADER h) {
-    wbuf_int(w, h.size);
-    wbuf_int(w, h.flags);
-    wbuf_int(w, h.nodesize);
+    wbuf_uint(w, h.size);
+    wbuf_uint(w, h.flags);
+    wbuf_uint(w, h.nodesize);
     wbuf_DISKOFF(w, h.freelist);
     wbuf_DISKOFF(w, h.unused_memory);
     wbuf_int(w, h.n_named_roots);
@@ -132,17 +135,17 @@ static inline void wbuf_LOGGEDBRTHEADER (struct wbuf *w, LOGGEDBRTHEADER h) {
 	int i;
 	for (i=0; i<h.n_named_roots; i++) {
 	    wbuf_DISKOFF(w, h.u.many.roots[i]);
-	    wbuf_bytes  (w, h.u.many.names[i], 1+strlen(h.u.many.names[i]));
+	    wbuf_bytes  (w, h.u.many.names[i], (u_int32_t)(1+strlen(h.u.many.names[i])));
 	}
     }
 }
 
 static inline void wbuf_INTPAIRARRAY (struct wbuf *w, INTPAIRARRAY h) {
     u_int32_t i;
-    wbuf_int(w, h.size);
+    wbuf_uint(w, h.size);
     for (i=0; i<h.size; i++) {
-	wbuf_int(w, h.array[i].a);
-	wbuf_int(w, h.array[i].b);
+	wbuf_uint(w, h.array[i].a);
+	wbuf_uint(w, h.array[i].b);
     }
 }
 

newbrt/ybt.c

@@ -20,12 +20,12 @@ int toku_dbt_set_value (DBT *ybt, bytevec val, ITEMLEN vallen, void **staticptrp
     int r = ENOSYS;
     if (ybt->flags==DB_DBT_MALLOC) {
     domalloc:
-	ybt->data = toku_malloc(vallen);
+	ybt->data = toku_malloc((size_t)vallen);
 	if (!ybt->data && vallen > 0) { r = errno; goto cleanup; }
     } else if (ybt->flags==DB_DBT_REALLOC) {
 	if (ybt->data==0) goto domalloc;
 	/* tmp is used to prevent a memory leak if realloc fails */
-        void* tmp = toku_realloc(ybt->data, vallen);
+	void* tmp = toku_realloc(ybt->data, (size_t)vallen);
 	if (!tmp && vallen > 0) { r = errno; goto cleanup; }
         ybt->data = tmp;
     } else if (ybt->flags==DB_DBT_USERMEM) {
@@ -36,10 +36,9 @@ int toku_dbt_set_value (DBT *ybt, bytevec val, ITEMLEN vallen, void **staticptrp
 	void *staticptr=*staticptrp;
 	//void *old=staticptr;
 	if (staticptr==0) { 
-	    staticptr = toku_malloc(vallen);
+	    staticptr = toku_malloc((size_t)vallen);
             if (!staticptr && vallen > 0) { r = errno; goto cleanup; }
-        }
-	else {
+        } else {
 	    /* tmp is used to prevent a memory leak if realloc fails */
             void* tmp = toku_realloc(staticptr, vallen);
             if (!tmp && vallen > 0) { r = errno; goto cleanup; }
@@ -51,7 +50,7 @@ int toku_dbt_set_value (DBT *ybt, bytevec val, ITEMLEN vallen, void **staticptrp
     }
     ybt->size = vallen;
     if (ybt->size>0) {
-	memcpy(ybt->data, val, vallen);
+	memcpy(ybt->data, val, (size_t)vallen);
     }
     r = 0;
 cleanup:

newbrt/yerror.h

@@ -5,10 +5,4 @@
 
 enum pma_errors { BRT_OK=0, BRT_ALREADY_THERE = -2, BRT_KEYEMPTY=-3 };
 
-enum typ_tag { TYP_BRTNODE = 0xdead0001,
-	       TYP_CACHETABLE, TYP_PAIR, /* for cachetables */
-	       TYP_PMA,
-               TYP_TOKULOGGER,
-	       TYP_TOKUTXN
-};
 #endif

src/Makefile

@@ -27,28 +27,36 @@ SHARED=-shared $(EXPORTMAP)
 RPATHNAME=
 endif
 
+build:
+	cd range_tree;$(MAKE) build
+	cd lock_tree;$(MAKE) build
+	$(MAKE) $(LIBRARY) $(LIBNAME).a
+	cd tests;$(MAKE) build
+
 .PHONY: install
-install: locktree $(LIBRARY) $(LIBNAME).a
+install: $(LIBRARY) $(LIBNAME).a
 	cp $(LIBRARY) ../lib/
 	cp $(LIBNAME).a ../lib
 
-locktree:
-	cd lock_tree && make
-
-check: $(LIBRARY)
+check_globals: $(LIBRARY)
 	python tokuglobals.py $(LIBRARY)
 
+check_tests:
+	cd tests;$(MAKE) check
+check: $(LIBRARY) check_globals check_tests
+
 strip: $(LIBRARY)
 	strip $(LIBRARY)
 
 clean:
 	rm -rf $(LIBRARY) $(LIBNAME).a *.o *.gcno *.gcda *.gcov
-	cd tests && make clean
-	cd lock_tree && make clean
+	cd tests && $(MAKE) clean
+	cd lock_tree && $(MAKE) clean
+	cd range_tree && $(MAKE) clean
 
 ydb.o: ../include/db.h ../newbrt/cachetable.h ../newbrt/brt.h ../newbrt/log.c
 
-DBBINS = ydb.o errors.o elocks.o ../newbrt/brt.o ../newbrt/brt-serialize.o ../newbrt/brt-verify.o ../newbrt/cachetable.o ../newbrt/fifo.o ../newbrt/key.o ../newbrt/memory.o ../newbrt/mempool.o ../newbrt/pma.o ../newbrt/ybt.o ../newbrt/primes.o ../newbrt/log.o ../newbrt/fingerprint.o ../newbrt/log_code.o ../newbrt/roll.o ../newbrt/toku_assert.o ../newbrt/recover.o
+DBBINS = ydb.o errors.o elocks.o ../newbrt/brt.o ../newbrt/brt-serialize.o ../newbrt/brt-verify.o ../newbrt/cachetable.o ../newbrt/fifo.o ../newbrt/key.o ../newbrt/memory.o ../newbrt/mempool.o ../newbrt/gpma.o ../newbrt/ybt.o ../newbrt/primes.o ../newbrt/log.o ../newbrt/fingerprint.o ../newbrt/log_code.o ../newbrt/roll.o ../newbrt/toku_assert.o ../newbrt/recover.o
 
 RANGETREE_BINS = range_tree/rangetree.o range_tree/tokuredblack.o
 LOCKTREE_BINS  = lock_tree/locktree.o lock_tree/rth.o lock_tree/lth.o lock_tree/idlth.o lock_tree/db_id.o $(RANGETREE_BINS)

src/lock_tree/Makefile

@@ -33,14 +33,15 @@ LT_LOG         = $(LT_OVERLAP)
 LT_BINS=$(LT_OVERLAP) $(LT_NOOVERLAP) locktree.o
 BINS=rth.o lth.o idlth.o db_id.o
 
-.PHONY: install logformat range_tree
-install: range_tree $(BINS) $(LT_BINS)
+.PHONY: install logformat
 
-range_tree:
-	cd ../range_tree && make
+build: $(LT_BINS)
+	cd tests; $(MAKE) build
+
+check:
+	cd tests; $(MAKE) check
 
 clean:
-	cd ../range_tree && make clean
 	rm -rf $(BINS) $(LT_BINS)
 	rm -rf *.gcno *.gcda *.gcov
 	cd tests && make clean

src/lock_tree/locktree.h

@@ -18,7 +18,6 @@
    we defer to the db panic handler. Pass in another parameter to do this.
 */
 
-#include <assert.h>
 #include <db.h>
 #include <brttypes.h>
 #include <rangetree.h>
@@ -26,6 +25,8 @@
 #include <rth.h>
 #include <idlth.h>
 
+#include "toku_assert.h"
+
 /** Errors returned by lock trees */
 typedef enum {
     TOKU_LT_INCONSISTENT=-1,  /**< The member data are in an inconsistent 

src/lock_tree/tests/Makefile

@@ -30,7 +30,7 @@ TLOG_TESTS = $(patsubst %.c,%.tlog,$(SRCS))
 LIN_TESTS  = $(patsubst %.c,%.lin,$(SRCS))
 TLIN_TESTS = $(patsubst %.c,%.tlin,$(SRCS))
 
-ALL_TESTS = $(LIN_TESTS) $(TLIN_TESTS) $(TLOG_TESTS) $(LOG_TESTS)
+ALL_TESTS = $(LIN_TESTS) $(TLIN_TESTS) $(TLOG_TESTS) #$(LOG_TESTS)
 
 RUN_LOG_TESTS  = $(patsubst %.log,%.logrun,$(LOG_TESTS))
 RUN_TLOG_TESTS = $(patsubst %.tlog,%.tlogrun,$(TLOG_TESTS))
@@ -40,9 +40,9 @@ RUN_ALL_TESTS  = $(RUN_LIN_TESTS) $(RUN_TLIN_TESTS) $(RUN_TLOG_TESTS) $(RUN_LOG_
 
 .PHONY: check tests default all check.lin check.tlin check.log check.tlog tests.lin tests.tlin tests.tlog tests.log
 
-default: check.tlog
+default: build
 
-all: make_libs $(ALL_TESTS)
+build all: $(ALL_TESTS)
 
 check: check.lin check.tlin check.tlog #check.log
 	@ echo $@ ok
@@ -50,19 +50,19 @@ tests: tests.lin tests.tlin tests.tlog #tests.log
 	@ echo $@ ok
 tests.lin: make_libs $(LIN_TESTS)
 	@ echo $@ ok
-check.lin: make_libs $(RUN_LIN_TESTS)
+check.lin: $(RUN_LIN_TESTS)
 	@ echo $@ ok
-tests.tlin: make_libs $(TLIN_TESTS)
+tests.tlin: $(TLIN_TESTS)
 	@ echo $@ ok
-check.tlin: make_libs $(RUN_TLIN_TESTS)
+check.tlin: $(RUN_TLIN_TESTS)
 	@ echo $@ ok
-tests.tlog: make_libs $(TLOG_TESTS)
+tests.tlog: $(TLOG_TESTS)
 	@ echo $@ ok
-check.tlog: make_libs $(RUN_TLOG_TESTS)
+check.tlog: $(RUN_TLOG_TESTS)
 	@ echo $@ ok
-tests.log: make_libs $(LOG_TESTS)
+tests.log: $(LOG_TESTS)
 	@ echo $@ ok
-check.log: make_libs $(RUN_LOG_TESTS)
+check.log: $(RUN_LOG_TESTS)
 	@ echo $@ ok
 
 # Need these rule so that Make knows about all the file names
@@ -99,8 +99,7 @@ endif
 %.tlogrun: %.tlog
 	$(MAYBEATSIGN) $(VGRIND) ./$< $(VERBVERBOSE)
 
-libs:
-	cd .. && make
+NEWBRT_BINS     = ../../../newbrt/toku_assert.o
 
 RT_LINEAR_BINS  = ../../range_tree/linear.o
 RT_TLINEAR_BINS = ../../range_tree/linear.o
@@ -115,19 +114,17 @@ LT_TLINEAR     = $(LT_NOOVERLAP) $(LT_BINS) $(RT_TLINEAR_BINS)
 LT_TLOG        = $(LT_NOOVERLAP) $(LT_BINS) $(RT_TLOG_BINS)
 LT_LOG         = $(LT_OVERLAP)   $(LT_BINS) $(RT_LOG_BINS)
 
+foo:
+	echo ../locktree.h test.h $(LT_LINEAR)
+
 %.lin: %.c ../locktree.h test.h $(LT_LINEAR)
-	cc  -DDIR=\"dir.$<.lin\"  $(CPPFLAGS) $(CFLAGS) $< -o $@ $(LT_LINEAR)
+	cc  -DDIR=\"dir.$<.lin\"  $(CPPFLAGS) $(CFLAGS) $< -o $@ $(LT_LINEAR)  $(NEWBRT_BINS)
 %.tlin: %.c ../locktree.h test.h $(LT_TLINEAR)
-	cc  -DDIR=\"dir.$<.tlin\" $(CPPFLAGS) $(CFLAGS) $< -o $@ $(LT_TLINEAR) -DTOKU_RT_NOOVERLAPS
+	cc  -DDIR=\"dir.$<.tlin\" $(CPPFLAGS) $(CFLAGS) $< -o $@ $(LT_TLINEAR) $(NEWBRT_BINS) -DTOKU_RT_NOOVERLAPS
 %.tlog: %.c ../locktree.h test.h $(LT_TLOG)
-	cc  -DDIR=\"dir.$<.tlog\" $(CPPFLAGS) $(CFLAGS) $< -o $@ $(LT_TLOG)    -DTOKU_RT_NOOVERLAPS
+	cc  -DDIR=\"dir.$<.tlog\" $(CPPFLAGS) $(CFLAGS) $< -o $@ $(LT_TLOG)    $(NEWBRT_BINS) -DTOKU_RT_NOOVERLAPS
 %.log: %.c ../locktree.h test.h  $(LT_LOG)
-	cc  -DDIR=\"dir.$<.log\"  $(CPPFLAGS) $(CFLAGS) $< -o $@ $(LT_LOG)
-
-.PHONY: make_libs
-
-make_libs:
-	cd .. && make
+	cc  -DDIR=\"dir.$<.log\"  $(CPPFLAGS) $(CFLAGS) $< -o $@ $(LT_LOG)     $(NEWBRT_BINS)
 
 clean:
 	rm -f $(ALL_TESTS) *.o *.gcno *.gcda *.gcov

src/range_tree/Makefile

@@ -24,8 +24,16 @@ ifneq ($(OSX),)
 CFLAGS+=-fno-common
 endif
 
+BINS = linear.o log_nooverlap.o tokuredblack.o rangetree.o #log.o 
+
+build: $(BINS)
+	cd tests;$(MAKE) build
+
+check:
+	cd tests;$(MAKE) check
+
 .PHONY: install
-install: linear.o log_nooverlap.o tokuredblack.o rangetree.o #log.o 
+install: $(BINS)
 
 clean:
 	rm -rf *.o *.gcno *.gcda *.gcov