Skip to content

M
mariadb
Commit 83d416fa authored by Bradley C. Kuszmaul's avatar Bradley C. Kuszmaul
Browse files
Options

Merge the 558 branch back to the main with 

{{{
svn merge -r3061:3225 https://svn.tokutek.com/tokudb/tokudb.558
}}}

Fixes #630.
Addresses #558.


git-svn-id: file:///svn/tokudb@3226 c7de825b-a66e-492c-adef-691d508d4ae1
parent 48d81cd4
Hide whitespace changes
Inline Side-by-side
Showing with 2883 additions and 2104 deletions
Makefile
View file @ 83d416fa
default: build
TAGS: */*.c */*.h
etags */*.c */*.h src/lock_tree/*.c src/lock_tree/*.h src/range_tree/*.c src/range_tree/*.h
SRCDIRS = newbrt src src/tests src/range_tree src/range_tree/tests src/lock_tree src/lock_tree/tests cxx cxx/tests \
utils db-benchmark-test db-benchmark-test-cxx
SRCDIRS = newbrt src cxx utils db-benchmark-test db-benchmark-test-cxx
BUILDDIRS = $(SRCDIRS) man/texi
build:
for d in $(BUILDDIRS); do (cd $$d; $(MAKE) -k); done
src.dir: newbrt.dir
cxx.dir: src.dir
db-benchmark-test.dir: src.dir
db-benchmark-test-cxx.dir: cxx.dir
utils.dir: src.dir
%.dir:
cd $(patsubst %.dir, %, $@);$(MAKE) build
build: $(patsubst %,%.dir, $(SRCDIRS))
CHECKS = $(patsubst %,checkdir_%,$(SRCDIRS))
......@@ -16,11 +26,7 @@ CHECKS = $(patsubst %,checkdir_%,$(SRCDIRS))
#check:
# for d in $(SRCDIRS); do (cd $$d; $(MAKE) -k check); done
checkdir_%: build
cd $(patsubst checkdir_%,%,$@) ; $(MAKE) -k check
checkdir_src/%: build
cd $(patsubst checkdir_%,%,$@) ; $(MAKE) -k check
checkdir_cxx/%: build
checkdir_%:
cd $(patsubst checkdir_%,%,$@) ; $(MAKE) -k check
check: $(CHECKS)
......@@ -45,7 +51,7 @@ check-coverage: check-coverage-newbrt check-coverage-src-tests check-coverage-ut
check-coverage-range-tree-tests check-coverage-lock-tree-tests
check-coverage-newbrt:
(cd newbrt; $(MAKE) -k check DTOOL="")
(cd newbrt; $(MAKE) -k check VGRIND="")
check-coverage-src-tests:
(cd src/tests; $(MAKE) -k check.tdb VGRIND="")
check-coverage-utils:
......
cxx/Makefile
View file @ 83d416fa
......@@ -9,7 +9,11 @@ OBJS = $(patsubst %.cpp, %.o, $(SRCS))
LIBNAME = libtokudb_cxx
default: install
default: install build
build: $(LIBNAME).a
cd tests; $(MAKE) build
check:
cd tests; $(MAKE) check
install: $(LIBNAME).a
cp $< ../lib/
$(OBJS): ../include/db_cxx.h
......@@ -19,5 +23,6 @@ $(LIBNAME).a: $(OBJS)
$(AR) rv $@ $(OBJS)
clean:
rm -f $(OBJS) $(LIBNAME).a $(LIBNAME).so *.gcno *.gcda *.gcov
cd tests; $(MAKE) clean
cxx/tests/Makefile
View file @ 83d416fa
......@@ -13,7 +13,8 @@ else
VGRIND=valgrind --quiet --error-exitcode=1 --leak-check=yes
endif
all: $(TARGETS)
default: build
build all: $(TARGETS)
$(TARGETS): $(DBCXX)
$(DBCXX):
......@@ -61,4 +62,4 @@ check_permissions:
rm -f test.db
./db_create test.db 1 1
chmod -w test.db
./db_create test.db 2 2; let exitcode=$$?; if [ $$exitcode -ne 0 ] ; then exit 0; else exit 1; fi
\ No newline at end of file
./db_create test.db 2 2; let exitcode=$$?; if [ $$exitcode -ne 0 ] ; then exit 0; else exit 1; fi
db-benchmark-test-cxx/Makefile
View file @ 83d416fa
......@@ -22,7 +22,8 @@ TARGET_BDB = db-benchmark-test-bdb
TARGET_TDB = db-benchmark-test-tokudb
TARGETS = $(TARGET_BDB) $(TARGET_TDB)
default: $(TARGETS)
default: build
build: $(TARGETS)
check: check-default
......
db-benchmark-test/Makefile
View file @ 83d416fa
......@@ -25,7 +25,8 @@ TARGET_BDB = db-benchmark-test-bdb
TARGET_TDB = db-benchmark-test-tokudb
TARGETS = $(TARGET_BDB) $(TARGET_TDB)
default: $(TARGETS)
default: build
build: $(TARGETS)
check: check-default check-xfast
......
newbrt/Makefile
View file @ 83d416fa
......@@ -27,10 +27,10 @@ ifeq ($(CYGWIN),cygwin)
else
FPICFLAGS = -fPIC
# valgrind is not present on cygwin
DTOOL = valgrind --quiet --error-exitcode=1 --leak-check=yes
VGRIND = valgrind --quiet --error-exitcode=1 --leak-check=yes
endif
CFLAGS = -Wall -W $(OPTFLAGS) -g3 -ggdb3 $(GCOV_FLAGS) $(PROF_FLAGS) -Werror $(FPICFLAGS) -Wshadow -fvisibility=hidden
CFLAGS = -Wall -W -Wcast-align -Wbad-function-cast -Wextra -Wmissing-noreturn -Wmissing-format-attribute $(OPTFLAGS) -g3 -ggdb3 $(GCOV_FLAGS) $(PROF_FLAGS) -Werror $(FPICFLAGS) -Wshadow -fvisibility=hidden
LDFLAGS = $(OPTFLAGS) -g $(GCOV_FLAGS) $(PROF_FLAGS)
CPPFLAGS += -D_FILE_OFFSET_BITS=64 -D_LARGEFILE64_SOURCE
......@@ -42,21 +42,27 @@ endif
# When debugging, try: valgrind --show-reachable=yes --leak-check=full ./brt-test
default: bins libs tdb-recover tdb_logprint
build default: bins libs tdb-recover tdb_logprint
# Put these one-per-line so that if we insert a new one the svn diff can understand it better.
# Also keep them sorted.
REGRESSION_TESTS = \
ybt-test \
pma-test \
test-gpma-glassbox \
brt-serialize-test \
cachetable-test \
cachetable-test2 \
fifo-test \
fifo-test-exp \
test-gpma-blackbox \
test-gpma-glassbox \
test-brt-delete-both \
brt-test \
brt-test0 \
brt-test1 \
brt-test2 \
brt-test3 \
brt-test4 \
brt-test-named-db \
brt-test-cursor \
brt-test-cursor-2 \
log-test \
......@@ -84,13 +90,13 @@ BINS = $(REGRESSION_TESTS) \
tdb_logprint: LDFLAGS+=-lz
tdb_logprint.o: log-internal.h brttypes.h yerror.h log.h kv-pair.h
tdb_logprint: log_code.o memory.o log.o brt-serialize.o fifo.o pma.o ybt.o fingerprint.o mempool.o primes.o toku_assert.o roll.o brt.o cachetable.o brt-verify.o key.o
tdb_logprint: log_code.o memory.o log.o brt-serialize.o fifo.o gpma.o ybt.o fingerprint.o mempool.o primes.o toku_assert.o roll.o brt.o cachetable.o brt-verify.o key.o
tdb-recover: LDFLAGS+=-lz
recover.o: log_header.h log-internal.h log.h yerror.h brttypes.h kv-pair.h memory.h key.h
tdb-recover: tdb-recover.o recover.o log_code.o memory.o log.o brt-serialize.o fifo.o pma.o ybt.o fingerprint.o mempool.o primes.o toku_assert.o cachetable.o brt.o brt-verify.o key.o roll.o
tdb-recover: tdb-recover.o recover.o log_code.o memory.o log.o brt-serialize.o fifo.o gpma.o ybt.o fingerprint.o mempool.o primes.o toku_assert.o cachetable.o brt.o brt-verify.o key.o roll.o
roll.o: log_header.h log-internal.h log.h yerror.h brttypes.h kv-pair.h memory.h key.h cachetable.h pma.h
roll.o: log_header.h log-internal.h log.h yerror.h brttypes.h kv-pair.h memory.h key.h cachetable.h gpma.h
log_code.o: log_header.h wbuf.h log-internal.h
log_header.h: log_code.c
......@@ -103,7 +109,6 @@ bins: $(BINS)
CHECKS = \
test_oexcl \
ybt-test \
pma-test \
cachetable-test \
cachetable-test2 \
brt-serialize-test \
......@@ -111,8 +116,12 @@ CHECKS = \
brt-test \
brt-test-cursor \
brt-test-cursor-2 \
brt-test0 \
brt-test1 \
brt-test2 \
brt-test3 \
brt-test4 \
brt-test-named-db \
fifo-test \
test_toku_malloc_plain_free \
test-primes \
......@@ -122,20 +131,20 @@ CHECKS = \
# Put check_benchmarktest_256 first because it is long-running (and therefore on the critical path, so get it started)
check: bins check_benchmarktest_256 $(patsubst %,check_%,$(CHECKS))
check_benchmarktest_256: benchmark-test
$(DTOOL) ./benchmark-test $(VERBVERBOSE) --valsize 256 --verify 1
$(VGRIND) ./benchmark-test $(VERBVERBOSE) --valsize 256 --verify 1
check_test-assert: test-assert
@# no arguments, should err
$(DTOOL) ./test-assert > /dev/null 2>&1 ; test \($$?\)
$(VGRIND) ./test-assert > /dev/null 2>&1 ; test \($$?\)
@# one argument, not "ok" should err
@echo Expect an abort message:
($(DTOOL) ./test-assert notok) > test-assert.out 2>&1 ; test \($$?\)
($(VGRIND) ./test-assert notok) > test-assert.out 2>&1 ; test \($$?\)
@fgrep failed test-assert.out > /dev/null
@rm test-assert.out
@# one argument, "ok" should not error
$(DTOOL) ./test-assert ok
$(DVGRIND) ./test-assert ok
check_%: %
$(DTOOL) ./$< $(VERBVERBOSE)
$(VGRIND) ./$< $(VERBVERBOSE)
check-fanout:
let BRT_FANOUT=4; \
......@@ -144,26 +153,31 @@ check-fanout:
let BRT_FANOUT=BRT_FANOUT+1; \
done
log-test log-test2 log-test3 log-test4 log-test5 log-test6 pma-test benchmark-test brt-test brt-test3 brt-test4 brt-test-cursor brt-test-cursor-2 test-brt-delete-both brt-serialize-test brtdump test-inc-split test-del-inorder: LDFLAGS+=-lz
log-test log-test2 log-test3 log-test4 log-test5 log-test6 benchmark-test brt-test brt-test0 brt-test1 brt-test2 brt-test3 brt-test4 brt-test-named-db brt-test-cursor brt-test-cursor-2 test-brt-delete-both brt-serialize-test brtdump test-inc-split test-del-inorder: LDFLAGS+=-lz
# pma: PROF_FLAGS=-fprofile-arcs -ftest-coverage
BRT_INTERNAL_H_INCLUDES = brt-internal.h cachetable.h fifo.h pma.h brt.h brttypes.h yerror.h ybt.h log.h ../include/db.h kv-pair.h memory.h crc.h
BRT_INTERNAL_H_INCLUDES = brt-internal.h cachetable.h fifo.h gpma.h brt.h brt-search.h brttypes.h yerror.h ybt.h log.h ../include/db.h kv-pair.h memory.h crc.h mempool.h
key.o: brttypes.h key.h
list-test: list-test.o toku_assert.o
test-brt-delete-both: ybt.o brt.o fifo.o pma.o memory.o brt-serialize.o cachetable.o ybt.o key.o primes.o toku_assert.o log.o mempool.o brt-verify.o fingerprint.o log_code.o roll.o
test-inc-split: test-inc-split.o brt.o brt-test-helpers.o toku_assert.o key.o ybt.o cachetable.o memory.o log.o pma.o log_code.o fifo.o fingerprint.o brt-serialize.o brt-verify.o roll.o primes.o mempool.o
test-del-inorder: test-del-inorder.o brt.o brt-test-helpers.o toku_assert.o key.o ybt.o cachetable.o memory.o log.o pma.o log_code.o fifo.o fingerprint.o brt-serialize.o brt-verify.o roll.o primes.o mempool.o
pma-test.o: $(BRT_INTERNAL_H_INCLUDES) pma-internal.h pma.h list.h mempool.h
pma-test: pma.o memory.o key.o ybt.o log.o mempool.o fingerprint.o brt-serialize.o fifo.o primes.o toku_assert.o log_code.o roll.o brt.o cachetable.o brt-verify.o
pma.o: pma.h yerror.h pma-internal.h memory.h key.h ybt.h brttypes.h log.h ../include/db.h log_header.h
test-brt-delete-both: ybt.o brt.o fifo.o gpma.o memory.o brt-serialize.o cachetable.o ybt.o key.o primes.o toku_assert.o log.o mempool.o brt-verify.o fingerprint.o log_code.o roll.o
test-inc-split: test-inc-split.o brt.o brt-test-helpers.o toku_assert.o key.o ybt.o cachetable.o memory.o log.o gpma.o log_code.o fifo.o fingerprint.o brt-serialize.o brt-verify.o roll.o primes.o mempool.o
test-del-inorder: test-del-inorder.o brt.o brt-test-helpers.o toku_assert.o key.o ybt.o cachetable.o memory.o log.o gpma.o log_code.o fifo.o fingerprint.o brt-serialize.o brt-verify.o roll.o primes.o mempool.o
# pma-test.o: $(BRT_INTERNAL_H_INCLUDES) pma-internal.h gpma.h list.h mempool.h
# pma-test: pma.o memory.o key.o ybt.o log.o mempool.o fingerprint.o brt-serialize.o fifo.o primes.o toku_assert.o log_code.o roll.o brt.o cachetable.o brt-verify.o
pma.o: gpma.h yerror.h pma-internal.h memory.h key.h ybt.h brttypes.h log.h ../include/db.h log_header.h
test-gpma-glassbox.o: test-gpma-glassbox.c gpma.h gpma-internal.h toku_assert.h memory.h
test-gpma-glassbox: test-gpma-glassbox.o toku_assert.o memory-debug.o gpma.o
test-gpma-blackbox: test-gpma-blackbox.o toku_assert.o memory.o gpma.o
test-gpma-blackbox.o: test-gpma-blackbox.c gpma.h memory.h toku_assert.h
gpma.o: gpma.c gpma.h
ybt.o: ybt.h brttypes.h ../include/db.h
ybt-test: ybt-test.o ybt.o memory.o toku_assert.o
ybt-test.o: ybt.h ../include/db.h
cachetable.o: cachetable.h hashfun.h memory.h
brt-test3 brt-test4 brt-test-cursor brt-test-cursor-2 brt-test: ybt.o brt.o fifo.o pma.o memory.o brt-serialize.o cachetable.o ybt.o key.o primes.o toku_assert.o log.o mempool.o brt-verify.o fingerprint.o log_code.o roll.o
brt-test0 brt-test1 brt-test2 brt-test3 brt-test4 brt-test-named-db brt-test-cursor brt-test-cursor-2 brt-test: ybt.o brt.o fifo.o gpma.o memory.o brt-serialize.o cachetable.o ybt.o key.o primes.o toku_assert.o log.o mempool.o brt-verify.o fingerprint.o log_code.o roll.o
log.o: log_header.h log-internal.h log.h wbuf.h crc.h brttypes.h $(BRT_INTERNAL_H_INCLUDES)
logformat: logformat.o toku_assert.o
brt-test3.o brt-test4.o brt-test-cursor.o brt-test-cursor-2.o brt-test.o brt.o: brt.h ../include/db.h fifo.h pma.h brttypes.h cachetable.h memory.h
brt-test0.o brt-test1.o brt-test2.o brt-test3.o brt-test4.o brt-test-named-db.o brt-test-cursor.o brt-test-cursor-2.o brt-test.o brt.o: brt.h brt-search.h ../include/db.h fifo.h gpma.h brttypes.h cachetable.h memory.h
brt-serialize-test.o: $(BRT_INTERNAL_H_INCLUDES)
brt.o: $(BRT_INTERNAL_H_INCLUDES) key.h log_header.h
fifo.o: fifo.h brttypes.h
......@@ -171,14 +185,15 @@ memory.o: memory.h
primes.o: primes.h toku_assert.h
fifo-test-exp fifo-test: fifo.o memory.o toku_assert.o ybt.o
brt-serialize.o: $(BRT_INTERNAL_H_INCLUDES) key.h wbuf.h rbuf.h
brt-bigtest: memory.o ybt.o brt.o pma.o cachetable.o key.o fifo.o brt-serialize.o
brt-bigtest.o: brt.h ../include/db.h
log-test6 log-test5 log-test4 log-test3 log-test2 log-test: log.o memory.o toku_assert.o roll.o log_code.o brt-serialize.o brt.o cachetable.o pma.o ybt.o fifo.o key.o fingerprint.o brt-verify.o mempool.o primes.o
brt-bigtest: memory.o ybt.o brt.o gpma.o cachetable.o key.o fifo.o brt-serialize.o
brt-bigtest.o: brt.h brt-search.h ../include/db.h
log-test6 log-test5 log-test4 log-test3 log-test2 log-test: log.o memory.o toku_assert.o roll.o log_code.o brt-serialize.o brt.o cachetable.o gpma.o ybt.o fifo.o key.o fingerprint.o brt-verify.o mempool.o primes.o
brt-verify.o: $(BRT_INTERNAL_H_INCLUDES)
fingerprint.o: $(BRT_INTERNAL_H_INCLUDES)
mempool.o: toku_assert.h mempool.h
toku_assert.o: toku_assert.h
brt-serialize-test: brt-serialize-test.o brt-serialize.o memory.o fifo.o pma.o key.o ybt.o brt.o cachetable.o primes.o toku_assert.o log.o mempool.o brt-verify.o fingerprint.o log_code.o roll.o
brt-serialize-test: brt-serialize-test.o brt-serialize.o memory.o fifo.o gpma.o key.o ybt.o brt.o cachetable.o primes.o toku_assert.o log.o mempool.o brt-verify.o fingerprint.o log_code.o roll.o
test_toku_malloc_plain_free: memory.o toku_assert.o
......@@ -188,14 +203,14 @@ cachetable-test: cachetable.o memory.o cachetable-test.o primes.o toku_assert.o
cachetable-test2.o: cachetable.h memory.h
cachetable-test2: cachetable.o memory.o cachetable-test2.o primes.o toku_assert.o
benchmark-test: benchmark-test.o ybt.o memory.o brt.o pma.o cachetable.o key.o fifo.o brt-serialize.o primes.o toku_assert.o log.o mempool.o brt-verify.o fingerprint.o log_code.o roll.o
benchmark-test.o: brt.h ../include/db.h
benchmark-test: benchmark-test.o ybt.o memory.o brt.o gpma.o cachetable.o key.o fifo.o brt-serialize.o primes.o toku_assert.o log.o mempool.o brt-verify.o fingerprint.o log_code.o roll.o
benchmark-test.o: brt.h brt-search.h ../include/db.h
test-primes: test-primes.o toku_assert.o primes.o toku_assert.o toku_assert.o
test-assert: test-assert.o toku_assert.o
brtdump: brtdump.o brt-serialize.o memory.o pma.o key.o fingerprint.o log.o log_code.o roll.o cachetable.o primes.o toku_assert.o brt.o ybt.o fifo.o mempool.o brt-verify.o
brtdump: brtdump.o brt-serialize.o memory.o gpma.o key.o fingerprint.o log.o log_code.o roll.o cachetable.o primes.o toku_assert.o brt.o ybt.o fifo.o mempool.o brt-verify.o
test_oexcl: test_oexcl.o toku_assert.o
......
newbrt/benchmark-test.c
View file @ 83d416fa
......@@ -9,6 +9,7 @@
#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/time.h>
#include <unistd.h>
......
newbrt/brt-internal.h
View file @ 83d416fa
......@@ -3,12 +3,16 @@
#ident "Copyright (c) 2007, 2008 Tokutek Inc. All rights reserved."
#include "toku_assert.h"
#include "cachetable.h"
#include "fifo.h"
#include "pma.h"
#include "yerror.h"
#include "gpma.h"
#include "brt.h"
#include "crc.h"
#include "list.h"
#include "mempool.h"
#include "kv-pair.h"
#ifndef BRT_FANOUT
#define BRT_FANOUT 16
......@@ -77,8 +81,9 @@ struct brtnode {
However, in the absense of duplicate keys, child 1's keys *are* > childkeys[0]. */
} n;
struct leaf {
PMA buffer;
GPMA buffer;
unsigned int n_bytes_in_buffer; /* How many bytes to represent the PMA (including the per-key overheads, but not including the overheads for the node. */
struct mempool buffer_mempool;
} l;
} u;
};
......@@ -101,11 +106,6 @@ struct brt_header {
unsigned int flags;
};
enum brt_header_flags {
TOKU_DB_DUP = 1,
TOKU_DB_DUPSORT = 2,
};
struct brt {
CACHEFILE cf;
char *database_name;
......@@ -125,7 +125,7 @@ struct brt {
/* serialization code */
void toku_serialize_brtnode_to(int fd, DISKOFF off, DISKOFF size, BRTNODE node);
int toku_deserialize_brtnode_from (int fd, DISKOFF off, BRTNODE *brtnode, int flags, int nodesize, int (*bt_compare)(DB *, const DBT*, const DBT*), int (*dup_compare)(DB *, const DBT *, const DBT *), DB *db, FILENUM filenum);
int toku_deserialize_brtnode_from (int fd, DISKOFF off, BRTNODE *brtnode, unsigned int flags, int nodesize);
unsigned int toku_serialize_brtnode_size(BRTNODE node); /* How much space will it take? */
int toku_keycompare (bytevec key1, ITEMLEN key1len, bytevec key2, ITEMLEN key2len);
......@@ -160,6 +160,7 @@ extern CACHEKEY* toku_calculate_root_offset_pointer (BRT brt);
static const BRTNODE null_brtnode=0;
extern u_int32_t toku_calccrc32_kvpair (const void *key, int keylen, const void *val, int vallen);
extern u_int32_t toku_calccrc32_kvpair_struct (const struct kv_pair *kvp);
extern u_int32_t toku_calccrc32_cmd (int type, TXNID xid, const void *key, int keylen, const void *val, int vallen);
extern u_int32_t toku_calccrc32_cmdstruct (BRT_CMD cmd);
......@@ -191,6 +192,34 @@ int toku_testsetup_insert_to_nonleaf (BRT brt, DISKOFF diskoff, enum brt_cmd_typ
int toku_set_func_fsync (int (*fsync_function)(int));
/* allocate a kv pair from a kv memory pool */
//static inline struct kv_pair *kv_pair_malloc_mempool(const void *key, int keylen, const void *val, int vallen, struct mempool *mp) {
// struct kv_pair *kv = toku_mempool_malloc(mp, sizeof (struct kv_pair) + keylen + vallen, 4);
// if (kv)
// kv_pair_init(kv, key, keylen, val, vallen);
// return kv;
//}
int toku_brtnode_compress_kvspace (GPMA pma, struct mempool *mp);
static inline struct kv_pair *brtnode_malloc_kv_pair (GPMA pma, struct mempool *mp, const void *key, unsigned int keylen, const void *val, unsigned int vallen) {
struct kv_pair *kv = toku_mempool_malloc(mp, sizeof (struct kv_pair) + keylen + vallen, 4);
if (kv == 0) {
if (0 == toku_brtnode_compress_kvspace (pma, mp)) {
kv = toku_mempool_malloc(mp, sizeof (struct kv_pair) + keylen + vallen, 4);
toku_verify_gpma(pma);
assert(kv);
}
}
kv_pair_init(kv, key, keylen, val, vallen);
return kv;
}
// used for the leaf compare fun
struct lc_pair {
BRT t;
int compare_both; // compare_both is set if it is a DUPSORT database and both keys are needed (e.g, for DB_DELETE_ANY)
};
int toku_brtleaf_compare_fun (u_int32_t alen __attribute__((__unused__)), void *aval, u_int32_t blen __attribute__((__unused__)), void *bval, void *lc /*this is (struct lc_pair *) cast to (void*). */) ;
#endif
newbrt/brt-search.h
View file @ 83d416fa
#ifndef BRT_SEARCH_H
#define BRT_SEARCH_H
enum {
enum brt_search_direction_e {
BRT_SEARCH_LEFT = 1, /* search left -> right, finds min xy as defined by the compare function */
BRT_SEARCH_RIGHT = 2, /* search right -> left, finds max xy as defined by the compare function */
BRT_SEARCH_ONE = 4, /* look into only one subtree, used for point queries */
};
struct brt_search;
......@@ -20,7 +19,7 @@ typedef int (*brt_search_compare_func_t)(struct brt_search */*so*/, DBT */*x*/,
typedef struct brt_search {
brt_search_compare_func_t compare;
int direction;
enum brt_search_direction_e direction;
DBT *k;
DBT *v;
void *context;
......
newbrt/brt-serialize-test.c
View file @ 83d416fa
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2007, 2008 Tokutek Inc. All rights reserved."
#include "assert.h"
#include "toku_assert.h"
#include "brt-internal.h"
#include "kv-pair.h"
#include <fcntl.h>
#include <string.h>
......@@ -26,7 +27,7 @@ static void test_serialize(void) {
sn.thisnodename = sn.nodesize*20;
sn.disk_lsn.lsn = 789;
sn.log_lsn.lsn = 123456;
sn.layout_version = 2;
sn.layout_version = 3;
sn.height = 1;
sn.rand4fingerprint = randval;
sn.local_fingerprint = 0;
......@@ -49,14 +50,14 @@ static void test_serialize(void) {
BNC_NBYTESINBUF(&sn, 1) = 1*(BRT_CMD_OVERHEAD+KEY_VALUE_OVERHEAD+2+5);
sn.u.n.n_bytes_in_buffers = 3*(BRT_CMD_OVERHEAD+KEY_VALUE_OVERHEAD+2+5);
toku_serialize_brtnode_to(fd, sn.nodesize*20, sn.nodesize, &sn); assert(r==0);
toku_serialize_brtnode_to(fd, sn.nodesize*(DISKOFF)20, (DISKOFF)sn.nodesize, &sn); assert(r==0);
r = toku_deserialize_brtnode_from(fd, nodesize*20, &dn, sn.flags, nodesize, 0, 0, 0, (FILENUM){0});
r = toku_deserialize_brtnode_from(fd, nodesize*(DISKOFF)20, &dn, sn.flags, nodesize);
assert(r==0);
assert(dn->thisnodename==nodesize*20);
assert(dn->disk_lsn.lsn==123456);
assert(dn->layout_version ==2);
assert(dn->layout_version ==3);
assert(dn->height == 1);
assert(dn->rand4fingerprint==randval);
assert(dn->u.n.n_children==2);
......@@ -100,6 +101,8 @@ static void test_serialize(void) {
toku_free(hello_string);
toku_fifo_free(&BNC_BUFFER(&sn,0));
toku_fifo_free(&BNC_BUFFER(&sn,1));
toku_free(sn.u.n.childinfos);
toku_free(sn.u.n.childkeys);
}
int main (int argc __attribute__((__unused__)), char *argv[] __attribute__((__unused__))) {
......
newbrt/brt-serialize.c
View file @ 83d416fa
......@@ -9,6 +9,8 @@
#include "key.h"
#include "rbuf.h"
#include "wbuf.h"
#include "kv-pair.h"
#include "mempool.h"
#include <unistd.h>
#include <stdio.h>
......@@ -57,10 +59,13 @@ static unsigned int toku_serialize_brtnode_size_slow(BRTNODE node) {
return size+hsize+csize;
} else {
unsigned int hsize=0;
PMA_ITERATE(node->u.l.buffer,
key __attribute__((__unused__)), keylen,
data __attribute__((__unused__)), datalen,
(hsize+=PMA_ITEM_OVERHEAD+KEY_VALUE_OVERHEAD+keylen+datalen));
GPMA_ITERATE(node->u.l.buffer,
idx, vlen, vdata,
({
struct kv_pair *p=vdata;
assert(vlen==sizeof(*p)+kv_pair_keylen(p)+kv_pair_vallen(p));
hsize+=PMA_ITEM_OVERHEAD+KEY_VALUE_OVERHEAD+kv_pair_keylen(p)+kv_pair_vallen(p);
}));
assert(hsize==node->u.l.n_bytes_in_buffer);
hsize+=4; /* the PMA size */
hsize+=4; /* add n entries in buffer table. */
......@@ -110,12 +115,13 @@ void toku_serialize_brtnode_to(int fd, DISKOFF off, DISKOFF size, BRTNODE node)
wbuf_int(&w, node->layout_version);
wbuf_ulonglong(&w, node->log_lsn.lsn);
//printf("%s:%d %lld.calculated_size=%d\n", __FILE__, __LINE__, off, calculated_size);
wbuf_int(&w, calculated_size);
wbuf_int(&w, node->flags);
wbuf_int(&w, node->height);
wbuf_uint(&w, calculated_size);
wbuf_uint(&w, node->flags);
wbuf_uint(&w, node->height);
//printf("%s:%d %lld rand=%08x sum=%08x height=%d\n", __FILE__, __LINE__, node->thisnodename, node->rand4fingerprint, node->subtree_fingerprint, node->height);
wbuf_int(&w, node->rand4fingerprint);
wbuf_int(&w, node->local_fingerprint);
// printf("%s:%d wrote %08x for node %lld\n", __FILE__, __LINE__, node->local_fingerprint, (long long)node->thisnodename);
//printf("%s:%d local_fingerprint=%8x\n", __FILE__, __LINE__, node->local_fingerprint);
//printf("%s:%d w.ndone=%d n_children=%d\n", __FILE__, __LINE__, w.ndone, node->n_children);
if (node->height>0) {
......@@ -169,15 +175,19 @@ void toku_serialize_brtnode_to(int fd, DISKOFF off, DISKOFF size, BRTNODE node)
}
} else {
//printf(" n_entries=%d\n", toku_pma_n_entries(node->u.l.buffer));
wbuf_int(&w, toku_pma_n_entries(node->u.l.buffer));
wbuf_int(&w, toku_pma_index_limit(node->u.l.buffer));
PMA_ITERATE_IDX(node->u.l.buffer, idx,
key, keylen, data, datalen,
({
wbuf_int(&w, idx);
wbuf_bytes(&w, key, keylen);
wbuf_bytes(&w, data, datalen);
}));
wbuf_int(&w, toku_gpma_n_entries(node->u.l.buffer));
wbuf_int(&w, toku_gpma_index_limit(node->u.l.buffer));
GPMA_ITERATE(node->u.l.buffer, idx, vlen, vdata,
({
struct kv_pair *p=vdata;
assert((char*)node->u.l.buffer_mempool.base<= (char*)p && (char*)p < (char*)node->u.l.buffer_mempool.base+node->u.l.buffer_mempool.size );
int keylen=kv_pair_keylen(p);
int datalen=kv_pair_vallen(p);
assert(vlen==sizeof(*p)+keylen+datalen);
wbuf_int(&w, idx);
wbuf_bytes(&w, kv_pair_key(p), keylen);
wbuf_bytes(&w, kv_pair_val(p), datalen);
}));
}
assert(w.ndone<=w.size);
#ifdef CRC_ATEND
......@@ -205,10 +215,7 @@ void toku_serialize_brtnode_to(int fd, DISKOFF off, DISKOFF size, BRTNODE node)
toku_free(buf);
}
int toku_deserialize_brtnode_from (int fd, DISKOFF off, BRTNODE *brtnode, int flags, int nodesize,
int (*bt_compare)(DB *, const DBT *, const DBT *),
int (*dup_compare)(DB *, const DBT *, const DBT *),
DB *db, FILENUM filenum) {
int toku_deserialize_brtnode_from (int fd, DISKOFF off, BRTNODE *brtnode, unsigned int flags, int nodesize) {
TAGMALLOC(BRTNODE, result);
struct rbuf rc;
int i;
......@@ -262,7 +269,7 @@ int toku_deserialize_brtnode_from (int fd, DISKOFF off, BRTNODE *brtnode, int fl
}
}
result->layout_version = rbuf_int(&rc);
if (result->layout_version!=2) {
if (result->layout_version!=3) {
r=DB_BADFORMAT;
goto died1;
}
......@@ -278,6 +285,7 @@ int toku_deserialize_brtnode_from (int fd, DISKOFF off, BRTNODE *brtnode, int fl
result->height = rbuf_int(&rc);
result->rand4fingerprint = rbuf_int(&rc);
result->local_fingerprint = rbuf_int(&rc);
// printf("%s:%d read %08x\n", __FILE__, __LINE__, result->local_fingerprint);
result->dirty = 0;
//printf("height==%d\n", result->height);
if (result->height>0) {
......@@ -365,27 +373,32 @@ int toku_deserialize_brtnode_from (int fd, DISKOFF off, BRTNODE *brtnode, int fl
int n_in_buf = rbuf_int(&rc);
int index_limit = rbuf_int(&rc);
result->u.l.n_bytes_in_buffer = 0;
r=toku_pma_create(&result->u.l.buffer, bt_compare, db, filenum, nodesize, index_limit);
r=toku_gpma_create(&result->u.l.buffer, index_limit);
if (r!=0) {
if (0) { died_21: toku_pma_free(&result->u.l.buffer); }
if (0) { died_21: toku_gpma_free(&result->u.l.buffer, 0, 0); }
goto died1;
}
toku_pma_set_dup_mode(result->u.l.buffer, flags);
toku_pma_set_dup_compare(result->u.l.buffer, dup_compare);
//printf("%s:%d r PMA= %p\n", __FILE__, __LINE__, result->u.l.buffer);
toku_verify_counts(result);
{
void *mp = toku_malloc(nodesize);
if (mp==0) return ENOMEM;
toku_mempool_init(&result->u.l.buffer_mempool, mp, nodesize);
}
u_int32_t actual_sum = 0;
for (i=0; i<n_in_buf; i++) {
bytevec key; ITEMLEN keylen;
bytevec val; ITEMLEN vallen;
int idx = rbuf_int(&rc);
DBT keydbt, datadbt;
rbuf_bytes(&rc, &key, &keylen); /* Returns a pointer into the rbuf. */
rbuf_bytes(&rc, &val, &vallen);
result->u.l.n_bytes_in_buffer += keylen + vallen + KEY_VALUE_OVERHEAD + PMA_ITEM_OVERHEAD;
r = toku_pma_set_at_index(result->u.l.buffer, idx, toku_fill_dbt(&keydbt, key, keylen), toku_fill_dbt(&datadbt, val, vallen));
actual_sum += result->rand4fingerprint*toku_calccrc32_kvpair(key, keylen, val, vallen);
struct kv_pair *pair = brtnode_malloc_kv_pair(result->u.l.buffer, &result->u.l.buffer_mempool, key, keylen, val, vallen);
assert(pair);
int pairlen = kv_pair_size(pair);
toku_gpma_set_at_index(result->u.l.buffer, idx, pairlen, pair);
actual_sum += result->rand4fingerprint*toku_calccrc32_kvpair_struct(pair);
// printf("%s:%d rand4=%08x actual=%08x this=%08x expect=%08x\n", __FILE__, __LINE__, result->rand4fingerprint, actual_sum, toku_calccrc32_kvpair_struct(pair), result->local_fingerprint);
}
if (r!=0) goto died_21;
......@@ -425,8 +438,13 @@ void toku_verify_counts (BRTNODE node) {
if (node->height==0) {
assert(node->u.l.buffer);
unsigned int sum=0;
PMA_ITERATE(node->u.l.buffer, key __attribute__((__unused__)), keylen, data __attribute__((__unused__)), datalen,
sum+=(PMA_ITEM_OVERHEAD + KEY_VALUE_OVERHEAD + keylen + datalen));
unsigned int count=0;
GPMA_ITERATE(node->u.l.buffer, idx, dlen, dvadata,
({
count++;
sum+=(PMA_ITEM_OVERHEAD + dlen);
}));
assert(count==toku_gpma_n_entries(node->u.l.buffer));
assert(sum==node->u.l.n_bytes_in_buffer);
} else {
unsigned int sum = 0;
......
newbrt/brt-test-cursor-2.c
View file @ 83d416fa
......@@ -3,7 +3,6 @@
#include "brt.h"
#include "key.h"
#include "pma.h"
#include "brt-internal.h"
#include "memory.h"
#include "toku_assert.h"
......
newbrt/brt-test-cursor.c
View file @ 83d416fa
......@@ -3,7 +3,6 @@
#include "brt.h"
#include "key.h"
#include "pma.h"
#include "brt-internal.h"
#include "memory.h"
#include "toku_assert.h"
......
newbrt/brt-test-helpers.c
View file @ 83d416fa
......@@ -71,24 +71,29 @@ int toku_testsetup_insert_to_leaf (BRT brt, DISKOFF diskoff, char *key, int keyl
if (r!=0) return r;
BRTNODE node=node_v;
assert(node->height==0);
DBT k,v;
int replaced_v_size;
enum pma_errors pma_status =
toku_pma_insert_or_replace(node->u.l.buffer,
toku_fill_dbt(&k, key, keylen),
toku_fill_dbt(&v, val, vallen),
&replaced_v_size,
(TOKULOGGER)0, (TXNID)0,
toku_cachefile_filenum(brt->cf),
node->thisnodename, node->rand4fingerprint,
&node->local_fingerprint,
&node->log_lsn);
assert(pma_status==BRT_OK);
if (replaced_v_size>=0) {
node->u.l.n_bytes_in_buffer += v.size - replaced_v_size;
struct kv_pair *kv = kv_pair_malloc(key, keylen, val, vallen);
struct lc_pair lc = {brt, node->flags & TOKU_DB_DUPSORT};
u_int32_t storedlen;
void *storeddata;
u_int32_t idx;
r = toku_gpma_lookup_item(node->u.l.buffer, kv_pair_size(kv), kv, toku_brtleaf_compare_fun, &lc, &storedlen, &storeddata, &idx);
if (r==0) {
// It's already there. So now we have to remove it and put the new one back in.
node->u.l.n_bytes_in_buffer -= PMA_ITEM_OVERHEAD + storedlen;
node->local_fingerprint -= toku_crc32(toku_null_crc, storeddata, storedlen);
toku_mempool_mfree(&node->u.l.buffer_mempool, storeddata, storedlen);
// Now put the new kv in.
toku_gpma_set_at_index(node->u.l.buffer, idx, kv_pair_size(kv), kv);
} else {
node->u.l.n_bytes_in_buffer += k.size + v.size + KEY_VALUE_OVERHEAD + PMA_ITEM_OVERHEAD;
r = toku_gpma_insert(node->u.l.buffer, kv_pair_size(kv), kv, toku_brtleaf_compare_fun, &lc, 0, 0, 0);
assert(r==0);
}
node->u.l.n_bytes_in_buffer += PMA_ITEM_OVERHEAD + kv_pair_size(kv);
node->local_fingerprint += toku_crc32(toku_null_crc, kv, kv_pair_size(kv));
node->dirty=1;
*subtree_fingerprint = node->local_fingerprint;
r = toku_unpin_brtnode(brt, node_v);
......
newbrt/brt-test-named-db.c 0 → 100644
View file @ 83d416fa
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2007, 2008 Tokutek Inc. All rights reserved."
#include "brt.h"
#include "key.h"
#include "test.h"
#include "toku_assert.h"
#include <unistd.h>
static TOKUTXN const null_txn = 0;
static DB * const null_db = 0;
static void test_named_db (void) {
const char *n0 = "brt-test-named-db-0.brt";
CACHETABLE ct;
BRT t0;
int r;
DBT k,v;
if (verbose) printf("test_named_db\n");
unlink(n0);
toku_memory_check_all_free();
r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER); assert(r==0);
r = toku_open_brt(n0, "db1", 1, &t0, 1<<12, ct, null_txn, toku_default_compare_fun, null_db); assert(r==0);
toku_brt_insert(t0, toku_fill_dbt(&k, "good", 5), toku_fill_dbt(&v, "day", 4), null_txn); assert(r==0);
r = toku_close_brt(t0); assert(r==0);
r = toku_cachetable_close(&ct); assert(r==0);
toku_memory_check_all_free();
toku_memory_check_all_free();
r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER); assert(r==0);
r = toku_open_brt(n0, "db1", 0, &t0, 1<<12, ct, null_txn, toku_default_compare_fun, null_db); assert(r==0);
{
r = toku_brt_lookup(t0, toku_fill_dbt(&k, "good", 5), toku_init_dbt(&v));
assert(r==0);
assert(v.size==4);
assert(strcmp(v.data,"day")==0);
}
r = toku_close_brt(t0); assert(r==0);
r = toku_cachetable_close(&ct); assert(r==0);
toku_memory_check_all_free();
}
int main (int argc , const char *argv[]) {
default_parse_args(argc, argv);
test_named_db();
toku_malloc_cleanup();
if (verbose) printf("test_named_db ok\n");
return 0;
}
newbrt/brt-test.c
View file @ 83d416fa
......@@ -3,7 +3,7 @@
#include "brt.h"
#include "key.h"
#include "pma.h"
#include "gpma.h"
#include "brt-internal.h"
#include "memory.h"
#include "toku_assert.h"
......@@ -21,93 +21,6 @@
static TOKUTXN const null_txn = 0;
static DB * const null_db = 0;
static void test0 (void) {
BRT t;
int r;
CACHETABLE ct;
char fname[]="testbrt.brt";
if (verbose) printf("%s:%d test0\n", __FILE__, __LINE__);
toku_memory_check=1;
toku_memory_check_all_free();
r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
assert(r==0);
if (verbose) printf("%s:%d test0\n", __FILE__, __LINE__);
unlink(fname);
r = toku_open_brt(fname, 0, 1, &t, 1024, ct, null_txn, toku_default_compare_fun, null_db);
assert(r==0);
//printf("%s:%d test0\n", __FILE__, __LINE__);
//printf("%s:%d n_items_malloced=%lld\n", __FILE__, __LINE__, n_items_malloced);
r = toku_close_brt(t); assert(r==0);
//printf("%s:%d n_items_malloced=%lld\n", __FILE__, __LINE__, n_items_malloced);
r = toku_cachetable_close(&ct);
assert(r==0);
toku_memory_check_all_free();
}
static void test1 (void) {
BRT t;
int r;
CACHETABLE ct;
char fname[]="testbrt.brt";
DBT k,v;
toku_memory_check=1;
toku_memory_check_all_free();
r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
assert(r==0);
unlink(fname);
r = toku_open_brt(fname, 0, 1, &t, 1024, ct, null_txn, toku_default_compare_fun, null_db);
assert(r==0);
toku_brt_insert(t, toku_fill_dbt(&k, "hello", 6), toku_fill_dbt(&v, "there", 6), null_txn);
{
r = toku_brt_lookup(t, toku_fill_dbt(&k, "hello", 6), toku_init_dbt(&v));
assert(r==0);
assert(strcmp(v.data, "there")==0);
assert(v.size==6);
}
r = toku_close_brt(t); assert(r==0);
r = toku_cachetable_close(&ct); assert(r==0);
toku_memory_check_all_free();
if (verbose) printf("test1 ok\n");
}
static void test2 (int memcheck) {
BRT t;
int r;
int i;
CACHETABLE ct;
char fname[]="testbrt.brt";
toku_memory_check=memcheck;
if (verbose) printf("%s:%d checking\n", __FILE__, __LINE__);
toku_memory_check_all_free();
r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER); assert(r==0);
unlink(fname);
r = toku_open_brt(fname, 0, 1, &t, 1024, ct, null_txn, toku_default_compare_fun, null_db);
if (verbose) printf("%s:%d did setup\n", __FILE__, __LINE__);
assert(r==0);
for (i=0; i<2048; i++) {
DBT k,v;
char key[100],val[100];
snprintf(key,100,"hello%d",i);
snprintf(val,100,"there%d",i);
toku_brt_insert(t, toku_fill_dbt(&k, key, 1+strlen(key)), toku_fill_dbt(&v, val, 1+strlen(val)), null_txn);
//printf("%s:%d did insert %d\n", __FILE__, __LINE__, i);
if (0) {
brt_flush(t);
{
int n = toku_get_n_items_malloced();
if (verbose) printf("%s:%d i=%d n_items_malloced=%d\n", __FILE__, __LINE__, i, n);
if (n!=3) toku_print_malloced_items();
assert(n==3);
}
}
}
if (verbose) printf("%s:%d inserted\n", __FILE__, __LINE__);
r = toku_close_brt(t); assert(r==0);
r = toku_cachetable_close(&ct); assert(r==0);
toku_memory_check_all_free();
if (verbose) printf("test2 ok\n");
}
static void test5 (void) {
int r;
BRT t;
......@@ -240,44 +153,6 @@ static void test_multiple_files (void) {
test_multiple_files_of_size (1<<20);
}
static void test_named_db (void) {
const char *n0 = "test0.brt";
const char *n1 = "test1.brt";
CACHETABLE ct;
BRT t0;
int r;
DBT k,v;
if (verbose) printf("test_named_db\n");
unlink(n0);
unlink(n1);
toku_memory_check_all_free();
r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER); assert(r==0);
r = toku_open_brt(n0, "db1", 1, &t0, 1<<12, ct, null_txn, toku_default_compare_fun, null_db); assert(r==0);
toku_brt_insert(t0, toku_fill_dbt(&k, "good", 5), toku_fill_dbt(&v, "day", 4), null_txn); assert(r==0);
r = toku_close_brt(t0); assert(r==0);
r = toku_cachetable_close(&ct); assert(r==0);
toku_memory_check_all_free();
toku_memory_check_all_free();
r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER); assert(r==0);
r = toku_open_brt(n0, "db1", 0, &t0, 1<<12, ct, null_txn, toku_default_compare_fun, null_db); assert(r==0);
{
r = toku_brt_lookup(t0, toku_fill_dbt(&k, "good", 5), toku_init_dbt(&v));
assert(r==0);
assert(v.size==4);
assert(strcmp(v.data,"day")==0);
}
r = toku_close_brt(t0); assert(r==0);
r = toku_cachetable_close(&ct); assert(r==0);
toku_memory_check_all_free();
}
static void test_multiple_dbs (void) {
const char *n0 = "test0.brt";
const char *n1 = "test1.brt";
......@@ -1640,20 +1515,9 @@ static void brt_blackbox_test (void) {
test_cursor_last_empty(); toku_memory_check_all_free();
test_multiple_brts_one_db_one_file(); toku_memory_check_all_free();
test_dump_empty_db(); toku_memory_check_all_free();
test_named_db();
toku_memory_check_all_free();
test_multiple_dbs();
toku_memory_check_all_free();
if (verbose) printf("test0 A\n");
test0();
if (verbose) printf("test0 B\n");
test0(); /* Make sure it works twice. */
if (verbose) printf("test1\n");
test1();
if (verbose) printf("test2 checking memory\n");
test2(1);
if (verbose) printf("test2 faster\n");
test2(0);
if (verbose) printf("test5\n");
test5();
if (verbose) printf("test_multiple_files\n");
......@@ -1672,14 +1536,6 @@ static void brt_blackbox_test (void) {
toku_brt_do_push_cmd = old_brt_do_push_cmd;
// test3(1<<19, 1<<20, 0);
// test3(1<<20, 1<<20, 0);
// test3(1<<20, 1<<21, 0);
// test3(1<<20, 1<<22, 0);
}
int main (int argc , const char *argv[]) {
......
newbrt/brt-test0.c 0 → 100644
View file @ 83d416fa
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2007, 2008 Tokutek Inc. All rights reserved."
#include "brt.h"
#include "key.h"
#include "test.h"
#include "toku_assert.h"
#include <unistd.h>
static TOKUTXN const null_txn = 0;
static DB * const null_db = 0;
static void test0 (void) {
BRT t;
int r;
CACHETABLE ct;
char fname[]="brt-test0.brt";
if (verbose) printf("%s:%d test0\n", __FILE__, __LINE__);
toku_memory_check=1;
toku_memory_check_all_free();
r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
assert(r==0);
if (verbose) printf("%s:%d test0\n", __FILE__, __LINE__);
unlink(fname);
r = toku_open_brt(fname, 0, 1, &t, 1024, ct, null_txn, toku_default_compare_fun, null_db);
assert(r==0);
//printf("%s:%d test0\n", __FILE__, __LINE__);
//printf("%s:%d n_items_malloced=%lld\n", __FILE__, __LINE__, n_items_malloced);
r = toku_close_brt(t); assert(r==0);
//printf("%s:%d n_items_malloced=%lld\n", __FILE__, __LINE__, n_items_malloced);
r = toku_cachetable_close(&ct);
assert(r==0);
toku_memory_check_all_free();
}
int main (int argc , const char *argv[]) {
default_parse_args(argc, argv);
if (verbose) printf("test0 A\n");
test0();
if (verbose) printf("test0 B\n");
test0(); /* Make sure it works twice. */
toku_malloc_cleanup();
if (verbose) printf("test0 ok\n");
return 0;
}
newbrt/brt-test1.c 0 → 100644
View file @ 83d416fa
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2007, 2008 Tokutek Inc. All rights reserved."
#include "brt.h"
#include "key.h"
#include "test.h"
#include "toku_assert.h"
#include <unistd.h>
static TOKUTXN const null_txn = 0;
static DB * const null_db = 0;
static void test1 (void) {
BRT t;
int r;
CACHETABLE ct;
char fname[]="brt-test1.brt";
DBT k,v;
toku_memory_check=1;
toku_memory_check_all_free();
r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
assert(r==0);
unlink(fname);
r = toku_open_brt(fname, 0, 1, &t, 1024, ct, null_txn, toku_default_compare_fun, null_db);
assert(r==0);
toku_brt_insert(t, toku_fill_dbt(&k, "hello", 6), toku_fill_dbt(&v, "there", 6), null_txn);
{
r = toku_brt_lookup(t, toku_fill_dbt(&k, "hello", 6), toku_init_dbt(&v));
assert(r==0);
assert(strcmp(v.data, "there")==0);
assert(v.size==6);
}
r = toku_close_brt(t); assert(r==0);
r = toku_cachetable_close(&ct); assert(r==0);
toku_memory_check_all_free();
if (verbose) printf("test1 ok\n");
}
int main (int argc , const char *argv[]) {
default_parse_args(argc, argv);
if (verbose) printf("test1\n");
test1();
toku_malloc_cleanup();
if (verbose) printf("test1 ok\n");
return 0;
}
newbrt/brt-test2.c 0 → 100644
View file @ 83d416fa
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2007, 2008 Tokutek Inc. All rights reserved."
#include "brt.h"
#include "key.h"
#include "test.h"
#include "toku_assert.h"
#include <unistd.h>
static TOKUTXN const null_txn = 0;
static DB * const null_db = 0;
static void test2 (int memcheck) {
BRT t;
int r;
int i;
CACHETABLE ct;
char fname[]="brt-test2.brt";
toku_memory_check=memcheck;
if (verbose) printf("%s:%d checking\n", __FILE__, __LINE__);
toku_memory_check_all_free();
r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER); assert(r==0);
unlink(fname);
r = toku_open_brt(fname, 0, 1, &t, 1024, ct, null_txn, toku_default_compare_fun, null_db);
if (verbose) printf("%s:%d did setup\n", __FILE__, __LINE__);
assert(r==0);
for (i=0; i<4096; i++) {
DBT k,v;
char key[100],val[100];
snprintf(key,100,"hello%d",i);
snprintf(val,100,"there%d",i);
toku_brt_insert(t, toku_fill_dbt(&k, key, 1+strlen(key)), toku_fill_dbt(&v, val, 1+strlen(val)), null_txn);
//printf("%s:%d did insert %d\n", __FILE__, __LINE__, i);
if (0) {
brt_flush(t);
{
int n = toku_get_n_items_malloced();
if (verbose) printf("%s:%d i=%d n_items_malloced=%d\n", __FILE__, __LINE__, i, n);
if (n!=3) toku_print_malloced_items();
assert(n==3);
}
}
}
if (verbose) printf("%s:%d inserted\n", __FILE__, __LINE__);
r = toku_close_brt(t); assert(r==0);
r = toku_cachetable_close(&ct); assert(r==0);
toku_memory_check_all_free();
if (verbose) printf("test2 ok\n");
}
int main (int argc , const char *argv[]) {
default_parse_args(argc, argv);
if (verbose) printf("test2 checking memory\n");
// test2(1);
if (verbose) printf("test2 faster\n");
test2(0);
toku_malloc_cleanup();
if (verbose) printf("test1 ok\n");
return 0;
}
newbrt/brt-test3.c
View file @ 83d416fa
......@@ -3,7 +3,6 @@
#include "brt.h"
#include "key.h"
#include "pma.h"
#include "brt-internal.h"
#include "memory.h"
#include "toku_assert.h"
......@@ -59,7 +58,7 @@ static void brt_blackbox_test (void) {
test3(2048, 1<<15, 1);
if (verbose) printf("test3 fast\n");
if (verbose) toku_pma_show_stats();
//if (verbose) toku_pma_show_stats();
test3(1<<15, 1024, 1);
if (verbose) printf("test3 fast\n");
......
newbrt/brt-test4.c
View file @ 83d416fa
......@@ -3,7 +3,6 @@
#include "brt.h"
#include "key.h"
#include "pma.h"
#include "brt-internal.h"
#include "memory.h"
#include "toku_assert.h"
......@@ -57,7 +56,7 @@ static void brt_blackbox_test (void) {
if (verbose) printf("test4 slow\n");
test4(2048, 1<<15, 1);
if (verbose) toku_pma_show_stats();
//if (verbose) toku_pma_show_stats();
test4(1<<15, 1024, 1);
......
newbrt/brt-verify.c
View file @ 83d416fa
......@@ -17,6 +17,16 @@
#include "brt-internal.h"
#include "toku_assert.h"
#include "kv-pair.h"
static void gpma_verify_fingerprint (GPMA pma, u_int32_t rand4fingerprint, u_int32_t fingerprint) {
u_int32_t actual_fingerprint=0;
GPMA_ITERATE(pma, idx, len, val,
actual_fingerprint+=rand4fingerprint*toku_calccrc32_kvpair_struct(val)
);
assert(actual_fingerprint==fingerprint);
}
static void verify_local_fingerprint (BRTNODE node) {
u_int32_t fp=0;
......@@ -29,7 +39,7 @@ static void verify_local_fingerprint (BRTNODE node) {
}));
assert(fp==node->local_fingerprint);
} else {
toku_pma_verify_fingerprint(node->u.l.buffer, node->rand4fingerprint, node->local_fingerprint);
gpma_verify_fingerprint(node->u.l.buffer, node->rand4fingerprint, node->local_fingerprint);
}
}
......
newbrt/brt.c
View file @ 83d416fa
......@@ -36,6 +36,8 @@
#include "brt-internal.h"
#include "key.h"
#include "log_header.h"
#include "kv-pair.h"
#include "mempool.h"
extern long long n_items_malloced;
......@@ -60,8 +62,14 @@ void toku_brtnode_free (BRTNODE *nodep) {
toku_free(node->u.n.childinfos);
} else {
if (node->u.l.buffer) // The buffer may have been freed already, in some cases.
toku_pma_free(&node->u.l.buffer);
toku_gpma_free(&node->u.l.buffer, 0, 0);
void *mpbase = toku_mempool_get_base(&node->u.l.buffer_mempool);
toku_mempool_fini(&node->u.l.buffer_mempool);
toku_free(mpbase);
}
toku_free(node);
*nodep=0;
}
......@@ -121,8 +129,8 @@ void toku_brtnode_flush_callback (CACHEFILE cachefile, DISKOFF nodename, void *b
// toku_pma_verify_fingerprint(brtnode->u.l.buffer, brtnode->rand4fingerprint, brtnode->subtree_fingerprint);
// }
if (0) {
printf("%s:%d toku_brtnode_flush_callback %p keep_me=%d height=%d", __FILE__, __LINE__, brtnode, keep_me, brtnode->height);
if (brtnode->height==0) printf(" pma=%p", brtnode->u.l.buffer);
printf("%s:%d toku_brtnode_flush_callback %p thisnodename=%lld keep_me=%d height=%d", __FILE__, __LINE__, brtnode, (long long)brtnode->thisnodename, keep_me, brtnode->height);
if (brtnode->height==0) printf(" pma=%p mempool-base=%p", brtnode->u.l.buffer, brtnode->u.l.buffer_mempool.base);
printf("\n");
}
//if (modified_lsn.lsn > brtnode->lsn.lsn) brtnode->lsn=modified_lsn;
......@@ -141,8 +149,7 @@ void toku_brtnode_flush_callback (CACHEFILE cachefile, DISKOFF nodename, void *b
int toku_brtnode_fetch_callback (CACHEFILE cachefile, DISKOFF nodename, void **brtnode_pv, long *sizep, void*extraargs, LSN *written_lsn) {
BRT t =(BRT)extraargs;
BRTNODE *result=(BRTNODE*)brtnode_pv;
int r = toku_deserialize_brtnode_from(toku_cachefile_fd(cachefile), nodename, result, t->flags, t->nodesize,
t->compare_fun, t->dup_compare, t->db, toku_cachefile_filenum(t->cf));
int r = toku_deserialize_brtnode_from(toku_cachefile_fd(cachefile), nodename, result, t->flags, t->nodesize);
if (r == 0) {
*sizep = brtnode_size(*result);
*written_lsn = (*result)->disk_lsn;
......@@ -202,6 +209,7 @@ int toku_unpin_brtnode (BRT brt, BRTNODE node) {
// node->log_lsn = toku_txn_get_last_lsn(txn);
// //if (node->log_lsn.lsn>33320) printf("%s:%d node%lld lsn=%lld\n", __FILE__, __LINE__, node->thisnodename, node->log_lsn.lsn);
// }
//toku_verify_counts(node);
return toku_cachetable_unpin(brt->cf, node->thisnodename, node->dirty, brtnode_size(node));
}
......@@ -254,7 +262,7 @@ static void initialize_brtnode (BRT t, BRTNODE n, DISKOFF nodename, int height)
n->thisnodename = nodename;
n->disk_lsn.lsn = 0; // a new one can always be 0.
n->log_lsn = n->disk_lsn;
n->layout_version = 2;
n->layout_version = 3;
n->height = height;
n->rand4fingerprint = random();
n->local_fingerprint = 0;
......@@ -267,10 +275,14 @@ static void initialize_brtnode (BRT t, BRTNODE n, DISKOFF nodename, int height)
n->u.n.childinfos=0;
n->u.n.childkeys=0;
} else {
int r = toku_pma_create(&n->u.l.buffer, t->compare_fun, t->db, toku_cachefile_filenum(t->cf), n->nodesize, 0);
assert(r==0);
toku_pma_set_dup_mode(n->u.l.buffer, t->flags & (TOKU_DB_DUP+TOKU_DB_DUPSORT));
toku_pma_set_dup_compare(n->u.l.buffer, t->dup_compare);
int r = toku_gpma_create(&n->u.l.buffer, 0);
assert(r==0);
{
void *mp = toku_malloc(n->nodesize);
assert(mp);
toku_mempool_init(&n->u.l.buffer_mempool, mp, n->nodesize);
}
static int rcount=0;
//printf("%s:%d n PMA= %p (rcount=%d)\n", __FILE__, __LINE__, n->u.l.buffer, rcount);
rcount++;
......@@ -314,6 +326,88 @@ static int insert_to_buffer_in_nonleaf (BRTNODE node, int childnum, DBT *k, DBT
return 0;
}
struct move_struct {
TOKULOGGER logger;
FILENUM filenum;
BRTNODE from,to;
struct gitem last_pair_remaining_in_from;
};
int move_between_mempools (u_int32_t len, void *odata, void **ndata, void *extra) {
struct move_struct *ms=extra;
assert(ms->from->height==0);
assert(ms->to->height==0);
assert(len==(unsigned)kv_pair_size(odata));
void *newitem=toku_mempool_malloc(&ms->to->u.l.buffer_mempool, len, 4);
assert(newitem);
memcpy(newitem, odata, len);
toku_mempool_mfree(&ms->from->u.l.buffer_mempool, odata, len);
*ndata = newitem;
assert(len==(unsigned)kv_pair_size(newitem));
return 0;
}
int note_move_items_within_or_between (u_int32_t nitems, u_int32_t *froms, u_int32_t *tos, BRTNODE from, BRTNODE to, FILENUM filenum, TOKULOGGER logger, u_int32_t old_N, u_int32_t new_N) {
INTPAIRARRAY ipa;
MALLOC_N(nitems, ipa.array);
if (ipa.array==0) return errno;
u_int32_t i;
for (i=0; i<nitems; i++) {
ipa.array[i].a=froms[i];
ipa.array[i].b=tos [i];
}
ipa.size=nitems;
if (logger) {
LSN lsn;
int r = toku_log_pmadistribute(logger, &lsn, 0, filenum, from->thisnodename, to->thisnodename, ipa, old_N, new_N);
if (r!=0) return r;
from->log_lsn=lsn;
to->log_lsn =lsn;
}
toku_free(ipa.array);
return 0;
}
static int note_move_items_within (u_int32_t nitems, u_int32_t *froms, u_int32_t *tos, struct gitem *items __attribute__((__unused__)), u_int32_t old_N, u_int32_t new_N, void *extra) {
struct move_struct *ms=extra;
assert(nitems>0);
ms->last_pair_remaining_in_from=items[nitems-1];
return note_move_items_within_or_between(nitems, froms, tos, ms->from, ms->from, ms->filenum, ms->logger, old_N, new_N);
}
static int note_move_items_between (u_int32_t nitems, u_int32_t *froms, u_int32_t *tos, struct gitem *items, u_int32_t old_N, u_int32_t new_N, void *extra) {
struct move_struct *ms=extra;
int r = note_move_items_within_or_between(nitems, froms, tos, ms->from, ms->to, ms->filenum, ms->logger, old_N, new_N);
if (r!=0) return r;
u_int32_t i;
u_int32_t diffsize = 0;
u_int32_t diff_fp = 0;
for (i=0; i<nitems; i++) {
diffsize += PMA_ITEM_OVERHEAD + items[i].len;
diff_fp += toku_calccrc32_kvpair_struct(items[i].data);
}
ms->from->local_fingerprint -= ms->from->rand4fingerprint * diff_fp;
ms->to->local_fingerprint += ms->to->rand4fingerprint * diff_fp;
ms->from->u.l.n_bytes_in_buffer -= diffsize;
ms->to->u.l.n_bytes_in_buffer += diffsize;
return 0;
}
struct delete_struct {
BRTNODE node;
};
static int brt_leaf_delete_callback (u_int32_t slotnum, u_int32_t len, void *data, void *extra) {
struct delete_struct *d = extra;
d->node->local_fingerprint -= d->node->rand4fingerprint*toku_calccrc32_kvpair_struct(data);
d->node->u.l.n_bytes_in_buffer -= PMA_ITEM_OVERHEAD + len;
toku_mempool_mfree(&d->node->u.l.buffer_mempool, data, len);
d->node->dirty=1;
// Should use slotnum for logging
slotnum=slotnum; //????
return 0;
}
static int brtleaf_split (TOKULOGGER logger, FILENUM filenum, BRT t, BRTNODE node, BRTNODE *nodea, BRTNODE *nodeb, DBT *splitk) {
BRTNODE B;
......@@ -329,10 +423,46 @@ static int brtleaf_split (TOKULOGGER logger, FILENUM filenum, BRT t, BRTNODE nod
//printf("%s:%d B is at %lld nodesize=%d\n", __FILE__, __LINE__, B->thisnodename, B->nodesize);
assert(node->height>0 || node->u.l.buffer!=0);
int r;
r = toku_pma_split(logger, filenum,
node->thisnodename, node->u.l.buffer, &node->u.l.n_bytes_in_buffer, node->rand4fingerprint, &node->local_fingerprint, &node->log_lsn,
splitk,
B->thisnodename, B->u.l.buffer, &B->u.l.n_bytes_in_buffer, B->rand4fingerprint, &B->local_fingerprint, &B->log_lsn);
struct move_struct ms = {.logger=logger, .filenum=filenum, .from=node, .to=B};
//toku_verify_gpma(node->u.l.buffer);
GPMA_ITERATE(node->u.l.buffer, idx, vlen, vdata,
({
struct kv_pair *p=vdata;
//printf("%s:%d %d:%p ", __FILE__, __LINE__, idx, p);
assert((char*)node->u.l.buffer_mempool.base<= (char*)p && (char*)p < (char*)node->u.l.buffer_mempool.base+node->u.l.buffer_mempool.size );
}));
r = toku_gpma_split(node->u.l.buffer, B->u.l.buffer, PMA_ITEM_OVERHEAD,
move_between_mempools,
note_move_items_within,
note_move_items_between,
&ms);
GPMA_ITERATE(node->u.l.buffer, idx, vlen, vdata,
({
struct kv_pair *p=vdata;
//printf("%s:%d %d:%p ", __FILE__, __LINE__, idx, p);
assert((char*)node->u.l.buffer_mempool.base<= (char*)p && (char*)p < (char*)node->u.l.buffer_mempool.base+node->u.l.buffer_mempool.size );
}));
GPMA_ITERATE(B->u.l.buffer, idx, vlen, vdata,
({
struct kv_pair *p=vdata;
//printf("%s:%d %d:%p\n", __FILE__, __LINE__, idx, p);
assert((char*)B->u.l.buffer_mempool.base<= (char*)p && (char*)p < (char*)B->u.l.buffer_mempool.base+node->u.l.buffer_mempool.size );
}));
//toku_verify_gpma(node->u.l.buffer);
//toku_verify_gpma(B->u.l.buffer);
if (splitk) {
memset(splitk, 0, sizeof *splitk);
struct kv_pair *kp=ms.last_pair_remaining_in_from.data;
if (node->flags&TOKU_DB_DUPSORT) {
splitk->size = kv_pair_keylen(kp)+kv_pair_vallen(kp);
splitk->data = kv_pair_malloc(kv_pair_key(kp), kv_pair_keylen(kp), kv_pair_val(kp), kv_pair_vallen(kp));
} else {
splitk->size = kv_pair_keylen(kp);
splitk->data = kv_pair_malloc(kv_pair_key(kp), kv_pair_keylen(kp), 0, 0);
}
splitk->flags=0;
}
assert(r == 0);
assert(node->height>0 || node->u.l.buffer!=0);
/* Remove it from the cache table, and free its storage. */
......@@ -785,9 +915,9 @@ static int handle_split_of_child (BRT t, BRTNODE node, int childnum,
//verify_local_fingerprint_nonleaf(childb);
//verify_local_fingerprint_nonleaf(node);
toku_verify_counts(node);
toku_verify_counts(childa);
toku_verify_counts(childb);
//toku_verify_counts(node);
//toku_verify_counts(childa);
//toku_verify_counts(childb);
r=toku_unpin_brtnode(t, childa);
assert(r==0);
......@@ -842,7 +972,7 @@ static int push_some_brt_cmds_down (BRT t, BRTNODE node, int childnum,
//printf("%s:%d pin %p\n", __FILE__, __LINE__, childnode_v);
child=childnode_v;
//verify_local_fingerprint_nonleaf(child);
toku_verify_counts(child);
//toku_verify_counts(child);
//printf("%s:%d height=%d n_bytes_in_buffer = {%d, %d, %d, ...}\n", __FILE__, __LINE__, child->height, child->n_bytes_in_buffer[0], child->n_bytes_in_buffer[1], child->n_bytes_in_buffer[2]);
if (child->height>0 && child->u.n.n_children>0) assert(BNC_DISKOFF(child, child->u.n.n_children-1)!=0);
if (debug) printf("%s:%d %*spush_some_brt_cmds_down to %lld\n", __FILE__, __LINE__, debug, "", child->thisnodename);
......@@ -966,6 +1096,44 @@ static int brtnode_maybe_push_down(BRT t, BRTNODE node, int *did_split, BRTNODE
return 0;
}
int toku_brtleaf_compare_fun (u_int32_t alen __attribute__((__unused__)), void *aval, u_int32_t blen __attribute__((__unused__)), void *bval, void *extra) {
struct lc_pair *p = extra;
BRT t = p->t;
DBT k1,k2;
int cmp = t->compare_fun (t->db,
toku_fill_dbt(&k1, kv_pair_key(aval), kv_pair_keylen(aval)),
toku_fill_dbt(&k2, kv_pair_key(bval), kv_pair_keylen(bval)));
if (cmp == 0 && p->compare_both ) {
return t->dup_compare(t->db,
toku_fill_dbt(&k1, kv_pair_val(aval), kv_pair_vallen(aval)),
toku_fill_dbt(&k2, kv_pair_val(bval), kv_pair_vallen(bval)));
} else {
return cmp;
}
}
int toku_brtnode_compress_kvspace (GPMA pma, struct mempool *memp) {
if (toku_mempool_get_frag_size(memp) == 0)
return -1;
void *newmem = toku_malloc(memp->size);
if (newmem == 0)
return -2;
struct mempool new_kvspace;
toku_mempool_init(&new_kvspace, newmem, memp->size);
GPMA_ITERATE(pma, idx, len, data,
({
void *newdata = toku_mempool_malloc(&new_kvspace, len, 4);
assert(newdata);
memcpy(newdata, data, len);
toku_gpma_set_at_index(pma, idx, len, newdata);
toku_verify_gpma(pma);
}));
toku_free(memp->base);
*memp = new_kvspace;
toku_verify_gpma(pma);
return 0;
}
static int brt_leaf_put_cmd (BRT t, BRTNODE node, BRT_CMD cmd,
int *did_split, BRTNODE *nodea, BRTNODE *nodeb, DBT *splitk,
int debug,
......@@ -976,30 +1144,52 @@ static int brt_leaf_put_cmd (BRT t, BRTNODE node, BRT_CMD cmd,
if (cmd->type == BRT_INSERT) {
DBT *k = cmd->u.id.key;
DBT *v = cmd->u.id.val;
int replaced_v_size;
enum pma_errors pma_status = toku_pma_insert_or_replace(node->u.l.buffer,
k, v, &replaced_v_size,
logger, cmd->xid,
filenum, node->thisnodename, node->rand4fingerprint, &node->local_fingerprint, &node->log_lsn);
assert(pma_status==BRT_OK);
//printf("replaced_v_size=%d\n", replaced_v_size);
if (replaced_v_size>=0) {
node->u.l.n_bytes_in_buffer += v->size - replaced_v_size;
} else {
node->u.l.n_bytes_in_buffer += k->size + v->size + KEY_VALUE_OVERHEAD + PMA_ITEM_OVERHEAD;
}
struct kv_pair *kv = brtnode_malloc_kv_pair(node->u.l.buffer, &node->u.l.buffer_mempool, k->data, k->size, v->data, v->size);
assert(kv);
u_int32_t storedlen;
void *storeddata;
u_int32_t idx;
struct lc_pair lc = {t, node->flags & TOKU_DB_DUPSORT}; // for put operations we compare both keys if they are both there
int r = toku_gpma_lookup_item(node->u.l.buffer, kv_pair_size(kv), kv, toku_brtleaf_compare_fun, &lc, &storedlen, &storeddata, &idx);
if (r==0) {
// It's already there. Note that it's gone and remove it from the mempool.
node->u.l.n_bytes_in_buffer -= PMA_ITEM_OVERHEAD + storedlen;
node->local_fingerprint -= node->rand4fingerprint*toku_calccrc32_kvpair_struct(storeddata);
BYTESTRING okbs = { kv_pair_keylen(storeddata), kv_pair_key(storeddata) };
BYTESTRING odbs = { kv_pair_vallen(storeddata), kv_pair_val(storeddata) };
r = toku_log_deleteinleaf(logger, &node->log_lsn, 0, cmd->xid, filenum, node->thisnodename, idx, okbs, odbs);
toku_mempool_mfree(&node->u.l.buffer_mempool, storeddata, storedlen);
// Now put the new kv in.
toku_gpma_set_at_index(node->u.l.buffer, idx, kv_pair_size(kv), kv);
} else {
// Insert it.
struct move_struct ms = {.logger=logger, .filenum=filenum, .from=node, .to=node};
r = toku_gpma_insert(node->u.l.buffer, kv_pair_size(kv), kv, toku_brtleaf_compare_fun, &lc, note_move_items_within, &ms, &idx);
if (r!=0) return r;
}
{
BYTESTRING kbs = { kv_pair_keylen(kv), kv_pair_key(kv) };
BYTESTRING dbs = { kv_pair_vallen(kv), kv_pair_val(kv) };
r = toku_log_insertinleaf(logger, &node->log_lsn, 0, cmd->xid, filenum, node->thisnodename, idx, kbs, dbs);
if (r!=0) return r;
}
node->u.l.n_bytes_in_buffer += PMA_ITEM_OVERHEAD + kv_pair_size(kv);
node->local_fingerprint += node->rand4fingerprint*toku_calccrc32_kvpair_struct(kv);
// printf("%s:%d rand4=%08x local_fingerprint=%08x this=%08x\n", __FILE__, __LINE__, node->rand4fingerprint, node->local_fingerprint, toku_calccrc32_kvpair_struct(kv));
node->dirty = 1;
// toku_pma_verify_fingerprint(node->u.l.buffer, node->rand4fingerprint, node->subtree_fingerprint);
// If it doesn't fit, then split the leaf.
if (toku_serialize_brtnode_size(node) > node->nodesize) {
int r = brtleaf_split (logger, filenum, t, node, nodea, nodeb, splitk);
r = brtleaf_split (logger, filenum, t, node, nodea, nodeb, splitk);
if (r!=0) return r;
//printf("%s:%d splitkey=%s\n", __FILE__, __LINE__, (char*)*splitkey);
split_count++;
*did_split = 1;
toku_verify_counts(*nodea); toku_verify_counts(*nodeb);
if (debug) printf("%s:%d %*snodeb->thisnodename=%lld nodeb->size=%d\n", __FILE__, __LINE__, debug, "", (*nodeb)->thisnodename, (*nodeb)->nodesize);
assert(toku_serialize_brtnode_size(*nodea)<=(*nodea)->nodesize);
assert(toku_serialize_brtnode_size(*nodeb)<=(*nodeb)->nodesize);
......@@ -1010,30 +1200,22 @@ static int brt_leaf_put_cmd (BRT t, BRTNODE node, BRT_CMD cmd,
}
return 0;
} else if (cmd->type == BRT_DELETE) {
u_int32_t delta;
int r = toku_pma_delete(node->u.l.buffer, cmd->u.id.key, (DBT*)0,
logger, cmd->xid, node->thisnodename,
node->rand4fingerprint, &node->local_fingerprint, &delta, &node->log_lsn);
if (r == BRT_OK) {
node->u.l.n_bytes_in_buffer -= delta;
node->dirty = 1;
}
*did_split = 0;
return BRT_OK;
} else if (cmd->type == BRT_DELETE_BOTH) {
u_int32_t delta;
int r = toku_pma_delete(node->u.l.buffer, cmd->u.id.key, cmd->u.id.val,
logger, cmd->xid, node->thisnodename,
node->rand4fingerprint, &node->local_fingerprint, &delta, &node->log_lsn);
if (r == BRT_OK) {
node->u.l.n_bytes_in_buffer -= delta;
node->dirty = 1;
}
} else if (cmd->type == BRT_DELETE || cmd->type == BRT_DELETE_BOTH) {
DBT *k = cmd->u.id.key;
DBT *v = cmd->u.id.val;
struct kv_pair *kv = kv_pair_malloc(k->data, k->size, v->data, v->size);
struct lc_pair lc = {t, (cmd->type == BRT_DELETE_BOTH) };
struct move_struct ms = {.logger=logger, .filenum=filenum, .from=node, .to=node};
struct delete_struct dp = {node};
int r = toku_gpma_delete_item(node->u.l.buffer,
kv_pair_size(kv), kv,
toku_brtleaf_compare_fun, &lc,
brt_leaf_delete_callback, &dp,
note_move_items_within, &ms);
toku_free(kv);
*did_split = 0;
return BRT_OK;
if (r==DB_NOTFOUND) return 0;
return r;
} else {
return EINVAL;
}
......@@ -1172,11 +1354,8 @@ static int brt_nonleaf_insert_cmd (BRT t, BRTNODE node, BRT_CMD cmd,
assert((*nodeb)->u.n.n_children>0);
assert(BNC_DISKOFF(*nodea, (*nodea)->u.n.n_children-1)!=0);
assert(BNC_DISKOFF(*nodeb, (*nodeb)->u.n.n_children-1)!=0);
toku_verify_counts(*nodea);
toku_verify_counts(*nodeb);
} else {
assert(toku_serialize_brtnode_size(node)<=node->nodesize);
toku_verify_counts(node);
}
//if (*did_split) {
// verify_local_fingerprint_nonleaf(*nodea);
......@@ -1243,11 +1422,8 @@ static int brt_nonleaf_delete_cmd (BRT t, BRTNODE node, BRT_CMD cmd,
assert((*nodeb)->u.n.n_children>0);
assert(BNC_DISKOFF(*nodea,(*nodea)->u.n.n_children-1)!=0);
assert(BNC_DISKOFF(*nodeb,(*nodeb)->u.n.n_children-1)!=0);
toku_verify_counts(*nodea);
toku_verify_counts(*nodeb);
} else {
assert(toku_serialize_brtnode_size(node)<=node->nodesize);
toku_verify_counts(node);
}
//if (*did_split) {
// verify_local_fingerprint_nonleaf(*nodea);
......@@ -1357,7 +1533,6 @@ static int setup_initial_brt_root_node (BRT t, DISKOFF offset, TOKULOGGER logger
toku_free(node);
return r;
}
toku_verify_counts(node);
// verify_local_fingerprint_nonleaf(node);
toku_log_newbrtnode(logger, (LSN*)0, 0, toku_cachefile_filenum(t->cf), offset, 0, t->h->nodesize, (t->flags&TOKU_DB_DUPSORT)!=0, node->rand4fingerprint);
toku_update_brtnode_loggerlsn(node, logger);
......@@ -1729,7 +1904,6 @@ static int brt_init_new_root(BRT brt, BRTNODE nodea, BRTNODE nodeb, DBT splitk,
BNC_NBYTESINBUF(newroot, 1)=0;
BNC_SUBTREE_FINGERPRINT(newroot, 0)=0;
BNC_SUBTREE_FINGERPRINT(newroot, 1)=0;
toku_verify_counts(newroot);
//verify_local_fingerprint_nonleaf(nodea);
//verify_local_fingerprint_nonleaf(nodeb);
r=toku_log_newbrtnode(logger, (LSN*)0, 0, toku_cachefile_filenum(brt->cf), newroot_diskoff, new_height, new_nodesize, (brt->flags&TOKU_DB_DUPSORT)!=0, newroot->rand4fingerprint);
......@@ -1898,8 +2072,8 @@ int toku_dump_brtnode (BRT brt, DISKOFF off, int depth, bytevec lorange, ITEMLEN
} else {
printf("%*sNode %lld nodesize=%d height=%d n_bytes_in_buffer=%d keyrange=%d %d\n",
depth, "", off, node->nodesize, node->height, node->u.l.n_bytes_in_buffer, lorange ? ntohl(*(int*)lorange) : 0, hirange ? ntohl(*(int*)hirange) : 0);
PMA_ITERATE(node->u.l.buffer, key, keylen, val __attribute__((__unused__)), vallen,
( keylen=keylen, vallen=vallen, printf(" (%d)%d ", keylen, ntohl(*(int*)key))));
//GPMA_ITERATE(node->u.l.buffer, idx, len, data,
// ( keylen=keylen, vallen=vallen, printf(" (%d)%d ", keylen, ntohl(*(int*)key))));
printf("\n");
}
r = toku_cachetable_unpin(brt->cf, off, 0, 0);
......@@ -2059,17 +2233,67 @@ static int brt_search_nonleaf_node(BRT brt, BRTNODE node, brt_search_t *search,
return r;
}
static int brt_search_leaf_node(BRTNODE node, brt_search_t *search, DBT *newkey, DBT *newval) {
PMA pma = node->u.l.buffer;
int r = toku_pma_search(pma, search, newkey, newval);
return r;
struct bessel_from_search_struct {
brt_search_t *search;
};
static int bessel_from_search_t (u_int32_t len __attribute__((__unused__)), void *data, void *extra) {
struct bessel_from_search_struct *bs = extra;
brt_search_t *search=bs->search;
DBT x,y;
struct kv_pair *kv = data;
int cmp = search->compare(search,
search->k ? toku_fill_dbt(&x, kv_pair_key(kv), kv_pair_keylen(kv)) : 0,
search->v ? toku_fill_dbt(&y, kv_pair_val(kv), kv_pair_vallen(kv)) : 0);
// For a left-to-right search, the search compare function returns 0 for all pairs < kv. We want the first value that is 1.
// To convert it to a bessel, we have to convert the 0 to a -1.
// For a right-to-left search, the search compare function returns 0 for all pairs > kv, and 1 for lesser values. We want the last value that is 1.
// To convert it to a bessel, we have to convert 0 to +1, and 1 to -1.
switch (search->direction) {
case BRT_SEARCH_LEFT: return cmp==0 ? -1 : +1;
case BRT_SEARCH_RIGHT: return cmp==0 ? +1 : -1; // Because the comparison runs backwards for right searches.
}
assert(0);
return 0;
}
static int brt_search_leaf_node(BRT brt, BRTNODE node, brt_search_t *search, DBT *newkey, DBT *newval) {
// Now we have to convert from brt_search_t to the bessel function with a direction. What a pain...
struct bessel_from_search_struct bs = {search};
int direction;
switch (search->direction) {
case BRT_SEARCH_LEFT: direction = +1; goto ok;
case BRT_SEARCH_RIGHT: direction = -1; goto ok;
}
return EINVAL; // This return and the goto are a hack to get both compile-time and run-time checking on enum
ok: ;
u_int32_t len;
void * data;
u_int32_t idx; // Don't need this
int r = toku_gpma_lookup_bessel(node->u.l.buffer,
bessel_from_search_t,
direction,
&bs,
&len, &data, &idx);
if (r!=0) return r;
struct kv_pair *kv = data;
if (newkey) {
r = toku_dbt_set_value(newkey, kv_pair_key(kv), kv_pair_keylen(kv), &brt->skey);
if (r!=0) return r;
}
if (newval) {
r = toku_dbt_set_value(newval, kv_pair_val(kv), kv_pair_vallen(kv), &brt->sval);
if (r!=0) return r;
}
return 0;
}
static int brt_search_node(BRT brt, BRTNODE node, brt_search_t *search, DBT *newkey, DBT *newval, BRT_SPLIT *split, TOKULOGGER logger, DISKOFFARRAY path_to_parent) {
if (node->height > 0)
return brt_search_nonleaf_node(brt, node, search, newkey, newval, split, logger, path_to_parent);
else
return brt_search_leaf_node(node, search, newkey, newval);
return brt_search_leaf_node(brt, node, search, newkey, newval);
}
int toku_brt_search(BRT brt, brt_search_t *search, DBT *newkey, DBT *newval, TOKULOGGER logger) {
......
newbrt/brt.h
View file @ 83d416fa
......@@ -69,4 +69,9 @@ int toku_brt_height_of_root(BRT, int *height); // for an open brt, return the cu
// Special hack for recovery
int toku_brt_nonleaf_expunge_xaction(BRT brt, DISKOFF diskoff, TXNID xid);
enum brt_header_flags {
TOKU_DB_DUP = 1,
TOKU_DB_DUPSORT = 2,
};
#endif
newbrt/brt2.c
View file @ 83d416fa
......@@ -219,7 +219,7 @@ static void initialize_brtnode (BRT t, BRTNODE n, DISKOFF nodename, int height)
n->thisnodename = nodename;
n->disk_lsn.lsn = 0; // a new one can always be 0.
n->log_lsn = n->disk_lsn;
n->layout_version = 2;
n->layout_version = 3;
n->height = height;
n->rand4fingerprint = random();
n->local_fingerprint = 0;
......
newbrt/brtdump.c
View file @ 83d416fa
......@@ -44,9 +44,7 @@ void print_item (bytevec val, ITEMLEN len) {
void dump_node (int f, DISKOFF off, struct brt_header *h) {
BRTNODE n;
int r = toku_deserialize_brtnode_from (f, off, &n, h->flags, h->nodesize,
toku_default_compare_fun, toku_default_compare_fun,
(DB*)0, (FILENUM){0});
int r = toku_deserialize_brtnode_from (f, off, &n, h->flags, h->nodesize);
assert(r==0);
assert(n!=0);
printf("brtnode\n");
......@@ -105,13 +103,13 @@ void dump_node (int f, DISKOFF off, struct brt_header *h) {
}
} else {
printf(" n_bytes_in_buffer=%d\n", n->u.l.n_bytes_in_buffer);
printf(" items_in_buffer =%d\n", toku_pma_n_entries(n->u.l.buffer));
PMA_ITERATE_IDX(n->u.l.buffer, idx, key, keylen, data, datalen,
printf(" items_in_buffer =%d\n", toku_gpma_n_entries(n->u.l.buffer));
GPMA_ITERATE(n->u.l.buffer, idx, len, data,
({
printf("%d: ", idx);
print_item(key, keylen);
print_item(kv_pair_key(data), kv_pair_keylen(data));
printf(" ");
print_item(data, datalen);
print_item(kv_pair_val(data), kv_pair_vallen(data));
printf("\n");
}));
}
......
newbrt/brttypes.h
View file @ 83d416fa
......@@ -23,7 +23,7 @@ typedef long long DISKOFF; /* Offset in a disk. -1 is the NULL pointer. */
typedef u_int64_t TXNID;
typedef struct {
int len;
u_int32_t len;
char *data;
} BYTESTRING;
......
newbrt/cachetable-test2.c
View file @ 83d416fa
......@@ -170,7 +170,7 @@ static void test_chaining (void) {
r = toku_cachetable_close(&ct); assert(r==0);
}
void usage (const char *progname) {
void __attribute__((__noreturn__)) usage (const char *progname) {
fprintf(stderr, "Usage:\n %s [-v] [-q]\n", progname);
exit(1);
}
......
newbrt/crc.h
View file @ 83d416fa
......@@ -8,7 +8,7 @@
// zlib crc32 has a bug: If len==0 then it should return oldcrc32, but crc32 returns 0.
static inline u_int32_t toku_crc32 (u_int32_t oldcrc32, const void *data, u_int32_t len) {
if (len==0) return oldcrc32;
else return crc32((unsigned long)oldcrc32, data, len);
else return crc32((unsigned long)oldcrc32, data, (uInt)len);
}
static const u_int32_t toku_null_crc = 0;
......
newbrt/fingerprint.c
View file @ 83d416fa
......@@ -21,6 +21,11 @@ u_int32_t toku_calccrc32_kvpair (const void *key, int keylen, const void *val, i
return toku_calc_more_crc32_kvpair(toku_null_crc, key, keylen, val, vallen);
}
u_int32_t toku_calccrc32_kvpair_struct (const struct kv_pair *kvp) {
return toku_calccrc32_kvpair(kv_pair_key_const(kvp), kv_pair_keylen(kvp),
kv_pair_val_const(kvp), kv_pair_vallen(kvp));
}
u_int32_t toku_calccrc32_cmd (int type, TXNID xid, const void *key, int keylen, const void *val, int vallen) {
unsigned char type_c = type;
unsigned int a = htonl(xid>>32);
......
newbrt/gpma-internal.h 0 → 100644
View file @ 83d416fa
#include "memory.h"
struct gpma {
enum typ_tag tag;
unsigned int N; /* How long is the array? Always a power of two >= 4. */
u_int32_t n_items_present; /* How many array elements are non-null. */
struct gitem *items; /* A malloced array. If any item's DATA is null, then it's not in use. */
double udt_step; /* upper density threshold step */
/* Each doubling decreases the density by density step.
* For example if array_len=256 and uplgN=8 then there are 5 doublings.
* Regions of size 8 are full. Regions of size 16 are 90% full.
* Regions of size 32 are 80% full. Regions of size 64 are 70% full.
* Regions of size 128 are 60% full. Regions of size 256 are 50% full.
* The density step is 0.10. */
double ldt_step; /* lower density threshold step */
};
#define GPMA_MIN_ARRAY_SIZE 4
/* density thresholds */
#define GPMA_LDT_HIGH 0.25
#define GPMA_LDT_LOW 0.40
#define GPMA_UDT_HIGH 1.00
#define GPMA_UDT_LOW 0.50
/* Expose these for testing purposes */
u_int32_t toku_gpma_find_index_bes (GPMA pma, gpma_besselfun_t besf, int direction, void *extra, int *found);
u_int32_t toku_gpma_find_index (GPMA pma, u_int32_t len, void *data, gpma_compare_fun_t compare, void *extra, int *found);
int toku_lg (unsigned int n);
u_int32_t toku_hyperceil (u_int32_t v);
int toku_max_int (int, int);
int toku_gpma_smooth_region (GPMA pma,
u_int32_t lo, u_int32_t hi,
u_int32_t count, // The number of nonnull values
u_int32_t idx, u_int32_t *newidxp, gpma_renumber_callback_t rcall, void *extra,
u_int32_t old_N);
int toku_make_space_at (GPMA pma, u_int32_t idx, u_int32_t *newidx, gpma_renumber_callback_t rcall, void *extra);
void toku_gpma_distribute (GPMA pma,
u_int32_t lo, u_int32_t hi,
u_int32_t count,
struct gitem *items, // some of these may be NULL data, be we leave space for them anyway.
/*out*/ u_int32_t *tos // the indices where the values end up (we fill this in)
);
int toku_smooth_deleted_region (GPMA pma, u_int32_t minidx, u_int32_t maxidx, gpma_renumber_callback_t renumberf, void *extra_for_renumberf);
newbrt/gpma.c 0 → 100644
View file @ 83d416fa
/* General PMA. */
#ident "Copyright (c) 2007 Tokutek Inc. All rights reserved."
#include <errno.h>
#include "gpma.h"
#include "yerror.h"
#include "toku_assert.h"
#include "memory.h"
// Need this for DB_KEYEXIST
#include "../include/db.h"
#include "gpma-internal.h"
// Find the ceiling of lg n. */
int toku_lg (unsigned int n) {
int result=0;
unsigned int two_to_result = 1;
while (two_to_result<n) {
result++;
two_to_result*=2;
}
return result;
}
/* find the smallest power of 2 >= n */
inline u_int32_t toku_hyperceil (u_int32_t v) {
u_int32_t n = 1;
while (n < v)
n *= 2;
return n;
}
/* Calculate densitysteps and uplgN, given N. */
static void calculate_parameters (GPMA pma) {
unsigned int N = toku_gpma_index_limit(pma);
int lgN = toku_lg(N);
int n_divisions = lgN;
//printf("uplgN = %d n_divisions=%d\n", pma->uplgN, n_divisions);
assert(n_divisions>0);
pma->udt_step = (GPMA_UDT_HIGH - GPMA_UDT_LOW)/n_divisions;
pma->ldt_step = (GPMA_LDT_HIGH - GPMA_LDT_LOW)/n_divisions;
}
int toku_gpma_create(GPMA*gpma, int initial_index_limit) {
if (initial_index_limit && (initial_index_limit&(initial_index_limit-1))) return EINVAL; // must be a power of two.
TAGMALLOC(GPMA, result);
if (result==0) return errno;
result->N = initial_index_limit ? initial_index_limit : GPMA_MIN_ARRAY_SIZE;
result->n_items_present=0;
calculate_parameters(result);
MALLOC_N(result->N, result->items);
if (result->items==0) { int r=errno; toku_free(result); return r; }
{
u_int32_t i;
for (i=0; i<result->N; i++) result->items[i].data=0;
}
*gpma=result;
return 0;
}
void toku_gpma_free(GPMA*gpmap, gpma_free_callback_t freeme,void*extra) {
u_int32_t i;
GPMA pma=*gpmap;
for (i=0; i<pma->N; i++) {
if (pma->items[i].data) {
if (freeme)
freeme(pma->items[i].len, pma->items[i].data, extra);
pma->items[i].data=0;
}
}
toku_free(pma->items);
toku_free(pma);
*gpmap=0;
}
u_int32_t toku_gpma_n_entries(GPMA pma) {
return pma->n_items_present;
}
u_int32_t toku_gpma_index_limit(GPMA pma) {
return pma->N;
}
// If direction==0 then find any match for which the bessel gives 0. *found is set to 1 iff something with 0. The return value is the place where the zero is (if found), or the place where it would go (if there's a value there, then that value goes after the zero.)
// If direction>0 then find the first match for which bessel gives >0. *found is set to 1 iff something with >0. The return value is the index of the leftmost such value (if found). In the not-found case, all items are <=0 and the return value is pma->N.
// If direction<0 then find the last match for which bessel gives <0. *found is set to 1 iff something with <0. The return value is the index of the rightmost such value (if found). In the not-found case, all items are >=0 and the return value is 0.
u_int32_t toku_gpma_find_index_bes (GPMA pma, gpma_besselfun_t besf, int direction, void *extra, int *found) {
if (direction==0) {
int lo=0, hi=pma->N;
while (lo<hi) {
int mi = (lo+hi)/2;
int look = mi;
while (look<hi && pma->items[look].data==0) look++;
if (look>=hi) {
// went too far, so mi is new hi
hi=mi;
} else {
int cmp = besf(pma->items[look].len, pma->items[look].data, extra);
if (cmp==0) {
/* We found a match. */
*found=1;
return look;
} else if (cmp>0) {
hi=mi;
} else {
lo=look+1;
}
}
}
*found = 0;
return lo;
} else if (direction<0) {
// Find the rightmost negative value.
#if 0
// Linear-time code, for ease of reading
u_int32_t i;
for (i=pma->N; i>0; i--) {
if (pma->items[i-1].data) {
int cmp = besf(pma->items[i-1].len, pma->items[i-1].data, extra);
if (cmp<0) {
*found=1;
return i-1;
}
}
}
*found=0;
return 0;
#else
// direction<0. Log-time code. For performance.
int lo=0, hi=pma->N;
int foundone=0;
int answer=lo;
while (lo<hi) {
int mi = (lo+hi)/2;
int look = mi;
while (look<hi && pma->items[look].data==0) look++;
if (look>=hi) {
// there was nothing in the right half
hi=mi;
} else {
int cmp = besf(pma->items[look].len, pma->items[look].data, extra);
if (cmp>=0) {
// look is too big.
hi=mi;
} else {
// look is is a good answer, so set lo to that. From now on we can only change lo if we find another good answer.
answer=look;
foundone=1;
lo=look+1;
}
}
}
*found = foundone;
return answer;
#endif
} else {
// Find the leftmost postive value.
#if 0
// Linear-time code, for ease of reading
u_int32_t i;
for (i=0; i<pma->N; i++) {
if (pma->items[i].data) {
int cmp = besf(pma->items[i].len, pma->items[i].data, extra);
if (cmp>0) {
*found=1;
return i;
}
}
}
*found=0;
return pma->N;
#else
// direction>0. Log-time code. For performance.
// The loop invariant is that if we found one, then hi is a good answer.
int lo=0, hi=pma->N;
int foundone=0;
while (lo<hi) {
int mi = (lo+hi)/2;
int look = mi;
while (look>lo && pma->items[look].data==0) look--;
if (look==lo && pma->items[look].data==0) {
// There was nothing in the left half.
lo = mi+1;
} else {
int cmp = besf(pma->items[look].len, pma->items[look].data, extra);
if (cmp<=0) {
// look is too small. That means mi is too small.
lo = mi+1;
} else {
// look is a good answer, so set hi to that. From now on we only change hi if we find another good answer.
hi = look;
foundone=1;
}
}
}
*found = foundone;
return hi;
#endif
}
}
// Convert a comparison function against a particular item to a besselfun.
struct convert_extra {
gpma_compare_fun_t comparef;
u_int32_t dlen;
void *dval;
void *extra;
};
static int bessel_from_compare (u_int32_t dlen, void *dval, void *extra) {
struct convert_extra *ce=extra;
return -ce->comparef(ce->dlen, ce->dval, dlen, dval, ce->extra);
}
// Find the place where (len,data) is stored. Return *found==0 iff the item is not actually there.
// Could return anything from 0 to N inclusive.
u_int32_t toku_gpma_find_index (GPMA pma, u_int32_t dlen, void *dval, gpma_compare_fun_t comparef, void *extra, int *found) {
struct convert_extra ce = {comparef, dlen, dval, extra};
return toku_gpma_find_index_bes(pma, bessel_from_compare, 0, &ce, found);
}
// the region from lo (inclusive) to hi (exclusive) is all empty.
// Distribute the data across it.
void toku_gpma_distribute (GPMA pma,
u_int32_t lo, u_int32_t hi,
u_int32_t count,
struct gitem *items, // some of these may be NULL data, be we leave space for them anyway.
/*out*/ u_int32_t *tos) // the indices where the values end up (we fill this in)
{
int width = hi-lo;
u_int32_t nplaced=0;
u_int32_t nused =0;
u_int32_t i;
assert(hi<=pma->N);
for (i=lo; i<hi; i++) {
// if nused/i <= (nitems)/width then place something here
// But don't do floating point divisions
if (nused*(u_int64_t)width <= count*(u_int64_t)i) {
tos[nplaced] = i;
pma->items[i] = items[nplaced++];
nused++;
}
}
assert(nplaced==count);
}
int toku_gpma_smooth_region (GPMA pma,
u_int32_t lo, u_int32_t hi,
u_int32_t count, // The number of nonnull values
u_int32_t idx, u_int32_t *newidxp, // set newidxp to 0 if you don't want to track a particular index
gpma_renumber_callback_t rcall, void *extra,
u_int32_t old_N) {
if (count==0) return 0;
int width = hi-lo;
u_int32_t *MALLOC_N(count, froms); if (!froms) return ENOMEM;
u_int32_t *MALLOC_N(count, tos); if (!tos) { toku_free(froms); return ENOMEM; }
u_int32_t nitems=0;
struct gitem *MALLOC_N(width, temp); if (!temp) { toku_free(tos); toku_free(froms); return ENOMEM; }
u_int32_t i;
u_int32_t idx_goes_to_tmp=pma->N+1; // too big, so we will notice a problem
u_int32_t newidx=pma->N+1;
for (i=lo; i<hi; i++) {
if (newidxp && idx==i) idx_goes_to_tmp=nitems;
if (pma->items[i].data) {
//printf("froms[%d]=%d (count=%d)\n", nitems, i, count);
froms[nitems]=i;
temp [nitems]=pma->items[i];
pma->items[i].data=0;
nitems++;
}
}
if (newidxp && idx==i) idx_goes_to_tmp = nitems;
// Now they are all compacted into temp. Spread them out again
u_int32_t nplaced=0;
u_int32_t nused =0;
u_int64_t nitems_to_place = newidxp ? (nitems+1) : nitems;
for (i=lo; i<hi; i++) {
// if nused/i < (nitems+1)/width then place something here
// But don't do floating point divisions
if (nused*(u_int64_t)width < nitems_to_place*(u_int64_t)(i-lo)) {
if (newidxp && nused==idx_goes_to_tmp) {
newidx=i;
} else {
tos[nplaced] = i;
pma->items[i] = temp[nplaced++];
}
nused++;
}
}
assert((newidxp ? nplaced+1 : nplaced) ==nused);
assert(nplaced==nitems);
int r = 0;
if (rcall) {
r = rcall(nitems, froms, tos, temp, old_N, pma->N, extra);
}
toku_free(temp);
toku_free(froms);
toku_free(tos);
if (newidxp) {
assert(newidx<pma->N);
*newidxp = newidx;
}
return r;
}
static int double_array (GPMA pma, u_int32_t idx, u_int32_t *newidx, gpma_renumber_callback_t rcall, void *extra) {
{
void *olditems = pma->items;
REALLOC_N(pma->N*2, pma->items);
if (pma->items==0) { pma->items=olditems; return errno; }
}
u_int32_t i;
for (i=pma->N; i<pma->N*2; i++) pma->items[i].data=0;
u_int32_t old_N = pma->N;
pma->N *= 2;
calculate_parameters(pma);
int r = toku_gpma_smooth_region(pma, 0, pma->N, pma->n_items_present, idx, newidx, rcall, extra, old_N);
if (r==ENOMEM) {
pma->N /= 2;
// Don't reallocate the memory downward. We'll just hope that the current memory array is OK.
}
return r;
}
int toku_make_space_at (GPMA pma, u_int32_t idx, u_int32_t *newidx, gpma_renumber_callback_t rcall, void *extra) {
if (idx!=pma->N) assert(pma->items[idx].data);
u_int32_t lo=idx;
u_int32_t hi=idx+1;
if (idx==pma->N) { lo--; hi--; }
double udt=GPMA_UDT_HIGH;
u_int32_t count = 2; // one for the item that is there, plus one for the new item.
u_int32_t width=1;
double one_over_width = 1.0;
while (1) {
assert(lo<hi); assert(hi<=pma->N); // Make those separate asserts so that we don't get false complaints from gcov.
double density = count*one_over_width;
//printf("%s:%d %d..%d density=%f udt=%f\n", __FILE__, __LINE__, lo, hi, density, udt);
if (density<=udt) break; // found a region that is good enough
// Otherwise the density isn't good.
u_int32_t N = pma->N;
assert(width<=N);
if (width<N) {
if (idx==N || width&idx) { // Grow the array downward.
u_int32_t i;
assert(lo>=width);
lo -= width;
for (i=0; i<width; i++) {
if (pma->items[lo+i].data) count++;
}
} else { // Grow the array upward.
u_int32_t i;
for (i=0; i<width; i++) {
if (pma->items[hi+i].data) count++;
}
hi += width;
}
width*=2;
one_over_width*=0.5;
udt -= pma->udt_step;
} else {
// The array must be resized. */
assert(0==lo); assert(hi==pma->N);
return double_array(pma, idx, newidx, rcall, extra);
}
}
return toku_gpma_smooth_region (pma, lo, hi, count, idx, newidx, rcall, extra, pma->N);
}
int toku_gpma_insert(GPMA pma,
u_int32_t len, void*data,
gpma_compare_fun_t compare, void *extra_for_compare,
gpma_renumber_callback_t rcall, void*extra_for_rcall, // if anything gets renumbered, let the caller know
u_int32_t *idxp
) {
int found;
u_int32_t idx = toku_gpma_find_index(pma, len, data, compare, extra_for_compare, &found);
if (found) return DB_KEYEXIST;
assert(idx<=toku_gpma_index_limit(pma));
if (idx==toku_gpma_index_limit(pma) || pma->items[idx].data) {
u_int32_t newidx;
int r = toku_make_space_at(pma, idx, &newidx, rcall, extra_for_rcall);
if (r!=0) return r;
idx=newidx;
assert(pma->items[idx].data==0);
}
pma->items[idx].data=data;
pma->items[idx].len =len;
pma->n_items_present++;
if (idxp) *idxp=idx;
return 0;
}
inline int toku_max_int (int a, int b) {
return a<b ? b : a;
}
inline unsigned int toku_max_uint (unsigned int a, unsigned int b) {
return a<b ? b : a;
}
static int shrink_pma (GPMA pma, gpma_renumber_callback_t renumberf, void *extra_for_renumberf) {
u_int32_t old_N = pma->N;
if (pma->n_items_present==0) {
pma->N=8;
void *olditems = pma->items;
REALLOC_N(pma->N, pma->items);
if (pma->items==0) { pma->items = olditems; return errno; }
return 0;
}
int r;
u_int32_t *MALLOC_N(pma->n_items_present, froms); if (froms==0) { r=errno; if (0) { L0: toku_free(froms); } return r; }
u_int32_t *MALLOC_N(pma->n_items_present, tos); if (tos==0) { r=errno; if (0) { L1: toku_free(tos); } goto L0; }
struct gitem *MALLOC_N(pma->n_items_present, items); if (items==0) { r=errno; if (0) { L2: toku_free(items); } goto L1; }
u_int32_t nplaced=0;
u_int32_t i;
for (i=0; i<pma->N; i++) {
if (pma->items[i].data) {
froms[nplaced] = i;
items[nplaced++] = pma->items[i];
pma->items[i].data = 0;
}
}
{
void *olditems = pma->items;
REALLOC_N(pma->N/2, pma->items);
if (pma->items==0) { r=errno; pma->items=olditems; goto L2; }
}
u_int32_t new_N = pma->N/2;
pma->N = new_N;
//printf("Shrunk to %d\n", pma->N);
toku_gpma_distribute(pma, 0, pma->N, pma->n_items_present, items, tos);
r = renumberf(pma->n_items_present, froms, tos, items, old_N, new_N, extra_for_renumberf);
goto L2;
}
// if minidx (inclusive) to maxidx (inclusive) gives a range of empty slots, find a big enough region and renumber everything.
int toku_smooth_deleted_region (GPMA pma, u_int32_t minidx, u_int32_t maxidx, gpma_renumber_callback_t renumberf, void *extra_for_renumberf) {
if (pma->N<=8) return 0;
u_int32_t lgN = toku_lg(pma->N);
u_int32_t lglgN = toku_lg(lgN);
u_int32_t n_steps = toku_max_uint(1, lgN-lglgN);
double increment = (GPMA_LDT_HIGH-GPMA_LDT_LOW)/n_steps;
u_int32_t initial_width = maxidx+1-minidx;
u_int32_t lg_initw = toku_lg(initial_width);
u_int32_t next_width = 1<<lg_initw;
double target = GPMA_LDT_LOW+increment*lg_initw;
u_int32_t count = 0;
u_int32_t lo=minidx;
u_int32_t hi=maxidx+1;
while (1) {
assert(next_width<=pma->N);
while (hi-lo < next_width) {
if (hi<pma->N) {
if (pma->items[hi].data) count++;
hi++;
} else {
assert(lo>0);
lo--;
if (pma->items[lo].data) count++;
}
}
// if count/(hi-lo) >= target then we are happy
if (count >= target*(hi-lo)) {
// we are happy with this width, spread things out.
return toku_gpma_smooth_region(pma, lo, hi, count, lo, 0, renumberf, extra_for_renumberf, pma->N);
}
if (next_width==pma->N) {
return shrink_pma(pma, renumberf, extra_for_renumberf);
}
next_width*=2;
}
}
int toku_gpma_delete_bessel (GPMA pma,
gpma_besselfun_t besself, void*extra_for_besself,
gpma_delete_callback_t deletef, void*extra_for_deletef, // for each deleted item, let the caller know
gpma_renumber_callback_t renumberf, void*extra_for_renumberf // if anything gets renumbered, let the caller know
) {
int r;
u_int32_t len;
void *data;
u_int32_t idx;
r = toku_gpma_lookup_bessel(pma, besself, 0, extra_for_besself, &len, &data, &idx);
// Find how many items there are to delete. Scan back and forward.
if (r!=0) return DB_NOTFOUND;
u_int32_t i;
int nitems=1;
u_int32_t maxidx=idx, minidx=idx;
for (i=idx+1; i<pma->N; i++) {
if (pma->items[i].data) {
if (besself(pma->items[i].len, pma->items[i].data, extra_for_besself)!=0)
break;
nitems++;
maxidx=i;
}
}
for (i=idx; i>0 ; i--) {
if (pma->items[i-1].data) {
if (besself(pma->items[i-1].len, pma->items[i-1].data, extra_for_besself)!=0)
break;
nitems++;
minidx=i-1;
}
}
pma->n_items_present -= nitems;
// Now we know the range and how many items will be deleted.
for (i=minidx; i<=maxidx; i++) {
if (pma->items[i].data) {
r = deletef(i, pma->items[i].len, pma->items[i].data, extra_for_deletef);
pma->items[i].data = 0;
if (r!=0) return r;
}
}
// Now we must find a region that is sufficiently densely packed and spread things out.
return toku_smooth_deleted_region(pma, minidx, maxidx, renumberf, extra_for_renumberf);
}
int toku_gpma_delete_item (GPMA pma,
u_int32_t len, void *data,
gpma_compare_fun_t comparef, void *extra_for_comparef,
gpma_delete_callback_t deletef, void *extra_for_deletef,
gpma_renumber_callback_t renumberf, void *extra_for_renumberf) {
struct convert_extra ce = { comparef, len, data, extra_for_comparef };
return toku_gpma_delete_bessel (pma, bessel_from_compare, &ce,
deletef, extra_for_deletef,
renumberf, extra_for_renumberf);
}
#if 0
// Delete anything for which the besselfun is zero.
// If things go wrong (e.g., the renumber_callback returns nonzero, or memory runs out
int toku_gpma_delete(GPMA pma,
gpma_besselfun_t besf,
gpma_delete_callback_t delcall, // call this on each deleted object
gpma_renumber_callback_t rcall, // if anything gets renumbered, let the caller know
void*extra) {
}
#endif
int toku_gpma_lookup_item (GPMA pma,
u_int32_t len, void *data, gpma_compare_fun_t comparef, void *extra, u_int32_t *resultlen, void **resultdata, u_int32_t *idxp) {
int found;
u_int32_t idx = toku_gpma_find_index(pma, len, data, comparef, extra, &found);
if (!found) return DB_NOTFOUND;
*resultlen = pma->items[idx].len;
*resultdata = pma->items[idx].data;
if (idxp) *idxp=idx;
return 0;
}
int toku_gpma_lookup_bessel(GPMA pma, gpma_besselfun_t besf, int direction, void*extra, u_int32_t *resultlen, void **resultdata, u_int32_t *idxp) {
int found;
u_int32_t idx = toku_gpma_find_index_bes(pma, besf, direction, extra, &found);
if (found) {
*resultlen =pma->items[idx].len;
*resultdata=pma->items[idx].data;
if (idxp) *idxp=idx;
return 0;
} else {
return DB_NOTFOUND;
}
}
// Split the pma, putting some right suffix into newpma. Try to split up so sum(lengths)+ overhead*N is equal.
// Move at least one element (if there is one)
// newpma is an empty pma
// If an error code is returned, then the pmas are likely to be all messed up. Probably all you can do is close them.
int toku_gpma_split (GPMA pma, GPMA newpma, u_int32_t overhead,
int (*realloc_data)(u_int32_t olen, void *odata, void **ndata, void *extra),
gpma_renumber_callback_t rcall,
gpma_renumber_callback_t rcall_across_pmas, // This one is called for everything that moved
void *extra) {
unsigned long totalweight=0;
u_int32_t old_N = pma->N;
{
u_int32_t i;
for (i=0; i<pma->N; i++) if (pma->items[i].data) totalweight += overhead +pma->items[i].len;
}
//toku_verify_gpma(pma);
if (totalweight==0) return 0; // Nothing there
unsigned long weight=0;
u_int32_t prev=0;
u_int32_t n_to_move=0;
u_int32_t i;
for (i=0; 1; i++) {
assert(i<pma->N);
if (pma->items[i].data) {
u_int32_t delta = 1 + pma->items[i].len;
if (weight+delta > totalweight/2) break; // prev is the last one to split.
weight += delta;
n_to_move++;
prev = i;
}
}
u_int32_t split_here = prev;
u_int32_t n_left = n_to_move;
u_int32_t n_right = pma->n_items_present - n_left;
#define MALLOC_N_ECK(n,v,l,lp) MALLOC_N(n,v); if (!v) { r=errno; if (0) { l: toku_free(v); } goto lp; }
int r;
if (0) { L0: return r; }
struct gitem *MALLOC_N_ECK(n_left, leftitems, L1,L0);
struct gitem *MALLOC_N_ECK(n_right, rightitems, L2,L1);
u_int32_t *MALLOC_N_ECK(n_left, leftfroms, L3,L2);
u_int32_t *MALLOC_N_ECK(n_right, rightfroms, L4,L3);
u_int32_t *MALLOC_N_ECK(n_left, lefttos, L5,L4);
u_int32_t *MALLOC_N_ECK(n_right, righttos, L6,L5);
{
u_int32_t n_moved=0;
for (i=0; i<=split_here; i++) {
if (pma->items[i].data) {
leftfroms[n_moved] = i;
leftitems[n_moved++] = pma->items[i];
pma->items[i].data = 0;
}
}
assert(n_moved==n_left);
}
{
u_int32_t n_moved=0;
for (i=split_here+1; i<pma->N; i++) {
if (pma->items[i].data) {
rightfroms[n_moved] = i;
rightitems[n_moved++] = pma->items[i];
pma->items[i].data = 0;
}
}
assert(n_moved==n_right);
}
for (i=0; i<n_right; i++) {
void *ndata;
//printf("%s:%d len=%d\n", __FILE__, __LINE__, rightitems[i].len);
r = realloc_data (rightitems[i].len, rightitems[i].data, &ndata, extra);
if (r!=0) { goto L6; } // At this point the PMA is all messed up, and there is no easy way to put it all back together again.
rightitems[i].data=ndata;
}
// Now we have split out the left and right stuff. All we have to do is put it back.
pma->N = toku_hyperceil(2*n_left);
newpma->N = toku_hyperceil(2*n_left);
REALLOC_N(pma->N, pma->items); if (!pma->items) return errno;
REALLOC_N(newpma->N, newpma->items); if (!pma->items) return errno;
for (i=0; i<pma->N; i++) pma->items[i].data=0;
for (i=0; i<newpma->N; i++) newpma->items[i].data=0;
toku_gpma_distribute(pma, 0, pma->N, n_left, leftitems, lefttos);
toku_gpma_distribute(newpma, 0, newpma->N, n_right, rightitems, righttos);
pma->n_items_present = n_left;
newpma->n_items_present = n_right;
//toku_verify_gpma(pma);
//toku_verify_gpma(newpma);
r = rcall(n_left, leftfroms, lefttos, leftitems, old_N, pma->N, extra);
if (r!=0) { goto L6; }
r = rcall_across_pmas(n_right, rightfroms, righttos, rightitems, 0, newpma->N, extra);
if (r!=0) { goto L6; }
r=0;
goto L6; // free all that stuff
}
int toku_gpma_valididx (GPMA pma, u_int32_t idx) {
return (idx<pma->N) && pma->items[idx].data;
}
int toku_gpma_get_from_index(GPMA pma, u_int32_t idx, u_int32_t *len, void **data) {
if (idx>=pma->N) return EINVAL;
void *d=pma->items[idx].data;
if (d==0) return DB_NOTFOUND;
*data=d;
*len =pma->items[idx].len;
return 0;
}
void toku_gpma_set_at_index (GPMA pma, u_int32_t idx, u_int32_t len, void *data) {
assert(idx<pma->N);
if (pma->items[idx].data==0)
pma->n_items_present++;
pma->items[idx].data=data;
pma->items[idx].len =len;
}
void toku_gpma_clear_at_index (GPMA pma, u_int32_t idx) {
assert(idx<pma->N);
if (pma->items[idx].data==0) {
pma->n_items_present--;
}
pma->items[idx].data = 0;
}
void toku_verify_gpma (GPMA pma) {
// The only thing we can really verify is that the n_items_present is OK.
u_int32_t i;
u_int32_t count=0;
for (i=0; i<pma->N; i++) {
if (pma->items[i].data) count++;
}
assert(count==pma->n_items_present);
#if 0
// We can also check that the lengths match up, but that's really brt-specific.
for (i=0; i<pma->N; i++) {
if (pma->items[i].data) {
struct foo {unsigned int a,b;} *foop = pma->items[i].data;
assert(sizeof(*foop)+foop->a+foop->b==pma->items[i].len);
}
}
#endif
}
int toku_resize_gpma_exactly (GPMA pma, u_int32_t newsize) {
void *old = pma->items;
REALLOC_N(newsize, pma->items);
if (pma->items==0) {
pma->items = old;
return errno;
}
u_int32_t i;
for (i=pma->N; i<newsize; i++) pma->items[i].data=0;
pma->N = newsize;
return 0;
}
newbrt/gpma.h 0 → 100644
View file @ 83d416fa
#ifndef GPMA_H
#define GPMA_H
#ident "Copyright (c) 2007 Tokutek Inc. All rights reserved."
// Need this to get the u_int32_t types and so forth
#include <sys/types.h>
typedef struct gpma *GPMA;
struct gitem {
u_int32_t len;
void *data;
};
typedef int (*gpma_compare_fun_t)(u_int32_t alen, void *aval, u_int32_t blen, void *bval, void*extra);
typedef int (*gpma_besselfun_t)(u_int32_t dlen, void *dval, void *extra); // return a number, not an error code.
typedef int (*gpma_delete_callback_t)(u_int32_t slotnum, u_int32_t deletelen, void*deletedata, void*extra); // return 0 if OK.
// If the pma moves things around and/or changes the size of the pma, it calls this function to indicate what happened.
typedef int (*gpma_renumber_callback_t)(u_int32_t nitems, // How many things moved
u_int32_t *froms, // An array of indices indicating where things moved from
u_int32_t *tos, // An array of indices indicating where thigns moved to
struct gitem *items, // The actual items that were moved
u_int32_t old_N, // The old size of the array
u_int32_t new_N, // The new size of teh array
void *extra); // Context
typedef void (*gpma_free_callback_t)(u_int32_t len, void*freeme, void*extra);
// initial_index_limit must be zero or a power of two.
int toku_gpma_create (GPMA*, int initial_index_limit);
/* Return 0 if OK, and sets the referenced GPMA to NULL. */
void toku_gpma_free (GPMA*, gpma_free_callback_t, void*);
// How many items are present
u_int32_t toku_gpma_n_entries (GPMA);
// What is the maximum index limit
u_int32_t toku_gpma_index_limit (GPMA);
// Require that the item not be already present, according ot the compare function
// The data in the DBT is passed in.
int toku_gpma_insert (GPMA,
u_int32_t len, void*data,
gpma_compare_fun_t comparef, void*extra_for_comparef,
gpma_renumber_callback_t renumberf, void*extra_for_renumberf, // if anything gets renumbered, let the caller know
u_int32_t *indexp // Where did the item get stored?
);
// Delete anything for which the besselfun is zero. The besselfun must be monotonically increasing compared to the comparison function.
// That is, if two othings compare to be < then their besselfun's must yield <=, and if the compare to be = their besselfuns must be =, and if they are > then their besselfuns must be >=
// Note the delete_callback would be responsible for calling free on the object.
int toku_gpma_delete_bessel (GPMA,
gpma_besselfun_t,
void*extra_for_besself,
gpma_delete_callback_t,
void*extra_for_deletef,
gpma_renumber_callback_t, // if anything gets renumbered, let the caller know
void*extra_for_renumberf);
// Delete any items for which the compare function says things are zero.
// For each item deleted, invoke deletef.
// For any items moved around, invoke renumberf.
int toku_gpma_delete_item (GPMA,
u_int32_t len, void *data,
gpma_compare_fun_t comparef, void *extra_for_comparef,
gpma_delete_callback_t deletef, void *extra_for_deletef,
gpma_renumber_callback_t renumberf, void *extra_for_renumberf);
// Look up a particular item, using the compare function. Find some X such that compf(len,data, X.len, X.data)==0
// (Note that the len and data passed here are always passed as the first pair of arguments to compf. )
// The item being looked up is the second pair of arguments.
int toku_gpma_lookup_item (GPMA, u_int32_t len, void *data, gpma_compare_fun_t compf, void*extra, u_int32_t *resultlen, void **resultdata, u_int32_t *idx);
// Lookup something according to the besselfun.
// If direction==0 then return something for which the besselfun is zero (or return DB_NOTFOUND).
// If direction>0 then return the first thing for which the besselfun is positive (or return DB_NOTFOUND).
// If direction<0 then return the last thing for which the besselfun is negative (or return DB_NOTFOUND).
int toku_gpma_lookup_bessel (GPMA, gpma_besselfun_t, int direction, void*extra, u_int32_t *len, void **data, u_int32_t *idx);
void toku_gpma_iterate (GPMA, void(*)(u_int32_t len, void*data, void*extra), void*extra);
#define GPMA_ITERATE(table,idx,vallen,val,body) ({ \
u_int32_t idx; \
for (idx=0; idx<toku_gpma_index_limit(table); idx++) { \
u_int32_t vallen; void*val; \
if (0==toku_gpma_get_from_index(table, idx, &vallen, &val)) { \
body; \
} } })
int toku_gpma_valididx (GPMA, u_int32_t idx);
int toku_gpma_get_from_index (GPMA, u_int32_t idx, u_int32_t *len, void **data);
// Whatever is in the slot gets overwritten. Watch out that you free the thing before overwriting it.
void toku_gpma_set_at_index (GPMA, u_int32_t idx, u_int32_t len, void*data);
void toku_gpma_clear_at_index (GPMA, u_int32_t idx);
int toku_gpma_move_inside_pma_by_renumbering (GPMA,
u_int32_t nitems,
u_int32_t *froms, u_int32_t *tos);
int toku_gpma_split (GPMA pma, GPMA newpma, u_int32_t overhead,
int (*realloc_data)(u_int32_t len, void *odata, void **ndata, void *extra),
gpma_renumber_callback_t rcall,
gpma_renumber_callback_t rcall_across_pmas, // This one is called for everything that moved
void *extra);
void toku_verify_gpma (GPMA pma);
// Change the size of the PMA. Anything beyond the oldsize is discarded (if the newsize is smaller) or zerod (if the newsize is larger)
int toku_resize_gpma_exactly (GPMA pma, u_int32_t newsize);
#endif
newbrt/kv-pair.h
View file @ 83d416fa
......@@ -57,7 +57,7 @@ static inline const void *kv_pair_key_const(const struct kv_pair *pair) {
return pair->key;
}
static inline unsigned int kv_pair_keylen(struct kv_pair *pair) {
static inline unsigned int kv_pair_keylen(const struct kv_pair *pair) {
return pair->keylen;
}
......@@ -69,7 +69,7 @@ static inline const void *kv_pair_val_const(const struct kv_pair *pair) {
return pair->key + pair->keylen;
}
static inline unsigned int kv_pair_vallen(struct kv_pair *pair) {
static inline unsigned int kv_pair_vallen(const struct kv_pair *pair) {
return pair->vallen;
}
......
newbrt/log-internal.h
View file @ 83d416fa
......@@ -7,6 +7,7 @@
#include <stdio.h>
#include <pthread.h>
#include <sys/types.h>
#include <string.h>
// Locking for the logger
// For most purposes we use the big ydb lock.
......
newbrt/log.c
View file @ 83d416fa
......@@ -494,7 +494,7 @@ int toku_fread_BYTESTRING (FILE *f, BYTESTRING *bs, u_int32_t *crc, u_int32_t *l
int r=toku_fread_u_int32_t(f, (u_int32_t*)&bs->len, crc, len);
if (r!=0) return r;
bs->data = toku_malloc(bs->len);
int i;
u_int32_t i;
for (i=0; i<bs->len; i++) {
r=toku_fread_u_int8_t(f, (u_int8_t*)&bs->data[i], crc, len);
if (r!=0) {
......@@ -574,7 +574,7 @@ int toku_logprint_BYTESTRING (FILE *outf, FILE *inf, const char *fieldname, u_in
int r = toku_fread_BYTESTRING(inf, &bs, crc, len);
if (r!=0) return r;
fprintf(outf, " %s={len=%d data=\"", fieldname, bs.len);
int i;
u_int32_t i;
for (i=0; i<bs.len; i++) {
switch (bs.data[i]) {
case '"': fprintf(outf, "\\\""); break;
......
newbrt/log.h
View file @ 83d416fa
......@@ -7,7 +7,7 @@
#include "../include/db.h"
#include "brttypes.h"
#include "kv-pair.h"
#include "memory.h"
struct logbytes;
struct logbytes {
......@@ -33,8 +33,6 @@ LSN toku_logger_last_lsn(TOKULOGGER);
int toku_logger_set_lg_max (TOKULOGGER logger, u_int32_t);
int toku_logger_get_lg_max (TOKULOGGER logger, u_int32_t *);
int toku_logger_log_phys_add_or_delete_in_leaf (DB *db, TOKUTXN txn, DISKOFF diskoff, int is_add, const struct kv_pair *pair);
int toku_logger_commit (TOKUTXN txn, int no_sync);
int toku_logger_txn_begin (TOKUTXN /*parent*/,TOKUTXN *, TOKULOGGER /*logger*/);
......
newbrt/logformat.c
View file @ 83d416fa
......@@ -41,9 +41,9 @@ int logformat_version_number = 0;
const struct logtype rollbacks[] = {
{"fcreate", 'F', FA{{"BYTESTRING", "fname", 0},
NULLFIELD}},
{"fclose", 'c', FA{{"FILENUM", "filenum", 0},
{"BYTESTRING", "fname", 0},
NULLFIELD}},
// {"fclose", 'c', FA{{"FILENUM", "filenum", 0},
// {"BYTESTRING", "fname", 0},
// NULLFIELD}},
{"deleteatleaf", 'd', FA{{"FILENUM", "filenum", 0}, // Note a delete for rollback. The delete takes place in a leaf.
{"BYTESTRING", "key", 0},
{"BYTESTRING", "data", 0},
......@@ -175,6 +175,8 @@ const struct logtype logtypes[] = {
{"DISKOFF", "old_diskoff", 0},
{"DISKOFF", "new_diskoff", 0},
{"INTPAIRARRAY", "fromto", 0},
{"u_int32_t", "old_N", 0},
{"u_int32_t", "new_N", 0},
NULLFIELD}},
{0,0,FA{NULLFIELD}}
};
......@@ -199,7 +201,7 @@ const struct logtype logtypes[] = {
} })
void fprintf2 (FILE *f1, FILE *f2, const char *format, ...) {
static void __attribute__((format (printf, 3, 4))) fprintf2 (FILE *f1, FILE *f2, const char *format, ...) {
va_list ap;
int r;
va_start(ap, format);
......
newbrt/memory-debug.c 0 → 100644
View file @ 83d416fa
/* An implementation of memory that can be made to return NULL and ENOMEM on certain mallocs. */
#include "memory.h"
#include "toku_assert.h"
#include <errno.h>
#include <string.h>
int *toku_dead_mallocs=0;
int toku_malloc_counter=0;
void toku_malloc_cleanup(void) {}
void toku_free(void*x) {
free(x);
}
// if it fails, return 1, and set errno
static int does_malloc_fail (void) {
if (toku_dead_mallocs) {
int mnum = *toku_dead_mallocs;
if (mnum==toku_malloc_counter) {
toku_malloc_counter++;
toku_dead_mallocs++;
errno=ENOMEM;
return 1;
}
}
toku_malloc_counter++;
return 0;
}
void* toku_malloc(size_t n) {
if (does_malloc_fail()) return 0;
return malloc(n);
}
void *toku_tagmalloc(size_t size, enum typ_tag typtag) {
//printf("%s:%d tagmalloc\n", __FILE__, __LINE__);
void *r = toku_malloc(size);
if (!r) return 0;
assert(size>sizeof(int));
((int*)r)[0] = typtag;
return r;
}
void *toku_memdup (const void *v, size_t len) {
void *r=toku_malloc(len);
memcpy(r,v,len);
return r;
}
char *toku_strdup (const char *s) {
return toku_memdup(s, strlen(s)+1);
}
void *toku_realloc(void *p, size_t size) {
if (p==0) return toku_malloc(size);
if (does_malloc_fail()) return 0;
return realloc(p, size);
}
newbrt/memory.c
View file @ 83d416fa
......@@ -21,7 +21,7 @@ static int overflowed=0;
static void *items[items_limit];
static long sizes[items_limit];
static void note_did_malloc (void *p, long size) {
static void note_did_malloc (void *p, size_t size) {
static long long count=0;
WHEN_MEM_DEBUG(
if (n_items_malloced<items_limit) { items[n_items_malloced]=p; sizes[n_items_malloced]=size; }
......@@ -168,9 +168,10 @@ void *toku_malloc(size_t size) {
//if ((long)r==0x80523f8) { printf("%s:%d %p size=%ld\n", __FILE__, __LINE__, r, size); }
return r;
}
void *toku_tagmalloc(size_t size, int typtag) {
void *toku_tagmalloc(size_t size, enum typ_tag typtag) {
//printf("%s:%d tagmalloc\n", __FILE__, __LINE__);
void *r = toku_malloc(size);
if (!r) return 0;
assert(size>sizeof(int));
((int*)r)[0] = typtag;
return r;
......@@ -216,10 +217,14 @@ char *toku_strdup (const char *s) {
void toku_memory_check_all_free (void) {
if (n_items_malloced>0) {
printf("n_items_malloced=%lld\n", n_items_malloced);
if (toku_memory_check)
printf(" one item is %p size=%ld\n", items[0], sizes[0]);
exit(1);
fprintf(stderr, "n_items_malloced=%lld\n", n_items_malloced);
if (toku_memory_check) {
int i;
for (i=0; i<n_items_malloced && i< items_limit && i<10; i++) {
fprintf(stderr, " one item is %p size=%ld\n", items[i], sizes[i]);
}
}
return;
}
assert(n_items_malloced==0);
}
......
newbrt/memory.h
View file @ 83d416fa
......@@ -10,6 +10,14 @@
/* Generally: errno is set to 0 or a value to indicate problems. */
enum typ_tag { TYP_BRTNODE = 0xdead0001,
TYP_CACHETABLE, TYP_PAIR, /* for cachetables */
TYP_PMA,
TYP_GPMA,
TYP_TOKULOGGER,
TYP_TOKUTXN
};
/* Everything should call toku_malloc() instead of malloc(), and toku_calloc() instead of calloc() */
void *toku_calloc(size_t nmemb, size_t size);
void *toku_malloc(size_t size);
......@@ -17,7 +25,7 @@ void *toku_malloc(size_t size);
* This "tag" is useful if you are debugging and run across a void* that is
* really a (struct foo *), and you want to figure out what it is.
*/
void *toku_tagmalloc(size_t size, int typ);
void *toku_tagmalloc(size_t size, enum typ_tag typ);
void toku_free(void*);
/* toku_free_n() should be used if the caller knows the size of the malloc'd object. */
void toku_free_n(void*, size_t size);
......@@ -73,4 +81,9 @@ int toku_get_n_items_malloced(void); /* How many items are malloc'd but not free
void toku_print_malloced_items(void); /* Try to print some malloced-but-not-freed items. May be a noop. */
void toku_malloc_report (void); /* report on statistics about number of mallocs. Maybe a no-op. */
// For memory-debug.c Set this to an array of integers that say which mallocs should return NULL and ENOMEM.
// The array is terminated by a -1.
extern int *toku_dead_mallocs;
extern int toku_malloc_counter; // so you can reset it
#endif
newbrt/mempool.c
View file @ 83d416fa
......@@ -41,10 +41,10 @@ int toku_mempool_get_frag_size(struct mempool *mp) {
return mp->frag_size;
}
void *toku_mempool_malloc(struct mempool *mp, int size, int alignment) {
void *toku_mempool_malloc(struct mempool *mp, size_t size, int alignment) {
assert(mp->free_offset <= mp->size);
void *vp;
int offset = (mp->free_offset + (alignment-1)) & ~(alignment-1);
size_t offset = (mp->free_offset + (alignment-1)) & ~(alignment-1);
if (offset + size > mp->size) {
vp = 0;
} else {
......
newbrt/mempool.h
View file @ 83d416fa
......@@ -8,15 +8,17 @@
when the memory pool no longer has free space, the allocated chunks
must be relocated by the application to a new memory pool. */
#include <sys/types.h>
struct mempool;
typedef int (*mempool_compress_func)(struct mempool *mp, void *arg);
struct mempool {
void *base; /* the base address of the memory */
int free_offset; /* the offset of the memory pool free space */
int size; /* the size of the memory */
int frag_size; /* the size of the fragmented memory */
size_t free_offset; /* the offset of the memory pool free space */
size_t size; /* the size of the memory */
size_t frag_size; /* the size of the fragmented memory */
mempool_compress_func compress_func;
void *compress_arg;
};
......@@ -42,7 +44,7 @@ int toku_mempool_get_size(struct mempool *mp);
int toku_mempool_get_frag_size(struct mempool *mp);
/* allocate a chunk of memory from the memory pool suitably aligned */
void *toku_mempool_malloc(struct mempool *mp, int size, int alignment);
void *toku_mempool_malloc(struct mempool *mp, size_t size, int alignment);
/* free a previously allocated chunk of memory. the free only updates
a count of the amount of free space in the memory pool. the memory
......
newbrt/pma-internal.h
View file @ 83d416fa
......@@ -8,7 +8,7 @@ struct pma {
int dup_mode;
unsigned int N; /* How long is the array? Always a power of two >= 4. */
int n_pairs_present; /* How many array elements are non-null. */
struct kv_pair **pairs;
LEAFENTRY *pairs;
int uplgN; /* The smallest power of two >= lg(N) */
double udt_step; /* upper density threshold step */
/* Each doubling decreases the density by density step.
......@@ -26,10 +26,10 @@ struct pma {
struct mempool kvspace;
};
int toku_pmainternal_count_region (struct kv_pair *pairs[], int lo, int hi);
int toku_pmainternal_count_region (LEAFENTRY pairs[], int lo, int hi);
void toku_pmainternal_calculate_parameters (PMA pma);
int toku_pmainternal_smooth_region (TOKULOGGER, FILENUM, DISKOFF, struct kv_pair */*pairs*/[], int /*n*/, int /*idx*/, int /*base*/, PMA /*pma*/, int */*new_idx*/, LSN */*node_lsn*/);
int toku_pmainternal_printpairs (struct kv_pair *pairs[], int N);
int toku_pmainternal_smooth_region (TOKULOGGER, FILENUM, DISKOFF, LEAFENTRY/*pairs*/[], int /*n*/, int /*idx*/, int /*base*/, PMA /*pma*/, int */*new_idx*/, LSN */*node_lsn*/);
int toku_pmainternal_printpairs (LEAFENTRY pairs[], int N);
int toku_pmainternal_make_space_at (TOKULOGGER, FILENUM, DISKOFF, PMA pma, int idx, unsigned int *new_index, LSN *node_lsn);
int toku_pmainternal_find (PMA pma, DBT *); // The DB is so the comparison fuction can be called.
void toku_print_pma (PMA pma); /* useful for debugging, so keep the name short. I.e., not pmainternal_print_pma() */
......
newbrt/pma-test.c
View file @ 83d416fa
......@@ -1149,98 +1149,6 @@ static void test_pma_split(void) {
test_pma_split_varkey(); local_memory_check_all_free();
}
/*
* test the toku_pma_bulk_insert function by creating n kv pairs and bulk
* inserting them into an empty pma. verify that the pma contains all
* of the kv pairs.
*/
static void test_pma_bulk_insert_n(int n) {
PMA pma;
int r;
int i;
DBT *keys, *vals;
u_int32_t rand4fingerprint = random();
u_int32_t sum = 0;
u_int32_t expect_fingerprint = 0;
if (verbose) printf("test_pma_bulk_insert_n: %d\n", n);
r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0, 0);
assert(r == 0);
/* init n kv pairs */
keys = toku_malloc(n * sizeof (DBT));
assert(keys);
vals = toku_malloc(n * sizeof (DBT));
assert(vals);
/* init n kv pairs */
for (i=0; i<n; i++) {
char kstring[11];
char *k; int klen;
int *v; int vlen;
snprintf(kstring, sizeof kstring, "%.10d", i);
klen = strlen(kstring) + 1;
k = toku_malloc(klen);
assert(k);
strcpy(k, kstring);
toku_fill_dbt(&keys[i], k, klen);
vlen = sizeof (int);
v = toku_malloc(vlen);
assert(v);
*v = i;
toku_fill_dbt(&vals[i], v, vlen);
expect_fingerprint += rand4fingerprint*toku_calccrc32_kvpair (k, klen, v, vlen);
}
/* bulk insert n kv pairs */
r = toku_pma_bulk_insert(null_logger, null_filenum, (DISKOFF)0, pma, keys, vals, n, rand4fingerprint, &sum, 0);
assert(r == 0);
assert(sum==expect_fingerprint);
toku_pma_verify(pma);
toku_pma_verify_fingerprint(pma, rand4fingerprint, sum);
/* verify */
if (0) toku_print_pma(pma);
assert(n == toku_pma_n_entries(pma));
for (i=0; i<n; i++) {
DBT val;
toku_init_dbt(&val); val.flags = DB_DBT_MALLOC;
r = toku_pma_lookup(pma, &keys[i], &val);
assert(r == 0);
assert(vals[i].size == val.size);
assert(memcmp(vals[i].data, val.data, val.size) == 0);
toku_free(val.data);
}
/* cleanup */
for (i=0; i<n; i++) {
toku_free(keys[i].data);
toku_free(vals[i].data);
}
r = toku_pma_free(&pma);
assert(r == 0);
toku_free(keys);
toku_free(vals);
}
static void test_pma_bulk_insert(void) {
test_pma_bulk_insert_n(0); local_memory_check_all_free();
test_pma_bulk_insert_n(1); local_memory_check_all_free();
test_pma_bulk_insert_n(2); local_memory_check_all_free();
test_pma_bulk_insert_n(3); local_memory_check_all_free();
test_pma_bulk_insert_n(4); local_memory_check_all_free();
test_pma_bulk_insert_n(5); local_memory_check_all_free();
test_pma_bulk_insert_n(8); local_memory_check_all_free();
test_pma_bulk_insert_n(32); local_memory_check_all_free();
}
static void test_pma_insert_or_replace(void) {
PMA pma;
int r;
......@@ -2486,7 +2394,6 @@ static void pma_tests (void) {
test_pma_cursor(); local_memory_check_all_free();
test_pma_split(); local_memory_check_all_free();
test_pma_bulk_insert(); local_memory_check_all_free();
test_pma_insert_or_replace(); local_memory_check_all_free();
test_pma_delete();
test_pma_already_there(); local_memory_check_all_free();
......
newbrt/pma.c deleted 100644 → 0
View file @ 48d81cd4
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2007, 2008 Tokutek Inc. All rights reserved."
/* An in-memory Packed Memory Array dictionary.
The keys and values are arrays of bytes, but are not necessarily kept in scan order.
Only the pointers are kept.
*/
#include "key.h"
#include "memory.h"
#include "toku_assert.h"
#include "../include/db.h"
#include <stdio.h>
#include <errno.h>
/* Only needed for testing. */
#include <string.h>
#include <inttypes.h>
#include "kv-pair.h"
#include "pma-internal.h"
#include "log.h"
#include "log_header.h"
/* get KEY_VALUE_OVERHEAD */
#include "brt-internal.h"
/**************************** static functions forward declarations. *********************/
/* resize the pma array to asksize. zero all array entries starting from startx.*/
static int pma_resize_array(TOKULOGGER, FILENUM, DISKOFF, PMA pma, int asksize, int startx, LSN *node_lsn);
static int old_pma_resize_array(PMA pma, int asksize, int startx) {
return pma_resize_array((TOKULOGGER)0, (FILENUM){0}, (DISKOFF)0, pma, asksize, startx, (LSN*)0);
}
/* extract pairs from the pma in the window delimited by lo and hi.*/
static struct kv_pair_tag *pma_extract_pairs(PMA pma, int count, unsigned int lo, unsigned int hi);
/*
* a deletion occured at index "here" in the pma. rebalance the windows around "here". if
* necessary, shrink the pma.
*/
static void pma_delete_at(PMA pma, int here);
/**************************** end of static functions forward declarations. *********************/
static inline int kv_pair_inuse(struct kv_pair *pair) {
return pair != 0;
}
struct kv_pair_tag {
struct kv_pair *pair;
int oldtag, newtag;
};
#ifndef PMA_USE_MEMPOOL
#define PMA_USE_MEMPOOL 1
#endif
#if PMA_USE_MEMPOOL
/* allocate a kv pair from the pma kv memory pool */
static struct kv_pair *kv_pair_malloc_mempool(const void *key, int keylen, const void *val, int vallen, struct mempool *mp) {
struct kv_pair *kv = toku_mempool_malloc(mp, sizeof (struct kv_pair) + keylen + vallen, 4);
if (kv)
kv_pair_init(kv, key, keylen, val, vallen);
return kv;
}
/* compress all of the kv pairs to the left edge of the memory pool and
update the pma index with the new kv pair locations */
static int pma_compress_kvspace(PMA pma) {
if (toku_mempool_get_frag_size(&pma->kvspace) == 0)
return -1;
void *mp = toku_malloc(pma->kvspace.size);
if (mp == 0)
return -2;
struct mempool new_kvspace;
toku_mempool_init(&new_kvspace, mp, pma->kvspace.size);
unsigned int i;
for (i=0; i<pma->N; i++) {
struct kv_pair *kv = pma->pairs[i];
if (kv_pair_inuse(kv)) {
struct kv_pair *newkv = toku_mempool_malloc(&new_kvspace, kv_pair_size(kv), 4);
assert(newkv);
memcpy(newkv, kv, kv_pair_size(kv));
pma->pairs[i] = newkv;
}
}
toku_free(pma->kvspace.base);
pma->kvspace = new_kvspace;
return 0;
}
#endif
/* malloc space for a kv pair from the pma memory pool and initialize it.
if the allocation fails, try to compress the memory pool and try again. */
static struct kv_pair *pma_malloc_kv_pair(PMA pma __attribute__((unused)), const void *k, int ksize, const void *v, int vsize) {
#if PMA_USE_MEMPOOL
struct kv_pair *kv = kv_pair_malloc_mempool(k, ksize, v, vsize, &pma->kvspace);
if (kv == 0) {
if (0 == pma_compress_kvspace(pma))
kv = kv_pair_malloc_mempool(k, ksize, v, vsize, &pma->kvspace);
}
#else
struct kv_pair *kv = kv_pair_malloc(k, ksize, v, vsize);
#endif
return kv;
}
static void pma_mfree_kv_pair(PMA pma __attribute__((unused)), struct kv_pair *kv) {
#if PMA_USE_MEMPOOL
toku_mempool_mfree(&pma->kvspace, kv, kv_pair_size(kv));
#else
kv_pair_free(kv);
#endif
}
int toku_pma_n_entries (PMA pma) {
return pma->n_pairs_present;
}
unsigned int toku_pma_index_limit (PMA pma) {
return pma->N;
}
int toku_pmanode_valid (PMA pma, unsigned int i) {
assert(i<toku_pma_index_limit(pma));
return kv_pair_inuse(pma->pairs[i]);
}
bytevec toku_pmanode_key (PMA pma, unsigned int i) {
struct kv_pair *pair;
assert(i<toku_pma_index_limit(pma));
pair = pma->pairs[i];
assert(kv_pair_inuse(pair));
return kv_pair_key(pair);
}
ITEMLEN toku_pmanode_keylen (PMA pma, unsigned int i) {
struct kv_pair *pair;
assert(i<toku_pma_index_limit(pma));
pair = pma->pairs[i];
assert(kv_pair_inuse(pair));
return kv_pair_keylen(pair);
}
bytevec toku_pmanode_val (PMA pma, unsigned int i) {
struct kv_pair *pair;
assert(i<toku_pma_index_limit(pma));
pair = pma->pairs[i];
assert(kv_pair_inuse(pair));
return kv_pair_val(pair);
}
ITEMLEN toku_pmanode_vallen (PMA pma, unsigned int i) {
struct kv_pair *pair;
assert(i<toku_pma_index_limit(pma));
pair = pma->pairs[i];
assert(kv_pair_inuse(pair));
return kv_pair_vallen(pair);
}
/* Could pick the same one every time if we wanted. */
int toku_pma_random_pick(PMA pma, bytevec *key, ITEMLEN *keylen, bytevec *val, ITEMLEN *vallen) {
#if 1
unsigned int i;
/* For now a simple implementation where we simply start at the beginning and look. */
for (i=0; i<toku_pma_index_limit(pma); i++) {
struct kv_pair *pair = pma->pairs[i];
if (kv_pair_inuse(pair)) {
*key = kv_pair_key(pair);
*keylen = kv_pair_keylen(pair);
*val = kv_pair_val(pair);
*vallen = kv_pair_vallen(pair);
return 0;
}
}
return DB_NOTFOUND;
#else
/* Maybe we should pick a random item to remove in order to reduce the unbalancing. */
int i;
int l = toku_pma_index_limit(pma);
int r = random()%l;
/* For now a simple implementation where we simply start at the beginning and look. */
for (i=0; i<l; i++) {
int ir=(i+r)%l;
struct kv_pair *pair = pma->pairs[ir];
if (kv_pair_inuse(pair)) {
*key = kv_pair_key(pair);
*keylen = kv_pair_keylen(pair);
*val = kv_pair_val(pair);
*vallen = kv_pair_vallen(pair);
return 0;
}
}
return DB_NOTFOUND;
#endif
}
static int pma_count_finds=0;
static int pma_count_divides=0;
static int pma_count_scans=0;
void toku_pma_show_stats (void) {
printf("%d finds, %d divides, %d scans\n", pma_count_finds, pma_count_divides, pma_count_scans);
}
static int pma_compare_dbt_kv(PMA pma, DBT *k, DBT *v, struct kv_pair *kv) {
DBT k2, v2;
int cmp = pma->compare_fun(pma->db, k, toku_fill_dbt(&k2, kv_pair_key(kv), kv_pair_keylen(kv)));
if (cmp == 0 && v)
cmp = pma->dup_compare_fun(pma->db, v, toku_fill_dbt(&v2, kv_pair_val(kv), kv_pair_vallen(kv)));
return cmp;
}
/* search the index for a matching key and maybe value */
// this is just as fast as the iterative loop, since the compiler recognizes the tail calls.
static unsigned int pma_search(PMA pma, DBT *k, DBT *v, int lo, int hi, int *found) {
assert(0 <= lo && lo <= hi);
if (lo >= hi) {
*found = 0;
return lo;
} else {
int mi = (lo + hi)/2;
assert(lo <= mi && mi < hi);
int omi = mi;
while (mi < hi && !kv_pair_inuse(pma->pairs[mi]))
mi++;
if (mi >= hi)
return pma_search(pma, k, v, lo, omi, found);
int cmp = pma_compare_dbt_kv(pma, k, v, pma->pairs[mi]);
if (cmp > 0)
return pma_search(pma, k, v, mi+1, hi, found);
if (cmp < 0)
return pma_search(pma, k, v, lo, mi, found);
/* we have a match, try to find a better match on the left tree */
int here = pma_search(pma, k, v, lo, mi, found);
if (*found == 0)
here = mi;
*found = 1;
return here;
}
}
static unsigned int pma_search_func(PMA pma, brt_search_t *search, int lo, int hi, int *found) {
assert(0 <= lo && lo <= hi);
if (lo >= hi) {
*found = 0;
return lo;
} else {
int mi = (lo + hi)/2;
assert(lo <= mi && mi < hi);
int omi = mi;
while (mi < hi && !kv_pair_inuse(pma->pairs[mi]))
mi++;
if (mi >= hi)
return pma_search_func(pma, search, lo, omi, found);
struct kv_pair *kv = pma->pairs[mi];
DBT x, y;
int cmp = search->compare(search, search->k ? toku_fill_dbt(&x, kv_pair_key(kv), kv_pair_keylen(kv)) : 0,
search->v ? toku_fill_dbt(&y, kv_pair_val(kv), kv_pair_vallen(kv)) : 0);
if (cmp == 0) {
if (search->direction == BRT_SEARCH_LEFT)
return pma_search_func(pma, search, mi+1, hi, found);
else
return pma_search_func(pma, search, lo, mi, found);
}
/* we have a match, try to find a better match on the left or right subtrees */
int here;
if (search->direction == BRT_SEARCH_LEFT)
here = pma_search_func(pma, search, lo, mi, found);
else
here = pma_search_func(pma, search, mi+1, hi, found);
if (*found == 0)
here = mi;
*found = 1;
return here;
}
}
// Return the smallest index such that no lower index contains a larger key.
// This will be in the range 0 (inclusive) to toku_pma_index_limit(pma) (inclusive).
// Thus the returned index may not be a valid index into the array if it is == toku_pma_index_limit(pma)
// For example: if the array is empty, that means we return 0.
// For example: if the array is full of small keys, that means we return toku_pma_index_limit(pma), which is off the end of teh array.
// For example: if the array is full of large keys, then we return 0.
int toku_pmainternal_find (PMA pma, DBT *k) {
int found;
int lo = pma_search(pma, k, 0, 0, pma->N, &found);
return lo;
}
//int min (int i, int j) { if (i<j) return i; else return j; }
//int max (int i, int j) { if (i<j) return j; else return i; }
//double lg (int n) { return log((double)n)/log(2.0); }
int toku_pmainternal_printpairs (struct kv_pair *pairs[], int N) {
int count=0;
int i;
printf("{");
for (i=0; i<N; i++) {
if (i!=0) printf(" ");
if (kv_pair_inuse(pairs[i])) {
printf("%s", (char*)kv_pair_key(pairs[i]));
count++;
}
else printf("_");
}
printf("}");
return count;
}
void toku_print_pma (PMA pma) {
int count;
printf("N=%d n_present=%d ", toku_pma_index_limit(pma), pma->n_pairs_present);
count=toku_pmainternal_printpairs(pma->pairs, toku_pma_index_limit(pma));
printf("\n");
assert(count==pma->n_pairs_present);
}
/* Smooth the data, and return the location of the null.
* The sourcepairs are dense. The destpairs are sized to leave some holes.
* The destpairs are all initialized with null.
*/
static int distribute_data (struct kv_pair *destpairs[], int dcount,
struct kv_pair_tag sourcepairs[], int scount,
PMA pma) {
int null_location = -1;
unsigned long long numerator=0;
unsigned long long have_placed=0;
int i;
assert(scount<=dcount);
assert(dcount<(1<<30)); // so that long long will be enough to do everything precisely
for (i=0; i<dcount; i++) {
numerator+=scount;
if (numerator>dcount*have_placed) {
struct kv_pair *pair = sourcepairs[have_placed].pair;
assert(have_placed<(unsigned int)scount);
destpairs[i] = pair;
if (pma) sourcepairs[have_placed].newtag = destpairs+i-pma->pairs;
if (pair==0) {
assert(null_location==-1);
null_location=i;
}
have_placed++;
}
}
return null_location;
}
static int pma_log_distribute (TOKULOGGER logger, FILENUM filenum, DISKOFF old_diskoff, DISKOFF new_diskoff, int n_pairs, struct kv_pair_tag *pairs, LSN *oldnode_lsn, LSN*newnode_lsn) {
INTPAIRARRAY ipa;
MALLOC_N(n_pairs, ipa.array);
if (ipa.array==0) return errno;
int j=0;
int i;
for (i=0; i<n_pairs; i++) {
if (pairs[i].pair!=0) {
ipa.array[j].a = pairs[i].oldtag;
ipa.array[j].b = pairs[i].newtag;
j++;
}
}
ipa.size=j;
int r=toku_log_pmadistribute(logger, (LSN*)0, 0, filenum, old_diskoff, new_diskoff, ipa);
if (logger && oldnode_lsn) *oldnode_lsn = toku_logger_last_lsn(logger);
if (logger && newnode_lsn) *newnode_lsn = toku_logger_last_lsn(logger);
// if (0 && pma) {
// printf("Pma state:\n");
// PMA_ITERATE_IDX (pma, pidx, key, keylen, data, datalen,
// printf(" %d:(%d:%s) (%d:%s)\n", pidx, keylen, (char*)key, datalen, (char*)data));
// }
toku_free(ipa.array);
return r;
}
/* spread the non-empty pairs around. There are n of them. Create an empty slot just before the IDXth
element, and return that slot's index in the smoothed array. */
int toku_pmainternal_smooth_region (TOKULOGGER logger, FILENUM filenum, DISKOFF diskoff, struct kv_pair *pairs[], int n, int idx, int base, PMA pma, int *new_idx, LSN *node_lsn) {
int i;
int n_present=0;
for (i=0; i<n; i++) {
if (kv_pair_inuse(pairs[i])) n_present++;
}
n_present++; // Save one for the blank guy.
{
#define USE_MALLOC_IN_SMOOTH 1
#if USE_MALLOC_IN_SMOOTH
struct kv_pair_tag *MALLOC_N(n_present, tmppairs);
#else
struct kv_pair_tag tmppairs[n_present];
#endif
int n_saved=0;
int newidx;
for (i=0; i<n; i++) {
if (i==idx) {
tmppairs[n_saved++].pair = 0;
}
if (kv_pair_inuse(pairs[i])) {
tmppairs[n_saved].oldtag = base + i;
tmppairs[n_saved++].pair = pairs[i];
}
pairs[i] = 0;
}
if (idx==n) {
tmppairs[n_saved++].pair = 0;
}
//printf(" temp="); printpairs(tmppairs, n_saved);
assert(n_saved==n_present);
/* Now the tricky part. Distribute the data. */
newidx=distribute_data (pairs, n,
tmppairs, n_saved, pma);
int r = pma_log_distribute(logger, filenum, diskoff, diskoff,
n_saved,
tmppairs,
node_lsn, node_lsn);
if (r!=0) goto cleanup;
*new_idx = newidx;
cleanup:
#if USE_MALLOC_IN_SMOOTH
toku_free(tmppairs);
#endif
return 0;
}
}
int toku_lg (int n) {
int result=0;
int two_to_result = 1;
while (two_to_result<n) {
result++;
two_to_result*=2;
}
return result;
}
/* Calculate densitysteps and uplgN, given N. */
void toku_pmainternal_calculate_parameters (PMA pma) {
int N = toku_pma_index_limit(pma);
int lgN = toku_lg(N);
int n_divisions=0;
//printf("N=%d lgN=%d\n", N, lgN);
while (N/2>=lgN) {
n_divisions++;
N/=2;
}
pma->uplgN=N;
//printf("uplgN = %d n_divisions=%d\n", pma->uplgN, n_divisions);
assert(n_divisions>0);
pma->udt_step = (PMA_UDT_HIGH - PMA_UDT_LOW)/n_divisions;
pma->ldt_step = (PMA_LDT_HIGH - PMA_LDT_LOW)/n_divisions;
}
int toku_pmainternal_count_region (struct kv_pair *pairs[], int lo, int hi) {
int n=0;
while (lo<hi) {
if (kv_pair_inuse(pairs[lo])) n++;
lo++;
}
return n;
}
/* find the smallest power of 2 >= n */
static unsigned int pma_array_size(PMA pma __attribute__((unused)), int asksize) {
int n = PMA_MIN_ARRAY_SIZE;
while (n < asksize)
n *= 2;
return n;
}
int toku_pma_create(PMA *pma, pma_compare_fun_t compare_fun, DB *db, FILENUM filenum, int maxsize, int initial_n_pairs /* set to zero to get default. */) {
int error;
TAGMALLOC(PMA, result);
if (result==0) return -1;
result->dup_mode = 0;
result->n_pairs_present = 0;
result->pairs = 0;
result->compare_fun = compare_fun;
result->dup_compare_fun = 0;
result->db = db;
result->filenum = filenum;
result->skey = 0;
result->sval = 0;
result->N = initial_n_pairs ? initial_n_pairs : PMA_MIN_ARRAY_SIZE;
result->pairs = 0;
{
unsigned int n = pma_array_size(result, result->N);
error = toku_resize_pma_exactly(result, 0, n);
if (error) {
toku_free(result);
return -1;
}
toku_pmainternal_calculate_parameters(result);
}
if (maxsize == 0)
maxsize = 4*1024;
maxsize = maxsize + maxsize/4;
#if PMA_USE_MEMPOOL
void *mpbase = toku_malloc(maxsize); assert(mpbase);
toku_mempool_init(&result->kvspace, mpbase, maxsize);
#endif
*pma = result;
assert((unsigned long)result->pairs[result->N]==0xdeadbeefL);
return 0;
}
int toku_resize_pma_exactly (PMA pma, int oldsize, int newsize) {
pma->N = newsize;
if (pma->pairs == 0)
pma->pairs = toku_malloc((1 + pma->N) * sizeof (struct kv_pair *));
else
pma->pairs = toku_realloc(pma->pairs, (1 + pma->N) * sizeof (struct kv_pair *));
if (pma->pairs == 0)
return -1;
pma->pairs[pma->N] = (void *) 0xdeadbeef;
unsigned int i;
for (i=oldsize; i<pma->N; i++) {
pma->pairs[i] = 0;
}
return 0;
}
static int pma_resize_array_nolog(PMA pma, int asksize, int startz, unsigned int *oldn, unsigned int *newn) {
unsigned int oldN = pma->N;
unsigned int n = pma_array_size(pma, asksize);
int r = toku_resize_pma_exactly(pma, startz, n);
if (r!=0) return r;
toku_pmainternal_calculate_parameters(pma);
*oldn = oldN;
*newn = n;
return 0;
}
static int pma_resize_array(TOKULOGGER logger, FILENUM filenum, DISKOFF offset, PMA pma, int asksize, int startz, LSN *node_lsn) {
unsigned int oldN, n;
int r = pma_resize_array_nolog(pma, asksize, startz, &oldN, &n);
if (r!=0) return r;
toku_log_resizepma (logger, (LSN*)0, 0, filenum, offset, oldN, n);
if (logger && node_lsn) *node_lsn = toku_logger_last_lsn(logger);
return 0;
}
int toku_pma_set_compare(PMA pma, pma_compare_fun_t compare_fun) {
pma->compare_fun = compare_fun;
return 0;
}
int toku_pma_set_dup_mode(PMA pma, int dup_mode) {
if (!(dup_mode == 0 || dup_mode == (TOKU_DB_DUP+TOKU_DB_DUPSORT)))
return EINVAL;
pma->dup_mode = dup_mode;
return 0;
}
int toku_pma_set_dup_compare(PMA pma, pma_compare_fun_t dup_compare_fun) {
pma->dup_compare_fun = dup_compare_fun;
return 0;
}
/* find the next matching key in the pma starting from index here */
static int pma_next_key(PMA pma, DBT *k, DBT *v, int here, int n, int *found) {
assert(0 <= here);
*found = 0;
while (here < n && !kv_pair_inuse(pma->pairs[here]))
here += 1;
if (here < n) {
int cmp = pma_compare_dbt_kv(pma, k, v, pma->pairs[here]);
if (cmp == 0)
*found = 1;
}
return here;
}
/* Make some space for a key to go at idx (the thing currently at idx should end up at to the right.) */
/* (Making space may involve moving things around, including the hole at index.) */
int toku_pmainternal_make_space_at (TOKULOGGER logger, FILENUM filenum, DISKOFF offset, PMA pma, int idx, unsigned int *new_index, LSN *node_lsn) {
/* Within a range LO to HI we have a limit of how much packing we will tolerate.
* We allow the entire array to be 50% full.
* We allow a region of size lgN to be full.
* At sizes in between, we interpolate.
*/
unsigned int size=pma->uplgN;
int lo=idx;
int hi=idx;
double udt=PMA_UDT_HIGH;
while (1) {
/* set hi-lo equal size, make sure it is a supserset of (hi,lo). */
lo=idx-size/2;
hi=idx+size/2;
//printf("lo=%d hi=%d\n", lo, hi);
if (lo<0) { hi-=lo; lo=0; }
else if ((unsigned)hi>toku_pma_index_limit(pma)) { lo-=(hi-toku_pma_index_limit(pma)); hi=toku_pma_index_limit(pma); }
else { ; /* nothing */ }
//printf("lo=%d hi=%d\n", lo, hi);
assert(0<=lo); assert(lo<hi); assert((unsigned)hi<=toku_pma_index_limit(pma)); assert((unsigned)(hi-lo)==size); // separate into separate assertions so that gcov doesn't see branches not taken.
assert(udt>0.499); assert(udt<=1);
if (udt<0.5001) { assert(lo==0); assert((unsigned)hi==toku_pma_index_limit(pma)); }
{
int count = (1+ /* Don't forget space for the new guy. */
toku_pmainternal_count_region(pma->pairs, lo, hi));
double density = (double) count / (double) (hi - lo);
if (density <= udt)
break;
if (lo==0 && (unsigned)hi==toku_pma_index_limit(pma)) {
/* The array needs to be doubled in size. */
assert(size==toku_pma_index_limit(pma));
size*=2;
// printf("pma_make_space_realloc %d to %d hi %d\n", pma->N, size, hi);
pma_resize_array(logger, filenum, offset, pma, size, hi, node_lsn);
hi=size;
//printf("doubled N\n");
break;
}
}
udt-=pma->udt_step;
size*=2;
}
//printf("%s:%d Smoothing from %d to %d to density %f\n", __FILE__, __LINE__, lo, hi, density);
{
int sub_new_index;
int r = toku_pmainternal_smooth_region(logger, filenum, offset, pma->pairs+lo, hi-lo, idx-lo, lo, pma, &sub_new_index, node_lsn);
if (r!=0) return r;
*new_index=sub_new_index+lo;
return 0;
}
}
enum pma_errors toku_pma_lookup (PMA pma, DBT *k, DBT *v) {
int found;
unsigned int here = pma_search(pma, k, 0, 0, pma->N, &found);
struct kv_pair *kv = pma->pairs[here];
if (found && kv_pair_inuse(kv))
return toku_dbt_set_value(v, kv->key + kv->keylen, kv->vallen, &pma->sval);
else
return DB_NOTFOUND;
}
int toku_pma_search(PMA pma, brt_search_t *search, DBT *foundk, DBT *foundv) {
int found;
unsigned int here = pma_search_func(pma, search, 0, pma->N, &found);
struct kv_pair *kv = pma->pairs[here];
if (found && kv_pair_inuse(kv)) {
int r = 0;
if (foundk)
r = toku_dbt_set_value(foundk, kv_pair_key(kv), kv_pair_keylen(kv), &pma->skey);
if (r == 0 && foundv)
r = toku_dbt_set_value(foundv, kv_pair_val(kv), kv_pair_vallen(kv), &pma->sval);
return r;
} else
return DB_NOTFOUND;
}
/* returns 0 if OK.
* You must have freed all the cursors, otherwise returns nonzero and does nothing. */
int toku_pma_free (PMA *pmap) {
PMA pma=*pmap;
if (pma->n_pairs_present > 0) {
unsigned int i;
for (i=0; i < pma->N; i++) {
struct kv_pair *kv = pma->pairs[i];
if (kv_pair_inuse(kv)) {
pma_mfree_kv_pair(pma, kv);
pma->pairs[i] = 0;
pma->n_pairs_present--;
}
}
}
assert(pma->n_pairs_present == 0);
#if PMA_USE_MEMPOOL
void *mpbase = toku_mempool_get_base(&pma->kvspace);
toku_mempool_fini(&pma->kvspace);
toku_free(mpbase);
#endif
toku_free(pma->pairs);
if (pma->skey) toku_free(pma->skey);
if (pma->sval) toku_free(pma->sval);
toku_free(pma);
*pmap=0;
return 0;
}
/* Copies keylen and datalen */
/* returns an error if the key is already present. */
int toku_pma_insert (PMA pma, DBT *k, DBT *v, TOKULOGGER logger, TXNID xid, FILENUM filenum, DISKOFF diskoff, u_int32_t rand4fingerprint, u_int32_t *fingerprint, LSN *node_lsn) {
int found;
unsigned int idx = pma_search(pma, k, pma->dup_mode & TOKU_DB_DUPSORT ? v : 0, 0, pma->N, &found);
if (found)
return BRT_ALREADY_THERE; /* It is already here. Return an error. */
if (kv_pair_inuse(pma->pairs[idx])) {
unsigned int newidx;
int r = toku_pmainternal_make_space_at (logger, filenum, diskoff, pma, idx, &newidx, (LSN*)0); /* returns the new idx. */
if (r!=0) return r;
idx = newidx;
}
assert(idx < pma->N);
assert(!kv_pair_inuse(pma->pairs[idx]));
pma->pairs[idx] = pma_malloc_kv_pair(pma, k->data, k->size, v->data, v->size);
assert(pma->pairs[idx]);
pma->n_pairs_present++;
*fingerprint += rand4fingerprint*toku_calccrc32_kvpair(k->data, k->size, v->data, v->size);
struct kv_pair *pair = pma->pairs[idx];
if (logger) {
{
TOKUTXN txn;
int r;
if ((r=toku_txnid2txn(logger, xid, &txn))) return r;
if (txn) {
const BYTESTRING key = { pair->keylen, toku_memdup(kv_pair_key_const(pair), pair->keylen) };
const BYTESTRING data = { pair->vallen, toku_memdup(kv_pair_val_const(pair), pair->vallen) };
if ((r = toku_logger_save_rollback_insertatleaf(txn, pma->filenum, key, data))) {
toku_free(key.data); toku_free(data.data);
return r;
}
}
}
{
const BYTESTRING key = { pair->keylen, kv_pair_key(pair) };
const BYTESTRING data = { pair->vallen, kv_pair_val(pair) };
int r = toku_log_insertinleaf (logger, (LSN*)0, 0, xid, pma->filenum, diskoff, idx, key, data);
if (r!=0) return r;
if (node_lsn) *node_lsn = toku_logger_last_lsn(logger);
}
}
return 0;
}
static int pma_delete_dup (PMA pma, DBT *k, DBT *v, u_int32_t rand4sem, u_int32_t *fingerprint, u_int32_t *deleted_size) {
/* find the left most matching key in the pma */
int found;
unsigned int lefthere = pma_search(pma, k, v, 0, pma->N, &found);
int rightfound = found, righthere = lefthere;
while (rightfound) {
struct kv_pair *kv = pma->pairs[righthere];
if (kv_pair_inuse(kv)) {
*deleted_size += PMA_ITEM_OVERHEAD+ KEY_VALUE_OVERHEAD + kv_pair_keylen(kv) + kv_pair_vallen(kv);
*fingerprint -= rand4sem*toku_calccrc32_kvpair (kv_pair_key_const(kv), kv_pair_keylen(kv), kv_pair_val_const(kv), kv_pair_vallen(kv));
pma_mfree_kv_pair(pma, kv);
pma->pairs[righthere] = 0;
pma->n_pairs_present--;
}
/* find the next matching key in the pma */
righthere = pma_next_key(pma, k, v, righthere+1, pma->N, &rightfound);
}
if (found) {
/* check the density of the region centered around the deleted pairs */
pma_delete_at(pma, (lefthere + righthere) / 2);
}
return found ? BRT_OK : DB_NOTFOUND;
}
static int pma_log_delete (PMA pma, const char *key, int keylen, const char *val, int vallen,
DISKOFF diskoff, int idx, TOKULOGGER logger, TXNID xid, LSN *node_lsn) {
{
const BYTESTRING deletedkey = { keylen, (char*)key };
const BYTESTRING deleteddata = { vallen, (char*)val };
int r=toku_log_deleteinleaf(logger, (LSN*)0, 0, xid, pma->filenum, diskoff, idx, deletedkey, deleteddata);
if (r!=0) return r;
}
if (logger) {
TOKUTXN txn;
int r=toku_txnid2txn(logger, xid, &txn);
if (r!=0) return r;
if (txn) {
const BYTESTRING deletedkey = { keylen, toku_memdup(key, keylen) };
const BYTESTRING deleteddata = { vallen, toku_memdup(val, vallen) };
r=toku_logger_save_rollback_deleteatleaf(txn, pma->filenum, deletedkey, deleteddata);
if (r!=0) { toku_free(deletedkey.data); toku_free(deleteddata.data); return r;
}
}
if (node_lsn) *node_lsn = toku_logger_last_lsn(logger);
}
return 0;
}
static int pma_delete_nodup (PMA pma, DBT *k, DBT *v,
TOKULOGGER logger, TXNID xid, DISKOFF diskoff,
u_int32_t rand4sem, u_int32_t *fingerprint, u_int32_t *deleted_size, LSN *node_lsn) {
/* find the left most matching key in the pma */
int found;
unsigned int here;
here = pma_search(pma, k, v, 0, pma->N, &found);
struct kv_pair *kv = pma->pairs[here];
if (!found || !kv_pair_inuse(kv))
return DB_NOTFOUND;
int r=pma_log_delete(pma, kv_pair_key_const(kv), kv_pair_keylen(kv), kv_pair_val_const(kv), kv_pair_vallen(kv),
diskoff, here, logger, xid, node_lsn);
if (r!=0) return r;
*deleted_size = PMA_ITEM_OVERHEAD + KEY_VALUE_OVERHEAD + kv_pair_keylen(kv) + kv_pair_vallen(kv);
*fingerprint -= rand4sem*toku_calccrc32_kvpair (kv_pair_key_const(kv), kv_pair_keylen(kv), kv_pair_val_const(kv), kv_pair_vallen(kv));
pma_mfree_kv_pair(pma, kv);
pma->pairs[here] = 0;
pma->n_pairs_present--;
pma_delete_at(pma, here);
return BRT_OK;
}
int toku_pma_delete (PMA pma, DBT *k, DBT *v,
TOKULOGGER logger, TXNID xid, DISKOFF diskoff,
u_int32_t rand4sem, u_int32_t *fingerprint, u_int32_t *deleted_size, LSN *node_lsn) {
u_int32_t my_deleted_size;
if (!deleted_size)
deleted_size = &my_deleted_size;
*deleted_size = 0;
if (pma->dup_mode & TOKU_DB_DUPSORT)
return pma_delete_dup(pma, k, v, rand4sem, fingerprint, deleted_size);
else
return pma_delete_nodup(pma, k, v, logger, xid, diskoff, rand4sem, fingerprint, deleted_size, node_lsn);
}
static void pma_delete_at(PMA pma, int here) {
int count;
struct kv_pair_tag *newpairs;
unsigned int lgN = pma->uplgN;
unsigned int size = lgN;
double ldt = PMA_LDT_HIGH;
/* check the density of regions from lg(N) size to the entire array */
for (;;) {
int lo, hi;
double density;
/* select a region centered on here */
lo = here - size/2;
hi = here + size/2;
if (lo < 0) {
hi -= lo;
lo = 0;
if ((unsigned)hi > pma->N)
hi = pma->N;
} else if ((unsigned)hi > pma->N) {
lo -= hi - pma->N;
hi = pma->N;
if (lo < 0)
lo = 0;
}
assert(lo <= hi);
/* compute the density of the region */
count = toku_pmainternal_count_region(pma->pairs, lo, hi);
density = (double) count / ((double) (hi - lo));
/* rebalance if the density exceeds the lower threadshold */
if (0) printf("check size %d h %d density %d/%d %f %d-%d ldt %f\n", size,
lgN, count, hi-lo, density, lo, hi, ldt);
if (density >= ldt) {
if (size == lgN)
return;
if (0) printf("delete_at_rebalance %d over %d %d\n", count, lo, hi);
newpairs = pma_extract_pairs(pma, count, lo, hi);
distribute_data(pma->pairs + lo, hi - lo, newpairs, count, pma);
toku_free(newpairs);
return;
}
ldt -= pma->ldt_step;
size *= 2;
if (0 == lo && pma->N == (unsigned)hi)
break;
}
/* shrink */
size = pma_array_size(pma, count + count/4);
if (size == pma->N)
return;
if (0) printf("shrink %d from %d to %d\n", count, pma->N, size);
newpairs = pma_extract_pairs(pma, count, 0, pma->N);
assert(newpairs);
old_pma_resize_array(pma, size, 0);
distribute_data(pma->pairs, pma->N, newpairs, count, pma);
toku_free(newpairs);
}
int toku_pma_insert_or_replace (PMA pma, DBT *k, DBT *v,
int *replaced_v_size, /* If it is a replacement, set to the size of the old value, otherwise set to -1. */
TOKULOGGER logger, TXNID xid, FILENUM filenum, DISKOFF diskoff,
u_int32_t rand4fingerprint, u_int32_t *fingerprint,
LSN *node_lsn) {
//printf("%s:%d v->size=%d\n", __FILE__, __LINE__, v->size);
int r;
int found;
unsigned int idx = pma_search(pma, k, pma->dup_mode & TOKU_DB_DUPSORT ? v : 0, 0, pma->N, &found);
if (found) {
struct kv_pair *kv = pma->pairs[idx];
*replaced_v_size = kv->vallen;
*fingerprint -= rand4fingerprint*toku_calccrc32_kvpair(kv_pair_key_const(kv), kv_pair_keylen(kv), kv_pair_val_const(kv), kv_pair_vallen(kv));
r=pma_log_delete(pma, kv_pair_key(kv), kv->keylen, kv_pair_val(kv), kv->vallen, diskoff, idx, logger, xid, node_lsn);
if (r!=0) return r;
if (v->size == (unsigned int) kv_pair_vallen(kv)) {
memcpy(kv_pair_val(kv), v->data, v->size);
} else {
pma_mfree_kv_pair(pma, kv);
pma->pairs[idx] = pma_malloc_kv_pair(pma, k->data, k->size, v->data, v->size);
assert(pma->pairs[idx]);
}
/* idx is live here */
goto logit_and_update_fingerprint;
}
if (kv_pair_inuse(pma->pairs[idx])) {
unsigned int newidx;
r = toku_pmainternal_make_space_at (logger, filenum, diskoff, pma, idx, &newidx, node_lsn); /* returns the new idx. */
if (r!=0) return r;
idx=newidx;
}
assert(!kv_pair_inuse(pma->pairs[idx]));
//printf("%s:%d v->size=%d\n", __FILE__, __LINE__, v->size);
pma->pairs[idx] = pma_malloc_kv_pair(pma, k->data, k->size, v->data, v->size);
assert(pma->pairs[idx]);
pma->n_pairs_present++;
*replaced_v_size = -1;
//printf("%s:%d txn=%p\n", __FILE__, __LINE__, txn);
logit_and_update_fingerprint:
r=0;
if (logger) {
{
TOKUTXN txn;
if ((r=toku_txnid2txn(logger, xid, &txn))) return r;
if (txn) {
const BYTESTRING key = { k->size, toku_memdup(k->data, k->size) };
const BYTESTRING data = { v->size, toku_memdup(v->data, v->size) };
if ((r = toku_logger_save_rollback_insertatleaf(txn, pma->filenum, key, data))) {
toku_free(key.data); toku_free(data.data);
return r;
}
}
}
{
const BYTESTRING key = { k->size, k->data };
const BYTESTRING data = { v->size, v->data };
r = toku_log_insertinleaf (logger, (LSN*)0, 0, xid, pma->filenum, diskoff, idx, key, data);
if (logger && node_lsn) *node_lsn = toku_logger_last_lsn(logger);
if (r!=0) return r;
/* We don't record the insert here for rollback. The insert should have been logged at the top-level. */
}
}
*fingerprint += rand4fingerprint*toku_calccrc32_kvpair(k->data, k->size, v->data, v->size);
return r;
}
void toku_pma_iterate (PMA pma, void(*f)(bytevec,ITEMLEN,bytevec,ITEMLEN, void*), void*v) {
unsigned int i;
for (i=0; i<toku_pma_index_limit(pma); i++) {
struct kv_pair *pair = pma->pairs[i];
if (pair) {
f(pair->key, pair->keylen,
pair->key + pair->keylen, pair->vallen, v);
}
}
}
static struct kv_pair_tag *pma_extract_pairs(PMA pma, int npairs, unsigned int lo, unsigned int hi) {
struct kv_pair_tag *pairs;
unsigned int i;
int lastpair;
pairs = toku_malloc(npairs * sizeof (struct kv_pair_tag));
if (pairs == 0)
return 0;
lastpair = 0;
for (i=lo; i<hi; i++) {
assert(i < pma->N);
if (pma->pairs[i] != 0) {
assert(pma->pairs[i] != (void*)0xdeadbeef);
pairs[lastpair].pair = pma->pairs[i];
pairs[lastpair].oldtag = i;
pma->pairs[i] = 0;
lastpair += 1;
}
}
assert(lastpair == npairs);
return pairs;
}
#if PMA_USE_MEMPOOL
static void __pma_relocate_kvpairs(PMA pma) {
unsigned int i;
for (i=0; i<pma->N; i++) {
struct kv_pair *kv = pma->pairs[i];
if (kv) {
pma->pairs[i] = kv_pair_malloc_mempool(kv_pair_key(kv), kv_pair_keylen(kv), kv_pair_val(kv),
kv_pair_vallen(kv), &pma->kvspace);
assert(pma->pairs[i]);
}
}
}
#endif
int toku_pma_split(TOKULOGGER logger, FILENUM filenum,
DISKOFF diskoff, PMA pma, unsigned int *pma_size_p, u_int32_t rand4fp, u_int32_t *fingerprint_p, LSN *lsn,
DBT *splitk,
DISKOFF newdiskoff, PMA newpma, unsigned int *newpma_size_p, u_int32_t newrand4fp, u_int32_t *newfingerprint_p, LSN *newlsn) {
int error;
int npairs;
struct kv_pair_tag *pairs;
int i;
int n;
int spliti;
/* extract the pairs */
npairs = toku_pma_n_entries(pma);
if (npairs == 0) {
if (splitk)
memset(splitk, 0, sizeof *splitk);
return 0;
}
/* TODO move pairs to the stack */
pairs = pma_extract_pairs(pma, npairs, 0, pma->N);
assert(pairs);
assert(toku_pma_n_entries(newpma) == 0);
/* debug check the kv length sum */
unsigned int sumlen = 0;
for (i=0; i<npairs; i++)
sumlen += kv_pair_keylen(pairs[i].pair) + kv_pair_vallen(pairs[i].pair) + PMA_ITEM_OVERHEAD + KEY_VALUE_OVERHEAD;
if (pma_size_p)
assert(*pma_size_p == sumlen);
unsigned int runlen = 0;
for (i=0; i<npairs;) {
runlen += kv_pair_keylen(pairs[i].pair) + kv_pair_vallen(pairs[i].pair) + PMA_ITEM_OVERHEAD + KEY_VALUE_OVERHEAD;
i++;
if (2*runlen >= sumlen)
break;
}
spliti = i;
unsigned int revised_leftpmasize = runlen;
unsigned int revised_rightpmasize = sumlen-runlen;
u_int32_t revised_left_fingerprint;
u_int32_t revised_right_fingerprint;
{
u_int32_t sum = 0;
for (i=spliti; i<npairs; i++) {
sum+=toku_calccrc32_kvpair(kv_pair_key_const(pairs[i].pair), kv_pair_keylen(pairs[i].pair),
kv_pair_val_const(pairs[i].pair), kv_pair_vallen(pairs[i].pair));
}
revised_left_fingerprint = -rand4fp * sum;
revised_right_fingerprint = newrand4fp * sum;
}
if (splitk) {
struct kv_pair *a = pairs[spliti-1].pair;
if (pma->dup_mode & TOKU_DB_DUPSORT) {
splitk->data = kv_pair_malloc(kv_pair_key(a), kv_pair_keylen(a), kv_pair_val(a), kv_pair_vallen(a));
splitk->size = kv_pair_keylen(a) + kv_pair_vallen(a);
} else {
splitk->data = kv_pair_malloc(kv_pair_key(a), kv_pair_keylen(a), 0, 0);
splitk->size = kv_pair_keylen(a);
}
splitk->flags = 0;
}
/* put the second half of pairs into the right pma */
/* Do this first, so that the logging will move the stuff out of the left pma first, and then later when we redistribute in the left PMA, we won't overwrite something. */
n = npairs - spliti;
error = pma_resize_array(logger, filenum, newdiskoff, newpma, n + n/4, 0, newlsn);
assert(error == 0);
distribute_data(newpma->pairs, toku_pma_index_limit(newpma), &pairs[spliti], n, newpma);
{
int r = pma_log_distribute(logger, filenum, diskoff, newdiskoff, n, &pairs[spliti], lsn, newlsn);
if (r!=0) { toku_free(pairs); return r; }
}
#if PMA_USE_MEMPOOL
__pma_relocate_kvpairs(newpma);
// If it's in an mpool, we must free those pairs.
for (i=spliti; i<npairs; i++) {
pma_mfree_kv_pair(pma, pairs[i].pair);
}
#endif
newpma->n_pairs_present = n;
/* put the first half of pairs into the left pma */
n = spliti;
// Since the new array is smaller than the old one, during recovery we need to do the resize after moving the elements.
// But we must actually do the resize first here so we can determine the size.
unsigned int oldn_for_logging = 0, newn_for_logging = 0;
error = pma_resize_array_nolog(pma, n + n/4, 0, // zeros the elements
&oldn_for_logging, &newn_for_logging);
assert(error == 0);
distribute_data(pma->pairs, toku_pma_index_limit(pma), &pairs[0], n, pma);
{
int r = pma_log_distribute(logger, filenum, diskoff, diskoff, spliti, &pairs[0], lsn, lsn);
if (r!=0) { toku_free(pairs); return r; }
r = toku_log_resizepma(logger, (LSN*)0, 0, filenum, diskoff, oldn_for_logging, newn_for_logging);
if (r!=0) { toku_free(pairs); return r; }
if (logger && lsn) *lsn = toku_logger_last_lsn(logger);
}
// Don't have to relocate kvpairs, because these ones are still there.
pma->n_pairs_present = spliti;
toku_free(pairs);
/* The remaining cursors are in the left pma */
if (fingerprint_p) *fingerprint_p += revised_left_fingerprint;
if (newfingerprint_p) *newfingerprint_p += revised_right_fingerprint;
if (pma_size_p) *pma_size_p = revised_leftpmasize;
if (newpma_size_p) *newpma_size_p = revised_rightpmasize;
return 0;
}
static void __pma_bulk_cleanup(struct pma *pma, struct kv_pair_tag *pairs, int n) {
int i;
for (i=0; i<n; i++)
if (pairs[i].pair)
pma_mfree_kv_pair(pma, pairs[i].pair);
}
int toku_pma_bulk_insert(TOKULOGGER logger, FILENUM filenum, DISKOFF diskoff, PMA pma, DBT *keys, DBT *vals, int n_newpairs, u_int32_t rand4fp, u_int32_t *sum, LSN *node_lsn) {
struct kv_pair_tag *newpairs;
int i;
int error;
u_int32_t delta=0;
if (n_newpairs == 0)
return 0;
if (toku_pma_n_entries(pma) > 0)
return -2;
/* TODO put newpairs on the stack */
newpairs = toku_malloc(n_newpairs * sizeof (struct kv_pair_tag));
if (newpairs == 0) {
error = -3; return error;
}
for (i=0; i<n_newpairs; i++) {
delta += rand4fp*toku_calccrc32_kvpair (keys[i].data, keys[i].size, vals[i].data, vals[i].size);
#if PMA_USE_MEMPOOL
newpairs[i].pair = kv_pair_malloc_mempool(keys[i].data, keys[i].size,
vals[i].data, vals[i].size, &pma->kvspace);
#else
newpairs[i].pair = kv_pair_malloc(keys[i].data, keys[i].size, vals[i].data, vals[i].size);
#endif
if (newpairs[i].pair == 0) {
__pma_bulk_cleanup(pma, newpairs, i);
toku_free(newpairs);
error = -4; return error;
}
}
error = pma_resize_array(logger, filenum, diskoff, pma, n_newpairs + n_newpairs/4, 0, node_lsn);
if (error) {
__pma_bulk_cleanup(pma, newpairs, n_newpairs);
toku_free(newpairs);
error = -5; return error;
}
distribute_data(pma->pairs, toku_pma_index_limit(pma), newpairs, n_newpairs, pma);
pma->n_pairs_present = n_newpairs;
toku_free(newpairs);
*sum += delta;
return 0;
}
/* verify that the keys in the pma index are sorted subject to the pma mode
* no duplications, duplicates, sorted duplicates.
*/
void toku_pma_verify(PMA pma) {
unsigned int i;
struct kv_pair *kv;
/* find the first key in the index */
for (i=0; i<pma->N; i++) {
kv = pma->pairs[i];
if (kv_pair_inuse(kv)) {
i += 1;
break;
}
}
/* compare the current key with the next key in the index */
struct kv_pair *nextkv;
for (; i<pma->N; i++) {
nextkv = pma->pairs[i];
if (kv_pair_inuse(nextkv)) {
DBT kv_dbt, nextkv_dbt;
toku_fill_dbt(&kv_dbt, kv_pair_key(kv), kv_pair_keylen(kv));
toku_fill_dbt(&nextkv_dbt, kv_pair_key(nextkv), kv_pair_keylen(nextkv));
int r = pma->compare_fun(pma->db, &kv_dbt, &nextkv_dbt);
if (pma->dup_mode == 0)
assert(r < 0);
else if (pma->dup_mode & TOKU_DB_DUPSORT)
assert(r <= 0);
if (r == 0 && (pma->dup_mode & TOKU_DB_DUPSORT)) {
toku_fill_dbt(&kv_dbt, kv_pair_val(kv), kv_pair_vallen(kv));
toku_fill_dbt(&nextkv_dbt, kv_pair_val(nextkv), kv_pair_vallen(nextkv));
r = pma->dup_compare_fun(pma->db, &kv_dbt, &nextkv_dbt);
assert(r <= 0);
}
kv = nextkv;
}
}
#if PMA_USE_MEMPOOL
/* verify all kv pairs are in the memory pool */
for (i=0; i<pma->N; i++) {
kv = pma->pairs[i];
if (kv_pair_inuse(kv)) {
assert(toku_mempool_inrange(&pma->kvspace, kv, kv_pair_size(kv)));
}
}
#endif
}
void toku_pma_verify_fingerprint (PMA pma, u_int32_t rand4fingerprint, u_int32_t fingerprint) {
u_int32_t actual_fingerprint=0;
PMA_ITERATE(pma, kv, kl, dv, dl,
actual_fingerprint+=rand4fingerprint*toku_calccrc32_kvpair(kv,kl,dv,dl)
);
assert(actual_fingerprint==fingerprint);
}
// If the index is wrong or there is a value already, return nonzero
// There should be no cursors, but if there were they wouldn't need to be updated.
int toku_pma_set_at_index (PMA pma, unsigned int idx, DBT *key, DBT *value) {
if (idx>=pma->N) return -1;
if (kv_pair_inuse(pma->pairs[idx])) return -1;
pma->pairs[idx] = pma_malloc_kv_pair(pma, key->data, key->size, value->data, value->size);
pma->n_pairs_present++;
return 0;
}
int toku_pma_clear_at_index (PMA pma, unsigned int idx) {
if (idx>=pma->N) return -1;
if (!kv_pair_inuse(pma->pairs[idx])) return -1;
pma_mfree_kv_pair(pma, pma->pairs[idx]);
pma->pairs[idx]=0;
pma->n_pairs_present--;
return 0;
}
// Move from a to b
static void pma_move (PMA pmaa, int idxa, u_int32_t randa, u_int32_t *fingerprinta, u_int32_t *n_in_bufa,
PMA pmab, int idxb, u_int32_t randb, u_int32_t *fingerprintb, u_int32_t *n_in_bufb) {
if (pmaa==pmab) {
assert(pmab->pairs[idxb]==0);
pmab->pairs[idxb] = pmaa->pairs[idxa];
pmaa->pairs[idxa] = 0;
} else {
struct kv_pair *pair = pmaa->pairs[idxa];
u_int32_t fdiff = toku_calccrc32_kvpair(kv_pair_key_const(pair), kv_pair_keylen(pair), kv_pair_val_const(pair), kv_pair_vallen(pair));
u_int32_t sizediff = PMA_ITEM_OVERHEAD + KEY_VALUE_OVERHEAD + kv_pair_keylen(pair) + kv_pair_vallen(pair);
*fingerprinta -= randa*fdiff; *fingerprintb += randb*fdiff;
*n_in_bufa -= sizediff; *n_in_bufb += sizediff;
pmab->pairs[idxb] = pma_malloc_kv_pair(pmab, kv_pair_key_const(pair), kv_pair_keylen(pair), kv_pair_val_const(pair), kv_pair_vallen(pair));
pma_mfree_kv_pair(pmaa, pair);
pmaa->pairs[idxa] = 0;
pmaa->n_pairs_present--;
pmab->n_pairs_present++;
}
}
// assume no cursors
// Move stuff from pmaa to pmab
int toku_pma_move_indices (PMA pma_from, PMA pma_to, INTPAIRARRAY fromto,
u_int32_t rand_from, u_int32_t *fingerprint_from,
u_int32_t rand_to, u_int32_t *fingerprint_to,
u_int32_t *n_in_buf_from, u_int32_t *n_in_buf_to
) {
u_int32_t i;
for (i=0; i<fromto.size; i++) {
// First handle the case for sliding something left. We can simply move it.
{
int a=fromto.array[i].a;
int b=fromto.array[i].b;
if (b==a) continue;
if (b<a) {
pma_move(pma_from, a, rand_from, fingerprint_from, n_in_buf_from,
pma_to, b, rand_to, fingerprint_to, n_in_buf_to);
continue;
}
}
// Otherwise slide things to the right We have to find the rightmost thing that slides right and move it first.
{
// We must slide things to the right.
// Find the next index that does want to go to the left
u_int32_t j;
for (j=i+1; j<fromto.size && fromto.array[j].a < fromto.array[j].b; j++) {
/*nothing */
}
// everything from i (inclusive) to j (exclusive) wants to slide to the right.
u_int32_t jdown;
for (jdown=j-1; 1; jdown--) {
int a=fromto.array[jdown].a;
int b=fromto.array[jdown].b;
if (a!=b) {
pma_move(pma_from, a, rand_from, fingerprint_from, n_in_buf_from,
pma_to, b, rand_to, fingerprint_to, n_in_buf_to);
}
if (i==jdown) break; // Do it this way so everything can be unsigned and we won't try to go negative.
}
i=j-1;
}
}
return 0;
}
static void reverse_fromto (INTPAIRARRAY fromto) {
u_int32_t i;
for (i=0; i<fromto.size; i++) {
int tmp = fromto.array[i].a;
fromto.array[i].a = fromto.array[i].b;
fromto.array[i].b = tmp;
}
}
int toku_pma_move_indices_back (PMA pma_backto, PMA pma_backfrom, INTPAIRARRAY fromto,
u_int32_t rand_backto, u_int32_t *fingerprint_backto,
u_int32_t rand_backfrom, u_int32_t *fingerprint_backfrom,
u_int32_t *n_in_buf_backto, u_int32_t *n_in_buf_backfrom
) {
int r;
reverse_fromto(fromto);
r = toku_pma_move_indices(pma_backfrom, pma_backto, fromto,
rand_backfrom, fingerprint_backfrom,
rand_backto, fingerprint_backto,
n_in_buf_backfrom, n_in_buf_backto
);
reverse_fromto(fromto);
return r;
}
newbrt/pma.h deleted 100644 → 0
View file @ 48d81cd4
#ifndef PMA_H
#define PMA_H
#ident "Copyright (c) 2007 Tokutek Inc. All rights reserved."
#include "brttypes.h"
#include "ybt.h"
#include "yerror.h"
#include "../include/db.h"
#include "log.h"
#include "brt-search.h"
/* An in-memory Packed Memory Array dictionary. */
/* There is a built-in-cursor. */
/* All functions return 0 on success. */
typedef struct pma *PMA;
// typedef struct pma_cursor *PMA_CURSOR;
/* compare 2 DBT's. return a value < 0, = 0, > 0 if a < b, a == b, a > b respectively */
typedef int (*pma_compare_fun_t)(DB *, const DBT *a, const DBT *b);
int toku_pma_create(PMA *, pma_compare_fun_t compare_fun, DB *, FILENUM filenum, int maxsize, int initial_n_pairs);
int toku_pma_set_compare(PMA pma, pma_compare_fun_t compare_fun);
/* set the duplicate mode
0 -> no duplications, TOKU_DB_DUP, TOKU_DB_DUPSORT */
int toku_pma_set_dup_mode(PMA pma, int mode);
/* set the duplicate compare function */
int toku_pma_set_dup_compare(PMA pma, pma_compare_fun_t dup_compare_fun);
/* verify the integrity of a pma */
void toku_pma_verify(PMA pma);
/* returns 0 if OK.
* You must have freed all the cursors, otherwise returns nonzero and does nothing. */
int toku_pma_free (PMA *);
int toku_pma_n_entries (PMA);
/* Returns an error if the key is already present. */
/* The values returned should not be modified.by the caller. */
/* Any cursors should be updated. */
/* Duplicates the key and keylen. */
//enum pma_errors toku_pma_insert (PMA, bytevec key, ITEMLEN keylen, bytevec data, ITEMLEN datalen);
// The DB pointer is there so that the comparison function can be called.
enum pma_errors toku_pma_insert (PMA, DBT*, DBT*, TOKULOGGER, TXNID, FILENUM, DISKOFF, u_int32_t /*random for fingerprint */, u_int32_t */*fingerprint*/, LSN *node_lsn);
/* This returns an error if the key is NOT present. */
int pma_replace (PMA, bytevec key, ITEMLEN keylen, bytevec data, ITEMLEN datalen);
/* Delete pairs from the pma.
* If val is 0 then delete all pairs from the pma that match the key.
* If val is not 0 then only delete the pair that matches both the key and the val.
* (This even works if there is no such pair (in which case DB_NOTFOUND is returned, and
* no changes are made.)
* The case where val!=0 should work for both DUP and NODUP dictionaries.
* For NODUP dictionaries, the value is deleted only if both the key and the value match.
*/
int toku_pma_delete (PMA, DBT */*key*/, DBT */*val*/,
TOKULOGGER, TXNID, DISKOFF,
u_int32_t /*random for fingerprint*/, u_int32_t */*fingerprint*/, u_int32_t *deleted_size, LSN*);
int toku_pma_delete_fixupsize (PMA, DBT */*key*/, DBT */*val*/,
TOKULOGGER, TXNID, DISKOFF,
u_int32_t /*random for fingerprint*/, u_int32_t */*fingerprint*/, LSN*, u_int32_t *n_bytes_in_buffer_including_overheads);
int toku_pma_strong_insert_or_replace (PMA pma, DBT *k, DBT *v,
TOKULOGGER, TXNID, FILENUM, DISKOFF,
u_int32_t rand4fingerprint, u_int32_t *fingerprint,
LSN *node_lsn,
u_int32_t *n_bytes_in_buffer_including_overheads);
int toku_pma_insert_or_replace_ws (PMA pma, DBT *k, DBT *v,
TOKULOGGER, TXNID, FILENUM, DISKOFF,
u_int32_t rand4fingerprint, u_int32_t *fingerprint,
LSN *node_lsn,
u_int32_t *n_bytes_in_buffer_including_overheads,
int weak_p);
int toku_pma_insert_or_replace (PMA /*pma*/, DBT */*k*/, DBT */*v*/,
int */*replaced_v_size*/, /* If it is a replacement, set to the size of the old value, otherwise set to -1. */
TOKULOGGER, TXNID, FILENUM, DISKOFF,
u_int32_t /*random for fingerprint*/, u_int32_t */*fingerprint*/,
LSN */*node_lsn*/);
//?? __attribute__((deprecated));
/* Exposes internals of the PMA by returning a pointer to the guts.
* Don't modify the returned data. Don't free it. */
enum pma_errors toku_pma_lookup (PMA, DBT*, DBT*);
int toku_pma_search(PMA, brt_search_t *, DBT *, DBT *);
/*
* The kv pairs in PMA are split into two (nearly) equal sized sets.
* THe ones in the left half are left in PMA, the ones in the right half are put into NEWPMA.
* The size is determined by the sum of the sizes of the keys and values.
* The NEWPMA must be empty.
*
* DISKOFF - the disk offset of the node containing the PMA to be split. (Needed for logging)
* PMA - the pma to be split.
* PMA_SIZE - a variable containing the size of the disk image of the PMA.
* RAND4SUM - the random number for fingerprinting
* FINGERPRINT - the current fingerprint of the PMA.
*
* NEWDISKOFF, NEWPMA, NEWPMASIZE, NEWRAND4SUM, NEWFINGERPRINT - The same information fo the pma to hold the stuff to be moved out of PMA.
*
* SPLITK filled in with the resulting pivot key.
* The original PMA gets keys <= pivot key
* The NEWPMA gets keys > pivot key
*/
int toku_pma_split(TOKULOGGER, FILENUM,
DISKOFF /*diskoff*/, PMA /*pma*/, unsigned int */*pma_size*/, u_int32_t /*rand4sum*/, u_int32_t */*fingerprint*/, LSN* /*lsn*/,
DBT */*splitk*/,
DISKOFF /*newdiskoff*/, PMA /*newpma*/, unsigned int */*newpma_size*/, u_int32_t /*newrand4sum*/, u_int32_t */*newfingerprint*/, LSN* /*newlsn*/);
/*
* Insert several key value pairs into an empty pma.
* Doesn't delete any existing keys (even if they are duplicates)
* Requires: The keys are sorted
*
* pma - the pma that the key value pairs will be inserted into.
* must be empty with no cursors.
* keys - an array of keys
* vals - an array of values
* n_newpairs - the number of key value pairs
*/
int toku_pma_bulk_insert(TOKULOGGER, FILENUM, DISKOFF, PMA pma, DBT *keys, DBT *vals, int n_newpairs, u_int32_t rand4sem, u_int32_t *fingerprint, LSN */*node_lsn*/);
int toku_pma_random_pick(PMA, bytevec *key, ITEMLEN *keylen, bytevec *data, ITEMLEN *datalen);
unsigned int toku_pma_index_limit(PMA); // How many slots are in the PMA right now?
int toku_pmanode_valid(PMA, unsigned int);
bytevec toku_pmanode_key(PMA, unsigned int);
ITEMLEN toku_pmanode_keylen(PMA, unsigned int);
bytevec toku_pmanode_val(PMA, unsigned int);
ITEMLEN toku_pmanode_vallen(PMA, unsigned int);
void toku_pma_iterate (PMA, void(*)(bytevec,ITEMLEN,bytevec,ITEMLEN, void*), void*);
#define PMA_ITERATE_IDX(table,idx,keyvar,keylenvar,datavar,datalenvar,body) ({ \
unsigned int idx; \
for (idx=0; idx<toku_pma_index_limit(table); idx++) { \
if (toku_pmanode_valid(table,idx)) { \
bytevec keyvar = toku_pmanode_key(table,idx); \
ITEMLEN keylenvar = toku_pmanode_keylen(table,idx); \
bytevec datavar = toku_pmanode_val(table, idx); \
ITEMLEN datalenvar = toku_pmanode_vallen(table, idx); \
body; \
} } })
#define PMA_ITERATE(table,keyvar,keylenvar,datavar,datalenvar,body) PMA_ITERATE_IDX(table, __i, keyvar, keylenvar, datavar, datalenvar, body)
void toku_pma_verify_fingerprint (PMA pma, u_int32_t rand4fingerprint, u_int32_t fingerprint);
// Set the PMA's size, without moving anything.
int toku_resize_pma_exactly (PMA pma, int oldsize, int newsize);
int toku_pma_set_at_index (PMA, unsigned int /*index*/, DBT */*key*/, DBT */*value*/); // If the index is wrong or there is a value already, return nonzero
int toku_pma_clear_at_index (PMA, unsigned int /*index*/); // If the index is wrong or there is a value already, return nonzero
// Requires: No open cursors on the pma.
// Return nonzero if the indices are somehow wrong.
int toku_pma_move_indices (PMA pma_from, PMA pma_to, INTPAIRARRAY fromto,
u_int32_t rand_from, u_int32_t *fingerprint_from,
u_int32_t rand_to, u_int32_t *fingerprint_to,
u_int32_t *n_in_buf_from, u_int32_t *n_in_buf_to);
// Move things backwards according to fromto.
int toku_pma_move_indices_back (PMA pma_backto, PMA pma_backfrom, INTPAIRARRAY fromto,
u_int32_t rand_backto, u_int32_t *fingerprint_backto,
u_int32_t rand_backfrom, u_int32_t *fingerprint_backfrom,
u_int32_t *n_in_buf_backto, u_int32_t *n_in_buf_backfrom
);
void toku_pma_show_stats (void);
#endif
newbrt/recover.c
View file @ 83d416fa
......@@ -12,6 +12,8 @@
#include "log-internal.h"
#include "log_header.h"
#include "toku_assert.h"
#include "kv-pair.h"
#include "gpma-internal.h"
#include <fcntl.h>
#include <stdlib.h>
......@@ -146,16 +148,22 @@ void toku_recover_newbrtnode (LSN lsn, FILENUM filenum,DISKOFF diskoff,u_int32_t
n->thisnodename = diskoff;
n->log_lsn = n->disk_lsn = lsn;
//printf("%s:%d %p->disk_lsn=%"PRId64"\n", __FILE__, __LINE__, n, n->disk_lsn.lsn);
n->layout_version = 2;
n->layout_version = 3;
n->height = height;
n->rand4fingerprint = rand4fingerprint;
n->flags = is_dup_sort ? TOKU_DB_DUPSORT : 0; // Don't have TOKU_DB_DUP ???
n->local_fingerprint = 0; // nothing there yet
n->dirty = 1;
if (height==0) {
r=toku_pma_create(&n->u.l.buffer, toku_dont_call_this_compare_fun, null_db, filenum, nodesize, 0);
r=toku_gpma_create(&n->u.l.buffer, 0);
assert(r==0);
n->u.l.n_bytes_in_buffer=0;
{
void *mp = toku_malloc(n->nodesize);
assert(mp);
toku_mempool_init(&n->u.l.buffer_mempool, mp, n->nodesize);
}
} else {
n->u.n.n_children = 0;
n->u.n.totalchildkeylens = 0;
......@@ -392,10 +400,11 @@ void toku_recover_insertinleaf (LSN lsn, TXNID UU(txnid), FILENUM filenum, DISKO
BRTNODE node = node_v;
assert(node->height==0);
VERIFY_COUNTS(node);
DBT key,data;
r = toku_pma_set_at_index(node->u.l.buffer, pmaidx, toku_fill_dbt(&key, keybs.data, keybs.len), toku_fill_dbt(&data, databs.data, databs.len));
assert(r==0);
struct kv_pair *kvp = brtnode_malloc_kv_pair(node->u.l.buffer, &node->u.l.buffer_mempool, keybs.data, keybs.len, databs.data, databs.len);
assert(pair);
toku_gpma_set_at_index(node->u.l.buffer, pmaidx, kv_pair_size(kvp), kvp);
node->local_fingerprint += node->rand4fingerprint*toku_calccrc32_kvpair(keybs.data, keybs.len, databs.data, databs.len);
// printf("%s:%d local_fingerprint=%08x (this=%08x)\n", __FILE__, __LINE__, node->local_fingerprint, toku_calccrc32_kvpair(keybs.data, keybs.len, databs.data, databs.len));
node->u.l.n_bytes_in_buffer += PMA_ITEM_OVERHEAD + KEY_VALUE_OVERHEAD + keybs.len + databs.len;
// PMA_ITERATE_IDX(node->u.l.buffer, idx, skey, keylen __attribute__((__unused__)), sdata, datalen __attribute__((__unused__)),
......@@ -421,8 +430,15 @@ void toku_recover_deleteinleaf (LSN lsn, TXNID UU(txnid), FILENUM filenum, DISKO
BRTNODE node = node_v;
assert(node->height==0);
VERIFY_COUNTS(node);
r = toku_pma_clear_at_index(node->u.l.buffer, pmaidx);
assert (r==0);
{
u_int32_t len;
void *data;
r = toku_gpma_get_from_index(node->u.l.buffer, pmaidx, &len, &data);
if (r==0) {
toku_mempool_mfree(&node->u.l.buffer_mempool, data, len);
}
}
toku_gpma_clear_at_index(node->u.l.buffer, pmaidx);
node->local_fingerprint -= node->rand4fingerprint*toku_calccrc32_kvpair(keybs.data, keybs.len, databs.data, databs.len);
node->u.l.n_bytes_in_buffer -= PMA_ITEM_OVERHEAD + KEY_VALUE_OVERHEAD + keybs.len + databs.len;
VERIFY_COUNTS(node);
......@@ -434,7 +450,7 @@ void toku_recover_deleteinleaf (LSN lsn, TXNID UU(txnid), FILENUM filenum, DISKO
}
// a newbrtnode should have been done before this
void toku_recover_resizepma (LSN lsn, FILENUM filenum, DISKOFF diskoff, u_int32_t oldsize, u_int32_t newsize) {
void toku_recover_resizepma (LSN lsn, FILENUM filenum, DISKOFF diskoff, u_int32_t oldsize __attribute__((__unused__)), u_int32_t newsize) {
struct cf_pair *pair = NULL;
int r = find_cachefile(filenum, &pair);
assert(r==0);
......@@ -444,7 +460,7 @@ void toku_recover_resizepma (LSN lsn, FILENUM filenum, DISKOFF diskoff, u_int32_
assert(r==0);
BRTNODE node = node_v;
assert(node->height==0);
r = toku_resize_pma_exactly (node->u.l.buffer, oldsize, newsize);
r = toku_resize_gpma_exactly (node->u.l.buffer, newsize);
assert(r==0);
VERIFY_COUNTS(node);
......@@ -454,7 +470,35 @@ void toku_recover_resizepma (LSN lsn, FILENUM filenum, DISKOFF diskoff, u_int32_
assert(r==0);
}
void toku_recover_pmadistribute (LSN lsn, FILENUM filenum, DISKOFF old_diskoff, DISKOFF new_diskoff, INTPAIRARRAY fromto) {
int move_indices (GPMA from, GPMA to, INTPAIRARRAY fromto,
u_int32_t a_rand, u_int32_t *a_fp,
u_int32_t b_rand, u_int32_t *b_fp,
u_int32_t *a_nbytes, u_int32_t *b_nbytes) {
struct gitem *MALLOC_N(fromto.size, items);
if (items==0) return errno;
u_int32_t i;
u_int32_t fp=0;
u_int32_t sizediff=0;
for (i=0; i<fromto.size; i++) {
int idx = fromto.array[i].a;
struct gitem item = from->items[idx];
items[i]=item;
from->items[idx].data = 0;
fp += toku_crc32(toku_null_crc, item.data, item.len);
sizediff += PMA_ITEM_OVERHEAD + item.len;
}
for (i=0; i<fromto.size; i++) {
to->items[fromto.array[i].b] = items[i];
}
*a_fp -= a_rand * fp;
*b_fp += b_rand * fp;
*a_nbytes -= sizediff;
*b_nbytes += sizediff;
toku_free(items);
return 0;
}
void toku_recover_pmadistribute (LSN lsn, FILENUM filenum, DISKOFF old_diskoff, DISKOFF new_diskoff, INTPAIRARRAY fromto, u_int32_t old_N __attribute__((__unused__)), u_int32_t new_N) {
struct cf_pair *pair = NULL;
int r = find_cachefile(filenum, &pair);
assert(r==0);
......@@ -466,21 +510,25 @@ void toku_recover_pmadistribute (LSN lsn, FILENUM filenum, DISKOFF old_diskoff,
assert(r==0);
BRTNODE nodea = node_va; assert(nodea->height==0);
BRTNODE nodeb = node_vb; assert(nodeb->height==0);
if (new_N!=toku_gpma_index_limit(nodeb->u.l.buffer)) {
r = toku_resize_gpma_exactly(nodeb->u.l.buffer, new_N);
assert(r==0);
}
{
unsigned int i;
//printf("{");
for (i=0; i<fromto.size; i++) {
//printf(" {%d %d}", fromto.array[i].a, fromto.array[i].b);
assert(fromto.array[i].a < toku_pma_index_limit(nodea->u.l.buffer));
assert(fromto.array[i].b < toku_pma_index_limit(nodeb->u.l.buffer));
assert(fromto.array[i].a < toku_gpma_index_limit(nodea->u.l.buffer));
assert(fromto.array[i].b < toku_gpma_index_limit(nodeb->u.l.buffer));
}
//printf("}\n");
}
r = toku_pma_move_indices (nodea->u.l.buffer, nodeb->u.l.buffer, fromto,
nodea->rand4fingerprint, &nodea->local_fingerprint,
nodeb->rand4fingerprint, &nodeb->local_fingerprint,
&nodea->u.l.n_bytes_in_buffer, &nodeb->u.l.n_bytes_in_buffer
);
r = move_indices (nodea->u.l.buffer, nodeb->u.l.buffer, fromto,
nodea->rand4fingerprint, &nodea->local_fingerprint,
nodeb->rand4fingerprint, &nodeb->local_fingerprint,
&nodea->u.l.n_bytes_in_buffer, &nodeb->u.l.n_bytes_in_buffer
);
// The bytes in buffer and fingerprint shouldn't change
// PMA_ITERATE_IDX(nodeb->u.l.buffer, idx, key, keylen __attribute__((__unused__)), data, datalen __attribute__((__unused__)),
......
newbrt/roll.c
View file @ 83d416fa
......@@ -26,6 +26,7 @@ int toku_rollback_fcreate (BYTESTRING bs_fname,
return 0;
}
#if 0
int toku_rollback_fclose (FILENUM filenum, BYTESTRING bs_fname, TOKUTXN txn) {
abort();
filenum=filenum;
......@@ -55,7 +56,7 @@ int toku_rollback_fclose (FILENUM filenum, BYTESTRING bs_fname, TOKUTXN txn) {
return 0;
#endif
}
#endif
//int toku_rollback_newbrtnode (struct logtype_newbrtnode *le, TOKUTXN txn) {
// // All that must be done is to put the node on the freelist.
......
newbrt/test-assert.c
View file @ 83d416fa
/* -*- mode: C; c-basic-offset: 4 -*- */
#include "toku_assert.h"
#include "brttypes.h"
#include <signal.h>
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
static void catch_abort (int sig __attribute__((__unused__))) {
static __attribute__((__noreturn__)) void catch_abort (int sig __attribute__((__unused__))) {
exit(1);
}
static BOOL foo (void) {
return TRUE;
}
int main (int argc, const char *argv[]) {
signal (SIGABRT, catch_abort);
if (argc!=2) { printf("argcount should be 2.\n"); exit(1); }
const char *str=argv[1];
assert(strcmp(str,"ok")==0);
assert(foo());
assert(0x8000000000000000UL);
assert(0x4000000000000000UL);
assert(argv[1]);
return 0;
}
newbrt/test-gpma-blackbox.c 0 → 100644
View file @ 83d416fa
// Black box tester, uses only the public interfaces.
#include "gpma.h"
#include "memory.h"
#include "toku_assert.h"
#include "../include/db.h"
#include <stdio.h>
#include <string.h>
int verbose;
static int count_frees=0;
static void free_callback (u_int32_t len __attribute__((__unused__)), void*freeme, void *extra) {
assert(extra==(void*)&verbose);
toku_free(freeme);
count_frees++;
}
static void test_create_and_free (void) {
int r;
GPMA pma;
r = toku_gpma_create(&pma, 0);
assert(r==0);
count_frees=0;
toku_gpma_free(&pma, free_callback, &verbose);
assert(count_frees==0);
}
static int compare_strings(u_int32_t alen, void *aval, u_int32_t blen, void *bval, void *extra __attribute__((__unused__))) {
assert(alen==strlen(aval)+1);
assert(blen==strlen(bval)+1);
return strcmp(aval, bval);
}
static int rcall_never (u_int32_t nitems __attribute__((__unused__)), u_int32_t *froms __attribute__((__unused__)), u_int32_t *tos __attribute__((__unused__)), struct gitem *items __attribute__((__unused__)), u_int32_t old_N __attribute__((__unused__)), u_int32_t new_N __attribute__((__unused__)), void *extra __attribute__((__unused__))) {
assert(0);
return 0;
}
static int rcall_ok (u_int32_t nitems __attribute__((__unused__)), u_int32_t *froms __attribute__((__unused__)), u_int32_t *tos __attribute__((__unused__)), struct gitem *items __attribute__((__unused__)), u_int32_t old_N __attribute__((__unused__)), u_int32_t new_N __attribute__((__unused__)), void *extra __attribute__((__unused__))) {
return 0;
}
static void test_insert_A (void) {
int r;
GPMA pma;
r = toku_gpma_create(&pma, 0);
assert(r==0);
char *k1,*k2,*k3;
r = toku_gpma_insert(pma, 6, k1=strdup("hello"),
compare_strings, 0,
rcall_never, "hello", 0);
assert(r==0);
assert(toku_gpma_n_entries(pma)==1);
r = toku_gpma_insert(pma, 6, k2=strdup("gello"),
compare_strings, 0,
rcall_ok, "gello", 0);
assert(r==0);
r = toku_gpma_insert(pma, 6, k3=strdup("fello"),
compare_strings, 0,
rcall_ok, "fello", 0);
assert(r==0);
void *k;
r = toku_gpma_insert(pma, 6, k=strdup("fello"),
compare_strings, 0,
rcall_ok, "fello", 0);
assert(r==DB_KEYEXIST);
toku_free(k);
//printf("size=%d\n", toku_gpma_index_limit(pma));
u_int32_t resultlen;
void *resultdata;
r = toku_gpma_lookup_item(pma, 6, "hello", compare_strings, 0, &resultlen, &resultdata, 0);
assert(r==0);
assert(strcmp(resultdata, "hello")==0);
assert(resultdata==k1);
r = toku_gpma_lookup_item(pma, 6, "gello", compare_strings, 0, &resultlen, &resultdata, 0);
assert(r==0);
assert(strcmp(resultdata, "gello")==0);
assert(resultdata==k2);
u_int32_t idx=999;
r = toku_gpma_lookup_item(pma, 6, "fello", compare_strings, 0, &resultlen, &resultdata, &idx);
assert(r==0);
assert(strcmp(resultdata, "fello")==0);
assert(resultdata==k3);
assert(idx!=999);
r = toku_gpma_lookup_item(pma, 6, "aello", compare_strings, 0, &resultlen, &resultdata, 0);
assert(r==DB_NOTFOUND);
r = toku_gpma_lookup_item(pma, 6, "fillo", compare_strings, 0, &resultlen, &resultdata, 0);
assert(r==DB_NOTFOUND);
r = toku_gpma_lookup_item(pma, 6, "gillo", compare_strings, 0, &resultlen, &resultdata, 0);
assert(r==DB_NOTFOUND);
r = toku_gpma_lookup_item(pma, 6, "hillo", compare_strings, 0, &resultlen, &resultdata, 0);
assert(r==DB_NOTFOUND);
r = toku_gpma_lookup_item(pma, 6, "zello", compare_strings, 0, &resultlen, &resultdata, 0);
assert(r==DB_NOTFOUND);
{
int bes (u_int32_t dlen __attribute__((__unused__)), void *dval, void *extra __attribute__((__unused__))) {
return strcmp(dval, "a"); // This will return 1 for everything. For dir<=0 we'll have DB_NOTFOUND, for dir>0 we'll have "fello"
}
r = toku_gpma_lookup_bessel(pma, bes, -1, 0, &resultlen, &resultdata, 0);
assert(r==DB_NOTFOUND);
r = toku_gpma_lookup_bessel(pma, bes, 0, 0, &resultlen, &resultdata, 0);
assert(r==DB_NOTFOUND);
r = toku_gpma_lookup_bessel(pma, bes, +1, 0, &resultlen, &resultdata, 0);
assert(r==0);
assert(strcmp(resultdata, "fello")==0);
}
{
int bes (u_int32_t dlen __attribute__((__unused__)), void *dval, void *extra __attribute__((__unused__))) {
return strcmp(dval, "z"); // This will return -1 for everything. For dir>=0 we'll have DB_NOTFOUND, for dir<0 we'll have "hello"
}
r = toku_gpma_lookup_bessel(pma, bes, -1, 0, &resultlen, &resultdata, 0); // find the rightmost thing
assert(r==0);
assert(strcmp(resultdata, "hello")==0);
r = toku_gpma_lookup_bessel(pma, bes, 0, 0, &resultlen, &resultdata, 0);
assert(r==DB_NOTFOUND);
r = toku_gpma_lookup_bessel(pma, bes, +1, 0, &resultlen, &resultdata, 0);
assert(r==DB_NOTFOUND);
}
count_frees=0;
toku_gpma_free(&pma, free_callback, &verbose);
assert(count_frees==3);
}
void test_split_internal (const char *strings[],
int expect_n_left,
u_int32_t *expect_froms_left,
u_int32_t *expect_tos_left,
int expect_n_right,
u_int32_t *expect_froms_right,
u_int32_t *expect_tos_right) {
GPMA pma1, pma2;
int r;
r = toku_gpma_create(&pma1, 0);
assert(r==0);
r = toku_gpma_create(&pma2, 0);
assert(r==0);
assert(0==toku_gpma_valididx(pma1, toku_gpma_index_limit(pma1))); // because it's off the end of the array
assert(0==toku_gpma_valididx(pma1, 0)); // because nothing is there
assert(0!=toku_gpma_get_from_index(pma1, toku_gpma_index_limit(pma1), 0, 0));
u_int32_t i;
u_int32_t current_estimate_of_N = toku_gpma_index_limit(pma1);
//printf("%s:%d N=%d\n", __FILE__, __LINE__, current_estimate_of_N);
for (i=0; strings[i]; i++) {
int rcall_a (u_int32_t nitems __attribute__((__unused__)), u_int32_t *froms __attribute__((__unused__)), u_int32_t *tos __attribute__((__unused__)), struct gitem *items __attribute__((__unused__)), u_int32_t old_N, u_int32_t new_N, void *extra __attribute__((__unused__))) {
//printf("%s:%d old_N=%d new_N=%d est=%d\n", __FILE__, __LINE__, old_N, new_N, current_estimate_of_N);
assert(old_N==current_estimate_of_N);
current_estimate_of_N = new_N;
//printf("est=%d\n", current_estimate_of_N);
return 0;
}
u_int32_t idx, len;
void *data;
r = toku_gpma_insert(pma1, 1+strlen(strings[i]), (char*)strings[i], compare_strings, 0, rcall_a, (char*)strings[i], &idx);
//printf("est=%d\n", current_estimate_of_N);
assert(r==0);
r = toku_gpma_get_from_index(pma1, idx, &len, &data);
assert(r==0);
assert(len==1+strlen(strings[i]));
assert(data==strings[i]);
}
u_int32_t n_strings = i;
{
int do_realloc (u_int32_t len, void *data, void**ndata, void *extra) {
assert(extra==0);
assert(len=1+strlen(data));
*ndata = data; // Don't have to do anything
return 0;
}
int did_n_left=-1;
int rcall0 (u_int32_t nitems, u_int32_t *froms, u_int32_t *tos, struct gitem *items, u_int32_t old_N, u_int32_t new_N, void *extra) {
//printf("%s:%d old_N=%d new_N=%d\n", __FILE__, __LINE__, old_N, new_N);
assert(old_N==current_estimate_of_N);
current_estimate_of_N = new_N;
assert(extra==0);
u_int32_t j;
if (expect_n_left>=0) assert(nitems==(u_int32_t)expect_n_left);
did_n_left=nitems;
//printf("inner moved:"); for (j=0; j<nitems; j++) printf(" %d->%d", froms[j], tos[j]); printf("\n");
for (j=0; j<nitems; j++) {
if (expect_froms_left) assert(expect_froms_left[j]==froms[j]);
if (expect_tos_left) assert(expect_tos_left [j]==tos[j]);
assert(items[j].len==1+strlen(items[j].data));
if (j>0) {
assert(froms[j-1]<froms[j] && tos[j-1]<tos[j]);
assert(strcmp(items[j-1].data, items[j].data)<0);
}
}
return 0;
}
int rcall1 (u_int32_t nitems, u_int32_t *froms, u_int32_t *tos, struct gitem *items, u_int32_t old_N, u_int32_t new_N __attribute__((__unused__)), void *extra) {
assert(old_N==0);
//printf("new_N=%d\n", new_N);
assert(extra==0);
u_int32_t j;
if (expect_n_right>=0) assert(nitems==(u_int32_t)expect_n_right);
assert(did_n_left+nitems==n_strings);
//printf("outer moved:"); for (j=0; j<nitems; j++) printf(" %d->%d", froms[j], tos[j]); printf("\n");
for (j=0; j<nitems; j++) {
if (expect_froms_right) assert(expect_froms_right[j]==froms[j]);
if (expect_tos_right) assert(expect_tos_right [j]==tos[j]);
assert(items[j].len==1+strlen(items[j].data));
if (j>0) {
assert(froms[j-1]<froms[j] && tos[j-1]<tos[j]);
assert(strcmp(items[j-1].data, items[j].data)<0);
}
}
return 0;
}
r = toku_gpma_split(pma1, pma2, 1, do_realloc, rcall0, rcall1, 0);
toku_verify_gpma(pma1);
toku_verify_gpma(pma2);
assert (r==0);
char *prevval=0;
int foundem_left[]={-1,-1,-1,-1};
int foundem_right[]={-1,-1,-1,-1};
GPMA_ITERATE(pma1, idx, vallen, val,
({
assert(toku_gpma_valididx(pma2, idx));
if (prevval!=0) assert(strcmp(prevval,val)<0);
prevval=val;
unsigned int j;
for (j=0; j<n_strings; j++) {
if (strings[j]==val) { // The strings are EQ
assert(foundem_left[j]==-1);
foundem_left[j]=idx;
}
}
}));
GPMA_ITERATE(pma2, idx, vallen, val,
({
assert(toku_gpma_valididx(pma2, idx));
if (prevval!=0) assert(strcmp(prevval,val)<0);
prevval=val;
unsigned int j;
for (j=0; j<n_strings; j++) {
if (strings[j]==val) { // The strings are EQ
assert(foundem_right[j]==-1);
foundem_right[j]=idx;
}
}
}));
{
unsigned int j;
for (j=0; j<sizeof(strings)/sizeof(*strings); j++) assert(foundem_left[j]>=0 || foundem_right[j]>=0);
}
}
toku_gpma_free(&pma1, 0, 0);
toku_gpma_free(&pma2, 0, 0);
}
void test_split (void) {
{
const char *strings[]={"the", "quick", "brown", "fox", 0};
u_int32_t expect_froms_l[]={1,3};
u_int32_t expect_tos_l []={0,2};
u_int32_t expect_froms_r[]={5,7};
u_int32_t expect_tos_r []={0,2};
test_split_internal(strings,
2,
expect_froms_l,
expect_tos_l,
2,
expect_froms_r,
expect_tos_r);
}
}
int delete_free_callback (u_int32_t slotnum __attribute__((__unused__)),
u_int32_t deletelen,
void *deletedata,
void *extra) {
assert(deletelen==6);
assert(extra==deletedata);
//printf("Freeing %s\n", (char*)deletedata);
toku_free(deletedata);
return 0;
}
void test_delete_n (int N) {
GPMA pma;
int r = toku_gpma_create(&pma, 0);
assert(r==0);
int i;
char *strings[N];
for (i=0; i<N; i++) {
char str[6];
snprintf(str, 6, "%05d", i);
strings[i]=strdup(str);
r = toku_gpma_insert(pma, 6, strings[i], compare_strings, 0, rcall_ok, strings[i], 0);
assert(r==0);
}
for (i=0; i<N; i++) {
int number_of_strings_left = N-i;
int rval = random()%number_of_strings_left;
//printf("deleting %s\n", strings[rval]);
r = toku_gpma_delete_item(pma, 6, strings[rval],
compare_strings, 0,
delete_free_callback, strings[rval],
rcall_ok, 0);
strings[rval] = strings[number_of_strings_left-1];
}
toku_gpma_free(&pma, 0, 0);
}
void test_delete (void) {
test_delete_n(3);
test_delete_n(100);
test_delete_n(300);
}
static int compare_this_string (u_int32_t dlen, void *dval, void *extra) {
assert(dlen==1+strlen(dval));
return strcmp(dval, extra);
}
static void test_bes (void) {
GPMA pma;
int r = toku_gpma_create(&pma, 0);
assert(r==0);
enum { N = 257 };
char *strings[N];
int i;
for (i=0; i<N; i++) {
char str[4];
snprintf(str, 4, "%03d", i);
strings[i]=strdup(str);
r = toku_gpma_insert(pma, 1+strlen(strings[i]), strings[i], compare_strings, 0, rcall_ok, strings[i], 0);
assert(r==0);
}
for (i=0; i+1<N; i++) {
u_int32_t len,idx;
void *data;
r = toku_gpma_lookup_bessel(pma, compare_this_string, +1, strings[i], &len, &data, &idx);
assert(r==0);
assert(len==1+strlen(strings[i+1]));
assert(data==strings[i+1]);
}
for (i=1; i<N; i++) {
u_int32_t len,idx;
void *data;
r = toku_gpma_lookup_bessel(pma, compare_this_string, -1, strings[i], &len, &data, &idx);
assert(r==0);
assert(len==1+strlen(strings[i-1]));
assert(data==strings[i-1]);
}
for (i=0; i<N; i++) toku_free(strings[i]);
toku_gpma_free(&pma, 0, 0);
}
int main (int argc, const char *argv[]) {
int i;
for (i = 1; i < argc; i++) {
const char *arg = argv[i];
if (0 == strcmp(arg, "-v") || 0 == strcmp(arg, "--verbose"))
verbose = 1;
else if (0 == strcmp(arg, "-q") || 0 == strcmp(arg, "--quiet"))
verbose = 0;
}
test_create_and_free();
test_insert_A();
test_split();
test_delete();
test_bes();
toku_malloc_cleanup();
return 0;
}
newbrt/test-gpma-glassbox.c 0 → 100644
View file @ 83d416fa
// glass box tester looks inside gpma.c
#include "gpma.h"
#include "gpma-internal.h"
#include "memory.h"
#include "toku_assert.h"
#include "../include/db.h"
#include <errno.h>
#include <string.h>
int verbose=0;
static int count_frees=0;
static void free_callback (u_int32_t len __attribute__((__unused__)), void*freeme, void *extra) {
assert(extra==(void*)&verbose);
toku_free(freeme);
count_frees++;
}
static int compare_strings(u_int32_t alen, void *aval, u_int32_t blen, void *bval, void *extra __attribute__((__unused__))) {
assert(alen==strlen(aval)+1);
assert(blen==strlen(bval)+1);
return strcmp(aval, bval);
}
static void test_lg (void) {
assert(toku_lg(1)==0);
assert(toku_lg(2)==1);
assert(toku_lg(3)==2);
assert(toku_lg(4)==2);
assert(toku_lg(5)==3);
assert(toku_lg(7)==3);
assert(toku_lg(8)==3);
assert(toku_hyperceil(0)==1);
assert(toku_hyperceil(1)==1);
assert(toku_hyperceil(2)==2);
assert(toku_hyperceil(3)==4);
assert(toku_hyperceil(4)==4);
assert(toku_hyperceil(5)==8);
assert(toku_hyperceil(7)==8);
assert(toku_hyperceil(8)==8);
assert(toku_max_int(-1,2)==2);
assert(toku_max_int(2,2)==2);
assert(toku_max_int(2,3)==3);
assert(toku_max_int(3,2)==3);
}
static void test_create_sizes (void) {
GPMA pma;
int r = toku_gpma_create(&pma, 0);
assert(r==0);
toku_gpma_free(&pma, free_callback, &verbose);
r = toku_gpma_create(&pma, 3);
assert(r==EINVAL);
}
static void test_create_badmalloc (void) {
int i;
// There are two mallocs inside toku_gpma_create. Make sure that we test the possiblity that either could fail.
for (i=0; i<2; i++) {
int killarray[2]={i,-1};
toku_dead_mallocs=killarray;
toku_malloc_counter=0;
int r;
GPMA pma;
r = toku_gpma_create(&pma, 0);
assert(r==ENOMEM);
toku_dead_mallocs=0; // killarray is no longer valid, so get rid of the ref to it.
}
}
static void test_find_index (void) {
int r;
GPMA pma;
r = toku_gpma_create(&pma, 16);
assert(r==0);
assert(toku_gpma_index_limit(pma)==16);
int found;
{
u_int32_t idx;
idx = toku_gpma_find_index(pma, 6, "hello", compare_strings, 0, &found);
assert(found==0);
assert(idx==0);
void *k;
toku_gpma_set_at_index(pma, 3, 6, k=toku_strdup("hello"));
assert(pma->items[3].len = 6);
assert(pma->items[3].data == k);
idx = toku_gpma_find_index(pma, 6, "hello", compare_strings, 0, &found);
assert(found);
assert(idx==3);
idx = toku_gpma_find_index(pma, 2, "a", compare_strings, 0, &found);
assert(!found);
assert(idx==0);
idx = toku_gpma_find_index(pma, 2, "z", compare_strings, 0, &found);
assert(!found);
assert(idx==4);
}
{
u_int32_t resultlen; void*resultdata;
int bes (u_int32_t dlen __attribute__((__unused__)), void *dval, void *extra __attribute__((__unused__))) {
return strcmp(dval, "a"); // This will return 1 for everything. For dir<=0 we'll have DB_NOTFOUND, for dir>0 we'll have "fello"
}
r = toku_gpma_lookup_bessel(pma, bes, -1, 0, &resultlen, &resultdata, 0);
assert(r==DB_NOTFOUND);
r = toku_gpma_lookup_bessel(pma, bes, 0, 0, &resultlen, &resultdata, 0);
assert(r==DB_NOTFOUND);
r = toku_gpma_lookup_bessel(pma, bes, +1, 0, &resultlen, &resultdata, 0);
assert(r==0);
assert(strcmp(resultdata, "hello")==0);
}
{
u_int32_t resultlen; void*resultdata;
int bes (u_int32_t dlen __attribute__((__unused__)), void *dval, void *extra __attribute__((__unused__))) {
return strcmp(dval, "z"); // This will return -1 for everything. For dir>=0 we'll have DB_NOTFOUND, for dir<0 we'll have "hello"
}
u_int32_t idx;
r = toku_gpma_lookup_bessel(pma, bes, -1, 0, &resultlen, &resultdata, &idx); // find the rightmost thing
assert(r==0);
assert(strcmp(resultdata, "hello")==0);
{
u_int32_t altlen; void*altdata;
r = toku_gpma_get_from_index(pma, idx, &altlen, &altdata);
assert(r==0);
assert(altlen==resultlen);
assert(altdata==resultdata);
}
r = toku_gpma_lookup_bessel(pma, bes, 0, 0, &resultlen, &resultdata, 0);
assert(r==DB_NOTFOUND);
r = toku_gpma_lookup_bessel(pma, bes, +1, 0, &resultlen, &resultdata, 0);
assert(r==DB_NOTFOUND);
}
count_frees=0;
toku_gpma_free(&pma, free_callback, &verbose);
assert(count_frees==1);
}
struct rcall_0_pair {
int idx;
int use_index_case;
};
static int rcall_0 (u_int32_t nitems, u_int32_t *froms, u_int32_t *tos, struct gitem *items, u_int32_t old_N, u_int32_t new_N, void *extra) {
assert(old_N==16);
assert(new_N==16);
struct rcall_0_pair *p = extra;
assert(nitems==3);
u_int32_t i;
for (i=0; i<3; i++) assert(froms[i]==i);
for (i=0; i<2; i++) { assert(tos[i]<tos[i+1]); }
assert(strcmp(items[0].data,"a")==0);
assert(strcmp(items[1].data,"b")==0);
switch (p->use_index_case) {
case 1:
switch (p->idx) {
case 0: assert(tos[0]==5); assert(tos[1]==9); assert(tos[2]==13); break;
case 1: assert(tos[0]==1); assert(tos[1]==9); assert(tos[2]==13); break;
case 2: assert(tos[0]==1); assert(tos[1]==5); assert(tos[2]==13); break;
case 3: assert(tos[0]==1); assert(tos[1]==5); assert(tos[2]== 9); break;
case 4: assert(tos[0]==1); assert(tos[1]==5); assert(tos[2]== 9); break;
default: assert(0);
}
break;
case 0:
assert(tos[0]==1); assert(tos[1]==6); assert(tos[2]==11);
break;
default: assert(0);
}
return 0;
}
static void test_smooth_region (void) {
int r;
GPMA pma;
int use_index_case;
for (use_index_case = 0; use_index_case<2; use_index_case++) {
int malloc_failnum;
for (malloc_failnum=0; malloc_failnum<4; malloc_failnum++) {
u_int32_t idx;
for (idx=0; idx<4; idx++) {
r = toku_gpma_create(&pma, 16);
assert(r==0);
int j;
for (j=0; j<3; j++) {
char str[]={'a'+j, 0};
pma->items[j].len = 2;
pma->items[j].data = toku_strdup(str);
}
toku_malloc_counter=0;
int killarray[2]={malloc_failnum,-1};
if (malloc_failnum<3) {
toku_dead_mallocs=killarray;
}
u_int32_t newidx;
struct rcall_0_pair r0 = {idx,use_index_case};
r = toku_gpma_smooth_region(pma, 0, 16, 3, idx, use_index_case ? &newidx : 0, rcall_0, &r0, pma->N);
if (malloc_failnum<3) assert(r==ENOMEM); else assert(r==0);
toku_dead_mallocs=0;
count_frees=0;
toku_gpma_free(&pma, free_callback, &verbose);
assert(count_frees==3);
}
}
}
}
static int rcall_1 (u_int32_t nitems, u_int32_t *froms, u_int32_t *tos, struct gitem *items, u_int32_t old_N, u_int32_t new_N, void *extra __attribute__((__unused__))) {
u_int32_t i;
assert(old_N==8);
assert(new_N==8);
for (i=0; i<nitems; i++) assert(froms[i]==i);
for (i=0; i<nitems-1; i++) { assert(tos[i]<tos[i+1]); }
assert(tos[0]==3); assert(tos[1]==6);
assert(strcmp(items[0].data,"a")==0);
assert(strcmp(items[1].data,"b")==0);
return 0;
}
static void test_make_space_at_up (void) {
int malloc_failnum;
for (malloc_failnum=0; malloc_failnum<4; malloc_failnum++) {
int r;
GPMA pma;
r = toku_gpma_create(&pma, 8);
assert(r==0);
assert(toku_gpma_n_entries(pma)==0);
int j;
for (j=0; j<2; j++) {
char str[]={'a'+j, 0};
pma->items[j].len = 2;
pma->items[j].data = toku_strdup(str);
}
u_int32_t newidx;
toku_malloc_counter=0;
int killarray[2]={malloc_failnum,-1};
if (malloc_failnum<3) {
toku_dead_mallocs=killarray;
}
r = toku_make_space_at(pma, 0, &newidx, rcall_1, 0);
toku_dead_mallocs=0;
if (malloc_failnum<3) assert(r==ENOMEM);
else {
assert(r==0);
assert(newidx==1);
assert(strcmp(pma->items[3].data, "a")==0);
assert(strcmp(pma->items[6].data, "b")==0);
}
count_frees=0;
toku_gpma_free(&pma, free_callback, &verbose);
assert(count_frees==2);
}
}
static int rcall_2 (u_int32_t nitems, u_int32_t *froms, u_int32_t *tos, struct gitem *items, u_int32_t old_N, u_int32_t new_N, void *extra __attribute__((__unused__))) {
assert(old_N==8);
assert(new_N==8);
assert(nitems==2);
assert(froms[0]==6); assert(froms[1]==7);
assert(tos[0]==1); assert(tos[1]==6);
assert(strcmp(items[0].data,"a")==0);
assert(strcmp(items[1].data,"b")==0);
return 0;
}
static void test_make_space_at_down (void) {
int r;
GPMA pma;
int size=8;
r = toku_gpma_create(&pma, size);
assert(r==0);
assert(toku_gpma_n_entries(pma)==0);
int j;
for (j=0; j<2; j++) {
char str[]={'a'+j, 0};
pma->items[size-2+j].len = 2;
pma->items[size-2+j].data = toku_strdup(str);
}
u_int32_t newidx;
r = toku_make_space_at(pma, 7, &newidx, rcall_2, 0);
assert(r==0);
assert(newidx==3);
assert(strcmp(pma->items[1].data, "a")==0);
assert(strcmp(pma->items[6].data, "b")==0);
count_frees=0;
toku_gpma_free(&pma, free_callback, &verbose);
assert(count_frees==2);
}
static int rcall_3 (u_int32_t nitems, u_int32_t *froms, u_int32_t *tos, struct gitem *items, u_int32_t old_N, u_int32_t new_N, void *extra __attribute__((__unused__))) {
assert(old_N==8);
assert(new_N==8);
assert(nitems==2);
assert(froms[0]==6); assert(froms[1]==7);
assert(tos[0]==1); assert(tos[1]==3);
assert(strcmp(items[0].data,"a")==0);
assert(strcmp(items[1].data,"b")==0);
return 0;
}
static void test_make_space_at_down_end (void) {
int no_rcall;
for (no_rcall=0; no_rcall<2; no_rcall++) {
int r;
GPMA pma;
int size=8;
r = toku_gpma_create(&pma, size);
assert(r==0);
assert(toku_gpma_n_entries(pma)==0);
int j;
for (j=0; j<2; j++) {
char str[]={'a'+j, 0};
pma->items[size-2+j].len = 2;
pma->items[size-2+j].data = toku_strdup(str);
}
u_int32_t newidx;
r = toku_make_space_at(pma, 8, &newidx, no_rcall ? 0 : rcall_3, 0);
assert(r==0);
assert(newidx==6);
assert(strcmp(pma->items[1].data, "a")==0);
assert(strcmp(pma->items[3].data, "b")==0);
count_frees=0;
toku_gpma_free(&pma, free_callback, &verbose);
assert(count_frees==2);
}
}
static int rcall_ok (u_int32_t nitems __attribute__((__unused__)), u_int32_t *froms __attribute__((__unused__)), u_int32_t *tos __attribute__((__unused__)), struct gitem *items __attribute__((__unused__)), u_int32_t old_N __attribute__((__unused__)), u_int32_t new_N __attribute__((__unused__)), void *extra __attribute__((__unused__))) {
return 0;
}
static __attribute__((__noreturn__)) int rcall_never (u_int32_t nitems __attribute__((__unused__)), u_int32_t *froms __attribute__((__unused__)), u_int32_t *tos __attribute__((__unused__)), struct gitem *items __attribute__((__unused__)), void *extra __attribute__((__unused__))) {
abort();
}
static void test_insert_malloc_fails (void) {
int malloc_failnum;
int killarray[2]={-1,-1};
for (malloc_failnum=0; malloc_failnum<8; malloc_failnum++) {
toku_dead_mallocs=killarray;
toku_dead_mallocs[0]=-1;
int n_inserted=0;
int r;
GPMA pma;
r = toku_gpma_create(&pma, 0);
assert(r==0);
toku_malloc_counter=0;
r = toku_gpma_insert(pma, 6, strdup("hello"),
compare_strings, 0, rcall_ok, "hello", 0);
assert(r==0);
assert(toku_gpma_n_entries(pma)==1);
n_inserted++;
toku_malloc_counter=0;
if (1<=malloc_failnum && malloc_failnum<5) {
toku_dead_mallocs[0]=malloc_failnum-1;
}
void *k;
r = toku_gpma_insert(pma, 6, k=strdup("gello"),
compare_strings, 0, rcall_ok, "gello", 0);
if (1<=malloc_failnum && malloc_failnum<4) {
assert(r==ENOMEM);
toku_free(k);
assert(toku_gpma_n_entries(pma)==1);
int countem=0;
u_int32_t i;
for (i=0; i<pma->N; i++) {
if (pma->items[i].data) {
countem++;
assert(strcmp("hello", pma->items[i].data)==0);
}
}
assert(countem==1);
} else {
assert(r==0);
assert(toku_gpma_n_entries(pma)==2);
n_inserted++;
r = toku_gpma_insert(pma, 6, k=strdup("fello"),
compare_strings, 0, rcall_ok, "fello", 0);
assert(pma->N==4);
n_inserted++;
toku_malloc_counter=0;
assert(pma->N==4);
if (4<=malloc_failnum && malloc_failnum<8) {
toku_dead_mallocs=killarray;
toku_dead_mallocs[0]=malloc_failnum-4;
}
r = toku_gpma_insert(pma, 6, k=strdup("fellp"),
compare_strings, 0, rcall_ok, "fellp", 0);
if (4<=malloc_failnum && malloc_failnum<8) {
assert(r==ENOMEM);
toku_free(k);
assert(pma->N==4);
} else {
assert(r==0);
n_inserted++;
assert(pma->N==8);
}
}
count_frees=0;
toku_gpma_free(&pma, free_callback, &verbose);
assert(count_frees==n_inserted);
}
toku_dead_mallocs=0;
}
static void test_distribute (void) {
GPMA pma;
int r = toku_gpma_create(&pma, 16);
assert(r==0);
struct gitem items[4] = {{2,"a"},{2,"b"},{2,"c"},{2,"d"}};
u_int32_t tos[4];
toku_gpma_distribute(pma, 0, 16, 4, items, tos);
toku_gpma_free(&pma, 0, 0);
}
static int rcall_4a (u_int32_t nitems, u_int32_t *froms, u_int32_t *tos, struct gitem *items, u_int32_t old_N, u_int32_t new_N, void *extra __attribute__((__unused__))) {
assert(old_N==16);
assert(new_N==8);
assert(nitems==3);
assert(froms[0]==0); assert(tos[0]==0);
assert(froms[1]==1); assert(tos[1]==3);
assert(froms[2]==2); assert(tos[2]==6);
assert(strcmp(items[0].data,"a")==0);
assert(strcmp(items[1].data,"b")==0);
assert(strcmp(items[2].data,"c")==0);
return 0;
}
static int rcall_4b (u_int32_t nitems, u_int32_t *froms, u_int32_t *tos, struct gitem *items, u_int32_t old_N, u_int32_t new_N, void *extra __attribute__((__unused__))) {
assert(old_N==8);
assert(new_N==8);
assert(nitems==3);
assert(froms[0]==1); assert(tos[0]==1);
assert(froms[1]==3); assert(tos[1]==3);
assert(froms[2]==6); assert(tos[2]==6);
assert(strcmp(items[0].data,"a")==0);
assert(strcmp(items[1].data,"b")==0);
assert(strcmp(items[2].data,"c")==0);
return 0;
}
static void test_smooth_deleted (void) {
GPMA pma;
int r = toku_gpma_create(&pma, 16);
assert(r==0);
pma->items[0] = (struct gitem){2, "a"};
pma->items[1] = (struct gitem){2, "b"};
pma->items[2] = (struct gitem){2, "c"};
pma->n_items_present=3;
r = toku_smooth_deleted_region(pma, 3, 3, rcall_4a, 0);
assert(r==0);
r = toku_smooth_deleted_region(pma, 4, 4, rcall_4b, 0);
assert(r==0);
toku_gpma_free(&pma, 0, 0);
r = toku_gpma_create(&pma, 16);
assert(r==0);
pma->items[4] = (struct gitem){2, "a"};
pma->n_items_present = 1;
r = toku_smooth_deleted_region(pma, 15, 15, rcall_ok, 0);
assert(pma->N==8);
int i;
for (i=0; i<8; i++) {
if (i==0) assert(pma->items[i].data && 0==strcmp(pma->items[i].data,"a"));
else assert(!pma->items[i].data);
}
toku_gpma_free(&pma, 0, 0);
r = toku_gpma_create(&pma, 16);
assert(r==0);
pma->items[7] = (struct gitem){2, "a"};
pma->n_items_present = 1;
r = toku_smooth_deleted_region(pma, 0, 0, rcall_ok, 0);
assert(pma->N==8);
for (i=0; i<8; i++) {
if (i==0) assert(pma->items[i].data && 0==strcmp(pma->items[i].data,"a"));
else assert(!pma->items[i].data);
}
toku_gpma_free(&pma, 0, 0);
r = toku_gpma_create(&pma, 32);
assert(r==0);
r = toku_smooth_deleted_region(pma, 6, 12, 0, 0);
assert(r==0);
toku_gpma_free(&pma, 0, 0);
}
int bes_first (u_int32_t dlen, void *dval, void *extra) {
assert(dlen==2);
assert(extra==0);
char *val=dval;
if (val[0]=='a') return -1;
else return 1;
}
// Test looking up something with direction = -1, where every element in the array return +1, except for the first
// So we are supposed to return the largest index that has negative value, which is index 0.
static void test_lookup_first (void) {
GPMA pma;
int r = toku_gpma_create(&pma, 4);
assert(r==0);
pma->items[0] = (struct gitem){2, "a"};
pma->items[1] = (struct gitem){2, "b"};
pma->items[2] = (struct gitem){2, "c"};
pma->items[3] = (struct gitem){2, "d"};
pma->n_items_present = 3;
int found;
u_int32_t idx = toku_gpma_find_index_bes(pma, bes_first, -1, 0, &found);
// We expect the answer to be found, and we expect index to be 0.
assert(found);
assert(idx==0);
toku_gpma_free(&pma, 0, 0);
}
int bes_last (u_int32_t dlen, void *dval, void *extra) {
assert(dlen==2);
assert(extra==0);
char *val=dval;
if (val[0]=='d') return 1;
else return -1;
}
static void test_lookup_last (void) {
GPMA pma;
int r = toku_gpma_create(&pma, 4);
assert(r==0);
pma->items[0] = (struct gitem){2, "a"};
pma->items[1] = (struct gitem){2, "b"};
pma->items[2] = (struct gitem){2, "c"};
pma->items[3] = (struct gitem){2, "d"};
pma->n_items_present = 3;
int found;
u_int32_t idx = toku_gpma_find_index_bes(pma, bes_first, +1, 0, &found);
// We expect the answer to be found, and we expect index to be 1.
assert(found);
assert(idx==1);
toku_gpma_free(&pma, 0, 0);
}
int main (int argc, const char *argv[]) {
int i;
for (i = 1; i < argc; i++) {
const char *arg = argv[i];
if (0 == strcmp(arg, "-v") || 0 == strcmp(arg, "--verbose"))
verbose = 1;
else if (0 == strcmp(arg, "-q") || 0 == strcmp(arg, "--quiet"))
verbose = 0;
}
test_lg();
test_create_sizes();
test_create_badmalloc();
test_find_index();
test_smooth_region();
test_make_space_at_up();
test_make_space_at_down();
test_make_space_at_down_end();
test_insert_malloc_fails();
test_distribute();
toku_malloc_cleanup();
test_smooth_deleted();
test_lookup_last();
test_lookup_first();
return 0;
}
newbrt/toku_assert.c
View file @ 83d416fa
......@@ -2,7 +2,7 @@
#include <stdio.h>
#include <stdlib.h>
void toku_do_assert(long expr,const char* expr_as_string,const char *function,const char*file,int line) {
void toku_do_assert(int expr,const char* expr_as_string,const char *function,const char*file,int line) {
if (expr==0) {
fprintf(stderr, "%s:%d %s: Assertion `%s' failed\n", file,line,function,expr_as_string);
abort();
......
newbrt/toku_assert.h
View file @ 83d416fa
......@@ -8,11 +8,11 @@
#error NDEBUG should not be set
#endif
void toku_do_assert(long,const char*/*expr_as_string*/,const char */*fun*/,const char*/*file*/,int/*line*/);
void toku_do_assert(int,const char*/*expr_as_string*/,const char */*fun*/,const char*/*file*/,int/*line*/);
#ifndef FAST_ASSERT
#define assert(expr) toku_do_assert((long)(expr), #expr, __FUNCTION__, __FILE__, __LINE__)
#define assert(expr) toku_do_assert((expr) != 0, #expr, __FUNCTION__, __FILE__, __LINE__)
#else
#define assert(expr) ({ long __assert_expr = (int)(expr); if (__assert_expr==0) toku_do_assert(0, #expr, __FUNCTION__, __FILE__, __LINE__); })
#define assert(expr) ({ if ((expr)==0) toku_do_assert(0, #expr, __FUNCTION__, __FILE__, __LINE__); })
#endif
#endif
newbrt/wbuf.h
View file @ 83d416fa
......@@ -34,7 +34,7 @@ static inline void wbuf_init (struct wbuf *w, void *buf, DISKOFF size) {
w->size=size;
w->ndone=0;
#ifdef CRC_INCR
w->crc32 = toku_crc32(0L, Z_NULL, 0);
w->crc32 = toku_crc32(toku_null_crc, Z_NULL, 0);
#endif
}
......@@ -47,7 +47,7 @@ static inline void wbuf_char (struct wbuf *w, int ch) {
#endif
}
static void wbuf_int (struct wbuf *w, unsigned int i) {
static void wbuf_int (struct wbuf *w, int32_t i) {
#if 0
wbuf_char(w, i>>24);
wbuf_char(w, i>>16);
......@@ -65,14 +65,17 @@ static void wbuf_int (struct wbuf *w, unsigned int i) {
w->ndone += 4;
#endif
}
static void wbuf_uint (struct wbuf *w, u_int32_t i) {
wbuf_int(w, (int32_t)i);
}
static inline void wbuf_literal_bytes(struct wbuf *w, bytevec bytes_bv, int nbytes) {
static inline void wbuf_literal_bytes(struct wbuf *w, bytevec bytes_bv, u_int32_t nbytes) {
const unsigned char *bytes=bytes_bv;
#if 0
{ int i; for (i=0; i<nbytes; i++) wbuf_char(w, bytes[i]); }
#else
assert(w->ndone + nbytes <= w->size);
memcpy(w->buf + w->ndone, bytes, nbytes);
memcpy(w->buf + w->ndone, bytes, (size_t)nbytes);
#ifdef CRC_INCR
w->crc32 = toku_crc32(w->crc32, &w->buf[w->ndone], nbytes);
#endif
......@@ -81,14 +84,14 @@ static inline void wbuf_literal_bytes(struct wbuf *w, bytevec bytes_bv, int nbyt
}
static void wbuf_bytes (struct wbuf *w, bytevec bytes_bv, int nbytes) {
wbuf_int(w, nbytes);
static void wbuf_bytes (struct wbuf *w, bytevec bytes_bv, u_int32_t nbytes) {
wbuf_uint(w, nbytes);
wbuf_literal_bytes(w, bytes_bv, nbytes);
}
static void wbuf_ulonglong (struct wbuf *w, unsigned long long ull) {
wbuf_int(w, ull>>32);
wbuf_int(w, ull&0xFFFFFFFF);
static void wbuf_ulonglong (struct wbuf *w, u_int64_t ull) {
wbuf_uint(w, (u_int32_t)(ull>>32));
wbuf_uint(w, (u_int32_t)(ull&0xFFFFFFFF));
}
static inline void wbuf_BYTESTRING (struct wbuf *w, BYTESTRING v) {
......@@ -100,11 +103,11 @@ static inline void wbuf_u_int8_t (struct wbuf *w, u_int8_t v) {
}
static inline void wbuf_u_int32_t (struct wbuf *w, u_int32_t v) {
wbuf_int(w, v);
wbuf_uint(w, v);
}
static inline void wbuf_DISKOFF (struct wbuf *w, DISKOFF off) {
wbuf_ulonglong(w, off);
wbuf_ulonglong(w, (u_int64_t)off);
}
static inline void wbuf_TXNID (struct wbuf *w, TXNID tid) {
......@@ -116,13 +119,13 @@ static inline void wbuf_LSN (struct wbuf *w, LSN lsn) {
}
static inline void wbuf_FILENUM (struct wbuf *w, FILENUM fileid) {
wbuf_int(w, fileid.fileid);
wbuf_uint(w, fileid.fileid);
}
static inline void wbuf_LOGGEDBRTHEADER (struct wbuf *w, LOGGEDBRTHEADER h) {
wbuf_int(w, h.size);
wbuf_int(w, h.flags);
wbuf_int(w, h.nodesize);
wbuf_uint(w, h.size);
wbuf_uint(w, h.flags);
wbuf_uint(w, h.nodesize);
wbuf_DISKOFF(w, h.freelist);
wbuf_DISKOFF(w, h.unused_memory);
wbuf_int(w, h.n_named_roots);
......@@ -132,17 +135,17 @@ static inline void wbuf_LOGGEDBRTHEADER (struct wbuf *w, LOGGEDBRTHEADER h) {
int i;
for (i=0; i<h.n_named_roots; i++) {
wbuf_DISKOFF(w, h.u.many.roots[i]);
wbuf_bytes (w, h.u.many.names[i], 1+strlen(h.u.many.names[i]));
wbuf_bytes (w, h.u.many.names[i], (u_int32_t)(1+strlen(h.u.many.names[i])));
}
}
}
static inline void wbuf_INTPAIRARRAY (struct wbuf *w, INTPAIRARRAY h) {
u_int32_t i;
wbuf_int(w, h.size);
wbuf_uint(w, h.size);
for (i=0; i<h.size; i++) {
wbuf_int(w, h.array[i].a);
wbuf_int(w, h.array[i].b);
wbuf_uint(w, h.array[i].a);
wbuf_uint(w, h.array[i].b);
}
}
......
newbrt/ybt.c
View file @ 83d416fa
......@@ -20,12 +20,12 @@ int toku_dbt_set_value (DBT *ybt, bytevec val, ITEMLEN vallen, void **staticptrp
int r = ENOSYS;
if (ybt->flags==DB_DBT_MALLOC) {
domalloc:
ybt->data = toku_malloc(vallen);
ybt->data = toku_malloc((size_t)vallen);
if (!ybt->data && vallen > 0) { r = errno; goto cleanup; }
} else if (ybt->flags==DB_DBT_REALLOC) {
if (ybt->data==0) goto domalloc;
/* tmp is used to prevent a memory leak if realloc fails */
void* tmp = toku_realloc(ybt->data, vallen);
void* tmp = toku_realloc(ybt->data, (size_t)vallen);
if (!tmp && vallen > 0) { r = errno; goto cleanup; }
ybt->data = tmp;
} else if (ybt->flags==DB_DBT_USERMEM) {
......@@ -36,10 +36,9 @@ int toku_dbt_set_value (DBT *ybt, bytevec val, ITEMLEN vallen, void **staticptrp
void *staticptr=*staticptrp;
//void *old=staticptr;
if (staticptr==0) {
staticptr = toku_malloc(vallen);
staticptr = toku_malloc((size_t)vallen);
if (!staticptr && vallen > 0) { r = errno; goto cleanup; }
}
else {
} else {
/* tmp is used to prevent a memory leak if realloc fails */
void* tmp = toku_realloc(staticptr, vallen);
if (!tmp && vallen > 0) { r = errno; goto cleanup; }
......@@ -51,7 +50,7 @@ int toku_dbt_set_value (DBT *ybt, bytevec val, ITEMLEN vallen, void **staticptrp
}
ybt->size = vallen;
if (ybt->size>0) {
memcpy(ybt->data, val, vallen);
memcpy(ybt->data, val, (size_t)vallen);
}
r = 0;
cleanup:
......
newbrt/yerror.h
View file @ 83d416fa
......@@ -5,10 +5,4 @@
enum pma_errors { BRT_OK=0, BRT_ALREADY_THERE = -2, BRT_KEYEMPTY=-3 };
enum typ_tag { TYP_BRTNODE = 0xdead0001,
TYP_CACHETABLE, TYP_PAIR, /* for cachetables */
TYP_PMA,
TYP_TOKULOGGER,
TYP_TOKUTXN
};
#endif
src/Makefile
View file @ 83d416fa
......@@ -27,28 +27,36 @@ SHARED=-shared $(EXPORTMAP)
RPATHNAME=
endif
build:
cd range_tree;$(MAKE) build
cd lock_tree;$(MAKE) build
$(MAKE) $(LIBRARY) $(LIBNAME).a
cd tests;$(MAKE) build
.PHONY: install
install: locktree $(LIBRARY) $(LIBNAME).a
install: $(LIBRARY) $(LIBNAME).a
cp $(LIBRARY) ../lib/
cp $(LIBNAME).a ../lib
locktree:
cd lock_tree && make
check: $(LIBRARY)
check_globals: $(LIBRARY)
python tokuglobals.py $(LIBRARY)
check_tests:
cd tests;$(MAKE) check
check: $(LIBRARY) check_globals check_tests
strip: $(LIBRARY)
strip $(LIBRARY)
clean:
rm -rf $(LIBRARY) $(LIBNAME).a *.o *.gcno *.gcda *.gcov
cd tests && make clean
cd lock_tree && make clean
cd tests && $(MAKE) clean
cd lock_tree && $(MAKE) clean
cd range_tree && $(MAKE) clean
ydb.o: ../include/db.h ../newbrt/cachetable.h ../newbrt/brt.h ../newbrt/log.c
DBBINS = ydb.o errors.o elocks.o ../newbrt/brt.o ../newbrt/brt-serialize.o ../newbrt/brt-verify.o ../newbrt/cachetable.o ../newbrt/fifo.o ../newbrt/key.o ../newbrt/memory.o ../newbrt/mempool.o ../newbrt/pma.o ../newbrt/ybt.o ../newbrt/primes.o ../newbrt/log.o ../newbrt/fingerprint.o ../newbrt/log_code.o ../newbrt/roll.o ../newbrt/toku_assert.o ../newbrt/recover.o
DBBINS = ydb.o errors.o elocks.o ../newbrt/brt.o ../newbrt/brt-serialize.o ../newbrt/brt-verify.o ../newbrt/cachetable.o ../newbrt/fifo.o ../newbrt/key.o ../newbrt/memory.o ../newbrt/mempool.o ../newbrt/gpma.o ../newbrt/ybt.o ../newbrt/primes.o ../newbrt/log.o ../newbrt/fingerprint.o ../newbrt/log_code.o ../newbrt/roll.o ../newbrt/toku_assert.o ../newbrt/recover.o
RANGETREE_BINS = range_tree/rangetree.o range_tree/tokuredblack.o
LOCKTREE_BINS = lock_tree/locktree.o lock_tree/rth.o lock_tree/lth.o lock_tree/idlth.o lock_tree/db_id.o $(RANGETREE_BINS)
......
src/lock_tree/Makefile
View file @ 83d416fa
......@@ -33,14 +33,15 @@ LT_LOG = $(LT_OVERLAP)
LT_BINS=$(LT_OVERLAP) $(LT_NOOVERLAP) locktree.o
BINS=rth.o lth.o idlth.o db_id.o
.PHONY: install logformat range_tree
install: range_tree $(BINS) $(LT_BINS)
.PHONY: install logformat
range_tree:
cd ../range_tree && make
build: $(LT_BINS)
cd tests; $(MAKE) build
check:
cd tests; $(MAKE) check
clean:
cd ../range_tree && make clean
rm -rf $(BINS) $(LT_BINS)
rm -rf *.gcno *.gcda *.gcov
cd tests && make clean
......
src/lock_tree/locktree.h
View file @ 83d416fa
......@@ -18,7 +18,6 @@
we defer to the db panic handler. Pass in another parameter to do this.
*/
#include <assert.h>
#include <db.h>
#include <brttypes.h>
#include <rangetree.h>
......@@ -26,6 +25,8 @@
#include <rth.h>
#include <idlth.h>
#include "toku_assert.h"
/** Errors returned by lock trees */
typedef enum {
TOKU_LT_INCONSISTENT=-1, /**< The member data are in an inconsistent
......
src/lock_tree/tests/Makefile
View file @ 83d416fa
......@@ -30,7 +30,7 @@ TLOG_TESTS = $(patsubst %.c,%.tlog,$(SRCS))
LIN_TESTS = $(patsubst %.c,%.lin,$(SRCS))
TLIN_TESTS = $(patsubst %.c,%.tlin,$(SRCS))
ALL_TESTS = $(LIN_TESTS) $(TLIN_TESTS) $(TLOG_TESTS) $(LOG_TESTS)
ALL_TESTS = $(LIN_TESTS) $(TLIN_TESTS) $(TLOG_TESTS) #$(LOG_TESTS)
RUN_LOG_TESTS = $(patsubst %.log,%.logrun,$(LOG_TESTS))
RUN_TLOG_TESTS = $(patsubst %.tlog,%.tlogrun,$(TLOG_TESTS))
......@@ -40,9 +40,9 @@ RUN_ALL_TESTS = $(RUN_LIN_TESTS) $(RUN_TLIN_TESTS) $(RUN_TLOG_TESTS) $(RUN_LOG_
.PHONY: check tests default all check.lin check.tlin check.log check.tlog tests.lin tests.tlin tests.tlog tests.log
default: check.tlog
default: build
all: make_libs $(ALL_TESTS)
build all: $(ALL_TESTS)
check: check.lin check.tlin check.tlog #check.log
@ echo $@ ok
......@@ -50,19 +50,19 @@ tests: tests.lin tests.tlin tests.tlog #tests.log
@ echo $@ ok
tests.lin: make_libs $(LIN_TESTS)
@ echo $@ ok
check.lin: make_libs $(RUN_LIN_TESTS)
check.lin: $(RUN_LIN_TESTS)
@ echo $@ ok
tests.tlin: make_libs $(TLIN_TESTS)
tests.tlin: $(TLIN_TESTS)
@ echo $@ ok
check.tlin: make_libs $(RUN_TLIN_TESTS)
check.tlin: $(RUN_TLIN_TESTS)
@ echo $@ ok
tests.tlog: make_libs $(TLOG_TESTS)
tests.tlog: $(TLOG_TESTS)
@ echo $@ ok
check.tlog: make_libs $(RUN_TLOG_TESTS)
check.tlog: $(RUN_TLOG_TESTS)
@ echo $@ ok
tests.log: make_libs $(LOG_TESTS)
tests.log: $(LOG_TESTS)
@ echo $@ ok
check.log: make_libs $(RUN_LOG_TESTS)
check.log: $(RUN_LOG_TESTS)
@ echo $@ ok
# Need these rule so that Make knows about all the file names
......@@ -99,8 +99,7 @@ endif
%.tlogrun: %.tlog
$(MAYBEATSIGN) $(VGRIND) ./$< $(VERBVERBOSE)
libs:
cd .. && make
NEWBRT_BINS = ../../../newbrt/toku_assert.o
RT_LINEAR_BINS = ../../range_tree/linear.o
RT_TLINEAR_BINS = ../../range_tree/linear.o
......@@ -115,19 +114,17 @@ LT_TLINEAR = $(LT_NOOVERLAP) $(LT_BINS) $(RT_TLINEAR_BINS)
LT_TLOG = $(LT_NOOVERLAP) $(LT_BINS) $(RT_TLOG_BINS)
LT_LOG = $(LT_OVERLAP) $(LT_BINS) $(RT_LOG_BINS)
foo:
echo ../locktree.h test.h $(LT_LINEAR)
%.lin: %.c ../locktree.h test.h $(LT_LINEAR)
cc -DDIR=\"dir.$<.lin\" $(CPPFLAGS) $(CFLAGS) $< -o $@ $(LT_LINEAR)
cc -DDIR=\"dir.$<.lin\" $(CPPFLAGS) $(CFLAGS) $< -o $@ $(LT_LINEAR) $(NEWBRT_BINS)
%.tlin: %.c ../locktree.h test.h $(LT_TLINEAR)
cc -DDIR=\"dir.$<.tlin\" $(CPPFLAGS) $(CFLAGS) $< -o $@ $(LT_TLINEAR) -DTOKU_RT_NOOVERLAPS
cc -DDIR=\"dir.$<.tlin\" $(CPPFLAGS) $(CFLAGS) $< -o $@ $(LT_TLINEAR) $(NEWBRT_BINS) -DTOKU_RT_NOOVERLAPS
%.tlog: %.c ../locktree.h test.h $(LT_TLOG)
cc -DDIR=\"dir.$<.tlog\" $(CPPFLAGS) $(CFLAGS) $< -o $@ $(LT_TLOG) -DTOKU_RT_NOOVERLAPS
cc -DDIR=\"dir.$<.tlog\" $(CPPFLAGS) $(CFLAGS) $< -o $@ $(LT_TLOG) $(NEWBRT_BINS) -DTOKU_RT_NOOVERLAPS
%.log: %.c ../locktree.h test.h $(LT_LOG)
cc -DDIR=\"dir.$<.log\" $(CPPFLAGS) $(CFLAGS) $< -o $@ $(LT_LOG)
.PHONY: make_libs
make_libs:
cd .. && make
cc -DDIR=\"dir.$<.log\" $(CPPFLAGS) $(CFLAGS) $< -o $@ $(LT_LOG) $(NEWBRT_BINS)
clean:
rm -f $(ALL_TESTS) *.o *.gcno *.gcda *.gcov
......
src/range_tree/Makefile
View file @ 83d416fa
......@@ -24,8 +24,16 @@ ifneq ($(OSX),)
CFLAGS+=-fno-common
endif
BINS = linear.o log_nooverlap.o tokuredblack.o rangetree.o #log.o
build: $(BINS)
cd tests;$(MAKE) build
check:
cd tests;$(MAKE) check
.PHONY: install
install: linear.o log_nooverlap.o tokuredblack.o rangetree.o #log.o
install: $(BINS)
clean:
rm -rf *.o *.gcno *.gcda *.gcov
......
src/range_tree/tests/Makefile
View file @ 83d416fa
......@@ -34,25 +34,25 @@ RUN_ALL_TESTS = $(RUN_LIN_TESTS) $(RUN_TLOG_TESTS) $(RUN_LOG_TESTS)
.PHONY: default all check tests check.lin check.tlog check.log tests.lin tests.log tests.tlog
default: check.tlog
default: build
all: make_libs $(ALL_TESTS)
all build: $(ALL_TESTS)
check: check.lin check.tlog #check.log
@ echo $@ ok
tests: tests.lin tests.tlog #tests.log
@ echo $@ ok
tests.lin: make_libs $(LIN_TESTS)
tests.lin: $(LIN_TESTS)
@ echo $@ ok
check.lin: make_libs $(RUN_LIN_TESTS)
check.lin: $(RUN_LIN_TESTS)
@ echo $@ ok
tests.tlog: make_libs $(TLOG_TESTS)
tests.tlog: $(TLOG_TESTS)
@ echo $@ ok
check.tlog: make_libs $(RUN_TLOG_TESTS)
check.tlog: $(RUN_TLOG_TESTS)
@ echo $@ ok
tests.log: make_libs $(LOG_TESTS)
tests.log: $(LOG_TESTS)
@ echo $@ ok
check.log: make_libs $(RUN_LOG_TESTS)
check.log: $(RUN_LOG_TESTS)
@ echo $@ ok
# Need these rule so that Make knows about all the file names
......@@ -99,11 +99,6 @@ HEADERS=../rangetree.h ../rangetree-internal.h test.h
%.log: %.c $(HEADERS) $(LOG_BINS)
cc -DDIR=\"dir.$<.log\" $(CFLAGS) $(CPPFLAGS) $< -o $@ $(LOG_BINS)
.PHONY: make_libs
make_libs:
cd .. && make
clean:
rm -f $(ALL_TESTS) *.o *.gcno *.gcda *.gcov
rm -rf dir.*.log dir.*.LIN
src/tests/Makefile
View file @ 83d416fa
......@@ -60,7 +60,7 @@ RUN_BDB_TESTS = $(patsubst %.bdb,%.bdbrun,$(BDB_TESTS))
RUN_ALL_TESTS = $(RUN_TDB_TESTS) $(RUN_BDB_TESTS)
all: $(ALL_TESTS)
all build: $(ALL_TESTS)
foo:
echo RUN_TDB_TESTS: $(RUN_TDB_TESTS)
......
src/ydb.c
View file @ 83d416fa
......@@ -6,7 +6,6 @@
const char *toku_patent_string = "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it.";
const char *toku_copyright_string = "Copyright (c) 2007, 2008 Tokutek Inc. All rights reserved.";
#include <assert.h>
#include <ctype.h>
#include <errno.h>
#include <libgen.h>
......@@ -22,8 +21,8 @@ const char *toku_copyright_string = "Copyright (c) 2007, 2008 Tokutek Inc. All
#include <sys/wait.h>
#include <unistd.h>
#include "toku_assert.h"
#include "ydb-internal.h"
#include "brt-internal.h"
#include "cachetable.h"
#include "log.h"
......
utils/Makefile
View file @ 83d416fa
......@@ -46,7 +46,7 @@ STATIC_UTILS=$(patsubst %,%_static,$(UTILS))
.PHONY: all clean test test_gen test_gen_hex test_load test_dump
all: $(UTILS) $(BDB_UTILS) $(STATIC_UTILS)
build all: $(UTILS) $(BDB_UTILS) $(STATIC_UTILS)
coverage: $(UTILS)
......@@ -62,7 +62,7 @@ coverage: $(UTILS)
strip: $(STATIC_UTILS)
strip $(STATIC_UTILS)
test: $(UTILS) $(BDB_UTILS) $(STATIC_UTILS) test_gen test_load test_dump test_nodup test_dupsort
check: $(UTILS) $(BDB_UTILS) $(STATIC_UTILS) test_gen test_load test_dump test_nodup test_dupsort
test-coverage: $(UTILS) test_gen test_load test_dump test_nodup test_dupsort
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7