Skip to content

M
MariaDB
Commit 6b8ee583 authored by Yoni Fogel's avatar Yoni Fogel
Browse files
Options

Addresses #337 

Addresses #293
Addresses #307
Adds range count to limit lock system memory from the dbenv, and correspondingly changes range trees and test files

git-svn-id: file:///svn/tokudb@2103 c7de825b-a66e-492c-adef-691d508d4ae1
parent aa9e2e14
Expand all Show whitespace changes
Inline Side-by-side
Showing with 337 additions and 125 deletions
src/Makefile
View file @ 6b8ee583
...@@ -5,8 +5,8 @@ LIBNAME=libdb ...@@ -5,8 +5,8 @@ LIBNAME=libdb
OPTFLAGS = -O2 OPTFLAGS = -O2
# GCOV_FLAGS = -fprofile-arcs -ftest-coverage # GCOV_FLAGS = -fprofile-arcs -ftest-coverage
CFLAGS = -W -Wall -Werror -g -fPIC $(OPTFLAGS) $(GCOV_FLAGS) CFLAGS = -W -Wall -Werror -g3 -ggdb3 -fPIC $(OPTFLAGS) $(GCOV_FLAGS)
CPPFLAGS = -I../include -I../newbrt CPPFLAGS = -I../include -I../newbrt -I./lock_tree/ -I./range_tree/
CPPFLAGS += -D_GNU_SOURCE -D_THREAD_SAFE -D_FILE_OFFSET_BITS=64 -D_LARGEFILE64_SOURCE CPPFLAGS += -D_GNU_SOURCE -D_THREAD_SAFE -D_FILE_OFFSET_BITS=64 -D_LARGEFILE64_SOURCE
...@@ -20,13 +20,19 @@ SHARED=-shared ...@@ -20,13 +20,19 @@ SHARED=-shared
endif endif
.PHONY: install logformat .PHONY: install logformat
install: logformat $(LIBNAME).$(LIBEXT) $(LIBNAME).a install: logformat rangetree locktree $(LIBNAME).$(LIBEXT) $(LIBNAME).a
cp $(LIBNAME).$(LIBEXT) ../lib/ cp $(LIBNAME).$(LIBEXT) ../lib/
cp $(LIBNAME).a ../lib cp $(LIBNAME).a ../lib
logformat: logformat:
(cd ../newbrt;make) (cd ../newbrt;make)
locktree:
cd lock_tree;make OSX=$(OSX) OPT=$(OPT) GCOV=$(GCOV)
rangetree:
cd range_tree;make OSX=$(OSX) OPT=$(OPT) GCOV=$(GCOV)
check: $(LIBNAME).$(LIBEXT) check: $(LIBNAME).$(LIBEXT)
python tokuglobals.py $(LIBNAME).$(LIBEXT) python tokuglobals.py $(LIBNAME).$(LIBEXT)
...@@ -36,9 +42,16 @@ strip: $(LIBNAME).$(LIBEXT) ...@@ -36,9 +42,16 @@ strip: $(LIBNAME).$(LIBEXT)
clean: clean:
rm -rf $(LIBNAME).$(LIBEXT) $(LIBNAME).a *.o *.gcno *.gcda *.gcov rm -rf $(LIBNAME).$(LIBEXT) $(LIBNAME).a *.o *.gcno *.gcda *.gcov
cd tests;make clean cd tests;make clean
cd lock_tree;make clean
cd range_tree;make clean
ydb.o: ../include/db.h ../newbrt/cachetable.h ../newbrt/brt.h ../newbrt/log.c ydb.o: ../include/db.h ../newbrt/cachetable.h ../newbrt/brt.h ../newbrt/log.c
DBBINS = ydb.o ../newbrt/brt.o ../newbrt/brt-serialize.o ../newbrt/brt-verify.o ../newbrt/cachetable.o ../newbrt/fifo.o ../newbrt/key.o ../newbrt/memory.o ../newbrt/mempool.o ../newbrt/pma.o ../newbrt/ybt.o ../newbrt/primes.o ../newbrt/log.o ../newbrt/fingerprint.o ../newbrt/log_code.o ../newbrt/roll.o
RANGETREE_BINS = range_tree/linear.o
LOCKTREE_BINS = lock_tree/rth.o lock_tree/locktree.o
DBBINS = ydb.o ../newbrt/brt.o ../newbrt/brt-serialize.o ../newbrt/brt-verify.o ../newbrt/cachetable.o ../newbrt/fifo.o ../newbrt/key.o ../newbrt/memory.o ../newbrt/mempool.o ../newbrt/pma.o ../newbrt/ybt.o ../newbrt/primes.o ../newbrt/log.o ../newbrt/fingerprint.o ../newbrt/log_code.o ../newbrt/roll.o $(RANGETREE_BINS) $(LOCKTREE_BINS)
$(LIBNAME).$(LIBEXT): $(DBBINS) $(LIBNAME).$(LIBEXT): $(DBBINS)
cc $(CPPFLAGS) $(DBBINS) $(SHARED) -o $@ $(CFLAGS) -lz cc $(CPPFLAGS) $(DBBINS) $(SHARED) -o $@ $(CFLAGS) -lz
......
src/lock_tree/Makefile
View file @ 6b8ee583
...@@ -4,9 +4,9 @@ ...@@ -4,9 +4,9 @@
LIBNAME=liblocktree LIBNAME=liblocktree
ifneq ($(OPT),) ifneq ($(OPT),)
OPTFLAGS = -O4 OPTFLAGS = -O3
else else
OPTFLAGS = -O0 -g3 -ggdb3 OPTFLAGS = -O2 -g3 -ggdb3
endif endif
ifneq ($(GCOV),) ifneq ($(GCOV),)
...@@ -17,7 +17,7 @@ endif ...@@ -17,7 +17,7 @@ endif
CFLAGS = -W -Wall -Wextra -Werror -fPIC $(OPTFLAGS) $(GCOV_FLAGS) CFLAGS = -W -Wall -Wextra -Werror -fPIC $(OPTFLAGS) $(GCOV_FLAGS)
CFLAGS += -Wbad-function-cast -Wcast-align -Wconversion -Waggregate-return CFLAGS += -Wbad-function-cast -Wcast-align -Wconversion -Waggregate-return
CFLAGS += -Wmissing-noreturn -Wmissing-format-attribute -Wunreachable-code CFLAGS += -Wmissing-noreturn -Wmissing-format-attribute
CPPFLAGS = -I. -I.. -I../range_tree -I../../include -I../../newbrt -L../range_tree CPPFLAGS = -I. -I.. -I../range_tree -I../../include -I../../newbrt -L../range_tree
CPPFLAGS += -D_GNU_SOURCE -D_THREAD_SAFE -D_FILE_OFFSET_BITS=64 -D_LARGEFILE64_SOURCE CPPFLAGS += -D_GNU_SOURCE -D_THREAD_SAFE -D_FILE_OFFSET_BITS=64 -D_LARGEFILE64_SOURCE
...@@ -37,19 +37,21 @@ install: $(LIBNAME).$(LIBEXT) $(LIBNAME).a ...@@ -37,19 +37,21 @@ install: $(LIBNAME).$(LIBEXT) $(LIBNAME).a
cp $(LIBNAME).a ../../lib cp $(LIBNAME).a ../../lib
clean: clean:
rm -rf $(LIBNAMELINEAR).$(LIBEXT) $(LIBNAMELINEAR).a rm -rf $(LIBNAME_LINEAR).$(LIBEXT) $(LIBNAME_LINEAR).a
rm -rf $(LIBNAME).$(LIBEXT) $(LIBNAME).a *.o *.gcno *.gcda *.gcov rm -rf $(LIBNAME_TLOG).$(LIBEXT) $(LIBNAME_TLOG).a
rm -rf $(LIBNAME_LOG).$(LIBEXT) $(LIBNAME_LOG).a
rm -rf *.gcno *.gcda *.gcov
cd tests;make clean cd tests;make clean
locktree.o: locktree.c locktree.h locktree.o: locktree.c locktree.h
gcc $(CFLAGS) $(CPPFLAGS) -c -DTOKU_LINEAR_RANGE_TREE $< -o $@ gcc $(CFLAGS) $(CPPFLAGS) -DTOKU_LT_LINEAR -c $< -o $@
rth.o: rth.c rth.h rth.o: rth.c rth.h
gcc $(CFLAGS) $(CPPFLAGS) -c -DTOKU_LINEAR_RANGE_TREE $< -o $@ gcc $(CFLAGS) $(CPPFLAGS) -c $< -o $@
$(LIBNAME).$(LIBEXT): locktree.o rth.o $(LIBNAME).$(LIBEXT): locktree.o rth.o
cc $(CPPFLAGS) $^ $(SHARED) -o $@ $(CFLAGS) -lrangetreelinear cc $(CPPFLAGS) $^ $(SHARED) -o $@ $(CFLAGS) -lrangetree_linear
$(LIBNAME).a: locktree.o rth.o $(LIBNAME).a: locktree.o rth.o
$(AR) rv $@ $^ $(AR) rv $@ $^
src/lock_tree/locktree.c
View file @ 6b8ee583
This diff is collapsed.
src/lock_tree/locktree.h
View file @ 6b8ee583
...@@ -46,15 +46,34 @@ ...@@ -46,15 +46,34 @@
#include <assert.h> #include <assert.h>
#include <db.h> #include <db.h>
#include <brttypes.h> #include <brttypes.h>
#if defined(TOKU_LT_LINEAR)
#define TOKU_RT_LINEAR
#elif defined(TOKU_LT_TLOG)
#define TOKU_RT_TLOG
#elif defined(TOKU_LT_LOG)
#define TOKU_RT_LOG
#else
#error Using an undefined LOCK TREE TYPE.
#endif
#include <rangetree.h> #include <rangetree.h>
#include <rth.h> #include <rth.h>
typedef enum {
TOKU_LT_INCONSISTENT=-1,
} TOKU_LT_ERROR;
char* toku_lt_strerror(TOKU_LT_ERROR r) __attribute__((const,pure));
/** \brief The lock tree structure */ /** \brief The lock tree structure */
typedef struct { typedef struct __toku_lock_tree {
/** The database for which this locktree will be handling locks */ /** The database for which this locktree will be handling locks */
DB* db; DB* db;
/** Whether the db supports duplicate */ /** Whether the db supports duplicate */
BOOL duplicates; BOOL duplicates;
/** Whether the duplicates flag can no longer be changed. */
BOOL dups_final;
toku_range_tree* mainread; /**< See design document */ toku_range_tree* mainread; /**< See design document */
toku_range_tree* borderwrite; /**< See design document */ toku_range_tree* borderwrite; /**< See design document */
toku_rth* rth; toku_rth* rth;
...@@ -62,16 +81,16 @@ typedef struct { ...@@ -62,16 +81,16 @@ typedef struct {
the range trees that this lock tree owns */ the range trees that this lock tree owns */
toku_range* buf; toku_range* buf;
u_int32_t buflen; /**< The length of buf */ u_int32_t buflen; /**< The length of buf */
/** The maximum amount of memory to be used for DBT payloads. */ /** The maximum number of ranges allowed. */
size_t payload_capacity; u_int32_t max_ranges;
/** The current amount of memory used for DBT payloads. */ /** The current number of ranges. */
size_t payload_used; u_int32_t* num_ranges;
/** The key compare function */ /** The key compare function */
int (*compare_fun)(DB*,const DBT*,const DBT*); int (*compare_fun)(DB*,const DBT*,const DBT*);
/** The data compare function */ /** The data compare function */
int (*dup_compare)(DB*,const DBT*,const DBT*); int (*dup_compare)(DB*,const DBT*,const DBT*);
/** The panic function */ /** The panic function */
int (*panic)(DB*); int (*panic)(DB*, int);
/** The user malloc function */ /** The user malloc function */
void* (*malloc) (size_t); void* (*malloc) (size_t);
/** The user free function */ /** The user free function */
...@@ -135,13 +154,23 @@ typedef struct { ...@@ -135,13 +154,23 @@ typedef struct {
instead. instead.
*/ */
int toku_lt_create(toku_lock_tree** ptree, DB* db, BOOL duplicates, int toku_lt_create(toku_lock_tree** ptree, DB* db, BOOL duplicates,
int (*panic)(DB*), size_t payload_capacity, int (*panic)(DB*, int), u_int32_t max_locks,
u_int32_t* num_locks,
int (*compare_fun)(DB*,const DBT*,const DBT*), int (*compare_fun)(DB*,const DBT*,const DBT*),
int (*dup_compare)(DB*,const DBT*,const DBT*), int (*dup_compare)(DB*,const DBT*,const DBT*),
void* (*user_malloc) (size_t), void* (*user_malloc) (size_t),
void (*user_free) (void*), void (*user_free) (void*),
void* (*user_realloc)(void*, size_t)); void* (*user_realloc)(void*, size_t));
/**
Set whether duplicates are allowed.
This can be called after create, but NOT after any locks or unlocks have
occurred.
Return: 0 on success.
Return: EINVAL if tree is NULL
Return: EDOM if it is too late to change.
*/
int toku_lt_set_dups(toku_lock_tree* tree, BOOL duplicates);
/** /**
Closes and frees a lock tree. Closes and frees a lock tree.
......
src/lock_tree/tests/Makefile
View file @ 6b8ee583
...@@ -39,31 +39,35 @@ CPPFLAGS += -I. -I../ -I../../range_tree -I../../../newbrt -I../../../include -l ...@@ -39,31 +39,35 @@ CPPFLAGS += -I. -I../ -I../../range_tree -I../../../newbrt -I../../../include -l
SRCS = $(wildcard *.c) SRCS = $(wildcard *.c)
LOG_TESTS = $(patsubst %.c,%.log,$(SRCS)) LOG_TESTS = $(patsubst %.c,%.log,$(SRCS))
TLOG_TESTS = $(patsubst %.c,%.tlog,$(SRCS))
LIN_TESTS = $(patsubst %.c,%.lin,$(SRCS)) LIN_TESTS = $(patsubst %.c,%.lin,$(SRCS))
ALL_TESTS = $(LOG_TESTS) $(LIN_TESTS) ALL_TESTS = $(LIN_TESTS) $(TLOG_TESTS) $(LOG_TESTS)
RUN_LOG_TESTS = $(patsubst %.log,%.logrun,$(LOG_TESTS)) RUN_LOG_TESTS = $(patsubst %.log,%.logrun,$(LOG_TESTS))
RUN_TLOG_TESTS = $(patsubst %.tlog,%.tlogrun,$(TLOG_TESTS))
RUN_LIN_TESTS = $(patsubst %.lin,%.linrun,$(LIN_TESTS)) RUN_LIN_TESTS = $(patsubst %.lin,%.linrun,$(LIN_TESTS))
RUN_ALL_TESTS = $(RUN_LOG_TESTS) $(RUN_LIN_TESTS) RUN_ALL_TESTS = $(RUN_LIN_TESTS) $(RUN_TLOG_TESTS) $(RUN_LOG_TESTS)
all: make_libs $(ALL_TESTS) all: make_libs $(ALL_TESTS)
.PHONY: check check.lin check.log tests.lin tests.log .PHONY: check check.lin check.log tests.lin tests.log check.tlog tests.tlog
check: check.lin check.log check: check.lin check.tlog check.log
@ echo ok @ echo ok
tests.lin: make_libs $(LIN_TESTS) tests.lin: make_libs $(LIN_TESTS)
check.lin: make_libs $(RUN_LIN_TESTS) check.lin: make_libs $(RUN_LIN_TESTS)
tests.tlog: make_libs $(TLOG_TESTS)
check.tlog: make_libs $(RUN_TLOG_TESTS)
tests.log: make_libs $(LOG_TESTS) tests.log: make_libs $(LOG_TESTS)
check.log: make_libs $(RUN_LOG_TESTS) check.log: make_libs $(RUN_LOG_TESTS)
# Need these rule so that Make knows about all the file names # Need these rule so that Make knows about all the file names
.PHONY: %.linrun %.logrun %.run .PHONY: %.linrun %.logrun %.run %.tlogrun
$(RUN_ALL_TESTS): $(RUN_ALL_TESTS):
$(ALL_TESTS): $(ALL_TESTS):
%.run: %.linrun %.logrun %.run: %.linrun %.tlogrun %.logrun
@ echo ok @ echo ok
ifeq ($(VERBOSE),2) ifeq ($(VERBOSE),2)
...@@ -87,14 +91,18 @@ endif ...@@ -87,14 +91,18 @@ endif
$(MAYBEATSIGN) $(SETENV) $(VGRIND) ./$< $(VERBVERBOSE) $(MAYBEATSIGN) $(SETENV) $(VGRIND) ./$< $(VERBVERBOSE)
%.logrun: %.log %.logrun: %.log
$(MAYBEATSIGN) $(SETENV) $(VGRIND) ./$< $(VERBVERBOSE) $(MAYBEATSIGN) $(SETENV) $(VGRIND) ./$< $(VERBVERBOSE)
%.tlogrun: %.tlog
$(MAYBEATSIGN) $(SETENV) $(VGRIND) ./$< $(VERBVERBOSE)
libs: libs:
cd ..;make cd ..;make
%.lin: %.c ../locktree.h test.h %.lin: %.c ../locktree.h test.h
$(SETENV) cc -DDIR=\"dir.$<.lin\" -DTOKU_LINEAR_RANGE_TREE $(CPPFLAGS) $(CFLAGS) $< -lrangetreelinear -llocktree -o $@ $(SETENV) cc -DDIR=\"dir.$<.lin\" -DTOKU_LT_LINEAR $(CPPFLAGS) $(CFLAGS) $< -lrangetree_linear -llocktree_linear -o $@
%.tlog: %.c ../locktree.h test.h
$(SETENV) cc -DDIR=\"dir.$<.log\" -DTOKU_LT_TLOG $(CPPFLAGS) $(CFLAGS) $< -lrangetree_tlog -llocktree_tlog -o $@
%.log: %.c ../locktree.h test.h %.log: %.c ../locktree.h test.h
$(SETENV) cc -DDIR=\"dir.$<.log\" -DTOKU_LOG_RANGE_TREE $(CPPFLAGS) $(CFLAGS) $< -lrangetree -llocktre -o $@ $(SETENV) cc -DDIR=\"dir.$<.log\" -DTOKU_LT_LOG $(CPPFLAGS) $(CFLAGS) $< -lrangetree_log -llocktree_log -o $@
.PHONY: make_libs .PHONY: make_libs
......
src/lock_tree/tests/test_00010_parameter_errors.c
View file @ 6b8ee583
...@@ -146,6 +146,12 @@ static void do_point_test(int (*acquire)(toku_lock_tree*, DB_TXN*, ...@@ -146,6 +146,12 @@ static void do_point_test(int (*acquire)(toku_lock_tree*, DB_TXN*,
CKERR(r); CKERR(r);
assert(lt); assert(lt);
r = toku_lt_unlock(NULL, (DB_TXN*)1);
CKERR2(r, EINVAL);
r = toku_lt_unlock(lt, NULL);
CKERR2(r, EINVAL);
r = acquire(NULL, txn, key, data); r = acquire(NULL, txn, key, data);
CKERR2(r, EINVAL); CKERR2(r, EINVAL);
......
src/range_tree/Makefile
View file @ 6b8ee583
# On OSX do: # On OSX do:
# make OSX=OSX # make OSX=OSX
LIBNAMELINEAR=librangetreelinear LIBNAME_LINEAR = librangetree_linear
LIBNAME=librangetree LIBNAME_TLOG = librangetree_tlog
LIBNAME_LOG = librangetree_log
ifneq ($(OPT),) ifneq ($(OPT),)
OPTFLAGS = -O4 OPTFLAGS = -O3
else else
OPTFLAGS = -O0 -g3 -ggdb3 OPTFLAGS = -O2 -g3 -ggdb3
endif endif
ifneq ($(GCOV),) ifneq ($(GCOV),)
...@@ -18,7 +19,7 @@ endif ...@@ -18,7 +19,7 @@ endif
CFLAGS = -W -Wall -Wextra -Werror -fPIC $(OPTFLAGS) $(GCOV_FLAGS) CFLAGS = -W -Wall -Wextra -Werror -fPIC $(OPTFLAGS) $(GCOV_FLAGS)
CFLAGS += -Wbad-function-cast -Wcast-align -Wconversion -Waggregate-return CFLAGS += -Wbad-function-cast -Wcast-align -Wconversion -Waggregate-return
CFLAGS += -Wmissing-noreturn -Wmissing-format-attribute -Wunreachable-code CFLAGS += -Wmissing-noreturn -Wmissing-format-attribute
CPPFLAGS = -I../../include -I../../newbrt CPPFLAGS = -I../../include -I../../newbrt
CPPFLAGS += -D_GNU_SOURCE -D_THREAD_SAFE -D_FILE_OFFSET_BITS=64 -D_LARGEFILE64_SOURCE CPPFLAGS += -D_GNU_SOURCE -D_THREAD_SAFE -D_FILE_OFFSET_BITS=64 -D_LARGEFILE64_SOURCE
...@@ -34,32 +35,44 @@ SHARED=-shared ...@@ -34,32 +35,44 @@ SHARED=-shared
endif endif
.PHONY: install logformat .PHONY: install logformat
install: $(LIBNAMELINEAR).$(LIBEXT) $(LIBNAMELINEAR).a #$(LIBNAME).$(LIBEXT) $(LIBNAME).a install: $(LIBNAME_LINEAR).$(LIBEXT) $(LIBNAME_LINEAR).a #$(LIBNAME_TLOG).$(LIBEXT) $(LIBNAME_TLOG).a $(LIBNAME_LOG).$(LIBEXT) $(LIBNAME_LOG).a
#cp $(LIBNAME).$(LIBEXT) ../../lib/ #cp $(LIBNAME_LOG).$(LIBEXT) ../../lib/
#cp $(LIBNAME).a ../../lib #cp $(LIBNAME_LOG).a ../../lib
cp $(LIBNAMELINEAR).$(LIBEXT) ../../lib/ #cp $(LIBNAME_TLOG).$(LIBEXT) ../../lib/
cp $(LIBNAMELINEAR).a ../../lib #cp $(LIBNAME_TLOG).a ../../lib
cp $(LIBNAME_LINEAR).$(LIBEXT) ../../lib/
cp $(LIBNAME_LINEAR).a ../../lib
clean: clean:
rm -rf $(LIBNAMELINEAR).$(LIBEXT) $(LIBNAMELINEAR).a rm -rf $(LIBNAME_LINEAR).$(LIBEXT) $(LIBNAME_LINEAR).a
rm -rf $(LIBNAME).$(LIBEXT) $(LIBNAME).a *.o *.gcno *.gcda *.gcov rm -rf $(LIBNAME_TLOG).$(LIBEXT) $(LIBNAME_TLOG).a
rm -rf $(LIBNAME_LOG).$(LIBEXT) $(LIBNAME_LOG).a
cd tests;make clean cd tests;make clean
linear.o: linear.c rangetree.h linear.o: linear.c rangetree.h
gcc $(CFLAGS) $(CPPFLAGS) -DTOKU_LINEAR_RANGE_TREE -c $< -o $@ gcc $(CFLAGS) $(CPPFLAGS) -DTOKU_RT_LINEAR -c $< -o $@
log.o: log.c rangetree.h log.o: log.c rangetree.h
gcc $(CFLAGS) $(CPPFLAGS) -DTOKU_LOG_RANGE_TREE -c $< -o $@ gcc $(CFLAGS) $(CPPFLAGS) -DTOKU_RT_LOG -c $< -o $@
$(LIBNAME).$(LIBEXT): log.o tlog.o: tlog.c rangetree.h
gcc $(CFLAGS) $(CPPFLAGS) -DTOKU_RT_TLOG -c $< -o $@
$(LIBNAME_TLOG).$(LIBEXT): tlog.o
cc $(CPPFLAGS) $< $(SHARED) -o $@ $(CFLAGS)
$(LIBNAME_LOG).$(LIBEXT): log.o
cc $(CPPFLAGS) $< $(SHARED) -o $@ $(CFLAGS) cc $(CPPFLAGS) $< $(SHARED) -o $@ $(CFLAGS)
$(LIBNAMELINEAR).$(LIBEXT): linear.o $(LIBNAME_LINEAR).$(LIBEXT): linear.o
cc $(CPPFLAGS) $< $(SHARED) -o $@ $(CFLAGS) cc $(CPPFLAGS) $< $(SHARED) -o $@ $(CFLAGS)
$(LIBNAME).a: log.o $(LIBNAME_TLOG).a: tlog.o
$(AR) rv $@ $<
$(LIBNAME_LOG).a: log.o
$(AR) rv $@ $< $(AR) rv $@ $<
$(LIBNAMELINEAR).a: linear.o $(LIBNAME_LINEAR).a: linear.o
$(AR) rv $@ $< $(AR) rv $@ $<
src/range_tree/linear.c
View file @ 6b8ee583
...@@ -33,7 +33,7 @@ static int __toku_rt_p_cmp(toku_range_tree* tree, ...@@ -33,7 +33,7 @@ static int __toku_rt_p_cmp(toku_range_tree* tree,
} }
static int __toku_rt_decrease_capacity(toku_range_tree* tree, u_int32_t _num) { static int __toku_rt_decrease_capacity(toku_range_tree* tree, u_int32_t _num) {
assert(tree); //TODO: SOME ATTRIBUTE TO REMOVE NEVER EXECUTABLE ERROR: assert(tree);
u_int32_t num = _num < minlen ? minlen : _num; u_int32_t num = _num < minlen ? minlen : _num;
if (tree->ranges_len >= num * 2) { if (tree->ranges_len >= num * 2) {
...@@ -50,7 +50,7 @@ static int __toku_rt_decrease_capacity(toku_range_tree* tree, u_int32_t _num) { ...@@ -50,7 +50,7 @@ static int __toku_rt_decrease_capacity(toku_range_tree* tree, u_int32_t _num) {
} }
static int __toku_rt_increase_capacity(toku_range_tree* tree, u_int32_t num) { static int __toku_rt_increase_capacity(toku_range_tree* tree, u_int32_t num) {
assert(tree); //TODO: SOME ATTRIBUTE TO REMOVE NEVER EXECUTABLE ERROR: assert(tree);
if (tree->ranges_len < num) { if (tree->ranges_len < num) {
u_int32_t temp_len = tree->ranges_len; u_int32_t temp_len = tree->ranges_len;
while (temp_len < num) temp_len *= 2; while (temp_len < num) temp_len *= 2;
...@@ -66,7 +66,7 @@ static int __toku_rt_increase_capacity(toku_range_tree* tree, u_int32_t num) { ...@@ -66,7 +66,7 @@ static int __toku_rt_increase_capacity(toku_range_tree* tree, u_int32_t num) {
static int __toku_rt_increase_buffer(toku_range_tree* tree, toku_range** buf, static int __toku_rt_increase_buffer(toku_range_tree* tree, toku_range** buf,
u_int32_t* buflen, u_int32_t num) { u_int32_t* buflen, u_int32_t num) {
assert(buf); assert(buf);
assert(buflen); //TODO: SOME ATTRIBUTE TO REMOVE NEVER EXECUTABLE ERROR: assert(buflen);
if (*buflen < num) { if (*buflen < num) {
u_int32_t temp_len = *buflen; u_int32_t temp_len = *buflen;
while (temp_len < num) temp_len *= 2; while (temp_len < num) temp_len *= 2;
...@@ -250,3 +250,8 @@ int toku_rt_get_allow_overlaps(toku_range_tree* tree, BOOL* allowed) { ...@@ -250,3 +250,8 @@ int toku_rt_get_allow_overlaps(toku_range_tree* tree, BOOL* allowed) {
*allowed = tree->allow_overlaps; *allowed = tree->allow_overlaps;
return 0; return 0;
} }
u_int32_t toku_rt_get_size(toku_range_tree *rt) {
assert(rt);
return rt->numelements;
}
src/range_tree/rangetree.h
View file @ 6b8ee583
...@@ -53,14 +53,20 @@ struct __toku_range_tree_internal { ...@@ -53,14 +53,20 @@ struct __toku_range_tree_internal {
void (*free) (void*); void (*free) (void*);
/** The user realloc function */ /** The user realloc function */
void* (*realloc)(void*, size_t); void* (*realloc)(void*, size_t);
#if defined(TOKU_LINEAR_RANGE_TREE) #if defined(TOKU_RT_LINEAR)
#if defined(TOKU_LOG_RANGE_TREE) #if defined(TOKU_RT_TLOG) || defined(TOKU_RT_LOG)
#error Choose just one range tree type. #error Choose just one range tree type.
#endif #endif
//Linear version only fields: //Linear version only fields:
toku_range* ranges; toku_range* ranges;
u_int32_t ranges_len; u_int32_t ranges_len;
#elif defined(TOKU_LOG_RANGE_TREE) #elif defined(TOKU_RT_TLOG)
#if defined(TOKU_RT_LOG)
#error Choose just one range tree type.
#endif
#error Not defined yet.
//TLog version only fields:
#elif defined(TOKU_RT_LOG)
#error Not defined yet. #error Not defined yet.
//Log version only fields: //Log version only fields:
#else #else
...@@ -225,4 +231,7 @@ int toku_rt_predecessor(toku_range_tree* tree, void* point, toku_range* pred, ...@@ -225,4 +231,7 @@ int toku_rt_predecessor(toku_range_tree* tree, void* point, toku_range* pred,
*/ */
int toku_rt_successor(toku_range_tree* tree, void* point, toku_range* succ, int toku_rt_successor(toku_range_tree* tree, void* point, toku_range* succ,
BOOL* wasfound); BOOL* wasfound);
u_int32_t toku_rt_get_size(toku_range_tree *);
#endif /* #if !defined(TOKU_RANGE_TREE_H) */ #endif /* #if !defined(TOKU_RANGE_TREE_H) */
src/range_tree/tests/Makefile
View file @ 6b8ee583
...@@ -31,31 +31,35 @@ CPPFLAGS += -L../ -I../ -I../../../newbrt -I../../../include -lpthread ...@@ -31,31 +31,35 @@ CPPFLAGS += -L../ -I../ -I../../../newbrt -I../../../include -lpthread
SRCS = $(wildcard *.c) SRCS = $(wildcard *.c)
LOG_TESTS = $(patsubst %.c,%.log,$(SRCS)) LOG_TESTS = $(patsubst %.c,%.log,$(SRCS))
TLOG_TESTS = $(patsubst %.c,%.tlog,$(SRCS))
LIN_TESTS = $(patsubst %.c,%.lin,$(SRCS)) LIN_TESTS = $(patsubst %.c,%.lin,$(SRCS))
ALL_TESTS = $(LOG_TESTS) $(LIN_TESTS) ALL_TESTS = $(LIN_TESTS) $(LOG_TESTS) $(TLOG_TESTS)
RUN_LOG_TESTS = $(patsubst %.log,%.logrun,$(LOG_TESTS)) RUN_LOG_TESTS = $(patsubst %.log,%.logrun,$(LOG_TESTS))
RUN_TLOG_TESTS = $(patsubst %.log,%.logrun,$(TLOG_TESTS))
RUN_LIN_TESTS = $(patsubst %.lin,%.linrun,$(LIN_TESTS)) RUN_LIN_TESTS = $(patsubst %.lin,%.linrun,$(LIN_TESTS))
RUN_ALL_TESTS = $(RUN_LOG_TESTS) $(RUN_LIN_TESTS) RUN_ALL_TESTS = $(RUN_LIN_TESTS) $(RUN_TLOG_TESTS) $(RUN_LOG_TESTS)
all: make_libs $(ALL_TESTS) all: make_libs $(ALL_TESTS)
.PHONY: check check.lin check.log tests.lin tests.log .PHONY: check check.lin check.log tests.lin tests.log tests.tlog
check: check.lin check.log check: check.lin check.tlog check.log
@ echo ok @ echo ok
tests.lin: make_libs $(LIN_TESTS) tests.lin: make_libs $(LIN_TESTS)
check.lin: make_libs $(RUN_LIN_TESTS) check.lin: make_libs $(RUN_LIN_TESTS)
tests.tlog: make_libs $(TLOG_TESTS)
check.tlog: make_libs $(RUN_TLOG_TESTS)
tests.log: make_libs $(LOG_TESTS) tests.log: make_libs $(LOG_TESTS)
check.log: make_libs $(RUN_LOG_TESTS) check.log: make_libs $(RUN_LOG_TESTS)
# Need these rule so that Make knows about all the file names # Need these rule so that Make knows about all the file names
.PHONY: %.linrun %.logrun %.run .PHONY: %.linrun %.logrun %.run %.tlogrun
$(RUN_ALL_TESTS): $(RUN_ALL_TESTS):
$(ALL_TESTS): $(ALL_TESTS):
%.run: %.linrun %.logrun %.run: %.linrun %.tlogrun %.logrun
@ echo ok @ echo ok
ifeq ($(VERBOSE),2) ifeq ($(VERBOSE),2)
...@@ -79,14 +83,18 @@ endif ...@@ -79,14 +83,18 @@ endif
$(MAYBEATSIGN) $(SETENV) $(VGRIND) ./$< $(VERBVERBOSE) $(MAYBEATSIGN) $(SETENV) $(VGRIND) ./$< $(VERBVERBOSE)
%.logrun: %.log %.logrun: %.log
$(MAYBEATSIGN) $(SETENV) $(VGRIND) ./$< $(VERBVERBOSE) $(MAYBEATSIGN) $(SETENV) $(VGRIND) ./$< $(VERBVERBOSE)
%.tlogrun: %.tlog
$(MAYBEATSIGN) $(SETENV) $(VGRIND) ./$< $(VERBVERBOSE)
libs: libs:
cd ..;make cd ..;make
%.lin: %.c ../rangetree.h test.h %.lin: %.c ../rangetree.h test.h
$(SETENV) cc -DDIR=\"dir.$<.lin\" -DTOKU_LINEAR_RANGE_TREE $(CPPFLAGS) $(CFLAGS) $< -lrangetreelinear -o $@ $(SETENV) cc -DDIR=\"dir.$<.lin\" -DTOKU_RT_LINEAR $(CPPFLAGS) $(CFLAGS) $< -lrangetree_linear -o $@
%.log: %.c ../rangetree.h test.h %.log: %.c ../rangetree.h test.h
$(SETENV) cc -DDIR=\"dir.$<.log\" -DTOKU_LOG_RANGE_TREE $(CPPFLAGS) $(CFLAGS) $< -lrangetree -o $@ $(SETENV) cc -DDIR=\"dir.$<.log\" -DTOKU_RT_LOG $(CPPFLAGS) $(CFLAGS) $< -lrangetree_log -o $@
%.tlog: %.c ../rangetree.h test.h
$(SETENV) cc -DDIR=\"dir.$<.log\" -DTOKU_RT_TLOG $(CPPFLAGS) $(CFLAGS) $< -lrangetree_tlog -o $@
.PHONY: make_libs .PHONY: make_libs
......
src/range_tree/tests/test.h
View file @ 6b8ee583
...@@ -72,3 +72,14 @@ void toku_free(void* p) { ...@@ -72,3 +72,14 @@ void toku_free(void* p) {
void* toku_realloc(void *ptr, size_t size) { void* toku_realloc(void *ptr, size_t size) {
return realloc(ptr, size); return realloc(ptr, size);
} }
int mallocced = 0;
int failon = -1;
void* fail_malloc(size_t size) {
if (++mallocced == failon) {
errno = ENOMEM;
return NULL;
}
return malloc(size);
}
src/range_tree/tests/test_00000_createclose.c
View file @ 6b8ee583
...@@ -2,17 +2,6 @@ ...@@ -2,17 +2,6 @@
#include "test.h" #include "test.h"
int mallocced = 0;
int failon = -1;
void* fail_malloc(size_t size) {
if (++mallocced == failon) {
errno = ENOMEM;
return NULL;
}
return malloc(size);
}
int main(int argc, const char *argv[]) { int main(int argc, const char *argv[]) {
int r; int r;
toku_range_tree *tree; toku_range_tree *tree;
......
src/ydb-internal.h
View file @ 6b8ee583
...@@ -14,6 +14,8 @@ struct db_header { ...@@ -14,6 +14,8 @@ struct db_header {
BRT *database_brts; // These BRT *database_brts; // These
}; };
struct __toku_lock_tree;
struct __toku_db_internal { struct __toku_db_internal {
DB *db; // A pointer back to the DB. DB *db; // A pointer back to the DB.
int freed; int freed;
...@@ -30,6 +32,7 @@ struct __toku_db_internal { ...@@ -30,6 +32,7 @@ struct __toku_db_internal {
DB *primary; // For secondary (associated) databases, what is the primary? NULL if not a secondary. DB *primary; // For secondary (associated) databases, what is the primary? NULL if not a secondary.
int(*associate_callback)(DB*, const DBT*, const DBT*, DBT*); // For secondary, the callback function for associate. NULL if not secondary int(*associate_callback)(DB*, const DBT*, const DBT*, DBT*); // For secondary, the callback function for associate. NULL if not secondary
int associate_is_immutable; // If this DB is a secondary then this field indicates that the index never changes due to updates. int associate_is_immutable; // If this DB is a secondary then this field indicates that the index never changes due to updates.
struct __toku_lock_tree* lt;
}; };
#if DB_VERSION_MAJOR == 4 && DB_VERSION_MINOR == 1 #if DB_VERSION_MAJOR == 4 && DB_VERSION_MINOR == 1
...@@ -57,6 +60,8 @@ struct __toku_db_env_internal { ...@@ -57,6 +60,8 @@ struct __toku_db_env_internal {
unsigned long cachetable_size; unsigned long cachetable_size;
CACHETABLE cachetable; CACHETABLE cachetable;
TOKULOGGER logger; TOKULOGGER logger;
u_int32_t max_locks;
u_int32_t num_locks;
}; };
struct __toku_db_txn_internal { struct __toku_db_txn_internal {
......
src/ydb.c
View file @ 6b8ee583
...@@ -28,6 +28,11 @@ const char *toku_copyright_string = "Copyright (c) 2007, 2008 Tokutek Inc. All ...@@ -28,6 +28,11 @@ const char *toku_copyright_string = "Copyright (c) 2007, 2008 Tokutek Inc. All
#include "log.h" #include "log.h"
#include "memory.h" #include "memory.h"
#define TOKU_LT_LINEAR
#include <locktree.h>
const u_int32_t __toku_env_default_max_locks = 1000;
/* the ydb big lock serializes access to the tokudb /* the ydb big lock serializes access to the tokudb
every call (including methods) into the tokudb library gets the lock every call (including methods) into the tokudb library gets the lock
no internal function should invoke a method through an object */ no internal function should invoke a method through an object */
...@@ -148,6 +153,9 @@ void toku_do_error_all_cases(const DB_ENV * env, int error, int include_stderrst ...@@ -148,6 +153,9 @@ void toku_do_error_all_cases(const DB_ENV * env, int error, int include_stderrst
} }
} }
static int do_error (DB_ENV *dbenv, int error, const char *string, ...)
__attribute__((__format__(__printf__, 3, 4)));
// Handle all the error cases (but don't do the default thing.) // Handle all the error cases (but don't do the default thing.)
static int do_error (DB_ENV *dbenv, int error, const char *string, ...) { static int do_error (DB_ENV *dbenv, int error, const char *string, ...) {
if (toku_logger_panicked(dbenv->i->logger)) dbenv->i->is_panicked=1; if (toku_logger_panicked(dbenv->i->logger)) dbenv->i->is_panicked=1;
...@@ -607,6 +615,10 @@ void toku_default_errcall(const DB_ENV *env, const char *errpfx, const char *msg ...@@ -607,6 +615,10 @@ void toku_default_errcall(const DB_ENV *env, const char *errpfx, const char *msg
#if _THREAD_SAFE #if _THREAD_SAFE
static void locked_env_err(const DB_ENV * env, int error, const char *fmt, ...)
__attribute__((__format__(__printf__, 3, 4)));
static void locked_env_err(const DB_ENV * env, int error, const char *fmt, ...) { static void locked_env_err(const DB_ENV * env, int error, const char *fmt, ...) {
ydb_lock(); ydb_lock();
va_list ap; va_list ap;
...@@ -725,6 +737,7 @@ static int toku_env_create(DB_ENV ** envp, u_int32_t flags) { ...@@ -725,6 +737,7 @@ static int toku_env_create(DB_ENV ** envp, u_int32_t flags) {
result->i->errcall = 0; result->i->errcall = 0;
result->i->errpfx = 0; result->i->errpfx = 0;
result->i->errfile = 0; result->i->errfile = 0;
result->i->max_locks = __toku_env_default_max_locks;
{ {
int r = toku_logger_create(&result->i->logger); int r = toku_logger_create(&result->i->logger);
...@@ -1616,6 +1629,31 @@ static int find_db_file(DB_ENV* dbenv, const char *fname, char** full_name_out) ...@@ -1616,6 +1629,31 @@ static int find_db_file(DB_ENV* dbenv, const char *fname, char** full_name_out)
return 0; return 0;
} }
static int toku_env_set_lk_max_locks(DB_ENV *dbenv, u_int32_t max) {
HANDLE_PANICKED_ENV(dbenv);
if (env_opened(dbenv)) return EINVAL;
if (!max) return EINVAL;
dbenv->i->max_locks = max;
return 0;
}
static int toku_env_get_lk_max_locks(DB_ENV *dbenv, u_int32_t *lk_maxp) {
HANDLE_PANICKED_ENV(dbenv);
if (!lk_maxp) return EINVAL;
*lk_maxp = dbenv->i->max_locks;
return 0;
}
static int toku_db_lt_panic(DB* db, int r) {
assert(db && db->i && db->dbenv && db->dbenv->i);
DB_ENV* env = db->dbenv;
env->i->is_panicked = 1;
if (r < 0) do_error(env, 0, toku_lt_strerror(r));
else do_error(env, r, "Error in locktree.\n");
return EINVAL;
}
static int toku_db_open(DB * db, DB_TXN * txn, const char *fname, const char *dbname, DBTYPE dbtype, u_int32_t flags, int mode) { static int toku_db_open(DB * db, DB_TXN * txn, const char *fname, const char *dbname, DBTYPE dbtype, u_int32_t flags, int mode) {
HANDLE_PANICKED_DB(db); HANDLE_PANICKED_DB(db);
// Warning. Should check arguments. Should check return codes on malloc and open and so forth. // Warning. Should check arguments. Should check return codes on malloc and open and so forth.
...@@ -1679,6 +1717,14 @@ static int toku_db_open(DB * db, DB_TXN * txn, const char *fname, const char *db ...@@ -1679,6 +1717,14 @@ static int toku_db_open(DB * db, DB_TXN * txn, const char *fname, const char *db
db->i->open_flags = flags; db->i->open_flags = flags;
db->i->open_mode = mode; db->i->open_mode = mode;
/* TODO: Only create lock tree if necessary! (lock subsystem?) */
r = toku_lt_create(&db->i->lt, db, FALSE,
toku_db_lt_panic, db->dbenv->i->max_locks,
&db->dbenv->i->num_locks,
db->i->brt->compare_fun, db->i->brt->dup_compare,
toku_malloc, toku_free, toku_realloc);
if (r!=0) goto error_cleanup;
r = toku_brt_open(db->i->brt, db->i->full_fname, fname, dbname, r = toku_brt_open(db->i->brt, db->i->full_fname, fname, dbname,
is_db_create, is_db_excl, is_db_unknown, is_db_create, is_db_excl, is_db_unknown,
db->dbenv->i->cachetable, db->dbenv->i->cachetable,
...@@ -1687,6 +1733,15 @@ static int toku_db_open(DB * db, DB_TXN * txn, const char *fname, const char *db ...@@ -1687,6 +1733,15 @@ static int toku_db_open(DB * db, DB_TXN * txn, const char *fname, const char *db
if (r != 0) if (r != 0)
goto error_cleanup; goto error_cleanup;
unsigned int brtflags;
BOOL dups;
/* Whether we have dups is only known starting now. */
toku_brt_get_flags(db->i->brt, &brtflags);
dups = (brtflags & TOKU_DB_DUPSORT || brtflags & TOKU_DB_DUP);
r = toku_lt_set_dups(db->i->lt, dups);
/* toku_lt_set_dups cannot return an error here. */
assert(r==0);
return 0; return 0;
error_cleanup: error_cleanup:
...@@ -1698,6 +1753,10 @@ static int toku_db_open(DB * db, DB_TXN * txn, const char *fname, const char *db ...@@ -1698,6 +1753,10 @@ static int toku_db_open(DB * db, DB_TXN * txn, const char *fname, const char *db
toku_free(db->i->full_fname); toku_free(db->i->full_fname);
db->i->full_fname = NULL; db->i->full_fname = NULL;
} }
if (db->i->lt) {
toku_lt_close(db->i->lt);
db->i->lt = NULL;
}
return r; return r;
} }
...@@ -1903,6 +1962,14 @@ static int toku_db_fd(DB *db, int *fdp) { ...@@ -1903,6 +1962,14 @@ static int toku_db_fd(DB *db, int *fdp) {
#if _THREAD_SAFE #if _THREAD_SAFE
static int __attribute__((unused)) locked_env_set_lk_max_locks(DB_ENV *dbenv, u_int32_t max) {
ydb_lock(); int r = toku_env_set_lk_max_locks(dbenv, max); ydb_unlock(); return r;
}
static int __attribute__((unused)) locked_env_get_lk_max_locks(DB_ENV *dbenv, u_int32_t *lk_maxp) {
ydb_lock(); int r = toku_env_get_lk_max_locks(dbenv, lk_maxp); ydb_unlock(); return r;
}
static int locked_db_associate (DB *primary, DB_TXN *txn, DB *secondary, static int locked_db_associate (DB *primary, DB_TXN *txn, DB *secondary,
int (*callback)(DB *secondary, const DBT *key, const DBT *data, DBT *result), u_int32_t flags) { int (*callback)(DB *secondary, const DBT *key, const DBT *data, DBT *result), u_int32_t flags) {
ydb_lock(); int r = toku_db_associate(primary, txn, secondary, callback, flags); ydb_unlock(); return r; ydb_lock(); int r = toku_db_associate(primary, txn, secondary, callback, flags); ydb_unlock(); return r;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7