Skip to content

M
MariaDB
Commit 9e06e600 authored by Rich Prohaska's avatar Rich Prohaska Committed by Yoni Fogel
Browse files
Options

merge 1032 to 1032b. addresses #1231 

git-svn-id: file:///svn/toku/tokudb.1032b@7815 c7de825b-a66e-492c-adef-691d508d4ae1
parent e9c99ea9
Show whitespace changes
Inline Side-by-side
Showing with 109 additions and 101 deletions
db-benchmark-test/Makefile
View file @ 9e06e600
#ident "Copyright (c) 2007 Tokutek Inc. All rights reserved."
# standard build: make
# build with Berkeley DB 4.1: make BDBDIR=/usr/local/BerkeleyDB.4.1
# build with TokuDB: make BDBDIR=~/svn/tokudb
# build with g++: make CC=g++
#ident "Copyright (c) 2007 Tokutek Inc. All rights reserved."
# -*- Mode: Makefile -*-
.DEFAULT_GOAL= build
TOKUROOT=../
INCLUDEDIRS=-I. -I../ -I$(TOKUROOT)newbrt -I../range_tree -I../lock_tree
DEPEND_COMPILE += \
../*.h \
../range_tree/*.h \
../lock_tree/*.h \
test.h \
#end
SKIP_NORETURN=1 #Do not add the -Wmissing-noreturn flag
HERE = db-benchmark-test
ifeq ($(SUMMARIZE),1)
SUMMARIZE_CMD = ;if test $$? = 0; then printf "%-60sPASS\n" $(HERE)/$@; else printf "%-60sFAIL\n" $(HERE)/$@ ; test 0 = 1; fi
QUIET = -q
else
SUMMARIZE_CMD =
QUIET =
endif
include $(TOKUROOT)include/Makefile.include
BENCHDBS = bench.bdb/ bench.tokudb ptest*.dir/ x.dir/ xfast.dir/
OPTFLAGS = -O2
CFLAGS = -Wall -Werror -g $(OPTFLAGS) $(GCOV_FLAGS) $(PROF_FLAGS)
# CFLAGS += -pg
ifeq ($(CC),icc)
CFLAGS += -diag-disable 981
endif
LDFLAGS += -lpthread
ifdef BDBDIR
BDB_CPPFLAGS = -I$(BDBDIR)/include
BDB_LDFLAGS = -L$(BDBDIR)/lib -ldb -Wl,-rpath,$(BDBDIR)/lib -lpthread
......@@ -32,57 +28,80 @@ else
BDB_CPPFLAGS =
BDB_LDFLAGS = -ldb
endif
TDB_CPPFLAGS = -I../include
TDB_LDFLAGS = -L../lib -ltokudb -Wl,-rpath,$(PWD)/../lib -lpthread -lz
TOKUDB=libtokudb.$(SOEXT)
TDB_DLINK_FILES = $(TOKUROOT)lib/$(TOKUDB)
TARGET_BDB = db-benchmark-test-bdb
TARGET_TDB = db-benchmark-test-tokudb
TARGETS = $(TARGET_BDB) scanscan-bdb $(TARGET_TDB) scanscan-tokudb
TARGET_BDB = db-benchmark-test-bdb$(BINSUF)
TARGET_TDB = db-benchmark-test-tokudb$(BINSUF)
SCANSCAN_TDB = scanscan-tokudb$(BINSUF)
TARGETS = $(TARGET_BDB) $(TARGET_TDB) $(SCANSCAN_TDB)
default: build
build: $(TARGETS)
build.tdb: $(TARGET_TDB) $(SCANSCAN_TDB)
check: check-default check-xfast check-x
check-default: $(TARGET_TDB)
$(VGRIND) ./$(TARGET_TDB) $(QUIET) $(SUMMARIZE_CMD)
$(MAYBEATSIGN)$(VGRIND) ./$(TARGET_TDB) $(QUIET) $(SUMMARIZE_CMD)
check-x: $(TARGET_TDB)
$(VGRIND) ./$(TARGET_TDB) $(QUIET) -x --xcount 1000 --periter 100000 --env x.dir 10 $(SUMMARIZE_CMD)
$(MAYBEATSIGN)$(VGRIND) ./$(TARGET_TDB) $(QUIET) -x --xcount 1000 --periter 100000 --env x.dir 10 $(SUMMARIZE_CMD)
# A fast transaction test that detects #455.
check-xfast: $(TARGET_TDB)
./$(TARGET_TDB) $(QUIET) --noserial -x --valsize 1000 --cachesize 8000000 --xcount 1000 --periter 20000 --env xfast.dir 1 $(SUMMARIZE_CMD)
$(MAYBEATSIGN)./$(TARGET_TDB) $(QUIET) --noserial -x --valsize 1000 --cachesize 8000000 --xcount 1000 --periter 20000 --env xfast.dir 1 $(SUMMARIZE_CMD)
clean:
rm -rf $(TARGETS) $(BENCHDBS) *.gcno *.gcda *.gcov
$(MAYBEATSIGN)rm -f $(TARGETS)
$(MAYBEATSIGN)rm -rf $(BENCHDBS)
$(TARGET_TDB) $(SCANSCAN_TDB): BIN_FROM_C_FLAGS += -DDIRSUF=tokudb
ifneq ($(PROF),)
USE_STATIC=1
endif
# A hack to make gprof work. See #515.
ifeq ($(PROF_FLAGS),-pg)
ifneq ($(USE_STATIC),)
$(TARGET_TDB) $(SCANSCAN_TDB): DLINK_FILES=
OFILES = \
../src/ydb.o ../src/errors.o ../src/elocks.o \
../newbrt/newbrt.o \
../src/lock_tree/locktree.o ../src/lock_tree/rth.o ../src/lock_tree/lth.o ../src/lock_tree/idlth.o ../src/lock_tree/db_id.o \
../src/range_tree/rangetree.o
db-benchmark-test-tokudb: db-benchmark-test.c
$(CC) $(CFLAGS) $(TDB_CPPFLAGS) $(OFILES) -lz $< -o $@ -DDIRSUF=tokudb
scanscan-tokudb: scanscan.c
$(CC) $(CFLAGS) $(TDB_CPPFLAGS) $(OFILES) -lz $< -o $@ -DDIRSUF=tokudb
$(TOKUROOT)src/ydb.lib \
$(TOKUROOT)src/range_tree/rangetree.lib \
$(TOKUROOT)src/lock_tree/locktree.lib \
$(TOKUROOT)newbrt/newbrt.lib \
$(LIBPORTABILITY) \
#end
$(TARGET_TDB): db-benchmark-test.c
$(MAYBEATSIGN)$(CC) $< $(BIN_FROM_C_FLAGS) $(OFILES)
$(SCANSCAN_TDB): scanscan.c
$(MAYBEATSIGN)$(CC) $< $(BIN_FROM_C_FLAGS) $(OFILES)
else
db-benchmark-test-tokudb: db-benchmark-test.c
$(CC) $(CFLAGS) $(TDB_CPPFLAGS) $(TDB_LDFLAGS) $< -o $@ -DDIRSUF=tokudb
scanscan-tokudb: scanscan.c
$(CC) $(CFLAGS) $(TDB_CPPFLAGS) $(TDB_LDFLAGS) $< -o $@ -DDIRSUF=tokudb
$(TARGET_TDB) $(SCANSCAN_TDB): DLINK_FILES=$(TDB_DLINK_FILES)
ifeq ($(SOEXT),dll) #Windows does not support rpath.
$(TOKUDB): $(TOKUROOT)lib/$(TOKUDB)
$(MAYBEATSIGN)cp $< $@
$(TOKUROOT)lib/$(TOKUDB):
$(MAYBEATSIGN)cd $(@D) && $(MAKE) $(@F).install
$(TARGET_TDB) $(SCANSCAN_TDB): $(TOKUDB)
endif
db-benchmark-test-bdb: db-benchmark-test.c
$(CC) $(CFLAGS) $(BDB_CPPFLAGS) $(BDB_LDFLAGS) $< -o $@ -DDIRSUF=bdb
scanscan-bdb: scanscan.c
$(TARGET_TDB) $(SCANSCAN_TDB): RPATH_DIRS=$(dir $(TDB_DLINK_FILES))
$(TARGET_TDB): db-benchmark-test.c
$(MAYBEATSIGN)$(CC) $< $(BIN_FROM_C_FLAGS)
$(SCANSCAN_TDB): scanscan.c
$(MAYBEATSIGN)$(CC) $< $(BIN_FROM_C_FLAGS)
endif
$(TARGET_BDB): db-benchmark-test.c
$(CC) $(CFLAGS) $(BDB_CPPFLAGS) $(BDB_LDFLAGS) $< -o $@ -DDIRSUF=bdb
PARGS =
ptest0 ptest1 ptest2 ptest3 ptest4 ptest5 ptest6 ptest7: db-benchmark-test-tokudb
./db-benchmark-test-tokudb --env $@.dir $(PARGS)
ptest%: $(TARGET_TDB)
./$< --env $@.dir $(PARGS)
parallel-test-8: ptest0 ptest1 ptest2 ptest3 ptest4 ptest5 ptest6 ptest7
db-benchmark-test/db-benchmark-test.c
View file @ 9e06e600
......@@ -2,12 +2,13 @@
#ident "Copyright (c) 2007, 2008 Tokutek Inc. All rights reserved."
/* Insert a bunch of stuff */
#include <portability.h>
#include <assert.h>
#include <errno.h>
#include <string.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/time.h>
#include <sys/stat.h>
#include <unistd.h>
#include <db.h>
......@@ -53,15 +54,13 @@ static void do_prelock(DB* db, DB_TXN* txn) {
#if !defined(NO_DB_PRELOCKED)
int r = db->pre_acquire_table_lock(db, txn);
assert(r==0);
#else
db = db; txn = txn;
#endif
}
}
#define STRINGIFY2(s) #s
#define STRINGIFY(s) STRINGIFY2(s)
const char *dbdir = "./bench." STRINGIFY(DIRSUF) "/"; /* DIRSUF is passed in as a -D argument to the compiler. */
const char *dbdir = "./bench." STRINGIFY(DIRSUF); /* DIRSUF is passed in as a -D argument to the compiler. */
char *dbfilename = "bench.db";
char *dbname;
......@@ -70,7 +69,7 @@ DB *db;
DB_TXN *tid=0;
static void setup (void) {
static void benchmark_setup (void) {
int r;
{
......@@ -79,8 +78,10 @@ static void setup (void) {
//printf("unlink_cmd=%s\n", unlink_cmd);
system(unlink_cmd);
}
if (strcmp(dbdir, ".") != 0)
mkdir(dbdir, 0755);
if (strcmp(dbdir, ".") != 0) {
r = os_mkdir(dbdir,S_IRWXU|S_IRGRP|S_IXGRP|S_IROTH|S_IXOTH);
assert(r == 0);
}
r = db_env_create(&dbenv, 0);
assert(r == 0);
......@@ -103,7 +104,7 @@ static void setup (void) {
}
{
r = dbenv->open(dbenv, dbdir, env_open_flags, 0644);
r = dbenv->open(dbenv, dbdir, env_open_flags, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
assert(r == 0);
}
......@@ -122,7 +123,7 @@ static void setup (void) {
r = db->set_flags(db, dupflags);
assert(r == 0);
}
r = db->open(db, tid, dbfilename, NULL, DB_BTREE, DB_CREATE, 0644);
r = db->open(db, tid, dbfilename, NULL, DB_BTREE, DB_CREATE, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
if (r!=0) fprintf(stderr, "errno=%d, %s\n", errno, strerror(errno));
assert(r == 0);
if (do_transactions) {
......@@ -135,7 +136,7 @@ static void setup (void) {
}
static void shutdown (void) {
static void benchmark_shutdown (void) {
int r;
if (do_transactions && singlex) {
......@@ -167,7 +168,7 @@ static void fill_array (unsigned char *data, int size) {
if (compressibility>0) {
int i;
for (i=0; i<size/compressibility; i++) {
data[i] = (unsigned char)random();
data[i] = (unsigned char) random();
}
}
}
......@@ -380,11 +381,11 @@ int main (int argc, const char *argv[]) {
if (!norandom) printf("random ");
printf("insertions of %d per batch%s\n", items_per_iteration, do_transactions ? " (with transactions)" : "");
}
setup();
benchmark_setup();
gettimeofday(&t1,0);
biginsert(total_n_items, &t1);
gettimeofday(&t2,0);
shutdown();
benchmark_shutdown();
gettimeofday(&t3,0);
if (verbose) {
printf("Shutdown %9.6fs\n", tdiff(&t3, &t2));
......
db-benchmark-test/scanscan.c
View file @ 9e06e600
/* Scan the bench.tokudb/bench.db over and over. */
#include <portability.h>
#include <assert.h>
#include <db.h>
#include <errno.h>
#include <stdlib.h>
#include <string.h>
#include <sys/resource.h>
#include <sys/time.h>
//#include <sys/resource.h>
//#include <sys/time.h>
#include <fcntl.h>
#include <unistd.h>
const char *pname;
......@@ -32,12 +34,8 @@ static void parse_args (int argc, const char *argv[]) {
if (specified_run_mode && run_mode!=RUN_VERIFY) goto two_modes;
run_mode = RUN_HWC;
} else if (strcmp(*argv, "--prelock")==0) prelock=1;
#if defined(DB_PRELOCKED)
else if (strcmp(*argv, "--prelockflag")==0) { prelockflag=1; lock_flag = DB_PRELOCKED; }
#endif
#if defined(DB_PRELOCKED_WRITE)
else if (strcmp(*argv, "--prelockwriteflag")==0) { prelockflag=1; lock_flag = DB_PRELOCKED_WRITE; }
#endif
else if (strcmp(*argv, "--nox")==0) { do_txns=0; }
else if (strcmp(*argv, "--count")==0) {
char *end;
......@@ -73,22 +71,21 @@ DB_TXN *tid=0;
#define STRINGIFY2(s) #s
#define STRINGIFY(s) STRINGIFY2(s)
const char *dbdir = "./bench." STRINGIFY(DIRSUF) "/"; /* DIRSUF is passed in as a -D argument to the compiler. */
const char *dbdir = "./bench." STRINGIFY(DIRSUF); /* DIRSUF is passed in as a -D argument to the compiler. */
int env_open_flags_yesx = DB_CREATE|DB_PRIVATE|DB_INIT_MPOOL|DB_INIT_TXN|DB_INIT_LOG|DB_INIT_LOCK;
int env_open_flags_nox = DB_CREATE|DB_PRIVATE|DB_INIT_MPOOL;
char *dbfilename = "bench.db";
static void setup (void) {
static void scanscan_setup (void) {
int r;
r = db_env_create(&env, 0); assert(r==0);
r = env->set_cachesize(env, 0, cachesize, 1); assert(r==0);
r = env->open(env, dbdir, do_txns? env_open_flags_yesx : env_open_flags_nox, 0644); assert(r==0);
r = env->open(env, dbdir, do_txns? env_open_flags_yesx : env_open_flags_nox, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH); assert(r==0);
r = db_create(&db, env, 0); assert(r==0);
if (do_txns) {
r = env->txn_begin(env, 0, &tid, 0); assert(r==0);
}
r = db->open(db, tid, dbfilename, NULL, DB_BTREE, 0, 0644); assert(r==0);
#if defined(DB_PRELOCKED)
r = db->open(db, tid, dbfilename, NULL, DB_BTREE, 0, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH); assert(r==0);
if (prelock) {
r = db->pre_acquire_read_lock(db,
tid,
......@@ -96,17 +93,17 @@ static void setup (void) {
db->dbt_pos_infty(), db->dbt_pos_infty());
assert(r==0);
}
#endif
}
static void shutdown (void) {
static void scanscan_shutdown (void) {
int r;
r = db->close(db, 0); assert(r==0);
if (do_txns) {
r = tid->commit(tid, 0); assert(r==0);
}
r = db->close(db, 0); assert(r==0);
r = env->close(env, 0); assert(r==0);
#if 0
#if 0 && defined TOKUDB
{
extern unsigned long toku_get_maxrss(void);
printf("maxrss=%.2fMB\n", toku_get_maxrss()/256.0);
......@@ -149,8 +146,6 @@ static void scanscan_hwc (void) {
}
}
#if defined(TOKUDB)
struct extra_count {
long long totalbytes;
int rowcounter;
......@@ -186,16 +181,12 @@ static void scanscan_lwc (void) {
}
}
#endif
struct extra_verify {
long long totalbytes;
int rowcounter;
DBT k,v; // the k and v are gotten using the old cursor
};
#if defined(TOKUDB)
static void checkbytes (DBT const *key, DBT const *data, void *extrav) {
struct extra_verify *e=extrav;
e->totalbytes += key->size + data->size;
......@@ -208,7 +199,6 @@ static void checkbytes (DBT const *key, DBT const *data, void *extrav) {
assert(e->v.data != data->data);
}
#endif
static void scanscan_verify (void) {
int r;
......@@ -230,14 +220,11 @@ static void scanscan_verify (void) {
c_get_flags |= lock_flag;
}
while (1) {
int r2 = dbc1->c_get(dbc1, &v.k, &v.v, c_get_flags);
#if defined(TOKUDB)
int r1 = dbc2->c_getf_next(dbc2, f_flags, checkbytes, &v);
int r1,r2;
r2 = dbc1->c_get(dbc1, &v.k, &v.v, c_get_flags);
r1 = dbc2->c_getf_next(dbc2, f_flags, checkbytes, &v);
assert(r1==r2);
if (r1) break;
#else
if (r2) break;
#endif
}
r = dbc1->c_close(dbc1); assert(r==0);
r = dbc2->c_close(dbc2); assert(r==0);
......@@ -252,17 +239,15 @@ int main (int argc, const char *argv[]) {
parse_args(argc,argv);
setup();
scanscan_setup();
switch (run_mode) {
case RUN_HWC: scanscan_hwc(); break;
#if defined(TOKUDB)
case RUN_LWC: scanscan_lwc(); break;
#endif
case RUN_VERIFY: scanscan_verify(); break;
default: assert(0);
case RUN_HWC: scanscan_hwc(); goto ok;
case RUN_LWC: scanscan_lwc(); goto ok;
case RUN_VERIFY: scanscan_verify(); goto ok;
}
shutdown();
assert(0);
ok:
scanscan_shutdown();
#if 0 && defined TOKUDB
if (0) {
......@@ -276,6 +261,7 @@ int main (int argc, const char *argv[]) {
extern void toku_print_trace_mem();
toku_print_trace_mem();
}
#endif
#if defined __linux__ && __linux__
char fname[256];
sprintf(fname, "/proc/%d/status", getpid());
......@@ -289,7 +275,6 @@ int main (int argc, const char *argv[]) {
}
fclose(f);
}
#endif
#endif
return 0;
}
src/ydb.c
View file @ 9e06e600
......@@ -771,6 +771,7 @@ static int toku_env_txn_stat(DB_ENV * env, DB_TXN_STAT ** statp, u_int32_t flags
return 1;
}
#if 0
#if DB_VERSION_MAJOR == 4 && DB_VERSION_MINOR == 1
static void toku_default_errcall(const char *errpfx, char *msg) {
fprintf(stderr, "YDB: %s: %s", errpfx, msg);
......@@ -781,6 +782,7 @@ static void toku_default_errcall(const DB_ENV *env, const char *errpfx, const ch
fprintf(stderr, "YDB: %s: %s", errpfx, msg);
}
#endif
#endif
static int locked_env_open(DB_ENV * env, const char *home, u_int32_t flags, int mode) {
toku_ydb_lock(); int r = toku_env_open(env, home, flags, mode); toku_ydb_unlock(); return r;
......@@ -862,7 +864,7 @@ static int toku_env_create(DB_ENV ** envp, u_int32_t flags) {
MALLOC(result);
if (result == 0) { r = ENOMEM; goto cleanup; }
memset(result, 0, sizeof *result);
result->err = toku_locked_env_err;
result->err = (void (*)(const DB_ENV * env, int error, const char *fmt, ...)) toku_locked_env_err;
result->open = locked_env_open;
result->close = locked_env_close;
result->txn_checkpoint = locked_env_txn_checkpoint;
......@@ -1138,8 +1140,9 @@ int txn_commit(DB_TXN * txn, u_int32_t flags) {
int log_compare(const DB_LSN * a, const DB_LSN * b) {
toku_ydb_lock();
fprintf(stderr, "%s:%d log_compare(%p,%p)\n", __FILE__, __LINE__, a, b);
abort();
assert(0);
toku_ydb_unlock();
return 0;
}
static int maybe_do_associate_create (DB_TXN*txn, DB*primary, DB*secondary) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7