Commit 8e2d40b9 authored by Rich Prohaska's avatar Rich Prohaska Committed by Yoni Fogel

merge tokudb.1844 to main close[t:1844]

git-svn-id: file:///svn/toku/tokudb@14180 c7de825b-a66e-492c-adef-691d508d4ae1
parent 6e14e934
......@@ -12,9 +12,9 @@ extern "C" {
#define TOKUDB_NATIVE_H 0
#define DB_VERSION_MAJOR 4
#define DB_VERSION_MINOR 6
#define DB_VERSION_PATCH 19
#define DB_VERSION_PATCH 21
#ifndef _TOKUDB_WRAP_H
#define DB_VERSION_STRING "Tokutek: TokuDB 4.6.19"
#define DB_VERSION_STRING "Tokutek: TokuDB 4.6.21"
#else
#define DB_VERSION_STRING_ydb "Tokutek: TokuDB (wrapped bdb)"
#endif
......@@ -63,6 +63,7 @@ typedef enum {
#define DB_PRIVATE 8388608
#define DB_RDONLY 32
#define DB_RECOVER 64
#define DB_RUNRECOVERY -30975
#define DB_THREAD 128
#define DB_TXN_NOSYNC 512
#define DB_LOCK_DEFAULT 1
......
......@@ -60,6 +60,7 @@ void print_defines (void) {
dodefine(DB_PRIVATE);
dodefine(DB_RDONLY);
dodefine(DB_RECOVER);
dodefine(DB_RUNRECOVERY);
dodefine(DB_THREAD);
dodefine(DB_TXN_NOSYNC);
......
This diff is collapsed.
......@@ -213,6 +213,9 @@ brtnode_put_cmd (BRT t, BRTNODE node, BRT_CMD cmd, enum reactivity *re, BOOL *di
static int
flush_this_child (BRT t, BRTNODE node, int childnum, enum reactivity *child_re, BOOL *did_io);
static void brt_verify_flags(BRT brt, BRTNODE node) {
assert(brt->flags == node->flags);
}
int toku_brt_debug_mode = 0;
......@@ -2502,10 +2505,9 @@ int toku_brt_root_put_cmd(BRT brt, BRT_CMD cmd, TOKULOGGER logger)
node=node_v;
VERIFY_NODE(brt, node);
assert(node->fullhash==fullhash);
brt_verify_flags(brt, node);
VERIFY_NODE(brt, node);
verify_local_fingerprint_nonleaf(node);
if ((r = push_something_at_root(brt, &node, rootp, cmd, logger))) {
toku_unpin_brtnode(brt, node); // ignore any error code on the unpin.
......
......@@ -13,6 +13,8 @@
#include "log_header.h"
#include "varray.h"
static int toku_recover_trace = 0;
//#define DO_VERIFY_COUNTS
#ifdef DO_VERIFY_COUNTS
#define VERIFY_COUNTS(n) toku_verify_counts(n)
......@@ -41,6 +43,8 @@ int toku_recover_init (void) {
assert(r == 0);
toku_logger_write_log_files(recover_logger, FALSE);
toku_logger_set_cachetable(recover_logger, recover_ct);
if (toku_recover_trace)
printf("%s:%d\n", __FUNCTION__, __LINE__);
return r;
}
......@@ -71,6 +75,9 @@ void toku_recover_cleanup (void) {
r = toku_cachetable_close(&recover_ct);
assert(r == 0);
if (toku_recover_trace)
printf("%s:%d\n", __FUNCTION__, __LINE__);
}
// Null function supplied to transaction commit and abort
......@@ -492,11 +499,10 @@ static int toku_delete_rolltmp_files (const char *log_dir) {
return result;
}
// Does this log environment need recovery?
// Effects: If there are no log files, or if there is a "null" checkpoint at the end of the log,
// Effects: If there are no log files, or if there is a "clean" checkpoint at the end of the log,
// then we don't need recovery to run. We skip the optional shutdown log entry.
// Returns: TRUE if we need recovery, FALSE if we do not need recovery.
static int tokudb_needs_recovery(const char *log_dir) {
int tokudb_needs_recovery(const char *log_dir) {
int needs_recovery;
int r;
TOKULOGCURSOR logcursor = NULL;
......@@ -569,8 +575,6 @@ static void abort_live_txn(void *v, void *UU(extra)) {
toku_txn_close_txn(txn);
}
static const int toku_recover_trace = 0;
static int really_do_recovery(const char *data_dir, const char *log_dir) {
int r;
......@@ -602,12 +606,14 @@ static int really_do_recovery(const char *data_dir, const char *log_dir) {
while (1) {
le = NULL;
r = toku_logcursor_prev(logcursor, &le);
if (toku_recover_trace) printf("%s:%d r=%d cmd=%c\n", __FUNCTION__, __LINE__, r, le ? le->cmd : '?');
if (toku_recover_trace)
printf("%s:%d r=%d cmd=%c\n", __FUNCTION__, __LINE__, r, le ? le->cmd : '?');
if (r != 0)
break;
logtype_dispatch_assign(le, toku_recover_backward_, r, &bs);
if (r != 0) {
if (toku_recover_trace) printf("%s:%d r=%d cmd=%c\n", __FUNCTION__, __LINE__, r, le ? le->cmd : '?');
if (toku_recover_trace)
printf("%s:%d r=%d cmd=%c\n", __FUNCTION__, __LINE__, r, le ? le->cmd : '?');
logtype_dispatch_args(le, toku_recover_);
break;
}
......@@ -617,7 +623,8 @@ static int really_do_recovery(const char *data_dir, const char *log_dir) {
while (1) {
le = NULL;
r = toku_logcursor_next(logcursor, &le);
if (toku_recover_trace) printf("%s:%d r=%d cmd=%c\n", __FUNCTION__, __LINE__, r, le ? le->cmd : '?');
if (toku_recover_trace)
printf("%s:%d r=%d cmd=%c\n", __FUNCTION__, __LINE__, r, le ? le->cmd : '?');
if (r != 0)
break;
logtype_dispatch_args(le, toku_recover_);
......
......@@ -15,7 +15,15 @@
#include "x1764.h"
int toku_recover_init(void);
void toku_recover_cleanup(void);
int tokudb_recover(const char *datadir, const char *logdir);
// Effect: Check the tokudb logs to determine whether or not we need to run recovery.
// If the log is empty or if there is a clean shutdown at the end of the log, then we
// dont need to run recovery.
// Returns: TRUE if we need recovery, otherwise FALSE.
int tokudb_needs_recovery(const char *logdir);
#endif // TOKURECOVER_H
......@@ -143,6 +143,9 @@ TDB_TESTS_THAT_SHOULD_FAIL= \
test_groupcommit_count \
test944 \
test_truncate_txn_abort \
test_log8 \
test_log9 \
test_log10 \
#\ ends prev line
ifneq ($(OS_CHOICE),windows)
TDB_TESTS_THAT_SHOULD_FAIL+= \
......
......@@ -23,7 +23,7 @@ checkpoint_test_1(u_int32_t flags, u_int32_t n, int snap_all) {
fflush(stdout);
}
dir_create();
env_startup(0);
env_startup(0, FALSE);
int run;
int r;
DICTIONARY_S db_control;
......@@ -61,7 +61,7 @@ checkpoint_test_2(u_int32_t flags, u_int32_t n) {
fflush(stdout);
}
dir_create();
env_startup(0);
env_startup(0, FALSE);
int run;
int r;
DICTIONARY_S db_control;
......
......@@ -39,6 +39,7 @@ Each iteration does:
#define NUM_DICTIONARIES 4 // any more than 3 is overkill to exercise linked list logic
static int oper_per_iter = 5001; // not-very-nice odd number (not a multiple of a power of two)
static int do_log_recover = 0;
static toku_pthread_t thread;
......@@ -219,7 +220,7 @@ run_test (int iter, int die) {
if (verbose)
printf("checkpoint_stress: iter = %d, cachesize (bytes) = 0x%08"PRIx64"\n", iter, cachebytes);
env_startup(cachebytes);
env_startup(cachebytes, do_log_recover);
// create array of dictionaries
// for each dictionary verify previous iterations and perform new inserts
......@@ -265,8 +266,9 @@ run_test (int iter, int die) {
static void
usage(char *progname) {
fprintf(stderr, "Usage:\n%s [-i n] [-q|-v]\n"
" \n%s [-h]\n", progname, progname);
fprintf(stderr, "Usage:\n%s [-c] [-C] [-i N] [-n N] [-l] [-q|-v]\n"
" \n%s [-h]\n", progname,
progname);
}
......@@ -279,7 +281,7 @@ test_main (int argc, char *argv[]) {
int c;
int crash = 0;
while ((c = getopt(argc, argv, "cChi:qvn:")) != -1) {
while ((c = getopt(argc, argv, "cChi:qvn:l:")) != -1) {
switch(c) {
case 'c':
crash = 1;
......@@ -293,6 +295,9 @@ test_main (int argc, char *argv[]) {
case 'n':
oper_per_iter = atoi(optarg);
break;
case 'l':
do_log_recover = 1;
break;
case 'v':
verbose++;
break;
......
......@@ -96,7 +96,7 @@ dir_create(void) {
// pass in zeroes for default cachesize
static void UU()
env_startup(int64_t bytes) {
env_startup(int64_t bytes, BOOL do_log_recover) {
int r;
r = db_env_create(&env, 0);
CKERR(r);
......@@ -108,7 +108,10 @@ env_startup(int64_t bytes) {
r = env->set_cachesize(env, bytes >> 30, bytes % (1<<30), 1);
CKERR(r);
}
r = env->open(env, ENVDIR, DB_INIT_LOCK|DB_INIT_LOG|DB_INIT_MPOOL|DB_INIT_TXN|DB_CREATE|DB_PRIVATE, S_IRWXU+S_IRWXG+S_IRWXO);
int envflags = DB_INIT_LOCK | DB_INIT_MPOOL | DB_INIT_TXN | DB_CREATE | DB_PRIVATE;
if (do_log_recover)
envflags += DB_INIT_LOG | DB_RECOVER;
r = env->open(env, ENVDIR, envflags, S_IRWXU+S_IRWXG+S_IRWXO);
CKERR(r);
env->set_errfile(env, stderr);
r = env->checkpointing_set_period(env, 0); //Disable auto-checkpointing.
......
......@@ -42,7 +42,7 @@ checkpoint_truncate_test(u_int32_t flags, u_int32_t n) {
fflush(stdout);
}
dir_create();
env_startup(0);
env_startup(0, FALSE);
DICTIONARY_S db_control;
init_dictionary(&db_control, flags, "control");
......
#include <sys/stat.h>
#include <sys/wait.h>
#include "test.h"
const int envflags = DB_INIT_MPOOL|DB_CREATE|DB_THREAD |DB_INIT_LOCK|DB_INIT_LOG|DB_INIT_TXN;
char *namea="a.db";
char *nameb="b.db";
static void
run_test (void) {
int r;
system("rm -rf " ENVDIR);
toku_os_mkdir(ENVDIR, S_IRWXU+S_IRWXG+S_IRWXO);
DB_ENV *env;
DB *dba, *dbb;
r = db_env_create(&env, 0); CKERR(r);
r = env->open(env, ENVDIR, envflags, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
r = db_create(&dba, env, 0); CKERR(r);
r = dba->set_flags(dba, DB_DUPSORT); CKERR(r);
r = dba->open(dba, NULL, namea, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); CKERR(r);
r = db_create(&dbb, env, 0); CKERR(r);
r = dbb->set_flags(dbb, DB_DUPSORT); CKERR(r);
r = dbb->open(dbb, NULL, nameb, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); CKERR(r);
DB_TXN *txn;
r = env->txn_begin(env, NULL, &txn, 0); CKERR(r);
{
DBT a={.data="a", .size=2};
DBT b={.data="b", .size=2};
r = dba->put(dba, txn, &a, &b, DB_YESOVERWRITE); CKERR(r);
r = env->txn_checkpoint(env, 0, 0, 0); CKERR(r);
r = dbb->put(dbb, txn, &b, &a, DB_YESOVERWRITE); CKERR(r);
}
r = txn->commit(txn, 0); CKERR(r);
abort();
}
static void
run_recover (void) {
DB_ENV *env;
int r;
r = db_env_create(&env, 0); CKERR(r);
r = env->open(env, ENVDIR, envflags + DB_RECOVER, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
r = env->close(env, 0); CKERR(r);
exit(0);
}
static void
run_no_recover (void) {
DB_ENV *env;
int r;
r = db_env_create(&env, 0); CKERR(r);
r = env->open(env, ENVDIR, envflags & ~DB_RECOVER, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
r = env->close(env, 0); CKERR(r);
exit(0);
}
const char *cmd;
BOOL do_commit=FALSE, do_abort=FALSE, do_explicit_abort=FALSE, do_recover_committed=FALSE, do_recover_aborted=FALSE, do_recover_only=FALSE, do_no_recover = FALSE;
static void test_parse_args (int argc, char *argv[]) {
int resultcode;
cmd = argv[0];
argc--; argv++;
while (argc>0) {
if (strcmp(argv[0], "-v") == 0) {
verbose++;
} else if (strcmp(argv[0],"-q")==0) {
verbose--;
if (verbose<0) verbose=0;
} else if (strcmp(argv[0], "--commit")==0) {
do_commit=TRUE;
} else if (strcmp(argv[0], "--abort")==0) {
do_abort=TRUE;
} else if (strcmp(argv[0], "--explicit-abort")==0) {
do_explicit_abort=TRUE;
} else if (strcmp(argv[0], "--recover-committed")==0) {
do_recover_committed=TRUE;
} else if (strcmp(argv[0], "--recover-aborted")==0) {
do_recover_aborted=TRUE;
} else if (strcmp(argv[0], "--recover-only") == 0) {
do_recover_only=TRUE;
} else if (strcmp(argv[0], "--no-recover") == 0) {
do_no_recover=TRUE;
} else if (strcmp(argv[0], "-h")==0) {
resultcode=0;
do_usage:
fprintf(stderr, "Usage:\n%s [-v|-q]* [-h] {--commit | --abort | --explicit-abort | --recover-committed | --recover-aborted } \n", cmd);
exit(resultcode);
} else {
fprintf(stderr, "Unknown arg: %s\n", argv[0]);
resultcode=1;
goto do_usage;
}
argc--;
argv++;
}
{
int n_specified=0;
if (do_commit) n_specified++;
if (do_abort) n_specified++;
if (do_explicit_abort) n_specified++;
if (do_recover_committed) n_specified++;
if (do_recover_aborted) n_specified++;
if (do_recover_only) n_specified++;
if (do_no_recover) n_specified++;
if (n_specified>1) {
printf("Specify only one of --commit or --abort or --recover-committed or --recover-aborted\n");
resultcode=1;
goto do_usage;
}
}
}
int
test_main (int argc, char *argv[]) {
test_parse_args(argc, argv);
if (do_commit) {
run_test();
} else if (do_recover_only) {
run_recover();
} else if (do_no_recover) {
run_no_recover();
}
return 0;
}
#include <sys/stat.h>
#include <sys/wait.h>
#include "test.h"
const int envflags = DB_INIT_MPOOL|DB_CREATE|DB_THREAD |DB_INIT_LOCK|DB_INIT_LOG|DB_INIT_TXN;
char *namea="a.db";
char *nameb="b.db";
static void
run_test (void) {
int r;
system("rm -rf " ENVDIR);
toku_os_mkdir(ENVDIR, S_IRWXU+S_IRWXG+S_IRWXO);
DB_ENV *env;
DB *dba, *dbb;
r = db_env_create(&env, 0); CKERR(r);
r = env->open(env, ENVDIR, envflags, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
r = db_create(&dba, env, 0); CKERR(r);
r = dba->open(dba, NULL, namea, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); CKERR(r);
r = db_create(&dbb, env, 0); CKERR(r);
r = dbb->open(dbb, NULL, nameb, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); CKERR(r);
DB_TXN *txn;
r = env->txn_begin(env, NULL, &txn, 0); CKERR(r);
{
DBT a={.data="a", .size=2};
DBT b={.data="b", .size=2};
r = dba->put(dba, txn, &a, &b, DB_YESOVERWRITE); CKERR(r);
r = env->txn_checkpoint(env, 0, 0, 0); CKERR(r);
r = dbb->put(dbb, txn, &b, &a, DB_YESOVERWRITE); CKERR(r);
}
r = txn->commit(txn, 0); CKERR(r);
abort();
}
static void
run_recover (void) {
DB_ENV *env;
int r;
r = db_env_create(&env, 0); CKERR(r);
r = env->open(env, ENVDIR, envflags + DB_RECOVER, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
r = env->close(env, 0); CKERR(r);
exit(0);
}
static void
run_no_recover (void) {
DB_ENV *env;
int r;
r = db_env_create(&env, 0); CKERR(r);
r = env->open(env, ENVDIR, envflags & ~DB_RECOVER, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
r = env->close(env, 0); CKERR(r);
exit(0);
}
const char *cmd;
BOOL do_commit=FALSE, do_abort=FALSE, do_explicit_abort=FALSE, do_recover_committed=FALSE, do_recover_aborted=FALSE, do_recover_only=FALSE, do_no_recover = FALSE;
static void test_parse_args (int argc, char *argv[]) {
int resultcode;
cmd = argv[0];
argc--; argv++;
while (argc>0) {
if (strcmp(argv[0], "-v") == 0) {
verbose++;
} else if (strcmp(argv[0],"-q")==0) {
verbose--;
if (verbose<0) verbose=0;
} else if (strcmp(argv[0], "--commit")==0) {
do_commit=TRUE;
} else if (strcmp(argv[0], "--abort")==0) {
do_abort=TRUE;
} else if (strcmp(argv[0], "--explicit-abort")==0) {
do_explicit_abort=TRUE;
} else if (strcmp(argv[0], "--recover-committed")==0) {
do_recover_committed=TRUE;
} else if (strcmp(argv[0], "--recover-aborted")==0) {
do_recover_aborted=TRUE;
} else if (strcmp(argv[0], "--recover-only") == 0) {
do_recover_only=TRUE;
} else if (strcmp(argv[0], "--no-recover") == 0) {
do_no_recover=TRUE;
} else if (strcmp(argv[0], "-h")==0) {
resultcode=0;
do_usage:
fprintf(stderr, "Usage:\n%s [-v|-q]* [-h] {--commit | --abort | --explicit-abort | --recover-committed | --recover-aborted } \n", cmd);
exit(resultcode);
} else {
fprintf(stderr, "Unknown arg: %s\n", argv[0]);
resultcode=1;
goto do_usage;
}
argc--;
argv++;
}
{
int n_specified=0;
if (do_commit) n_specified++;
if (do_abort) n_specified++;
if (do_explicit_abort) n_specified++;
if (do_recover_committed) n_specified++;
if (do_recover_aborted) n_specified++;
if (do_recover_only) n_specified++;
if (do_no_recover) n_specified++;
if (n_specified>1) {
printf("Specify only one of --commit or --abort or --recover-committed or --recover-aborted\n");
resultcode=1;
goto do_usage;
}
}
}
int
test_main (int argc, char *argv[]) {
test_parse_args(argc, argv);
if (do_commit) {
run_test();
} else if (do_recover_only) {
run_recover();
} else if (do_no_recover) {
run_no_recover();
}
return 0;
}
......@@ -120,6 +120,17 @@ do_x1_recover_only (void) {
exit(0);
}
static void
do_x1_no_recover (void) {
DB_ENV *env;
int r;
r = db_env_create(&env, 0); CKERR(r);
r = env->open(env, ENVDIR, envflags & ~DB_RECOVER, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
r = env->close(env, 0); CKERR(r);
exit(0);
}
const char *cmd;
static void
......@@ -162,7 +173,7 @@ do_test (void) {
do_test_internal(FALSE);
}
BOOL do_commit=FALSE, do_abort=FALSE, do_explicit_abort=FALSE, do_recover_committed=FALSE, do_recover_aborted=FALSE, do_recover_only=FALSE;
BOOL do_commit=FALSE, do_abort=FALSE, do_explicit_abort=FALSE, do_recover_committed=FALSE, do_recover_aborted=FALSE, do_recover_only=FALSE, do_no_recover = FALSE;
static void
x1_parse_args (int argc, char *argv[]) {
......@@ -176,17 +187,19 @@ x1_parse_args (int argc, char *argv[]) {
verbose--;
if (verbose<0) verbose=0;
} else if (strcmp(argv[0], "--commit")==0) {
do_commit=1;
do_commit=TRUE;
} else if (strcmp(argv[0], "--abort")==0) {
do_abort=1;
do_abort=TRUE;
} else if (strcmp(argv[0], "--explicit-abort")==0) {
do_explicit_abort=1;
do_explicit_abort=TRUE;
} else if (strcmp(argv[0], "--recover-committed")==0) {
do_recover_committed=1;
do_recover_committed=TRUE;
} else if (strcmp(argv[0], "--recover-aborted")==0) {
do_recover_aborted=1;
do_recover_aborted=TRUE;
} else if (strcmp(argv[0], "--recover-only") == 0) {
do_recover_only=1;
do_recover_only=TRUE;
} else if (strcmp(argv[0], "--no-recover") == 0) {
do_no_recover=TRUE;
} else if (strcmp(argv[0], "-h")==0) {
resultcode=0;
do_usage:
......@@ -208,6 +221,7 @@ x1_parse_args (int argc, char *argv[]) {
if (do_recover_committed) n_specified++;
if (do_recover_aborted) n_specified++;
if (do_recover_only) n_specified++;
if (do_no_recover) n_specified++;
if (n_specified>1) {
printf("Specify only one of --commit or --abort or --recover-committed or --recover-aborted\n");
resultcode=1;
......@@ -232,6 +246,8 @@ test_main (int argc, char *argv[])
do_x1_recover(FALSE);
} else if (do_recover_only) {
do_x1_recover_only();
} else if (do_no_recover) {
do_x1_no_recover();
} else {
do_test();
}
......
......@@ -302,29 +302,21 @@ static int do_recovery (DB_ENV *env) {
} else {
logdir = toku_strdup(env->i->dir);
}
#if 0
// want to do recovery in its own process
pid_t pid;
if ((pid=fork())==0) {
int r=tokudb_recover(datadir, logdir);
assert(r==0);
toku_free(logdir); // the child must also free.
exit(0);
}
int status;
waitpid(pid, &status, 0);
if (!WIFEXITED(status) || WEXITSTATUS(status)!=0) {
toku_free(logdir);
return toku_ydb_do_error(env, -1, "Recovery failed\n");
}
toku_free(logdir);
return 0;
#else
int r = tokudb_recover(datadir, logdir);
toku_free(logdir);
return r;
#endif
}
static int needs_recovery (DB_ENV *env) {
char *logdir;
if (env->i->lg_dir) {
logdir = construct_full_name(env->i->dir, env->i->lg_dir);
} else {
logdir = toku_strdup(env->i->dir);
}
BOOL recovery_needed = tokudb_needs_recovery(logdir);
toku_free(logdir);
return recovery_needed ? DB_RUNRECOVERY : 0;
}
static int toku_env_open(DB_ENV * env, const char *home, u_int32_t flags, int mode) {
......@@ -396,8 +388,11 @@ static int toku_env_open(DB_ENV * env, const char *home, u_int32_t flags, int mo
unused_flags &= ~DB_INIT_TXN & ~DB_INIT_LOG;
if (flags&DB_RECOVER) {
r=do_recovery(env);
if (r!=0) return r;
r = do_recovery(env);
if (r != 0) return r;
} else {
r = needs_recovery(env);
if (r != 0) return r;
}
if (flags & (DB_INIT_TXN | DB_INIT_LOG)) {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment