Skip to content

M
mariadb
Commit 3444202c authored by Barry Perlman's avatar Barry Perlman Committed by Yoni Fogel
Browse files
Options

Closes [t:2624] Added toku_os_close() which retries on EINTR. Closes [t:2621]... 

Closes [t:2624] Added toku_os_close() which retries on EINTR.  Closes [t:2621] Added toku_os_open(), toku_os_fopen(), toku_os_fdopen(), toku_os_fclose() and related override hooks  db_env_set_func_open(), db_env_set_func_fopen(), db_env_set_func_fdopen(), db_env_set_func_fclose(). Modified test program loader-cleanup-test to use new hooks to inject synthetic errors.  Marked some error cases in brtloader.c. with ###.

git-svn-id: file:///svn/toku/tokudb@20360 c7de825b-a66e-492c-adef-691d508d4ae1
parent a928de12
Show whitespace changes
Inline Side-by-side
Showing with 191 additions and 50 deletions
src/tests/loader-cleanup-test.c
View file @ 3444202c
...@@ -5,7 +5,7 @@ ...@@ -5,7 +5,7 @@
/* TODO: /* TODO:
* *
* When ready, add simulated errors on calls to pwrite() and malloc() * When ready, add simulated errors on calls to malloc()
* *
*/ */
...@@ -18,10 +18,7 @@ ...@@ -18,10 +18,7 @@
* - user calls loader->abort() * - user calls loader->abort()
* - user aborts transaction * - user aborts transaction
* - disk full (ENOSPC) * - disk full (ENOSPC)
* - crash * - crash (not tested in this test program)
*
* In the event of a crash, the verification of no temp files and
* no loader-generated iname file is done after recovery.
* *
* Mechanism: * Mechanism:
* This test is derived from the loader-stress-test. * This test is derived from the loader-stress-test.
...@@ -29,7 +26,7 @@ ...@@ -29,7 +26,7 @@
* The outline of the test is as follows: * The outline of the test is as follows:
* - use loader to create table * - use loader to create table
* - verify presence of temp files * - verify presence of temp files
* - commit / abort / enospc / crash * - commit / abort / inject error (simulated error from system call)
* - verify absence of temp files * - verify absence of temp files
* - verify absence of unwanted iname files (old inames if committed, new inames if aborted) * - verify absence of unwanted iname files (old inames if committed, new inames if aborted)
* *
...@@ -53,7 +50,11 @@ enum test_type {commit, // close loader, commit txn ...@@ -53,7 +50,11 @@ enum test_type {commit, // close loader, commit txn
abort_via_poll, // close loader, but poll function returns non-zero, abort txn abort_via_poll, // close loader, but poll function returns non-zero, abort txn
enospc_w, // close loader, but close fails due to enospc return from toku_os_write enospc_w, // close loader, but close fails due to enospc return from toku_os_write
enospc_f, // either loader->put() or loader->close() fails due to enospc return from do_fwrite() enospc_f, // either loader->put() or loader->close() fails due to enospc return from do_fwrite()
enospc_p}; // loader->close() fails due to enospc return from toku_os_pwrite() enospc_p, // loader->close() fails due to enospc return from toku_os_pwrite()
einval_fdo, // return einval from fdopen()
einval_fo, // return einval from fopen()
einval_o, // return einval from open()
enospc_fc}; // return enospc from fclose()
int abort_on_poll = 0; // set when test_loader() called with test_type of abort_via_poll int abort_on_poll = 0; // set when test_loader() called with test_type of abort_via_poll
...@@ -81,38 +82,70 @@ static void run_all_tests(void); ...@@ -81,38 +82,70 @@ static void run_all_tests(void);
static void free_inames(DBT* inames); static void free_inames(DBT* inames);
#define NUM_ENOSPC_TYPES 3 // how many different system calls are intercepted with error injection
#define NUM_ERR_TYPES 7
int fwrite_count = 0; int fwrite_count = 0;
int fwrite_enospc = 0;
int fwrite_count_nominal = 0; // number of fwrite calls for normal operation, initially zero int fwrite_count_nominal = 0; // number of fwrite calls for normal operation, initially zero
int fwrite_count_trigger = 0; // sequence number of fwrite call that will fail (zero disables induced failure) int fwrite_count_trigger = 0; // sequence number of fwrite call that will fail (zero disables induced failure)
int write_count = 0; int write_count = 0;
int write_enospc = 0;
int write_count_nominal = 0; // number of write calls for normal operation, initially zero int write_count_nominal = 0; // number of write calls for normal operation, initially zero
int write_count_trigger = 0; // sequence number of write call that will fail (zero disables induced failure) int write_count_trigger = 0; // sequence number of write call that will fail (zero disables induced failure)
int pwrite_count = 0; int pwrite_count = 0;
int pwrite_enospc = 0;
int pwrite_count_nominal = 0; // number of pwrite calls for normal operation, initially zero int pwrite_count_nominal = 0; // number of pwrite calls for normal operation, initially zero
int pwrite_count_trigger = 0; // sequence number of pwrite call that will fail (zero disables induced failure) int pwrite_count_trigger = 0; // sequence number of pwrite call that will fail (zero disables induced failure)
int fdopen_count = 0;
int fdopen_count_nominal = 0; // number of fdopen calls for normal operation, initially zero
int fdopen_count_trigger = 0; // sequence number of fdopen call that will fail (zero disables induced failure)
int fopen_count = 0;
int fopen_count_nominal = 0; // number of fopen calls for normal operation, initially zero
int fopen_count_trigger = 0; // sequence number of fopen call that will fail (zero disables induced failure)
int open_count = 0;
int open_count_nominal = 0; // number of open calls for normal operation, initially zero
int open_count_trigger = 0; // sequence number of open call that will fail (zero disables induced failure)
int fclose_count = 0;
int fclose_count_nominal = 0; // number of fclose calls for normal operation, initially zero
int fclose_count_trigger = 0; // sequence number of fclose call that will fail (zero disables induced failure)
const char * fwrite_str = "fwrite"; const char * fwrite_str = "fwrite";
const char * write_str = "write"; const char * write_str = "write";
const char * pwrite_str = "pwrite"; const char * pwrite_str = "pwrite";
const char * fdopen_str = "fdopen";
const char * fopen_str = "fopen";
const char * open_str = "open";
const char * fclose_str = "fclose";
static const char * static const char *
enospc_type_str (enum test_type t){ err_type_str (enum test_type t) {
const char * rval; const char * rval;
if (t == enospc_f) switch(t) {
rval = fwrite_str; case enospc_f:
else if (t == enospc_w) rval = fwrite_str; break;
rval = write_str; case enospc_w:
else if (t == enospc_p) rval = write_str; break;
rval = pwrite_str; case enospc_p:
else rval = pwrite_str; break;
case einval_fdo:
rval = fdopen_str; break;
case einval_fo:
rval = fopen_str; break;
case einval_o:
rval = open_str; break;
case enospc_fc:
rval = fclose_str; break;
default:
assert(0); assert(0);
}
return rval; return rval;
} }
...@@ -120,7 +153,6 @@ static size_t bad_fwrite (const void *ptr, size_t size, size_t nmemb, FILE *stre ...@@ -120,7 +153,6 @@ static size_t bad_fwrite (const void *ptr, size_t size, size_t nmemb, FILE *stre
fwrite_count++; fwrite_count++;
size_t r; size_t r;
if (fwrite_count_trigger == fwrite_count) { if (fwrite_count_trigger == fwrite_count) {
fwrite_enospc++;
errno = ENOSPC; errno = ENOSPC;
r = -1; r = -1;
} else { } else {
...@@ -138,7 +170,6 @@ bad_write(int fd, const void * bp, size_t len) { ...@@ -138,7 +170,6 @@ bad_write(int fd, const void * bp, size_t len) {
ssize_t r; ssize_t r;
write_count++; write_count++;
if (write_count_trigger == write_count) { if (write_count_trigger == write_count) {
write_enospc++;
errno = ENOSPC; errno = ENOSPC;
r = -1; r = -1;
} else { } else {
...@@ -152,7 +183,6 @@ bad_pwrite (int fd, const void *buf, size_t len, toku_off_t off) { ...@@ -152,7 +183,6 @@ bad_pwrite (int fd, const void *buf, size_t len, toku_off_t off) {
int r; int r;
pwrite_count++; pwrite_count++;
if (pwrite_count_trigger == pwrite_count) { if (pwrite_count_trigger == pwrite_count) {
pwrite_enospc++;
errno = ENOSPC; errno = ENOSPC;
r = -1; r = -1;
} else { } else {
...@@ -163,6 +193,66 @@ bad_pwrite (int fd, const void *buf, size_t len, toku_off_t off) { ...@@ -163,6 +193,66 @@ bad_pwrite (int fd, const void *buf, size_t len, toku_off_t off) {
static FILE *
bad_fdopen(int fd, const char * mode) {
FILE * rval;
fdopen_count++;
if (fdopen_count_trigger == fdopen_count) {
errno = EINVAL;
rval = NULL;
} else {
rval = fdopen(fd, mode);
}
return rval;
}
static FILE *
bad_fopen(const char *filename, const char *mode) {
FILE * rval;
fopen_count++;
if (fopen_count_trigger == fopen_count) {
errno = EINVAL;
rval = NULL;
} else {
rval = fopen(filename, mode);
}
return rval;
}
static int
bad_open(const char *path, int oflag, int mode) {
int rval;
open_count++;
if (open_count_trigger == open_count) {
errno = EINVAL;
rval = -1;
} else {
rval = open(path, oflag, mode);
}
return rval;
}
static int
bad_fclose(FILE * stream) {
int rval;
fclose_count++;
if (fclose_count_trigger == fclose_count) {
errno = ENOSPC;
rval = -1;
} else {
rval = fclose(stream);
}
return rval;
}
///////////////
// return number of temp files // return number of temp files
static int static int
count_temp(char * dirname) { count_temp(char * dirname) {
...@@ -449,12 +539,23 @@ static int poll_function (void *extra, float progress) { ...@@ -449,12 +539,23 @@ static int poll_function (void *extra, float progress) {
static void test_loader(enum test_type t, DB **dbs) static void test_loader(enum test_type t, DB **dbs)
{ {
int failed_put = 0; int failed_put = 0;
int error_injection; // are we expecting simulated errors from system calls?
if (t == commit ||
t == abort_txn ||
t == abort_loader ||
t == abort_via_poll)
error_injection = 0;
else
error_injection = 1;
if (t == abort_via_poll) if (t == abort_via_poll)
abort_on_poll = 1; abort_on_poll = 1;
else else
abort_on_poll = 0; abort_on_poll = 0;
int r; int r;
DB_TXN *txn; DB_TXN *txn;
DB_LOADER *loader; DB_LOADER *loader;
...@@ -494,7 +595,7 @@ static void test_loader(enum test_type t, DB **dbs) ...@@ -494,7 +595,7 @@ static void test_loader(enum test_type t, DB **dbs)
dbt_init(&key, &k, sizeof(unsigned int)); dbt_init(&key, &k, sizeof(unsigned int));
dbt_init(&val, &v, sizeof(unsigned int)); dbt_init(&val, &v, sizeof(unsigned int));
r = loader->put(loader, &key, &val); r = loader->put(loader, &key, &val);
if (t == enospc_f || t == enospc_w || t == enospc_p) if (error_injection)
failed_put = r; failed_put = r;
else else
CKERR(r); CKERR(r);
...@@ -527,10 +628,9 @@ static void test_loader(enum test_type t, DB **dbs) ...@@ -527,10 +628,9 @@ static void test_loader(enum test_type t, DB **dbs)
r = loader->close(loader); r = loader->close(loader);
assert(r); // not defined what close() returns when poll function returns non-zero assert(r); // not defined what close() returns when poll function returns non-zero
} }
else if ((t == enospc_f || t == enospc_w || t == enospc_p) else if (error_injection && !failed_put) {
&& !failed_put) { const char * type = err_type_str(t);
const char * type = enospc_type_str(t); printf("closing, but expecting failure from simulated error (enospc or einval)%s\n", type);
printf("closing, but expecting failure from enospc %s\n", type);
r = loader->close(loader); r = loader->close(loader);
if (!USE_PUTS) if (!USE_PUTS)
assert(r); assert(r);
...@@ -553,8 +653,22 @@ static void test_loader(enum test_type t, DB **dbs) ...@@ -553,8 +653,22 @@ static void test_loader(enum test_type t, DB **dbs)
fwrite_count_nominal = fwrite_count; // capture how many fwrites were required for normal operation fwrite_count_nominal = fwrite_count; // capture how many fwrites were required for normal operation
write_count_nominal = write_count; // capture how many writes were required for normal operation write_count_nominal = write_count; // capture how many writes were required for normal operation
pwrite_count_nominal = pwrite_count; // capture how many pwrites were required for normal operation pwrite_count_nominal = pwrite_count; // capture how many pwrites were required for normal operation
if (verbose) printf("Calls to fwrite nominal: %d, calls to write nominal: %d, calls to pwrite nominal: %d\n", fdopen_count_nominal = fdopen_count; // capture how many fdopens were required for normal operation
fwrite_count_nominal, write_count_nominal, pwrite_count_nominal); fopen_count_nominal = fopen_count; // capture how many fopens were required for normal operation
open_count_nominal = open_count; // capture how many opens were required for normal operation
fclose_count_nominal = fclose_count; // capture how many fcloses were required for normal operation
if (verbose) {
printf("Nominal calls: function calls (number of calls for normal operation)\n");
printf(" fwrite %d\n", fwrite_count_nominal);
printf(" write %d\n", write_count_nominal);
printf(" pwrite %d\n", pwrite_count_nominal);
printf(" fdopen %d\n", fdopen_count_nominal);
printf(" fopen %d\n", fopen_count_nominal);
printf(" open %d\n", open_count_nominal);
printf(" fclose %d\n", fclose_count_nominal);
}
r = txn->commit(txn, 0); r = txn->commit(txn, 0);
CKERR(r); CKERR(r);
if (!USE_PUTS) { if (!USE_PUTS) {
...@@ -589,7 +703,7 @@ static void run_test(enum test_type t, int trigger) ...@@ -589,7 +703,7 @@ static void run_test(enum test_type t, int trigger)
r = env->set_default_dup_compare(env, uint_dbt_cmp); CKERR(r); r = env->set_default_dup_compare(env, uint_dbt_cmp); CKERR(r);
r = env->set_generate_row_callback_for_put(env, put_multiple_generate); r = env->set_generate_row_callback_for_put(env, put_multiple_generate);
CKERR(r); CKERR(r);
// int envflags = DB_INIT_LOCK | DB_INIT_MPOOL | DB_INIT_TXN | DB_CREATE | DB_PRIVATE;
int envflags = DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN | DB_CREATE | DB_PRIVATE; int envflags = DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN | DB_CREATE | DB_PRIVATE;
r = env->open(env, ENVDIR, envflags, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r); r = env->open(env, ENVDIR, envflags, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
env->set_errfile(env, stderr); env->set_errfile(env, stderr);
...@@ -614,23 +728,46 @@ static void run_test(enum test_type t, int trigger) ...@@ -614,23 +728,46 @@ static void run_test(enum test_type t, int trigger)
generate_permute_tables(); generate_permute_tables();
fwrite_count_trigger = fwrite_count = fwrite_enospc = 0; fwrite_count_trigger = fwrite_count = 0;
write_count_trigger = write_count = write_enospc = 0; write_count_trigger = write_count = 0;
pwrite_count_trigger = pwrite_count = pwrite_enospc = 0; pwrite_count_trigger = pwrite_count = 0;
fdopen_count_trigger = fdopen_count = 0;
if (t == enospc_f) { fopen_count_trigger = fopen_count = 0;
fwrite_count_trigger = trigger; open_count_trigger = open_count = 0;
} fclose_count_trigger = fclose_count = 0;
else if (t == enospc_w) {
write_count_trigger = trigger; switch(t) {
} case commit:
else if (t == enospc_p) { case abort_txn:
pwrite_count_trigger = trigger; case abort_loader:
case abort_via_poll:
break;
case enospc_f:
fwrite_count_trigger = trigger; break;
case enospc_w:
write_count_trigger = trigger; break;
case enospc_p:
pwrite_count_trigger = trigger; break;
case einval_fdo:
fdopen_count_trigger = trigger; break;
case einval_fo:
fopen_count_trigger = trigger; break;
case einval_o:
open_count_trigger = trigger; break;
case enospc_fc:
fclose_count_trigger = trigger; break;
default:
assert(0);
} }
db_env_set_func_loader_fwrite(bad_fwrite); db_env_set_func_loader_fwrite(bad_fwrite);
db_env_set_func_write(bad_write); db_env_set_func_write(bad_write);
db_env_set_func_pwrite(bad_pwrite); db_env_set_func_pwrite(bad_pwrite);
db_env_set_func_fdopen(bad_fdopen);
db_env_set_func_fopen(bad_fopen);
db_env_set_func_open(bad_open);
db_env_set_func_fclose(bad_fclose);
test_loader(t, dbs); test_loader(t, dbs);
...@@ -638,6 +775,9 @@ static void run_test(enum test_type t, int trigger) ...@@ -638,6 +775,9 @@ static void run_test(enum test_type t, int trigger)
dbs[i]->close(dbs[i], 0); CKERR(r); dbs[i]->close(dbs[i], 0); CKERR(r);
dbs[i] = NULL; dbs[i] = NULL;
} }
if (verbose >= 3)
print_engine_status(env);
r = env->close(env, 0); CKERR(r); r = env->close(env, 0); CKERR(r);
toku_free(dbs); toku_free(dbs);
} }
...@@ -661,34 +801,35 @@ static void run_all_tests(void) { ...@@ -661,34 +801,35 @@ static void run_all_tests(void) {
if (verbose) printf("\n\nTesting loader with loader close and txn abort\n"); if (verbose) printf("\n\nTesting loader with loader close and txn abort\n");
run_test(abort_txn, 0); run_test(abort_txn, 0);
enum test_type et[NUM_ENOSPC_TYPES] = {enospc_f, enospc_w, enospc_p}; enum test_type et[NUM_ERR_TYPES] = {enospc_f, enospc_w, enospc_p, einval_fdo, einval_fo, einval_o, enospc_fc};
int * nomp[NUM_ENOSPC_TYPES] = {&fwrite_count_nominal, &write_count_nominal, &pwrite_count_nominal}; int * nomp[NUM_ERR_TYPES] = {&fwrite_count_nominal, &write_count_nominal, &pwrite_count_nominal,
&fdopen_count_nominal, &fopen_count_nominal, &open_count_nominal, &fclose_count_nominal};
int limit = NUM_DBS * 5; int limit = NUM_DBS * 5;
int j; int j;
for (j = 0; j<NUM_ENOSPC_TYPES; j++) { for (j = 0; j<NUM_ERR_TYPES; j++) {
enum test_type t = et[j]; enum test_type t = et[j];
const char * write_type = enospc_type_str(t); const char * write_type = err_type_str(t);
int nominal = *(nomp[j]); int nominal = *(nomp[j]);
printf("\nNow test with induced ENOSPC errors returned from %s, nominal = %d\n", write_type, nominal); printf("\nNow test with induced ENOSPC/EINVAL errors returned from %s, nominal = %d\n", write_type, nominal);
int i; int i;
// induce write error at beginning of process // induce write error at beginning of process
for (i = 1; i < limit && i < nominal+1; i++) { for (i = 1; i < limit && i < nominal+1; i++) {
trigger = i; trigger = i;
if (verbose) printf("\n\nTesting loader with enospc induced at %s count %d (of %d)\n", write_type, trigger, nominal); if (verbose) printf("\n\nTesting loader with enospc/einval induced at %s count %d (of %d)\n", write_type, trigger, nominal);
run_test(t, trigger); run_test(t, trigger);
} }
if (nominal > limit) { // if we didn't already test every possible case if (nominal > limit) { // if we didn't already test every possible case
// induce write error sprinkled through process // induce write error sprinkled through process
for (i = 2; i < 5; i++) { for (i = 2; i < 5; i++) {
trigger = nominal / i; trigger = nominal / i;
if (verbose) printf("\n\nTesting loader with enospc induced at %s count %d (of %d)\n", write_type, trigger, nominal); if (verbose) printf("\n\nTesting loader with enospc/einval induced at %s count %d (of %d)\n", write_type, trigger, nominal);
run_test(t, trigger); run_test(t, trigger);
} }
// induce write error at end of process // induce write error at end of process
for (i = 0; i < limit; i++) { for (i = 0; i < limit; i++) {
trigger = nominal - i; trigger = nominal - i;
assert(trigger > 0); assert(trigger > 0);
if (verbose) printf("\n\nTesting loader with enospc induced at %s count %d (of %d)\n", write_type, trigger, nominal); if (verbose) printf("\n\nTesting loader with enospc/einval induced at %s count %d (of %d)\n", write_type, trigger, nominal);
run_test(t, trigger); run_test(t, trigger);
} }
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7