Skip to content

M
MariaDB
Commit 19921b4e authored by Bradley C. Kuszmaul's avatar Bradley C. Kuszmaul Committed by Yoni Fogel
Browse files
Options

Fixes #2782. The brtloader uses temp dir. Note however that #2783 isn't... 

Fixes #2782.  The brtloader uses temp dir.   Note however that #2783 isn't fixed yet. close[t:2782].

git-svn-id: file:///svn/toku/tokudb@21827 c7de825b-a66e-492c-adef-691d508d4ae1
parent 47919cfd
Show whitespace changes
Inline Side-by-side
Showing with 43 additions and 343 deletions
src/loader.c
View file @ 19921b4e
...@@ -158,7 +158,7 @@ int toku_loader_create_loader(DB_ENV *env, ...@@ -158,7 +158,7 @@ int toku_loader_create_loader(DB_ENV *env,
loader->i->loader_flags = loader_flags; loader->i->loader_flags = loader_flags;
loader->i->temp_file_template = (char *)toku_malloc(MAX_FILE_SIZE); loader->i->temp_file_template = (char *)toku_malloc(MAX_FILE_SIZE);
int n = snprintf(loader->i->temp_file_template, MAX_FILE_SIZE, "%s/%s%s", env->i->real_data_dir, loader_temp_prefix, loader_temp_suffix); int n = snprintf(loader->i->temp_file_template, MAX_FILE_SIZE, "%s/%s%s", env->i->real_tmp_dir, loader_temp_prefix, loader_temp_suffix);
if ( !(n>0 && n<MAX_FILE_SIZE) ) { if ( !(n>0 && n<MAX_FILE_SIZE) ) {
rval = -1; rval = -1;
goto create_exit; goto create_exit;
......
src/tests/loader-stress-test.c
View file @ 19921b4e
...@@ -346,6 +346,7 @@ static void test_loader(DB **dbs) ...@@ -346,6 +346,7 @@ static void test_loader(DB **dbs)
char *free_me = NULL; char *free_me = NULL;
char *env_dir = ENVDIR; // the default env_dir. char *env_dir = ENVDIR; // the default env_dir.
char *tmp_subdir = "tmp.subdir";
static void run_test(void) static void run_test(void)
{ {
...@@ -358,8 +359,15 @@ static void run_test(void) ...@@ -358,8 +359,15 @@ static void run_test(void)
r = system(syscmd); CKERR(r); r = system(syscmd); CKERR(r);
} }
r = toku_os_mkdir(env_dir, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r); r = toku_os_mkdir(env_dir, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
{
char len = strlen(env_dir) + strlen(tmp_subdir) + 10;
char tmpdir[len];
r = snprintf(tmpdir, len, "%s/%s", env_dir, tmp_subdir);
assert(r<len);
r = toku_os_mkdir(tmpdir, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
r = db_env_create(&env, 0); CKERR(r); r = db_env_create(&env, 0); CKERR(r);
r = env->set_tmp_dir(env, tmp_subdir); CKERR(r);
}
r = env->set_default_bt_compare(env, uint_dbt_cmp); CKERR(r); r = env->set_default_bt_compare(env, uint_dbt_cmp); CKERR(r);
r = env->set_default_dup_compare(env, uint_dbt_cmp); CKERR(r); r = env->set_default_dup_compare(env, uint_dbt_cmp); CKERR(r);
if ( verbose ) printf("CACHESIZE = %d MB\n", CACHESIZE); if ( verbose ) printf("CACHESIZE = %d MB\n", CACHESIZE);
......
src/tests/loader-stress-test.cilk deleted 100644 → 0
View file @ 47919cfd
/* -*- mode: C; c-basic-offset: 4 -*- */
// /usr/local/cilk/bin/cilk++ -Wall -Wextra -Wcast-align -Wno-missing-noreturn -Werror -Wmissing-format-attribute -fvisibility=hidden -fPIC -Wshadow -O0 -g3 -ggdb3 -I. -I../ -I../../newbrt -I../range_tree -I../lock_tree -I../../linux -I../../toku_include -DBRT_LEVEL_STRADDLE_CALLBACK_LOGIC_NOT_READY=1 -D_SVID_SOURCE -D_FILE_OFFSET_BITS=64 -D_LARGEFILE64_SOURCE -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -I../../include -DENVDIR=\"dir.loader-stress-test.cilk\" loader-stress-test.cilk -L.. -ltokudb -Wl,-rpath,.. -o loader-stress-test
#ident "Copyright (c) 2010 Tokutek Inc. All rights reserved."
#ident "$Id$"
#include "test.h"
#include "toku_pthread.h"
#include <db.h>
#include <sys/stat.h>
DB_ENV *env;
enum {MAX_NAME=128};
enum {MAX_DBS=256};
int NUM_DBS=5;
int NUM_ROWS=100000;
int CHECK_RESULTS=0;
enum {MAGIC=311};
//
// Functions to create unique key/value pairs, extractors, checkers, ... for each of NUM_DBS
//
// a is the bit-wise permute table. For DB[i], permute bits as described in a[i] using 'twiddle32'
// inv is the inverse bit-wise permute of a[]. To get the original value from a twiddled value, twiddle32 (again) with inv[]
int a[MAX_DBS][32];
int inv[MAX_DBS][32];
// extract_functions is an array of extract functions. In our case, each entry is extract_i
#if defined(__cilkplusplus) || defined (__cplusplus)
extern "C" {
#endif
int (*extract_functions[MAX_DBS])(int i, DB *db, DBT *primary_key, DBT *primary_value, DBT *secondary_key, DBT *secondary_value, void *extra);
// rotate right and left functions
static inline unsigned int rotr32(const unsigned int x, const unsigned int num) {
const unsigned int n = num % 32;
return (x >> n) | ( x << (32 - n));
}
static inline unsigned int rotl32(const unsigned int x, const unsigned int num) {
const unsigned int n = num % 32;
return (x << n) | ( x >> (32 - n));
}
static void generate_permute_tables(void) {
int i, j, tmp;
for(int db=0;db<MAX_DBS;db++) {
for(i=0;i<32;i++) {
a[db][i] = i;
}
for(i=0;i<32;i++) {
j = random() % (i + 1);
tmp = a[db][j];
a[db][j] = a[db][i];
a[db][i] = tmp;
}
// if(db < NUM_DBS){ printf("a[%d] = ", db); for(i=0;i<32;i++) { printf("%2d ", a[db][i]); } printf("\n");}
for(i=0;i<32;i++) {
inv[db][a[db][i]] = i;
}
}
}
// permute bits of x based on permute table bitmap
static unsigned int twiddle32(unsigned int x, int db)
{
unsigned int b = 0;
for(int i=0;i<32;i++) {
b |= (( x >> i ) & 1) << a[db][i];
}
return b;
}
// permute bits of x based on inverse permute table bitmap
static unsigned int inv_twiddle32(unsigned int x, int db)
{
unsigned int b = 0;
for(int i=0;i<32;i++) {
b |= (( x >> i ) & 1) << inv[db][i];
}
return b;
}
// generate val from key, index
static unsigned int generate_val(int key, int i) {
return rotl32((key + MAGIC), i);
}
static unsigned int pkey_for_val(int key, int i) {
return rotr32(key, i) - MAGIC;
}
// primary table requires identity function
static int extract_primary(int i, DB *db,
DBT *primary_key, DBT *primary_value,
DBT *secondary_key, DBT *secondary_value,
void *extra)
{
i = i;
db = db;
extra = extra;
dbt_init(secondary_key, primary_key->data, primary_key->size);
dbt_init(secondary_value, primary_value->data, primary_value->size);
return 0; // as per spec
}
// for DB[i], extract key/value pairs from primary key
static int extract_i(int i, DB *db,
DBT *primary_key, DBT *primary_value,
DBT *secondary_key, DBT *secondary_value,
void *extra)
{
if ( i == 0 ) {
extract_primary(i,db,primary_key,primary_value,secondary_key,secondary_value,extra);
}
else
{
unsigned int *new_key = (unsigned int *)toku_malloc(sizeof(unsigned int));
unsigned int *new_val = (unsigned int *)toku_malloc(sizeof(unsigned int));
*new_key = twiddle32(*(unsigned int*)primary_key->data, i);
*new_val = generate_val(*(unsigned int*)primary_key->data, i);
dbt_init(secondary_key, new_key, sizeof(unsigned int));
dbt_init(secondary_value, new_val, sizeof(unsigned int));
}
return 0; // as per spec
}
#if defined(__cilkplusplus) || defined(__cplusplus)
} // extern "C"
#endif
//
// function to generate extract maps
//
static void generate_extract_funcs(void) {
generate_permute_tables();
for(int i=0;i<NUM_DBS;i++) {
extract_functions[i] = extract_i;
}
}
static void check_results(DB **dbs)
{
cilk_for(int j=0;j<NUM_DBS;j++){
DBT key, val;
unsigned int k=0, v=0;
dbt_init(&key, &k, sizeof(unsigned int));
dbt_init(&val, &v, sizeof(unsigned int));
int r;
unsigned int pkey_for_db_key;
DB_TXN *txn;
r = env->txn_begin(env, NULL, &txn, 0);
CKERR(r);
DBC *cursor;
r = dbs[j]->cursor(dbs[j], txn, &cursor, 0);
CKERR(r);
for(int i=0;i<NUM_ROWS;i++) {
r = cursor->c_get(cursor, &key, &val, DB_NEXT);
CKERR(r);
k = *(unsigned int*)key.data;
pkey_for_db_key = (j == 0) ? k : inv_twiddle32(k, j);
v = *(unsigned int*)val.data;
// test that we have the expected keys and values
assert((unsigned int)pkey_for_db_key == (unsigned int)pkey_for_val(v, j));
// printf(" DB[%d] key = %10u, val = %10u, pkey_for_db_key = %10u, pkey_for_val=%10d\n", j, v, k, pkey_for_db_key, pkey_for_val(v, j));
}
{printf("."); fflush(stdout);}
r = cursor->c_close(cursor);
CKERR(r);
r = txn->commit(txn, 0);
CKERR(r);
}
printf("\nCheck OK\n");
}
static void test_loader(DB **dbs)
{
int r;
DB_TXN *txn;
DB_LOADER *loader;
// create and initialize loader
r = env->txn_begin(env, NULL, &txn, 0);
CKERR(r);
r = env->create_loader(env, txn, &loader, NUM_DBS, dbs, extract_functions, NULL);
CKERR(r);
r = loader->set_duplicate_callback(loader, NULL);
CKERR(r);
r = loader->set_poll_function(loader, NULL);
CKERR(r);
// using loader->put, put values into DB
DBT key, val;
unsigned int k, v;
for(int i=1;i<=NUM_ROWS;i++) {
k = i;
v = generate_val(i, 0);
dbt_init(&key, &k, sizeof(unsigned int));
dbt_init(&val, &v, sizeof(unsigned int));
r = loader->put(loader, &key, &val);
CKERR(r);
if ( CHECK_RESULTS) { if((i%10000) == 0){printf("."); fflush(stdout);} }
}
if( CHECK_RESULTS ) {printf("\n"); fflush(stdout);}
// close the loader
r = loader->close(loader);
CKERR(r);
r = txn->commit(txn, 0);
CKERR(r);
// verify the DBs
if ( CHECK_RESULTS ) {
check_results(dbs);
}
}
static void run_test(void)
{
int r;
r = system("rm -rf " ENVDIR); CKERR(r);
r = toku_os_mkdir(ENVDIR, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
r = db_env_create(&env, 0); CKERR(r);
r = env->set_default_bt_compare(env, uint_dbt_cmp); CKERR(r);
r = env->set_default_dup_compare(env, uint_dbt_cmp); CKERR(r);
int envflags = DB_INIT_LOCK | DB_INIT_MPOOL | DB_INIT_TXN | DB_CREATE | DB_PRIVATE;
r = env->open(env, ENVDIR, envflags, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
env->set_errfile(env, stderr);
//Disable auto-checkpointing
r = env->checkpointing_set_period(env, 0); CKERR(r);
DBT desc;
dbt_init(&desc, "foo", sizeof("foo"));
char name[MAX_NAME*2];
DB **dbs = (DB**)toku_malloc(sizeof(DB*) * NUM_DBS);
assert(dbs != NULL);
for(int i=0;i<NUM_DBS;i++) {
r = db_create(&dbs[i], env, 0); CKERR(r);
r = dbs[i]->set_descriptor(dbs[i], 1, &desc, abort_on_upgrade); CKERR(r);
snprintf(name, sizeof(name), "db_%04x", i);
r = dbs[i]->open(dbs[i], NULL, name, NULL, DB_BTREE, DB_CREATE, 0666); CKERR(r);
}
generate_extract_funcs();
// printf("running test_loader()\n");
// -------------------------- //
test_loader(dbs);
// -------------------------- //
// printf("done test_loader()\n");
for(int i=0;i<NUM_DBS;i++) {
dbs[i]->close(dbs[i], 0); CKERR(r);
dbs[i] = NULL;
}
int x[5];
cilk_for(int j=0;j<NUM_DBS;j++) {
x[j] = j;
}
r = env->close(env, 0); CKERR(r);
toku_free(dbs);
}
// ------------ infrastructure ----------
static void do_args(int argc, char *argv[]);
int test_main(int argc, char **argv) {
do_args(argc, argv);
run_test();
return 0;
}
static void do_args(int argc, char *argv[]) {
int resultcode;
char *cmd = argv[0];
argc--; argv++;
while (argc>0) {
if (strcmp(argv[0], "-h")==0) {
resultcode=0;
do_usage:
fprintf(stderr, "Usage: -h -c -d <num_dbs> -r <num_rows>\n%s\n", cmd);
exit(resultcode);
} else if (strcmp(argv[0], "-d")==0) {
argc--; argv++;
NUM_DBS = atoi(argv[0]);
if ( NUM_DBS > MAX_DBS ) {
fprintf(stderr, "max value for -d field is %d\n", MAX_DBS);
resultcode=1;
goto do_usage;
}
} else if (strcmp(argv[0], "-r")==0) {
argc--; argv++;
NUM_ROWS = atoi(argv[0]);
} else if (strcmp(argv[0], "-c")==0) {
CHECK_RESULTS = 1;
} else {
fprintf(stderr, "Unknown arg: %s\n", argv[0]);
resultcode=1;
goto do_usage;
}
argc--;
argv++;
}
}
src/ydb-internal.h
View file @ 19921b4e
...@@ -70,8 +70,9 @@ struct __toku_db_env_internal { ...@@ -70,8 +70,9 @@ struct __toku_db_env_internal {
DB *persistent_environment; // Stores environment settings, can be used for upgrade DB *persistent_environment; // Stores environment settings, can be used for upgrade
OMT open_dbs; // Stores open db handles, sorted first by dname and then by numerical value of pointer to the db (arbitrarily assigned memory location) OMT open_dbs; // Stores open db handles, sorted first by dname and then by numerical value of pointer to the db (arbitrarily assigned memory location)
char *real_data_dir; // data dir used when the env is opened (relative to cwd) char *real_data_dir; // data dir used when the env is opened (relative to cwd, or absolute with leading /)
char *real_log_dir; // log dir used when the env is opened (relative to cwd) char *real_log_dir; // log dir used when the env is opened (relative to cwd, or absolute with leading /)
char *real_tmp_dir; // tmp dir used for temporary files (relative to cwd, or absoulte with leading /)
enum { enum {
FS_GREEN = 0, // green zone (we have lots of space) FS_GREEN = 0, // green zone (we have lots of space)
......
src/ydb.c
View file @ 19921b4e
...@@ -464,31 +464,34 @@ static int toku_c_count(DBC *cursor, db_recno_t *count, u_int32_t flags); ...@@ -464,31 +464,34 @@ static int toku_c_count(DBC *cursor, db_recno_t *count, u_int32_t flags);
static int toku_c_close(DBC * c); static int toku_c_close(DBC * c);
static void static void
env_setup_real_data_dir(DB_ENV *env) { env_setup_real_dir(DB_ENV *env, char **real_dir, const char *nominal_dir) {
toku_free(env->i->real_data_dir); toku_free(*real_dir);
env->i->real_data_dir = NULL; *real_dir = NULL;
assert(env->i->dir); assert(env->i->dir);
if (env->i->data_dir) if (nominal_dir)
env->i->real_data_dir = toku_construct_full_name(2, env->i->dir, env->i->data_dir); *real_dir = toku_construct_full_name(2, env->i->dir, nominal_dir);
else else
env->i->real_data_dir = toku_strdup(env->i->dir); *real_dir = toku_strdup(env->i->dir);
}
static void
env_setup_real_data_dir(DB_ENV *env) {
env_setup_real_dir(env, &env->i->real_data_dir, env->i->data_dir);
} }
static void static void
env_setup_real_log_dir(DB_ENV *env) { env_setup_real_log_dir(DB_ENV *env) {
toku_free(env->i->real_log_dir); env_setup_real_dir(env, &env->i->real_log_dir, env->i->lg_dir);
env->i->real_log_dir = NULL; }
if (env->i->lg_dir) { static void
assert(env->i->dir); env_setup_real_tmp_dir(DB_ENV *env) {
env->i->real_log_dir = toku_construct_full_name(2, env->i->dir, env->i->lg_dir); env_setup_real_dir(env, &env->i->real_tmp_dir, env->i->tmp_dir);
} else {
assert(env->i->dir);
env->i->real_log_dir = toku_strdup(env->i->dir);
}
} }
static int static int
ydb_do_recovery (DB_ENV *env) { ydb_do_recovery (DB_ENV *env) {
assert(env->i->real_log_dir); assert(env->i->real_log_dir);
...@@ -783,6 +786,7 @@ toku_env_open(DB_ENV * env, const char *home, u_int32_t flags, int mode) { ...@@ -783,6 +786,7 @@ toku_env_open(DB_ENV * env, const char *home, u_int32_t flags, int mode) {
env_setup_real_data_dir(env); env_setup_real_data_dir(env);
env_setup_real_log_dir(env); env_setup_real_log_dir(env);
env_setup_real_tmp_dir(env);
BOOL need_rollback_cachefile = FALSE; BOOL need_rollback_cachefile = FALSE;
if (flags & (DB_INIT_TXN | DB_INIT_LOG)) { if (flags & (DB_INIT_TXN | DB_INIT_LOG)) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7