Skip to content

M
mariadb
Commit f31a90e9 authored by Dave Wells's avatar Dave Wells Committed by Yoni Fogel
Browse files
Options

merge indexer to main refs #2843 [t:2843] 

git-svn-id: file:///svn/toku/tokudb@25593 c7de825b-a66e-492c-adef-691d508d4ae1
parent ff2b488e
Show whitespace changes
Inline Side-by-side
Showing with 5796 additions and 179 deletions
buildheader/db.h_4_1
View file @ f31a90e9
......@@ -54,6 +54,16 @@ struct __toku_loader {
int (*close)(DB_LOADER *loader); /* finish loading, free memory */
int (*abort)(DB_LOADER *loader); /* abort loading, free memory */
};
typedef struct __toku_indexer DB_INDEXER;
struct __toku_indexer_internal;
struct __toku_indexer {
struct __toku_indexer_internal *i;
int (*set_error_callback)(DB_INDEXER *indexer, void (*error_cb)(DB *db, int i, int err, DBT *key, DBT *val, void *error_extra), void *error_extra); /* set the error callback */
int (*set_poll_function)(DB_INDEXER *indexer, int (*poll_func)(void *extra, float progress), void *poll_extra); /* set the polling function */
int (*build)(DB_INDEXER *indexer); /* build the indexes */
int (*close)(DB_INDEXER *indexer); /* finish indexing, free memory */
int (*abort)(DB_INDEXER *indexer); /* abort indexing, free memory */
};
typedef struct __toku_engine_status {
char creationtime[26]; /* time of environment creation */
char startuptime[26]; /* time of engine startup */
......@@ -270,6 +280,7 @@ struct __toku_db_env {
int (*get_engine_status_text) (DB_ENV*, char*, int) /* Fill in status text */;
int (*get_iname) (DB_ENV* env, DBT* dname_dbt, DBT* iname_dbt) /* FOR TEST ONLY: lookup existing iname */;
int (*create_loader) (DB_ENV *env, DB_TXN *txn, DB_LOADER **blp, DB *src_db, int N, DB *dbs[/*N*/], uint32_t db_flags[/*N*/], uint32_t dbt_flags[/*N*/], uint32_t loader_flags);
int (*create_indexer) (DB_ENV *env, DB_TXN *txn, DB_INDEXER **idxrp, DB *src_db, int N, DB *dbs[/*N*/], uint32_t db_flags[/*N*/], uint32_t indexer_flags);
int (*put_multiple) (DB_ENV *env, DB *src_db, DB_TXN *txn,
const DBT *key, const DBT *val,
uint32_t num_dbs, DB **db_array, DBT *keys, DBT *vals, uint32_t *flags_array,
......@@ -299,7 +310,7 @@ struct __toku_db_env {
int (*set_redzone) (DB_ENV *env, int redzone) /* set the redzone limit in percent of total space */;
int (*set_lk_max_memory) (DB_ENV *env, uint64_t max);
int (*get_lk_max_memory) (DB_ENV *env, uint64_t *max);
void* __toku_dummy0[15];
void* __toku_dummy0[14];
char __toku_dummy1[64];
void *api1_internal; /* 32-bit offset=212 size=4, 64=bit offset=360 size=8 */
void* __toku_dummy2[7];
......@@ -394,7 +405,9 @@ struct __toku_db {
int (*flatten)(DB*, DB_TXN*) /* Flatten a dictionary, similar to (but faster than) a table scan */;
int (*optimize)(DB*) /* Run garbage collecion and promote all transactions older than oldest. Amortized (happens during flattening) */;
int (*get_fragmentation)(DB*,TOKU_DB_FRAGMENTATION);
void* __toku_dummy0[19];
int (*set_indexer)(DB*, DB_INDEXER*);
void (*get_indexer)(DB*, DB_INDEXER**);
void* __toku_dummy0[17];
char __toku_dummy1[96];
void *api_internal; /* 32-bit offset=236 size=4, 64=bit offset=376 size=8 */
void* __toku_dummy2[5];
......
buildheader/db.h_4_3
View file @ f31a90e9
......@@ -54,6 +54,16 @@ struct __toku_loader {
int (*close)(DB_LOADER *loader); /* finish loading, free memory */
int (*abort)(DB_LOADER *loader); /* abort loading, free memory */
};
typedef struct __toku_indexer DB_INDEXER;
struct __toku_indexer_internal;
struct __toku_indexer {
struct __toku_indexer_internal *i;
int (*set_error_callback)(DB_INDEXER *indexer, void (*error_cb)(DB *db, int i, int err, DBT *key, DBT *val, void *error_extra), void *error_extra); /* set the error callback */
int (*set_poll_function)(DB_INDEXER *indexer, int (*poll_func)(void *extra, float progress), void *poll_extra); /* set the polling function */
int (*build)(DB_INDEXER *indexer); /* build the indexes */
int (*close)(DB_INDEXER *indexer); /* finish indexing, free memory */
int (*abort)(DB_INDEXER *indexer); /* abort indexing, free memory */
};
typedef struct __toku_engine_status {
char creationtime[26]; /* time of environment creation */
char startuptime[26]; /* time of engine startup */
......@@ -272,6 +282,7 @@ struct __toku_db_env {
int (*get_iname) (DB_ENV* env, DBT* dname_dbt, DBT* iname_dbt) /* FOR TEST ONLY: lookup existing iname */;
void *app_private; /* 32-bit offset=44 size=4, 64=bit offset=88 size=8 */
int (*create_loader) (DB_ENV *env, DB_TXN *txn, DB_LOADER **blp, DB *src_db, int N, DB *dbs[/*N*/], uint32_t db_flags[/*N*/], uint32_t dbt_flags[/*N*/], uint32_t loader_flags);
int (*create_indexer) (DB_ENV *env, DB_TXN *txn, DB_INDEXER **idxrp, DB *src_db, int N, DB *dbs[/*N*/], uint32_t db_flags[/*N*/], uint32_t indexer_flags);
int (*put_multiple) (DB_ENV *env, DB *src_db, DB_TXN *txn,
const DBT *key, const DBT *val,
uint32_t num_dbs, DB **db_array, DBT *keys, DBT *vals, uint32_t *flags_array,
......@@ -301,7 +312,7 @@ struct __toku_db_env {
int (*set_redzone) (DB_ENV *env, int redzone) /* set the redzone limit in percent of total space */;
int (*set_lk_max_memory) (DB_ENV *env, uint64_t max);
int (*get_lk_max_memory) (DB_ENV *env, uint64_t *max);
void* __toku_dummy0[15];
void* __toku_dummy0[14];
char __toku_dummy1[96];
void *api1_internal; /* 32-bit offset=244 size=4, 64=bit offset=392 size=8 */
void* __toku_dummy2[7];
......@@ -404,7 +415,9 @@ struct __toku_db {
int (*flatten)(DB*, DB_TXN*) /* Flatten a dictionary, similar to (but faster than) a table scan */;
int (*optimize)(DB*) /* Run garbage collecion and promote all transactions older than oldest. Amortized (happens during flattening) */;
int (*get_fragmentation)(DB*,TOKU_DB_FRAGMENTATION);
void* __toku_dummy0[22];
int (*set_indexer)(DB*, DB_INDEXER*);
void (*get_indexer)(DB*, DB_INDEXER**);
void* __toku_dummy0[20];
char __toku_dummy1[96];
void *api_internal; /* 32-bit offset=248 size=4, 64=bit offset=400 size=8 */
void* __toku_dummy2[5];
......
buildheader/db.h_4_4
View file @ f31a90e9
......@@ -54,6 +54,16 @@ struct __toku_loader {
int (*close)(DB_LOADER *loader); /* finish loading, free memory */
int (*abort)(DB_LOADER *loader); /* abort loading, free memory */
};
typedef struct __toku_indexer DB_INDEXER;
struct __toku_indexer_internal;
struct __toku_indexer {
struct __toku_indexer_internal *i;
int (*set_error_callback)(DB_INDEXER *indexer, void (*error_cb)(DB *db, int i, int err, DBT *key, DBT *val, void *error_extra), void *error_extra); /* set the error callback */
int (*set_poll_function)(DB_INDEXER *indexer, int (*poll_func)(void *extra, float progress), void *poll_extra); /* set the polling function */
int (*build)(DB_INDEXER *indexer); /* build the indexes */
int (*close)(DB_INDEXER *indexer); /* finish indexing, free memory */
int (*abort)(DB_INDEXER *indexer); /* abort indexing, free memory */
};
typedef struct __toku_engine_status {
char creationtime[26]; /* time of environment creation */
char startuptime[26]; /* time of engine startup */
......@@ -272,6 +282,7 @@ struct __toku_db_env {
int (*get_iname) (DB_ENV* env, DBT* dname_dbt, DBT* iname_dbt) /* FOR TEST ONLY: lookup existing iname */;
void *app_private; /* 32-bit offset=44 size=4, 64=bit offset=88 size=8 */
int (*create_loader) (DB_ENV *env, DB_TXN *txn, DB_LOADER **blp, DB *src_db, int N, DB *dbs[/*N*/], uint32_t db_flags[/*N*/], uint32_t dbt_flags[/*N*/], uint32_t loader_flags);
int (*create_indexer) (DB_ENV *env, DB_TXN *txn, DB_INDEXER **idxrp, DB *src_db, int N, DB *dbs[/*N*/], uint32_t db_flags[/*N*/], uint32_t indexer_flags);
int (*put_multiple) (DB_ENV *env, DB *src_db, DB_TXN *txn,
const DBT *key, const DBT *val,
uint32_t num_dbs, DB **db_array, DBT *keys, DBT *vals, uint32_t *flags_array,
......@@ -301,7 +312,7 @@ struct __toku_db_env {
int (*set_redzone) (DB_ENV *env, int redzone) /* set the redzone limit in percent of total space */;
int (*set_lk_max_memory) (DB_ENV *env, uint64_t max);
int (*get_lk_max_memory) (DB_ENV *env, uint64_t *max);
void* __toku_dummy0[30];
void* __toku_dummy0[29];
char __toku_dummy1[128];
void *api1_internal; /* 32-bit offset=336 size=4, 64=bit offset=544 size=8 */
void* __toku_dummy2[7];
......@@ -406,7 +417,9 @@ struct __toku_db {
int (*flatten)(DB*, DB_TXN*) /* Flatten a dictionary, similar to (but faster than) a table scan */;
int (*optimize)(DB*) /* Run garbage collecion and promote all transactions older than oldest. Amortized (happens during flattening) */;
int (*get_fragmentation)(DB*,TOKU_DB_FRAGMENTATION);
void* __toku_dummy0[24];
int (*set_indexer)(DB*, DB_INDEXER*);
void (*get_indexer)(DB*, DB_INDEXER**);
void* __toku_dummy0[22];
char __toku_dummy1[96];
void *api_internal; /* 32-bit offset=256 size=4, 64=bit offset=416 size=8 */
void* __toku_dummy2[5];
......
buildheader/db.h_4_5
View file @ f31a90e9
......@@ -54,6 +54,16 @@ struct __toku_loader {
int (*close)(DB_LOADER *loader); /* finish loading, free memory */
int (*abort)(DB_LOADER *loader); /* abort loading, free memory */
};
typedef struct __toku_indexer DB_INDEXER;
struct __toku_indexer_internal;
struct __toku_indexer {
struct __toku_indexer_internal *i;
int (*set_error_callback)(DB_INDEXER *indexer, void (*error_cb)(DB *db, int i, int err, DBT *key, DBT *val, void *error_extra), void *error_extra); /* set the error callback */
int (*set_poll_function)(DB_INDEXER *indexer, int (*poll_func)(void *extra, float progress), void *poll_extra); /* set the polling function */
int (*build)(DB_INDEXER *indexer); /* build the indexes */
int (*close)(DB_INDEXER *indexer); /* finish indexing, free memory */
int (*abort)(DB_INDEXER *indexer); /* abort indexing, free memory */
};
typedef struct __toku_engine_status {
char creationtime[26]; /* time of environment creation */
char startuptime[26]; /* time of engine startup */
......@@ -271,11 +281,12 @@ struct __toku_db_env {
int (*get_engine_status_text) (DB_ENV*, char*, int) /* Fill in status text */;
int (*get_iname) (DB_ENV* env, DBT* dname_dbt, DBT* iname_dbt) /* FOR TEST ONLY: lookup existing iname */;
int (*create_loader) (DB_ENV *env, DB_TXN *txn, DB_LOADER **blp, DB *src_db, int N, DB *dbs[/*N*/], uint32_t db_flags[/*N*/], uint32_t dbt_flags[/*N*/], uint32_t loader_flags);
int (*create_indexer) (DB_ENV *env, DB_TXN *txn, DB_INDEXER **idxrp, DB *src_db, int N, DB *dbs[/*N*/], uint32_t db_flags[/*N*/], uint32_t indexer_flags);
void *app_private; /* 32-bit offset=52 size=4, 64=bit offset=104 size=8 */
int (*put_multiple) (DB_ENV *env, DB *src_db, DB_TXN *txn,
const DBT *key, const DBT *val,
uint32_t num_dbs, DB **db_array, DBT *keys, DBT *vals, uint32_t *flags_array,
void *extra) /* insert into multiple DBs */;
void *app_private; /* 32-bit offset=52 size=4, 64=bit offset=104 size=8 */
int (*set_generate_row_callback_for_put) (DB_ENV *env,
int (*generate_row_for_put)(DB *dest_db, DB *src_db,
DBT *dest_key, DBT *dest_val,
......@@ -301,7 +312,7 @@ struct __toku_db_env {
int (*set_redzone) (DB_ENV *env, int redzone) /* set the redzone limit in percent of total space */;
int (*set_lk_max_memory) (DB_ENV *env, uint64_t max);
int (*get_lk_max_memory) (DB_ENV *env, uint64_t *max);
void* __toku_dummy0[30];
void* __toku_dummy0[29];
char __toku_dummy1[128];
void *api1_internal; /* 32-bit offset=336 size=4, 64=bit offset=544 size=8 */
void* __toku_dummy2[8];
......@@ -406,7 +417,9 @@ struct __toku_db {
int (*flatten)(DB*, DB_TXN*) /* Flatten a dictionary, similar to (but faster than) a table scan */;
int (*optimize)(DB*) /* Run garbage collecion and promote all transactions older than oldest. Amortized (happens during flattening) */;
int (*get_fragmentation)(DB*,TOKU_DB_FRAGMENTATION);
void* __toku_dummy0[27];
int (*set_indexer)(DB*, DB_INDEXER*);
void (*get_indexer)(DB*, DB_INDEXER**);
void* __toku_dummy0[25];
char __toku_dummy1[96];
void *api_internal; /* 32-bit offset=268 size=4, 64=bit offset=440 size=8 */
void* __toku_dummy2[5];
......
buildheader/db.h_4_6
View file @ f31a90e9
......@@ -54,6 +54,16 @@ struct __toku_loader {
int (*close)(DB_LOADER *loader); /* finish loading, free memory */
int (*abort)(DB_LOADER *loader); /* abort loading, free memory */
};
typedef struct __toku_indexer DB_INDEXER;
struct __toku_indexer_internal;
struct __toku_indexer {
struct __toku_indexer_internal *i;
int (*set_error_callback)(DB_INDEXER *indexer, void (*error_cb)(DB *db, int i, int err, DBT *key, DBT *val, void *error_extra), void *error_extra); /* set the error callback */
int (*set_poll_function)(DB_INDEXER *indexer, int (*poll_func)(void *extra, float progress), void *poll_extra); /* set the polling function */
int (*build)(DB_INDEXER *indexer); /* build the indexes */
int (*close)(DB_INDEXER *indexer); /* finish indexing, free memory */
int (*abort)(DB_INDEXER *indexer); /* abort indexing, free memory */
};
typedef struct __toku_engine_status {
char creationtime[26]; /* time of environment creation */
char startuptime[26]; /* time of engine startup */
......@@ -272,11 +282,12 @@ struct __toku_db_env {
int (*get_engine_status_text) (DB_ENV*, char*, int) /* Fill in status text */;
int (*get_iname) (DB_ENV* env, DBT* dname_dbt, DBT* iname_dbt) /* FOR TEST ONLY: lookup existing iname */;
int (*create_loader) (DB_ENV *env, DB_TXN *txn, DB_LOADER **blp, DB *src_db, int N, DB *dbs[/*N*/], uint32_t db_flags[/*N*/], uint32_t dbt_flags[/*N*/], uint32_t loader_flags);
int (*create_indexer) (DB_ENV *env, DB_TXN *txn, DB_INDEXER **idxrp, DB *src_db, int N, DB *dbs[/*N*/], uint32_t db_flags[/*N*/], uint32_t indexer_flags);
void *app_private; /* 32-bit offset=52 size=4, 64=bit offset=104 size=8 */
int (*put_multiple) (DB_ENV *env, DB *src_db, DB_TXN *txn,
const DBT *key, const DBT *val,
uint32_t num_dbs, DB **db_array, DBT *keys, DBT *vals, uint32_t *flags_array,
void *extra) /* insert into multiple DBs */;
void *app_private; /* 32-bit offset=52 size=4, 64=bit offset=104 size=8 */
int (*set_generate_row_callback_for_put) (DB_ENV *env,
int (*generate_row_for_put)(DB *dest_db, DB *src_db,
DBT *dest_key, DBT *dest_val,
......@@ -302,7 +313,7 @@ struct __toku_db_env {
int (*set_redzone) (DB_ENV *env, int redzone) /* set the redzone limit in percent of total space */;
int (*set_lk_max_memory) (DB_ENV *env, uint64_t max);
int (*get_lk_max_memory) (DB_ENV *env, uint64_t *max);
void* __toku_dummy0[31];
void* __toku_dummy0[30];
char __toku_dummy1[144];
void *api1_internal; /* 32-bit offset=356 size=4, 64=bit offset=568 size=8 */
void* __toku_dummy2[8];
......@@ -409,7 +420,9 @@ struct __toku_db {
int (*flatten)(DB*, DB_TXN*) /* Flatten a dictionary, similar to (but faster than) a table scan */;
int (*optimize)(DB*) /* Run garbage collecion and promote all transactions older than oldest. Amortized (happens during flattening) */;
int (*get_fragmentation)(DB*,TOKU_DB_FRAGMENTATION);
void* __toku_dummy1[31];
int (*set_indexer)(DB*, DB_INDEXER*);
void (*get_indexer)(DB*, DB_INDEXER**);
void* __toku_dummy1[29];
char __toku_dummy2[80];
void *api_internal; /* 32-bit offset=276 size=4, 64=bit offset=464 size=8 */
void* __toku_dummy3[5];
......
buildheader/make_db_h.c
View file @ f31a90e9
......@@ -437,6 +437,18 @@ int main (int argc __attribute__((__unused__)), char *const argv[] __attribute__
printf(" int (*abort)(DB_LOADER *loader); /* abort loading, free memory */\n");
printf("};\n");
//indexer
printf("typedef struct __toku_indexer DB_INDEXER;\n");
printf("struct __toku_indexer_internal;\n");
printf("struct __toku_indexer {\n");
printf(" struct __toku_indexer_internal *i;\n");
printf(" int (*set_error_callback)(DB_INDEXER *indexer, void (*error_cb)(DB *db, int i, int err, DBT *key, DBT *val, void *error_extra), void *error_extra); /* set the error callback */\n");
printf(" int (*set_poll_function)(DB_INDEXER *indexer, int (*poll_func)(void *extra, float progress), void *poll_extra); /* set the polling function */\n");
printf(" int (*build)(DB_INDEXER *indexer); /* build the indexes */\n");
printf(" int (*close)(DB_INDEXER *indexer); /* finish indexing, free memory */\n");
printf(" int (*abort)(DB_INDEXER *indexer); /* abort indexing, free memory */\n");
printf("};\n");
//engine status info
printf("typedef struct __toku_engine_status {\n");
printf(" char creationtime[26]; /* time of environment creation */ \n");
......@@ -571,6 +583,7 @@ int main (int argc __attribute__((__unused__)), char *const argv[] __attribute__
"int (*get_engine_status_text) (DB_ENV*, char*, int) /* Fill in status text */",
"int (*get_iname) (DB_ENV* env, DBT* dname_dbt, DBT* iname_dbt) /* FOR TEST ONLY: lookup existing iname */",
"int (*create_loader) (DB_ENV *env, DB_TXN *txn, DB_LOADER **blp, DB *src_db, int N, DB *dbs[/*N*/], uint32_t db_flags[/*N*/], uint32_t dbt_flags[/*N*/], uint32_t loader_flags)",
"int (*create_indexer) (DB_ENV *env, DB_TXN *txn, DB_INDEXER **idxrp, DB *src_db, int N, DB *dbs[/*N*/], uint32_t db_flags[/*N*/], uint32_t indexer_flags)",
"int (*put_multiple) (DB_ENV *env, DB *src_db, DB_TXN *txn,\n"
" const DBT *key, const DBT *val,\n"
" uint32_t num_dbs, DB **db_array, DBT *keys, DBT *vals, uint32_t *flags_array,\n"
......@@ -652,6 +665,8 @@ int main (int argc __attribute__((__unused__)), char *const argv[] __attribute__
"int (*flatten)(DB*, DB_TXN*) /* Flatten a dictionary, similar to (but faster than) a table scan */",
"int (*optimize)(DB*) /* Run garbage collecion and promote all transactions older than oldest. Amortized (happens during flattening) */",
"int (*get_fragmentation)(DB*,TOKU_DB_FRAGMENTATION)",
"int (*set_indexer)(DB*, DB_INDEXER*)",
"void (*get_indexer)(DB*, DB_INDEXER**)",
NULL};
print_struct("db", 1, db_fields32, db_fields64, sizeof(db_fields32)/sizeof(db_fields32[0]), extra);
}
......
buildheader/tdb.h
View file @ f31a90e9
......@@ -54,6 +54,16 @@ struct __toku_loader {
int (*close)(DB_LOADER *loader); /* finish loading, free memory */
int (*abort)(DB_LOADER *loader); /* abort loading, free memory */
};
typedef struct __toku_indexer DB_INDEXER;
struct __toku_indexer_internal;
struct __toku_indexer {
struct __toku_indexer_internal *i;
int (*set_error_callback)(DB_INDEXER *indexer, void (*error_cb)(DB *db, int i, int err, DBT *key, DBT *val, void *error_extra), void *error_extra); /* set the error callback */
int (*set_poll_function)(DB_INDEXER *indexer, int (*poll_func)(void *extra, float progress), void *poll_extra); /* set the polling function */
int (*build)(DB_INDEXER *indexer); /* build the indexes */
int (*close)(DB_INDEXER *indexer); /* finish indexing, free memory */
int (*abort)(DB_INDEXER *indexer); /* abort indexing, free memory */
};
typedef struct __toku_engine_status {
char creationtime[26]; /* time of environment creation */
char startuptime[26]; /* time of engine startup */
......@@ -272,11 +282,12 @@ struct __toku_db_env {
int (*get_engine_status_text) (DB_ENV*, char*, int) /* Fill in status text */;
int (*get_iname) (DB_ENV* env, DBT* dname_dbt, DBT* iname_dbt) /* FOR TEST ONLY: lookup existing iname */;
int (*create_loader) (DB_ENV *env, DB_TXN *txn, DB_LOADER **blp, DB *src_db, int N, DB *dbs[/*N*/], uint32_t db_flags[/*N*/], uint32_t dbt_flags[/*N*/], uint32_t loader_flags);
int (*create_indexer) (DB_ENV *env, DB_TXN *txn, DB_INDEXER **idxrp, DB *src_db, int N, DB *dbs[/*N*/], uint32_t db_flags[/*N*/], uint32_t indexer_flags);
void *app_private;
int (*put_multiple) (DB_ENV *env, DB *src_db, DB_TXN *txn,
const DBT *key, const DBT *val,
uint32_t num_dbs, DB **db_array, DBT *keys, DBT *vals, uint32_t *flags_array,
void *extra) /* insert into multiple DBs */;
void *app_private;
int (*set_generate_row_callback_for_put) (DB_ENV *env,
int (*generate_row_for_put)(DB *dest_db, DB *src_db,
DBT *dest_key, DBT *dest_val,
......@@ -378,6 +389,8 @@ struct __toku_db {
int (*flatten)(DB*, DB_TXN*) /* Flatten a dictionary, similar to (but faster than) a table scan */;
int (*optimize)(DB*) /* Run garbage collecion and promote all transactions older than oldest. Amortized (happens during flattening) */;
int (*get_fragmentation)(DB*,TOKU_DB_FRAGMENTATION);
int (*set_indexer)(DB*, DB_INDEXER*);
void (*get_indexer)(DB*, DB_INDEXER**);
void *api_internal;
int (*close) (DB*, u_int32_t);
int (*cursor) (DB *, DB_TXN *, DBC **, u_int32_t);
......
include/db.h
View file @ f31a90e9
......@@ -54,6 +54,16 @@ struct __toku_loader {
int (*close)(DB_LOADER *loader); /* finish loading, free memory */
int (*abort)(DB_LOADER *loader); /* abort loading, free memory */
};
typedef struct __toku_indexer DB_INDEXER;
struct __toku_indexer_internal;
struct __toku_indexer {
struct __toku_indexer_internal *i;
int (*set_error_callback)(DB_INDEXER *indexer, void (*error_cb)(DB *db, int i, int err, DBT *key, DBT *val, void *error_extra), void *error_extra); /* set the error callback */
int (*set_poll_function)(DB_INDEXER *indexer, int (*poll_func)(void *extra, float progress), void *poll_extra); /* set the polling function */
int (*build)(DB_INDEXER *indexer); /* build the indexes */
int (*close)(DB_INDEXER *indexer); /* finish indexing, free memory */
int (*abort)(DB_INDEXER *indexer); /* abort indexing, free memory */
};
typedef struct __toku_engine_status {
char creationtime[26]; /* time of environment creation */
char startuptime[26]; /* time of engine startup */
......@@ -272,11 +282,12 @@ struct __toku_db_env {
int (*get_engine_status_text) (DB_ENV*, char*, int) /* Fill in status text */;
int (*get_iname) (DB_ENV* env, DBT* dname_dbt, DBT* iname_dbt) /* FOR TEST ONLY: lookup existing iname */;
int (*create_loader) (DB_ENV *env, DB_TXN *txn, DB_LOADER **blp, DB *src_db, int N, DB *dbs[/*N*/], uint32_t db_flags[/*N*/], uint32_t dbt_flags[/*N*/], uint32_t loader_flags);
int (*create_indexer) (DB_ENV *env, DB_TXN *txn, DB_INDEXER **idxrp, DB *src_db, int N, DB *dbs[/*N*/], uint32_t db_flags[/*N*/], uint32_t indexer_flags);
void *app_private;
int (*put_multiple) (DB_ENV *env, DB *src_db, DB_TXN *txn,
const DBT *key, const DBT *val,
uint32_t num_dbs, DB **db_array, DBT *keys, DBT *vals, uint32_t *flags_array,
void *extra) /* insert into multiple DBs */;
void *app_private;
int (*set_generate_row_callback_for_put) (DB_ENV *env,
int (*generate_row_for_put)(DB *dest_db, DB *src_db,
DBT *dest_key, DBT *dest_val,
......@@ -378,6 +389,8 @@ struct __toku_db {
int (*flatten)(DB*, DB_TXN*) /* Flatten a dictionary, similar to (but faster than) a table scan */;
int (*optimize)(DB*) /* Run garbage collecion and promote all transactions older than oldest. Amortized (happens during flattening) */;
int (*get_fragmentation)(DB*,TOKU_DB_FRAGMENTATION);
int (*set_indexer)(DB*, DB_INDEXER*);
void (*get_indexer)(DB*, DB_INDEXER**);
void *api_internal;
int (*close) (DB*, u_int32_t);
int (*cursor) (DB *, DB_TXN *, DBC **, u_int32_t);
......
newbrt/brt.c
View file @ f31a90e9
......@@ -2619,6 +2619,26 @@ toku_brt_load_recovery(TOKUTXN txn, char const * old_iname, char const * new_ina
return r;
}
// 2954
// this function handles the tasks needed to be recoverable
// - write to rollback log
// - write to recovery log
int
toku_brt_hot_index_recovery(TOKUTXN txn, FILENUMS filenums, int do_fsync, int do_log, LSN *hot_index_lsn)
{
int r = 0;
assert(txn);
TOKULOGGER logger = toku_txn_logger(txn);
// write to the rollback log
r = toku_logger_save_rollback_hot_index(txn, &filenums);
if ( r==0 && do_log && logger) {
TXNID xid = toku_txn_get_txnid(txn);
// write to the recovery log
r = toku_log_hot_index(logger, hot_index_lsn, do_fsync, xid, filenums);
}
return r;
}
static int brt_optimize (BRT brt, BOOL upgrade);
......@@ -2683,6 +2703,16 @@ toku_brt_load(BRT brt, TOKUTXN txn, char const * new_iname, int do_fsync, LSN *l
return r;
}
// 2954
// brt actions for logging hot index filenums
int
toku_brt_hot_index(BRT brt __attribute__ ((unused)), TOKUTXN txn, FILENUMS filenums, int do_fsync, LSN *lsn) {
int r = 0;
int do_log = 1;
r = toku_brt_hot_index_recovery(txn, filenums, do_fsync, do_log, lsn);
return r;
}
int
toku_brt_log_put (TOKUTXN txn, BRT brt, const DBT *key, const DBT *val) {
int r = 0;
......@@ -2728,7 +2758,7 @@ toku_brt_log_put_multiple (TOKUTXN txn, BRT src_brt, BRT *brts, int num_brts, co
}
int
toku_brt_maybe_insert (BRT brt, DBT *key, DBT *val, TOKUTXN txn, BOOL oplsn_valid, LSN oplsn, int do_logging, enum brt_msg_type type) {
toku_brt_maybe_insert (BRT brt, DBT *key, DBT *val, TOKUTXN txn, BOOL oplsn_valid, LSN oplsn, BOOL do_logging, enum brt_msg_type type) {
lazy_assert(type==BRT_INSERT || type==BRT_INSERT_NO_OVERWRITE);
int r = 0;
XIDS message_xids = xids_get_root_xids(); //By default use committed messages
......@@ -2766,12 +2796,26 @@ toku_brt_maybe_insert (BRT brt, DBT *key, DBT *val, TOKUTXN txn, BOOL oplsn_vali
if (oplsn_valid && oplsn.lsn <= (treelsn = toku_brt_checkpoint_lsn(brt)).lsn) {
r = 0;
} else {
BRT_MSG_S brtcmd = { type, message_xids, .u.id={key,val}};
r = toku_brt_root_put_cmd(brt, &brtcmd);
r = toku_brt_send_insert(brt, key, val, message_xids, type);
}
return r;
}
int
toku_brt_send_insert(BRT brt, DBT *key, DBT *val, XIDS xids, enum brt_msg_type type) {
BRT_MSG_S brtcmd = { type, xids, .u.id = { key, val }};
int r = toku_brt_root_put_cmd(brt, &brtcmd);
return r;
}
int
toku_brt_send_commit_any(BRT brt, DBT *key, XIDS xids) {
DBT val;
BRT_MSG_S brtcmd = { BRT_COMMIT_ANY, xids, .u.id = { key, toku_init_dbt(&val) }};
int r = toku_brt_root_put_cmd(brt, &brtcmd);
return r;
}
int toku_brt_delete(BRT brt, DBT *key, TOKUTXN txn) {
return toku_brt_maybe_delete(brt, key, txn, FALSE, ZERO_LSN, TRUE);
}
......@@ -2817,7 +2861,7 @@ toku_brt_log_del_multiple (TOKUTXN txn, BRT src_brt, BRT *brts, int num_brts, co
}
int
toku_brt_maybe_delete(BRT brt, DBT *key, TOKUTXN txn, BOOL oplsn_valid, LSN oplsn, int do_logging) {
toku_brt_maybe_delete(BRT brt, DBT *key, TOKUTXN txn, BOOL oplsn_valid, LSN oplsn, BOOL do_logging) {
int r;
XIDS message_xids = xids_get_root_xids(); //By default use committed messages
TXNID xid = toku_txn_get_txnid(txn);
......@@ -2848,13 +2892,19 @@ toku_brt_maybe_delete(BRT brt, DBT *key, TOKUTXN txn, BOOL oplsn_valid, LSN opls
if (oplsn_valid && oplsn.lsn <= (treelsn = toku_brt_checkpoint_lsn(brt)).lsn) {
r = 0;
} else {
DBT val;
BRT_MSG_S brtcmd = { BRT_DELETE_ANY, message_xids, .u.id={key, toku_init_dbt(&val)}};
r = toku_brt_root_put_cmd(brt, &brtcmd);
r = toku_brt_send_delete(brt, key, message_xids);
}
return r;
}
int
toku_brt_send_delete(BRT brt, DBT *key, XIDS xids) {
DBT val; toku_init_dbt(&val);
BRT_MSG_S brtcmd = { BRT_DELETE_ANY, xids, .u.id = { key, &val }};
int result = toku_brt_root_put_cmd(brt, &brtcmd);
return result;
}
struct omt_compressor_state {
struct mempool *new_kvspace;
OMT omt;
......
newbrt/brt.h
View file @ f31a90e9
......@@ -72,9 +72,15 @@ int toku_brt_optimize_for_upgrade (BRT brt) __attribute__ ((warn_unused_result)
// Effect: Insert a key and data pair into a brt if the oplsn is newer than the brt lsn. This function is called during recovery.
// Returns 0 if successful
int toku_brt_maybe_insert (BRT brt, DBT *k, DBT *v, TOKUTXN txn, BOOL oplsn_valid, LSN oplsn, int do_logging, enum brt_msg_type type) __attribute__ ((warn_unused_result));
int toku_brt_maybe_insert (BRT brt, DBT *k, DBT *v, TOKUTXN txn, BOOL oplsn_valid, LSN oplsn, BOOL do_logging, enum brt_msg_type type) __attribute__ ((warn_unused_result));
int toku_brt_load_recovery(TOKUTXN txn, char const * old_iname, char const * new_iname, int do_fsync, int do_log, LSN *load_lsn) __attribute__ ((warn_unused_result));
int toku_brt_load(BRT brt, TOKUTXN txn, char const * new_iname, int do_fsync, LSN *get_lsn) __attribute__ ((warn_unused_result));
// 2954
int toku_brt_hot_index_recovery(TOKUTXN txn, FILENUMS filenums, int do_fsync, int do_log, LSN *hot_index_lsn);
int toku_brt_hot_index(BRT brt, TOKUTXN txn, FILENUMS filenums, int do_fsync, LSN *lsn) __attribute__ ((warn_unused_result));
int toku_brt_log_put_multiple (TOKUTXN txn, BRT src_brt, BRT *brts, int num_brts, const DBT *key, const DBT *val) __attribute__ ((warn_unused_result));
int toku_brt_log_put (TOKUTXN txn, BRT brt, const DBT *key, const DBT *val) __attribute__ ((warn_unused_result));
int toku_brt_log_del_multiple (TOKUTXN txn, BRT src_brt, BRT *brts, int num_brts, const DBT *key, const DBT *val) __attribute__ ((warn_unused_result));
......@@ -86,16 +92,11 @@ int toku_brt_delete (BRT brt, DBT *k, TOKUTXN txn) __attribute__ ((warn_unused_
// Effect: Delete a key from a brt if the oplsn is newer than the brt lsn. This function is called during recovery.
// Returns 0 if successful
int toku_brt_maybe_delete (BRT brt, DBT *k, TOKUTXN txn, BOOL oplsn_valid, LSN oplsn, int do_logging) __attribute__ ((warn_unused_result));
int toku_brt_maybe_delete (BRT brt, DBT *k, TOKUTXN txn, BOOL oplsn_valid, LSN oplsn, BOOL do_logging) __attribute__ ((warn_unused_result));
// Effect: Delete a pair only if both k and v are equal according to the comparison function.
// Returns 0 if successful
int toku_brt_delete_both (BRT brt, DBT *k, DBT *v, TOKUTXN txn) __attribute__ ((warn_unused_result));
// Effect: Delete a pair only if both k and v are equal according to the comparison function and the
// oplsn is newer than the brt lsn. This function is called by recovery.
// Returns 0 if successful
int toku_brt_maybe_delete_both (BRT brt, DBT *k, DBT *v, TOKUTXN txn, BOOL oplsn_valid, LSN oplsn) __attribute__ ((warn_unused_result));
int toku_brt_send_insert(BRT brt, DBT *key, DBT *val, XIDS xids, enum brt_msg_type type) __attribute__ ((warn_unused_result));
int toku_brt_send_delete(BRT brt, DBT *key, XIDS xids) __attribute__ ((warn_unused_result));
int toku_brt_send_commit_any(BRT brt, DBT *key, XIDS xids) __attribute__ ((warn_unused_result));
int toku_brt_db_delay_closed (BRT brt, DB* db, int (*close_db)(DB*, u_int32_t), u_int32_t close_flags) __attribute__ ((warn_unused_result));
int toku_close_brt (BRT, char **error_string) __attribute__ ((warn_unused_result));
......@@ -155,7 +156,6 @@ int toku_brt_cursor_set_range_reverse(BRT_CURSOR cursor, DBT *key, BRT_GET_CALLB
int toku_brt_cursor_get_both_range(BRT_CURSOR cursor, DBT *key, DBT *val, BRT_GET_CALLBACK_FUNCTION getf, void *getf_v) __attribute__ ((warn_unused_result));
int toku_brt_cursor_get_both_range_reverse(BRT_CURSOR cursor, DBT *key, DBT *val, BRT_GET_CALLBACK_FUNCTION getf, void *getf_v) __attribute__ ((warn_unused_result));
int toku_brt_cursor_delete(BRT_CURSOR cursor, int flags, TOKUTXN) __attribute__ ((warn_unused_result));
int toku_brt_cursor_close (BRT_CURSOR curs) __attribute__ ((warn_unused_result));
BOOL toku_brt_cursor_uninitialized(BRT_CURSOR c) __attribute__ ((warn_unused_result));
......@@ -228,6 +228,12 @@ BOOL toku_brt_is_empty (BRT brt, BOOL *try_again) __attribute__ ((warn_unused_re
// Effect: Return TRUE iff the tree is empty. (However if *try_again is set to TRUE by toku_brt_is_empty, then the answer is inconclusive, and the function should
// be tried again. It's a good idea to release the big ydb lock in this case.
BOOL toku_brt_is_empty_fast (BRT brt);
// Effect: Return TRUE if there are no messages or leaf entries in the tree. If so, it's empty. If there are messages or leaf entries, we say it's not empty
// even though if we were to optimize the tree it might turn out that they are empty.
double get_tdiff(void) __attribute__((__visibility__("default")));
BOOL toku_brt_is_empty_fast (BRT brt) __attribute__ ((warn_unused_result));
// Effect: Return TRUE if there are no messages or leaf entries in the tree. If so, it's empty. If there are messages or leaf entries, we say it's not empty
// even though if we were to optimize the tree it might turn out that they are empty.
......
newbrt/le-cursor.c
View file @ f31a90e9
......@@ -72,7 +72,7 @@ le_cursor_next(LE_CURSOR le_cursor, DBT *key, DBT *val) {
}
int
is_key_right_of_le_cursor(LE_CURSOR le_cursor, DBT *key, int (*keycompare)(DB *, const DBT *, const DBT *), DB *db) {
is_key_right_of_le_cursor(LE_CURSOR le_cursor, const DBT *key, int (*keycompare)(DB *, const DBT *, const DBT *), DB *db) {
int result;
if (le_cursor->neg_infinity)
result = TRUE;
......
newbrt/le-cursor.h
View file @ f31a90e9
......@@ -2,6 +2,9 @@
#ident "Copyright (c) 2010 Tokutek Inc. All rights reserved."
#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
#ifndef LE_CURSOR_H
#define LE_CURSOR_H
// A leaf entry cursor (LE_CURSOR) is a special type of BRT_CURSOR that visits all of the leaf entries in a tree
// and returns the leaf entry to the caller. It maintains a copy of the key that it was last positioned over to
// speed up key comparisions with a given key. For example, the hot indexing could use the _key_right_of_cursor
......@@ -32,5 +35,6 @@ int le_cursor_next(LE_CURSOR le_cursor, DBT *key, DBT *val);
// The LE_CURSOR position is intialized to -infinity. Any key comparision with -infinity returns TRUE.
// When the cursor runs off the right edge of the tree, the LE_CURSOR position is set to +infinity. Any key comparision with +infinity
// returns FALSE.
int is_key_right_of_le_cursor(LE_CURSOR le_cursor, DBT *key, int (*keycompare)(DB *extra, const DBT *, const DBT *), DB *extra);
int is_key_right_of_le_cursor(LE_CURSOR le_cursor, const DBT *key, int (*keycompare)(DB *extra, const DBT *, const DBT *), DB *extra);
#endif
newbrt/log-internal.h
View file @ f31a90e9
......@@ -11,6 +11,7 @@
#include "toku_list.h"
#include "memarena.h"
#include "logfilemgr.h"
#include "txn.h"
#include <stdio.h>
#include <toku_pthread.h>
#include <sys/types.h>
......@@ -129,6 +130,7 @@ struct tokutxn {
u_int64_t snapshot_txnid64; /* this is the lsn of the snapshot */
TOKULOGGER logger;
TOKUTXN parent;
DB_TXN* container_db_txn; // reference to DB_TXN that contains this tokutxn
u_int64_t rollentry_raw_count; // the total count of every byte in the transaction and all its children.
OMT open_brts; // a collection of the brts that we touched. Indexed by filenum.
......@@ -150,6 +152,7 @@ struct tokutxn {
BOOL recovered_from_checkpoint;
ROLLBACK_LOG_NODE pinned_inprogress_rollback_log;
struct toku_list checkpoint_before_commit;
TXN_IGNORE ignore_errors; // 2954
};
struct txninfo {
......
newbrt/logformat.c
View file @ f31a90e9
......@@ -70,6 +70,9 @@ const struct logtype rollbacks[] = {
{"load", 'l', FA{{"BYTESTRING", "old_iname", 0},
{"BYTESTRING", "new_iname", 0},
NULLFIELD}},
// #2954
{"hot_index", 'h', FA{{"FILENUMS", "hot_index_filenums", 0},
NULLFIELD}},
{"dictionary_redirect", 'R', FA{{"FILENUM", "old_filenum", 0},
{"FILENUM", "new_filenum", 0},
NULLFIELD}},
......@@ -172,6 +175,10 @@ const struct logtype logtypes[] = {
{"BYTESTRING", "old_iname", 0},
{"BYTESTRING", "new_iname", 0},
NULLFIELD}},
// #2954
{"hot_index", 'h', FA{{"TXNID", "xid", 0},
{"FILENUMS", "hot_index_filenums", 0},
NULLFIELD}},
{0,0,FA{NULLFIELD}}
};
......@@ -528,6 +535,9 @@ generate_rollbacks (void) {
if ( strcmp(ft->type, "BYTESTRING") == 0 ) {
fprintf2(cf, hf, ", BYTESTRING *%s_ptr", ft->name);
}
else if ( strcmp(ft->type, "FILENUMS") == 0 ) {
fprintf2(cf, hf, ", FILENUMS *%s_ptr", ft->name);
}
else {
fprintf2(cf, hf, ", %s %s", ft->type, ft->name);
}
......@@ -548,6 +558,13 @@ generate_rollbacks (void) {
" };\n",
ft->name, ft->name, ft->name, ft->name);
}
if ( strcmp(ft->type, "FILENUMS") == 0 ) {
fprintf(cf, " FILENUMS %s = {\n"
" .num = %s_ptr->num,\n"
" .filenums = toku_memdup_in_rollback(log, %s_ptr->filenums, %s_ptr->num * (sizeof (FILENUM)))\n"
" };\n",
ft->name, ft->name, ft->name, ft->name);
}
});
{
int count=0;
......
newbrt/rbuf.h
View file @ f31a90e9
......@@ -131,6 +131,26 @@ static inline void rbuf_ma_FILENUM (struct rbuf *r, MEMARENA ma __attribute__((_
rbuf_FILENUM(r, filenum);
}
// 2954
// Don't try to use the same space, malloc it
static inline void rbuf_FILENUMS (struct rbuf *r, FILENUMS *filenums) {
filenums->num = rbuf_int(r);
u_int32_t newndone = r->ndone + ( filenums->num * sizeof(FILENUM) );
assert(newndone <= r->size);
filenums->filenums = (FILENUM *) toku_memdup(&r->buf[r->ndone], (size_t)filenums->num * sizeof(FILENUM));
assert(filenums->filenums);
r->ndone = newndone;
}
// 2954
static inline void rbuf_ma_FILENUMS (struct rbuf *r, MEMARENA ma __attribute__((__unused__)), FILENUMS *filenums) {
filenums->num = rbuf_int(r);
u_int32_t newndone = r->ndone + ( filenums->num * sizeof(FILENUM) );
assert(newndone <= r->size);
filenums->filenums = (FILENUM *) memarena_memdup(ma, &r->buf[r->ndone], (size_t)filenums->num * sizeof(FILENUM));
assert(filenums->filenums);
r->ndone = newndone;
}
// Don't try to use the same space, malloc it
static inline void rbuf_BYTESTRING (struct rbuf *r, BYTESTRING *bs) {
bs->len = rbuf_int(r);
......
newbrt/recover.c
View file @ f31a90e9
......@@ -977,6 +977,26 @@ static int toku_recover_backward_load(struct logtype_load *UU(l), RECOVER_ENV UU
return 0;
}
// #2954
static int toku_recover_hot_index(struct logtype_hot_index *UU(l), RECOVER_ENV UU(renv)) {
int r;
TOKUTXN txn = NULL;
r = toku_txnid2txn(renv->logger, l->xid, &txn);
assert(r == 0);
assert(txn!=NULL);
// just make an entry in the rollback log
// - set do_log = 0 -> don't write to recovery log
r = toku_brt_hot_index_recovery(txn, l->hot_index_filenums, 0, 0, (LSN*)NULL);
assert(r == 0);
return 0;
}
// #2954
static int toku_recover_backward_hot_index(struct logtype_hot_index *UU(l), RECOVER_ENV UU(renv)) {
// nothing
return 0;
}
// Effects: If there are no log files, or if there is a "clean" checkpoint at the end of the log,
// then we don't need recovery to run. Skip the shutdown log entry if there is one.
// Returns: TRUE if we need recovery, otherwise FALSE.
......
newbrt/roll.c
View file @ f31a90e9
......@@ -129,8 +129,11 @@ static int find_brt_from_filenum (OMTVALUE v, void *filenumvp) {
static int do_insertion (enum brt_msg_type type, FILENUM filenum, BYTESTRING key, BYTESTRING *data, TOKUTXN txn, LSN oplsn) {
CACHEFILE cf;
// 2954 - ignore messages for aborted hot-index
int r = toku_txn_ignore_contains(txn, filenum);
if ( r != ENOENT ) goto done; // ENOENT => filenum not in ignore list
//printf("%s:%d committing insert %s %s\n", __FILE__, __LINE__, key.data, data.data);
int r = toku_cachefile_of_filenum(txn->logger->ct, filenum, &cf);
r = toku_cachefile_of_filenum(txn->logger->ct, filenum, &cf);
if (r==ENOENT) { //Missing file on recovered transaction is not an error
assert(txn->recovered_from_checkpoint);
r = 0;
......@@ -378,6 +381,44 @@ toku_rollback_load (BYTESTRING UU(old_iname),
return 0;
}
//2954
int
toku_commit_hot_index (FILENUMS UU(hot_index_filenums),
TOKUTXN UU(txn),
YIELDF UU(yield),
void * UU(yield_v),
LSN UU(oplsn))
{
// nothing
return 0;
}
//2954
// function called by toku_omt_iterate to add hot_index filenums to
// each live txn's ignore list when a hot index is aborted
static int
live_txn_ignore(OMTVALUE vtxn, u_int32_t UU(idx) , void *vfn) {
TOKUTXN txn = vtxn;
FILENUMS *hot_index_filenums = vfn;
int r;
for (uint32_t i=0; i<hot_index_filenums->num;i++) {
r = toku_txn_ignore_add(txn, hot_index_filenums->filenums[i]);
if ( r != 0 ) return r;
}
return 0;
}
int
toku_rollback_hot_index (FILENUMS UU(hot_index_filenums),
TOKUTXN UU(txn),
YIELDF UU(yield),
void * UU(yield_v),
LSN UU(oplsn))
{
int r = toku_omt_iterate(txn->logger->live_txns, live_txn_ignore, &hot_index_filenums);
assert(r == 0);
return r;
}
int
toku_commit_dictionary_redirect (FILENUM UU(old_filenum),
......
newbrt/rollback.c
View file @ f31a90e9
......@@ -289,6 +289,7 @@ void toku_rollback_txn_close (TOKUTXN txn) {
note_txn_closing(txn);
xids_destroy(&txn->xids);
toku_txn_ignore_free(txn); // 2954
toku_free(txn);
return;
}
......
newbrt/tests/is_empty.c
View file @ f31a90e9
......@@ -37,7 +37,7 @@ static void test_it (int N) {
r = toku_logger_open_rollback(logger, ct, TRUE); CKERR(r);
TOKUTXN txn;
r = toku_txn_begin_txn((TOKUTXN)0, &txn, logger, TXN_SNAPSHOT_ROOT); CKERR(r);
r = toku_txn_begin_txn((DB_TXN*)NULL, (TOKUTXN)0, &txn, logger, TXN_SNAPSHOT_ROOT); CKERR(r);
r = toku_open_brt(FILENAME, 1, &brt, 1024, ct, txn, toku_builtin_compare_fun, NULL); CKERR(r);
......@@ -49,12 +49,12 @@ static void test_it (int N) {
unsigned int rands[N];
for (int i=0; i<N; i++) {
r = toku_txn_begin_txn((TOKUTXN)0, &txn, logger, TXN_SNAPSHOT_ROOT); CKERR(r);
r = toku_txn_begin_txn((DB_TXN*)NULL, (TOKUTXN)0, &txn, logger, TXN_SNAPSHOT_ROOT); CKERR(r);
r = toku_open_brt(FILENAME, 0, &brt, 1024, ct, txn, toku_builtin_compare_fun, NULL); CKERR(r);
r = toku_txn_commit_txn(txn, FALSE, do_yield, NULL, NULL, NULL); CKERR(r);
toku_txn_close_txn(txn);
r = toku_txn_begin_txn((TOKUTXN)0, &txn, logger, TXN_SNAPSHOT_ROOT); CKERR(r);
r = toku_txn_begin_txn((DB_TXN*)NULL, (TOKUTXN)0, &txn, logger, TXN_SNAPSHOT_ROOT); CKERR(r);
char key[100],val[300];
DBT k, v;
rands[i] = random();
......@@ -72,12 +72,12 @@ static void test_it (int N) {
if (verbose) printf("i=%d\n", i);
}
for (int i=0; i<N; i++) {
r = toku_txn_begin_txn((TOKUTXN)0, &txn, logger, TXN_SNAPSHOT_ROOT); CKERR(r);
r = toku_txn_begin_txn((DB_TXN*)NULL, (TOKUTXN)0, &txn, logger, TXN_SNAPSHOT_ROOT); CKERR(r);
r = toku_open_brt(FILENAME, 0, &brt, 1024, ct, txn, toku_builtin_compare_fun, NULL); CKERR(r);
r = toku_txn_commit_txn(txn, FALSE, do_yield, NULL, NULL, NULL); CKERR(r);
toku_txn_close_txn(txn);
r = toku_txn_begin_txn((TOKUTXN)0, &txn, logger, TXN_SNAPSHOT_ROOT); CKERR(r);
r = toku_txn_begin_txn((DB_TXN*)NULL, (TOKUTXN)0, &txn, logger, TXN_SNAPSHOT_ROOT); CKERR(r);
char key[100];
DBT k;
snprintf(key, sizeof(key), "key%x.%x", rands[i], i);
......@@ -102,7 +102,7 @@ static void test_it (int N) {
if (verbose) printf("d=%d\n", i);
}
r = toku_txn_begin_txn((TOKUTXN)0, &txn, logger, TXN_SNAPSHOT_ROOT); CKERR(r);
r = toku_txn_begin_txn((DB_TXN*)NULL, (TOKUTXN)0, &txn, logger, TXN_SNAPSHOT_ROOT); CKERR(r);
r = toku_open_brt(FILENAME, 0, &brt, 1024, ct, txn, toku_builtin_compare_fun, NULL); CKERR(r);
r = toku_txn_commit_txn(txn, FALSE, do_yield, NULL, NULL, NULL); CKERR(r);
toku_txn_close_txn(txn);
......
newbrt/tests/le-cursor-provdel.c
View file @ f31a90e9
......@@ -41,7 +41,7 @@ create_populate_tree(const char *logdir, const char *fname, int n) {
assert(error == 0);
TOKUTXN txn = NULL;
error = toku_txn_begin_txn(NULL, &txn, logger, TXN_SNAPSHOT_NONE);
error = toku_txn_begin_txn(NULL, NULL, &txn, logger, TXN_SNAPSHOT_NONE);
assert(error == 0);
BRT brt = NULL;
......@@ -53,7 +53,7 @@ create_populate_tree(const char *logdir, const char *fname, int n) {
toku_txn_close_txn(txn);
txn = NULL;
error = toku_txn_begin_txn(NULL, &txn, logger, TXN_SNAPSHOT_NONE);
error = toku_txn_begin_txn(NULL, NULL, &txn, logger, TXN_SNAPSHOT_NONE);
assert(error == 0);
// insert keys 0, 1, 2, .. (n-1)
......@@ -107,7 +107,7 @@ test_provdel(const char *logdir, const char *fname, int n) {
assert(error == 0);
TOKUTXN txn = NULL;
error = toku_txn_begin_txn(NULL, &txn, logger, TXN_SNAPSHOT_NONE);
error = toku_txn_begin_txn(NULL, NULL, &txn, logger, TXN_SNAPSHOT_NONE);
assert(error == 0);
BRT brt = NULL;
......@@ -119,7 +119,7 @@ test_provdel(const char *logdir, const char *fname, int n) {
toku_txn_close_txn(txn);
txn = NULL;
error = toku_txn_begin_txn(NULL, &txn, logger, TXN_SNAPSHOT_NONE);
error = toku_txn_begin_txn(NULL, NULL, &txn, logger, TXN_SNAPSHOT_NONE);
assert(error == 0);
// del keys 0, 2, 4, ...
......@@ -132,7 +132,7 @@ test_provdel(const char *logdir, const char *fname, int n) {
}
TOKUTXN cursortxn = NULL;
error = toku_txn_begin_txn(NULL, &cursortxn, logger, TXN_SNAPSHOT_NONE);
error = toku_txn_begin_txn(NULL, NULL, &cursortxn, logger, TXN_SNAPSHOT_NONE);
assert(error == 0);
LE_CURSOR cursor = NULL;
......
newbrt/tests/le-cursor-right.c
View file @ f31a90e9
......@@ -44,7 +44,7 @@ create_populate_tree(const char *logdir, const char *fname, int n) {
assert(error == 0);
TOKUTXN txn = NULL;
error = toku_txn_begin_txn(NULL, &txn, logger, TXN_SNAPSHOT_NONE);
error = toku_txn_begin_txn(NULL, NULL, &txn, logger, TXN_SNAPSHOT_NONE);
assert(error == 0);
BRT brt = NULL;
......@@ -56,7 +56,7 @@ create_populate_tree(const char *logdir, const char *fname, int n) {
toku_txn_close_txn(txn);
txn = NULL;
error = toku_txn_begin_txn(NULL, &txn, logger, TXN_SNAPSHOT_NONE);
error = toku_txn_begin_txn(NULL, NULL, &txn, logger, TXN_SNAPSHOT_NONE);
assert(error == 0);
// insert keys 0, 1, 2, .. (n-1)
......
newbrt/tests/le-cursor-walk.c
View file @ f31a90e9
......@@ -41,7 +41,7 @@ create_populate_tree(const char *logdir, const char *fname, int n) {
assert(error == 0);
TOKUTXN txn = NULL;
error = toku_txn_begin_txn(NULL, &txn, logger, TXN_SNAPSHOT_NONE);
error = toku_txn_begin_txn(NULL, NULL, &txn, logger, TXN_SNAPSHOT_NONE);
assert(error == 0);
BRT brt = NULL;
......@@ -53,7 +53,7 @@ create_populate_tree(const char *logdir, const char *fname, int n) {
toku_txn_close_txn(txn);
txn = NULL;
error = toku_txn_begin_txn(NULL, &txn, logger, TXN_SNAPSHOT_NONE);
error = toku_txn_begin_txn(NULL, NULL, &txn, logger, TXN_SNAPSHOT_NONE);
assert(error == 0);
// insert keys 0, 1, 2, .. (n-1)
......
newbrt/tests/test-leafentry-nested.c
View file @ f31a90e9
......@@ -5,6 +5,7 @@
#include "brttypes.h"
#include "includes.h"
#include "ule.h"
#include "ule-internal.h"
enum {MAX_SIZE = 256};
static XIDS nested_xids[MAX_TRANSACTION_RECORDS];
......
newbrt/tests/txn-ignore.c 0 → 100644
View file @ f31a90e9
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2010 Tokutek Inc. All rights reserved."
#include "test.h"
#include "includes.h"
#include "../brttypes.h"
#include "../txn.h"
/*
* a test of the txn filenums to ignore utilities:
* - toku_txn_ignore_create()
* - toku_txn_ignore_free()
* - toku_txn_ignore_add()
* - toku_txn_ignore_delete()
* - toku_txn_ignore_contains()
*/
int
test_main(int argc, const char *argv[]) {
default_parse_args(argc, argv);
TOKUTXN txn = (TOKUTXN) toku_malloc(sizeof(struct tokutxn));
int r;
r = toku_txn_ignore_init(txn); CKERR(r);
FILENUM f1 = {1};
FILENUM f2 = {2};
FILENUM f3 = {3};
FILENUM f4 = {4};
FILENUM f5 = {5};
FILENUM f6 = {6};
FILENUM f7 = {7};
FILENUM f8 = {8};
FILENUM f9 = {9};
r = toku_txn_ignore_add(txn, f1); CKERR(r);
r = toku_txn_ignore_add(txn, f3); CKERR(r);
r = toku_txn_ignore_add(txn, f5); CKERR(r);
r = toku_txn_ignore_add(txn, f7); CKERR(r);
r = toku_txn_ignore_add(txn, f9); CKERR(r);
r = toku_txn_ignore_remove(txn, f3); CKERR(r);
r = toku_txn_ignore_remove(txn, f2); assert( r == ENOENT );
r = toku_txn_ignore_contains(txn, f1); CKERR(r);
r = toku_txn_ignore_contains(txn, f2); assert( r == ENOENT );
r = toku_txn_ignore_contains(txn, f3); assert( r == ENOENT );
r = toku_txn_ignore_contains(txn, f4); assert( r == ENOENT );
r = toku_txn_ignore_contains(txn, f5); CKERR(r);
r = toku_txn_ignore_contains(txn, f6); assert( r == ENOENT );
r = toku_txn_ignore_contains(txn, f7); CKERR(r);
r = toku_txn_ignore_contains(txn, f8); assert( r == ENOENT );
r = toku_txn_ignore_contains(txn, f9); CKERR(r);
assert(txn->ignore_errors->fns_allocated == 8);
assert(txn->ignore_errors->filenums.num == 4);
r = toku_txn_ignore_add(txn, f2); CKERR(r);
r = toku_txn_ignore_add(txn, f3); CKERR(r);
r = toku_txn_ignore_add(txn, f4); CKERR(r);
r = toku_txn_ignore_add(txn, f6); CKERR(r);
r = toku_txn_ignore_add(txn, f8); CKERR(r);
assert(txn->ignore_errors->fns_allocated == 16);
assert(txn->ignore_errors->filenums.num == 9);
// check that dups are ignored
for (int i=0;i<10;i++) {
r = toku_txn_ignore_add(txn, f2); CKERR(r);
}
assert(txn->ignore_errors->fns_allocated == 16);
assert(txn->ignore_errors->filenums.num == 9);
toku_txn_ignore_free(txn);
toku_free(txn);
return 0;
}
newbrt/txn.c
View file @ f31a90e9
......@@ -26,19 +26,34 @@ toku_txn_get_status(TXN_STATUS s) {
*s = status;
}
int toku_txn_begin_txn (
int
toku_txn_begin_txn (
DB_TXN *container_db_txn,
TOKUTXN parent_tokutxn,
TOKUTXN *tokutxn,
TOKULOGGER logger,
TXN_SNAPSHOT_TYPE snapshot_type
)
{
return toku_txn_begin_with_xid(
parent_tokutxn,
int r;
r = toku_txn_begin_with_xid(parent_tokutxn,
tokutxn, logger,
0,
snapshot_type
);
if (r == 0) {
// container_db_txn set here, not in helper function toku_txn_begin_with_xid()
// because helper function is used by recovery, which does not have DB_TXN
(*tokutxn)->container_db_txn = container_db_txn; // internal struct points to container
}
return r;
}
DB_TXN *
toku_txn_get_container_db_txn (TOKUTXN tokutxn) {
DB_TXN * container = tokutxn->container_db_txn;
return container;
}
static int
......@@ -221,6 +236,10 @@ int toku_txn_begin_with_xid (
result->force_fsync_on_commit = FALSE;
result->recovered_from_checkpoint = FALSE;
toku_list_init(&result->checkpoint_before_commit);
// 2954
r = toku_txn_ignore_init(result);
if (r != 0) goto died;
*tokutxn = result;
status.begin++;
if (garbage_collection_debug) {
......@@ -549,3 +568,106 @@ verify_snapshot_system(TOKULOGGER logger) {
}
}
// routines for checking if rollback errors should be ignored because a hot index create was aborted
// 2954
// returns
// 0 on success
// ENOMEM if can't alloc memory
// EINVAL if txn = NULL
// -1 on other errors
int toku_txn_ignore_init(TOKUTXN txn)
{
if ( !txn ) return EINVAL;
TXN_IGNORE txni = toku_malloc(sizeof(TXN_IGNORE_S));
if ( txni == NULL ) return ENOMEM;
txni->fns_allocated = 0;
txni->filenums.num = 0;
txni->filenums.filenums = NULL;
txn->ignore_errors = txni;
return 0;
}
void toku_txn_ignore_free(TOKUTXN txn)
{
TXN_IGNORE txni = txn->ignore_errors;
toku_free(txni->filenums.filenums);
toku_free(txni);
txni = NULL;
}
// returns
// 0 on success
// ENOMEM if can't alloc memory
// EINVAL if txn = NULL
// -1 on other errors
int toku_txn_ignore_add(TOKUTXN txn, FILENUM filenum)
{
if ( !txn ) return EINVAL;
// check for dups
if ( toku_txn_ignore_contains(txn, filenum) == 0 ) return 0;
// alloc more space if needed
const int N = 2;
TXN_IGNORE txni = txn->ignore_errors;
if ( txni->filenums.num == txni->fns_allocated ) {
if ( txni->fns_allocated == 0 ) {
CALLOC_N(N, txni->filenums.filenums);
if ( txni->filenums.filenums == NULL ) return ENOMEM;
txni->fns_allocated = N;
}
else {
XREALLOC_N(txni->fns_allocated * N, txni->filenums.filenums);
if ( txni->filenums.filenums == NULL ) return ENOMEM;
txni->fns_allocated = txni->fns_allocated * N;
}
}
txni->filenums.num++;
txni->filenums.filenums[txni->filenums.num - 1].fileid = filenum.fileid;
return 0;
}
// returns
// 0 on success
// ENOENT if not found
// EINVAL if txn = NULL
// -1 on other errors
int toku_txn_ignore_remove(TOKUTXN txn, FILENUM filenum)
{
if ( !txn ) return EINVAL;
TXN_IGNORE txni = txn->ignore_errors;
int found_fn = 0;
if ( txni->filenums.num == 0 ) return ENOENT;
for(uint32_t i=0; i<txni->filenums.num; i++) {
if ( !found_fn ) {
if ( txni->filenums.filenums[i].fileid == filenum.fileid ) {
found_fn = 1;
}
}
else { // remove bubble in array
txni->filenums.filenums[i-1].fileid = txni->filenums.filenums[i].fileid;
}
}
if ( !found_fn ) return ENOENT;
txni->filenums.num--;
return 0;
}
// returns
// 0 on success
// ENOENT if not found
// EINVAL if txn = NULL
// -1 on other errors
int toku_txn_ignore_contains(TOKUTXN txn, FILENUM filenum)
{
if ( !txn ) return EINVAL;
TXN_IGNORE txni = txn->ignore_errors;
for(uint32_t i=0; i<txni->filenums.num; i++) {
if ( txni->filenums.filenums[i].fileid == filenum.fileid ) {
return 0;
}
}
return ENOENT;
}
newbrt/txn.h
View file @ f31a90e9
......@@ -10,12 +10,16 @@ extern "C" {
#endif
int toku_txn_begin_txn (
DB_TXN *container_db_txn,
TOKUTXN parent_tokutxn,
TOKUTXN *tokutxn,
TOKULOGGER logger,
TXN_SNAPSHOT_TYPE snapshot_type
);
DB_TXN * toku_txn_get_container_db_txn (TOKUTXN tokutxn);
// toku_txn_begin_with_xid is called from recovery and has no containing DB_TXN
int toku_txn_begin_with_xid (
TOKUTXN parent_tokutxn,
TOKUTXN *tokutxn,
......@@ -23,6 +27,7 @@ int toku_txn_begin_with_xid (
TXNID xid,
TXN_SNAPSHOT_TYPE snapshot_type
);
int toku_txn_load_txninfo (TOKUTXN txn, TXNINFO info);
int toku_txn_commit_txn (TOKUTXN txn, int nosync, YIELDF yield, void *yieldv,
......@@ -69,6 +74,18 @@ typedef struct {
TXNID xid2;
} XID_PAIR_S, *XID_PAIR;
// 2954
typedef struct tokutxn_filenum_ignore_errors {
uint32_t fns_allocated;
FILENUMS filenums;
} TXN_IGNORE_S, *TXN_IGNORE;
int toku_txn_ignore_init(TOKUTXN txn);
void toku_txn_ignore_free(TOKUTXN txn);
int toku_txn_ignore_add(TOKUTXN txn, FILENUM filenum);
int toku_txn_ignore_remove(TOKUTXN txn, FILENUM filenum);
int toku_txn_ignore_contains(TOKUTXN txn, FILENUM filenum);
#if defined(__cplusplus) || defined(__cilkplusplus)
};
#endif
......
newbrt/ule-internal.h 0 → 100644
View file @ f31a90e9
/* -*- mode: C; c-basic-offset: 4 -*- */
/* Purpose of this file is to provide the test programs with internal
* ule mechanisms that do not belong in the public interface.
*/
#ifndef TOKU_ULE_INTERNAL_H
#define TOKU_ULE_INTERNAL_H
#ident "$Id: ule.h 24600 2010-10-15 15:22:18Z dwells $"
#ident "Copyright (c) 2007-2010 Tokutek Inc. All rights reserved."
#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
#if defined(__cplusplus) || defined(__cilkplusplus)
extern "C" {
#endif
//1 does much slower debugging
#define ULE_DEBUG 0
/////////////////////////////////////////////////////////////////////////////////
// Following data structures are the unpacked format of a leafentry.
// * ule is the unpacked leaf entry, that contains an array of unpacked
// transaction records
// * uxr is the unpacked transaction record
//
//Types of transaction records.
enum {XR_INSERT = 1,
XR_DELETE = 2,
XR_PLACEHOLDER = 3};
typedef struct uxr { // unpacked transaction record
uint8_t type; // delete/insert/placeholder
uint32_t vallen; // number of bytes in value
void * valp; // pointer to value (Where is value really stored?)
TXNID xid; // transaction id
// Note: when packing ule into a new leafentry, will need
// to copy actual data from valp to new leafentry
} UXR_S, *UXR;
// Unpacked Leaf Entry is of fixed size because it's just on the
// stack and we care about ease of access more than the memory footprint.
typedef struct ule { // unpacked leaf entry
uint32_t num_puxrs; // how many of uxrs[] are provisional
uint32_t num_cuxrs; // how many of uxrs[] are committed
uint32_t keylen;
void * keyp;
UXR_S uxrs_static[MAX_TRANSACTION_RECORDS*2]; // uxrs[0] is oldest committed (txn commit time, not txn start time), uxrs[num_cuxrs] is outermost provisional value (if any exist/num_puxrs > 0)
UXR uxrs; //If num_cuxrs < MAX_TRANSACTION_RECORDS then &uxrs_static[0].
//Otherwise we use a dynamically allocated array of size num_cuxrs + 1 + MAX_TRANSATION_RECORD.
} ULE_S, *ULE;
void test_msg_modify_ule(ULE ule, BRT_MSG msg);
//////////////////////////////////////////////////////////////////////////////////////
//Functions exported for test purposes only (used internally for non-test purposes).
void le_unpack(ULE ule, LEAFENTRY le);
int le_pack(ULE ule, // data to be packed into new leafentry
size_t *new_leafentry_memorysize,
size_t *new_leafentry_disksize,
LEAFENTRY * const new_leafentry_p, // this is what this function creates
OMT omt,
struct mempool *mp,
void **maybe_free);
size_t le_memsize_from_ule (ULE ule);
void ule_cleanup(ULE ule);
#if defined(__cplusplus) || defined(__cilkplusplus)
}
#endif
#endif // TOKU_ULE_H
newbrt/ule.c
View file @ f31a90e9
......@@ -29,6 +29,31 @@
#include "xids.h"
#include "brt_msg.h"
#include "ule.h"
#include "ule-internal.h"
#define ULE_DEBUG 0
///////////////////////////////////////////////////////////////////////////////////
// Accessor functions used by outside world (e.g. indexer)
//
ULEHANDLE
toku_ule_create(void * le_p) {
ULE ule_p = toku_malloc(sizeof(ULE_S));
le_unpack(ule_p, le_p);
return (ULEHANDLE) ule_p;
}
void toku_ule_free(ULEHANDLE ule_p) {
ule_cleanup((ULE) ule_p);
toku_free(ule_p);
}
///////////////////////////////////////////////////////////////////////////////////
//
......@@ -84,9 +109,6 @@ static void ule_optimize(ULE ule, XIDS xids);
static inline BOOL uxr_type_is_insert(u_int8_t type);
static inline BOOL uxr_type_is_delete(u_int8_t type);
static inline BOOL uxr_type_is_placeholder(u_int8_t type);
static inline BOOL uxr_is_insert(UXR uxr);
static inline BOOL uxr_is_delete(UXR uxr);
static inline BOOL uxr_is_placeholder(UXR uxr);
static inline size_t uxr_pack_txnid(UXR uxr, uint8_t *p);
static inline size_t uxr_pack_type_and_length(UXR uxr, uint8_t *p);
static inline size_t uxr_pack_length_and_bit(UXR uxr, uint8_t *p);
......@@ -1650,6 +1672,49 @@ ule_add_placeholders(ULE ule, XIDS xids) {
}
}
int
ule_num_uxrs(ULE ule) {
return ule->num_cuxrs + ule->num_puxrs;
}
UXR
ule_get_uxr(ULE ule, int ith) {
invariant(0 <= ith && ith < ule_num_uxrs(ule));
return &ule->uxrs[ith];
}
int
ule_get_num_committed(ULE ule) {
return ule->num_cuxrs;
}
int
ule_get_num_provisional(ULE ule) {
return ule->num_puxrs;
}
int
ule_is_committed(ULE ule, int ith) {
invariant(0 <= ith && ith < ule_num_uxrs(ule));
return ith < (int) ule->num_cuxrs;
}
int
ule_is_provisional(ULE ule, int ith) {
invariant(0 <= ith && ith < ule_num_uxrs(ule));
return ith >= (int) ule->num_cuxrs;
}
void *
ule_get_key(ULE ule) {
return ule->keyp;
}
uint32_t
ule_get_keylen(ULE ule) {
return ule->keylen;
}
/////////////////////////////////////////////////////////////////////////////////
// This layer of abstraction (uxr_xxx) understands uxr and nothing else.
//
......@@ -1660,7 +1725,7 @@ uxr_type_is_insert(u_int8_t type) {
return rval;
}
static inline BOOL
BOOL
uxr_is_insert(UXR uxr) {
return uxr_type_is_insert(uxr->type);
}
......@@ -1671,7 +1736,7 @@ uxr_type_is_delete(u_int8_t type) {
return rval;
}
static inline BOOL
BOOL
uxr_is_delete(UXR uxr) {
return uxr_type_is_delete(uxr->type);
}
......@@ -1682,11 +1747,27 @@ uxr_type_is_placeholder(u_int8_t type) {
return rval;
}
static inline BOOL
BOOL
uxr_is_placeholder(UXR uxr) {
return uxr_type_is_placeholder(uxr->type);
}
void *
uxr_get_val(UXR uxr) {
return uxr->valp;
}
uint32_t
uxr_get_vallen(UXR uxr) {
return uxr->vallen;
}
TXNID
uxr_get_txnid(UXR uxr) {
return uxr->xid;
}
static int
le_iterate_get_accepted_index(TXNID *xids, uint32_t *index, uint32_t num_xids, LE_ITERATE_CALLBACK f, TOKUTXN context) {
uint32_t i;
......
newbrt/ule.h
View file @ f31a90e9
......@@ -16,43 +16,32 @@
extern "C" {
#endif
// opaque handles used by outside world (i.e. indexer)
typedef struct ule *ULEHANDLE;
typedef struct uxr *UXRHANDLE;
ULEHANDLE toku_ule_create(void * le_p);
void toku_ule_free(ULEHANDLE ule_p);
int ule_num_uxrs(ULEHANDLE ule);
int ule_get_num_committed(ULEHANDLE ule);
int ule_get_num_provisional(ULEHANDLE ule);
UXRHANDLE ule_get_uxr(ULEHANDLE ule, int ith);
int ule_is_committed(ULEHANDLE ule, int ith);
int ule_is_provisional(ULEHANDLE ule, int ith);
void *ule_get_key(ULEHANDLE ule);
uint32_t ule_get_keylen(ULEHANDLE ule);
BOOL uxr_is_insert(UXRHANDLE uxr);
BOOL uxr_is_delete(UXRHANDLE uxr);
BOOL uxr_is_placeholder(UXRHANDLE uxr);
void *uxr_get_val(UXRHANDLE uxr);
uint32_t uxr_get_vallen(UXRHANDLE uxr);
TXNID uxr_get_txnid(UXRHANDLE uxr);
//1 does much slower debugging
#define ULE_DEBUG 0
/////////////////////////////////////////////////////////////////////////////////
// Following data structures are the unpacked format of a leafentry.
// * ule is the unpacked leaf entry, that contains an array of unpacked
// transaction records
// * uxr is the unpacked transaction record
//
//Types of transaction records.
enum {XR_INSERT = 1,
XR_DELETE = 2,
XR_PLACEHOLDER = 3};
typedef struct { // unpacked transaction record
uint8_t type; // delete/insert/placeholder
uint32_t vallen; // number of bytes in value
void * valp; // pointer to value (Where is value really stored?)
TXNID xid; // transaction id
// Note: when packing ule into a new leafentry, will need
// to copy actual data from valp to new leafentry
} UXR_S, *UXR;
// Unpacked Leaf Entry is of fixed size because it's just on the
// stack and we care about ease of access more than the memory footprint.
typedef struct { // unpacked leaf entry
uint32_t num_puxrs; // how many of uxrs[] are provisional
uint32_t num_cuxrs; // how many of uxrs[] are committed
uint32_t keylen;
void * keyp;
UXR_S uxrs_static[MAX_TRANSACTION_RECORDS*2]; // uxrs[0] is oldest committed (txn commit time, not txn start time), uxrs[num_cuxrs] is outermost provisional value (if any exist/num_puxrs > 0)
UXR uxrs; //If num_cuxrs < MAX_TRANSACTION_RECORDS then uxrs = &uxrs_static[0].
//Otherwise we use a dynamically allocated array of size num_cuxrs + 1 + MAX_TRANSATION_RECORD.
//Felt that MAX_TRANSACTION_RECORD was good upper bound for number of committed UXRs we would be willing to allocate off stack.
} ULE_S, *ULE;
#define GARBAGE_COLLECTION_DEBUG 0
int apply_msg_to_leafentry(BRT_MSG msg,
LEAFENTRY old_leafentry, // NULL if there was no stored data.
......@@ -65,28 +54,11 @@ int apply_msg_to_leafentry(BRT_MSG msg,
OMT snapshot_xids,
OMT live_list_reverse);
void
test_msg_modify_ule(ULE ule, BRT_MSG msg);
//////////////////////////////////////////////////////////////////////////////////////
//Functions exported for test purposes only (used internally for non-test purposes).
void le_unpack(ULE ule, LEAFENTRY le);
int le_pack(ULE ule, // data to be packed into new leafentry
size_t *new_leafentry_memorysize,
size_t *new_leafentry_disksize,
LEAFENTRY * const new_leafentry_p, // this is what this function creates
OMT omt,
struct mempool *mp,
void **maybe_free);
size_t le_memsize_from_ule (ULE ule);
void ule_cleanup(ULE ule);
TXNID toku_get_youngest_live_list_txnid_for(TXNID xc, OMT live_list_reverse);
#if defined(__cplusplus) || defined(__cilkplusplus)
};
}
#endif
#endif // TOKU_ULE_H
......
newbrt/xids.c
View file @ f31a90e9
......@@ -158,6 +158,14 @@ xids_get_innermost_xid(XIDS xids) {
return rval;
}
TXNID
xids_get_outermost_xid(XIDS xids) {
TXNID rval = TXNID_NONE;
if (xids_get_num_xids(xids))
rval = xids_get_xid(xids, 0);
return rval;
}
void
xids_cpy(XIDS target, XIDS source) {
size_t size = xids_get_size(source);
......
newbrt/xids.h
View file @ f31a90e9
......@@ -46,6 +46,7 @@ u_int8_t xids_find_index_of_xid(XIDS xids, TXNID target_xid);
u_int8_t xids_get_num_xids(XIDS xids);
TXNID xids_get_innermost_xid(XIDS xids);
TXNID xids_get_outermost_xid(XIDS xids);
// return size in bytes
u_int32_t xids_get_size(XIDS xids);
......
src/Makefile
View file @ f31a90e9
......@@ -36,6 +36,8 @@ OBJS_RAW = \
errors \
dlmalloc \
loader \
indexer \
indexer-undo-do \
elocks \
#\end
#OBJS automatically defined.
......
src/elocks.c
View file @ f31a90e9
......@@ -63,9 +63,9 @@ ydbtime_destr(void *a) {
struct ydbtime *ydbtime = (struct ydbtime *) a;
int r;
// printf("%s %p\n", __FUNCTION__, a);
r = toku_pthread_mutex_lock(&ydb_big_lock.lock); assert(r == 0);
r = toku_pthread_mutex_lock(&ydb_big_lock.lock); resource_assert_zero(r);
toku_list_remove(&ydbtime->all_ydbtimes);
r = toku_pthread_mutex_unlock(&ydb_big_lock.lock); assert(r == 0);
r = toku_pthread_mutex_unlock(&ydb_big_lock.lock); resource_assert_zero(r);
toku_free(ydbtime);
}
......@@ -74,7 +74,7 @@ static uint64_t
get_tnow(void) {
struct timeval tv;
int r = gettimeofday(&tv, NULL);
assert(r == 0);
resource_assert_zero(r);
return tv.tv_sec * 1000000ULL + tv.tv_usec;
}
#endif
......@@ -83,7 +83,7 @@ static void
init_status(void) {
uint64_t cpuhz = 0;
#if YDB_LOCK_MISS_TIME
int r = toku_os_get_processor_frequency(&cpuhz); assert(r == 0);
int r = toku_os_get_processor_frequency(&cpuhz); resource_assert_zero(r);
#endif
status.ydb_lock_ctr = 0;
status.max_possible_sleep = MAX_SLEEP;
......@@ -108,10 +108,10 @@ toku_ydb_lock_get_status(SCHEDULE_STATUS statp) {
int
toku_ydb_lock_init(void) {
int r;
r = toku_pthread_mutex_init(&ydb_big_lock.lock, NULL); assert(r == 0);
r = toku_pthread_mutex_init(&ydb_big_lock.lock, NULL); resource_assert_zero(r);
#if YDB_LOCK_MISS_TIME
ydb_big_lock.waiters = 0;
r = toku_pthread_key_create(&ydb_big_lock.time_key, ydbtime_destr); assert(r == 0);
r = toku_pthread_key_create(&ydb_big_lock.time_key, ydbtime_destr); resource_assert_zero(r);
toku_list_init(&ydb_big_lock.all_ydbtimes);
#endif
init_status();
......@@ -122,21 +122,21 @@ int
toku_ydb_lock_destroy(void) {
int r;
#if YDB_LOCK_MISS_TIME
r = toku_pthread_key_delete(ydb_big_lock.time_key); assert(r == 0);
r = toku_pthread_key_delete(ydb_big_lock.time_key); resource_assert_zero(r);
while (!toku_list_empty(&ydb_big_lock.all_ydbtimes)) {
struct toku_list *list = toku_list_pop(&ydb_big_lock.all_ydbtimes);
struct ydbtime *ydbtime = toku_list_struct(list, struct ydbtime, all_ydbtimes);
ydbtime_destr(ydbtime);
}
#endif
r = toku_pthread_mutex_destroy(&ydb_big_lock.lock); assert(r == 0);
r = toku_pthread_mutex_destroy(&ydb_big_lock.lock); resource_assert_zero(r);
return r;
}
void
toku_ydb_lock(void) {
#if !YDB_LOCK_MISS_TIME
int r = toku_pthread_mutex_lock(&ydb_big_lock.lock); assert(r == 0);
int r = toku_pthread_mutex_lock(&ydb_big_lock.lock); resource_assert_zero(r);
#endif
#if YDB_LOCK_MISS_TIME
......@@ -147,10 +147,10 @@ toku_ydb_lock(void) {
if (!ydbtime) { // allocate the per thread timestamp if not yet allocated
new_ydbtime = TRUE;
ydbtime = toku_malloc(sizeof (struct ydbtime));
assert(ydbtime);
resource_assert(ydbtime);
memset(ydbtime, 0, sizeof (struct ydbtime));
r = toku_pthread_setspecific(ydb_big_lock.time_key, ydbtime);
assert(r == 0);
resource_assert_zero(r);
(void) toku_sync_fetch_and_increment_uint64(&status.total_clients);
}
if (ydbtime->tacquire) { // delay the thread if the lock acquire time is set and is less than the current time
......@@ -177,7 +177,7 @@ toku_ydb_lock(void) {
(void) toku_sync_fetch_and_increment_int32(&ydb_big_lock.waiters);
(void) toku_sync_fetch_and_increment_uint64(&status.total_waiters);
r = toku_pthread_mutex_lock(&ydb_big_lock.lock);
assert(r == 0);
resource_assert_zero(r);
(void) toku_sync_fetch_and_decrement_int32(&ydb_big_lock.waiters);
}
status.max_requested_sleep = u64max(status.max_requested_sleep, requested_sleep);
......@@ -188,25 +188,27 @@ toku_ydb_lock(void) {
#endif
status.ydb_lock_ctr++;
assert((status.ydb_lock_ctr & 0x01) == 1);
invariant((status.ydb_lock_ctr & 0x01) == 1);
}
void
toku_ydb_unlock(void) {
// low_priority causes the routine to sleep if there are others waiting for the ydb_lock
static void
ydb_unlock_internal(unsigned long useconds) {
status.ydb_lock_ctr++;
assert((status.ydb_lock_ctr & 0x01) == 0);
invariant((status.ydb_lock_ctr & 0x01) == 0);
#if !YDB_LOCK_MISS_TIME
int r = toku_pthread_mutex_unlock(&ydb_big_lock.lock); assert(r == 0);
int r = toku_pthread_mutex_unlock(&ydb_big_lock.lock); resource_assert_zero(r);
#endif
#if YDB_LOCK_MISS_TIME
struct ydbtime *ydbtime = toku_pthread_getspecific(ydb_big_lock.time_key);
assert(ydbtime);
resource_assert(ydbtime);
int r;
uint64_t theld;
int waiters = ydb_big_lock.waiters; // get the number of lock waiters (used to compute the lock acquisition time)
int do_sleep = 0;
if (waiters == 0) {
theld = 0;
} else {
......@@ -219,7 +221,7 @@ toku_ydb_unlock(void) {
disk_count += fsync_count - ydb_big_lock.start_fsync_count; // time waiting for fsyncs to complete
disk_time += fsync_time - ydb_big_lock.start_fsync_time;
if (disk_count == 0) {
theld = 0;
theld = 0; do_sleep = 1;
} else {
theld = disk_time ? disk_time : disk_count * 20000ULL; // if we decide not to compile in miss_time, then backoff to 20 milliseconds per cache miss
......@@ -235,7 +237,10 @@ toku_ydb_unlock(void) {
}
}
r = toku_pthread_mutex_unlock(&ydb_big_lock.lock); assert(r == 0);
r = toku_pthread_mutex_unlock(&ydb_big_lock.lock); resource_assert_zero(r);
if (do_sleep && useconds > 0)
usleep(useconds);
// we use a lower bound of 100 microseconds on the sleep time to avoid system call overhead for short sleeps
if (waiters == 0 || theld <= 100ULL)
......@@ -246,3 +251,12 @@ toku_ydb_unlock(void) {
}
void
toku_ydb_unlock(void) {
ydb_unlock_internal(0);
}
void
toku_ydb_unlock_and_yield(unsigned long useconds) {
ydb_unlock_internal(useconds);
}
src/export.map
View file @ f31a90e9
......@@ -37,6 +37,7 @@
toku_free;
toku_malloc;
toku_calloc;
toku_xmemdup;
toku_xrealloc;
toku_os_get_file_size;
......@@ -77,6 +78,7 @@
toku_test_db_redirect_dictionary;
toku_test_get_latest_lsn;
toku_test_get_checkpointing_user_data_status;
toku_indexer_set_test_only_flags;
local: *;
};
......
src/indexer-internal.h 0 → 100644
View file @ f31a90e9
/* -*- mode: C; c-basic-offset: 4 -*- */
/*
* Copyright (c) 2010 Tokutek Inc. All rights reserved."
* The technology is licensed by the Massachusetts Institute of Technology,
* Rutgers State University of New Jersey, and the Research Foundation of
* State University of New York at Stony Brook under United States of America
* Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
*/
#ifndef TOKU_INDEXER_INTERNAL_H
#define TOKU_INDEXER_INTERNAL_H
struct __toku_indexer_internal {
DB_ENV *env;
DB_TXN *txn;
DB *src_db;
int N;
DB **dest_dbs; /* [N] */
uint32_t *db_flags;
uint32_t indexer_flags;
void (*error_callback)(DB *db, int i, int err, DBT *key, DBT *val, void *error_extra);
void *error_extra;
int (*poll_func)(void *poll_extra, float progress);
void *poll_extra;
uint64_t estimated_rows; // current estimate of table size
uint64_t loop_mod; // how often to call poll_func
LE_CURSOR lec;
void *generate_extra;
FILENUM *fnums; /* [N] */
FILENUMS filenums;
// test functions
int (*undo_do)(DB_INDEXER *indexer, DB *hotdb, ULEHANDLE ule);
int (*test_is_xid_live)(DB_INDEXER *indexer, TXNID xid);
int (*test_maybe_lock_provisional_key)(DB_INDEXER *indexer, TXNID xid, DB *hotdb, DBT *key);
int (*test_delete_provisional)(DB_INDEXER *indexer, DB *hotdb, DBT *hotkey, XIDS xids);
int (*test_delete_committed)(DB_INDEXER *indexer, DB *hotdb, DBT *hotkey, XIDS xids);
int (*test_insert_provisional)(DB_INDEXER *indexer, DB *hotdb, DBT *hotkey, DBT *hotval, XIDS xids);
int (*test_insert_committed)(DB_INDEXER *indexer, DB *hotdb, DBT *hotkey, DBT *hotval, XIDS xids);
int (*test_commit_any)(DB_INDEXER *indexer, DB *db, DBT *key, XIDS xids);
// test flags
int test_only_flags;
};
int indexer_undo_do(DB_INDEXER *indexer, DB *hotdb, ULEHANDLE ule);
#endif
src/indexer-undo-do.c 0 → 100644
View file @ f31a90e9
/* -*- mode: C; c-basic-offset: 4 -*- */
/*
* Copyright (c) 2010 Tokutek Inc. All rights reserved."
* The technology is licensed by the Massachusetts Institute of Technology,
* Rutgers State University of New Jersey, and the Research Foundation of
* State University of New York at Stony Brook under United States of America
* Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
*/
#include <stdio.h>
#include <string.h>
#include <toku_portability.h>
#include "toku_assert.h"
#include "ydb-internal.h"
#include "le-cursor.h"
#include "indexer.h"
#include "brt-internal.h"
#include "toku_atomic.h"
#include "tokuconst.h"
#include "brt.h"
#include "mempool.h"
#include "leafentry.h"
#include "ule.h"
#include "xids.h"
#include "indexer-internal.h"
// internal functions
static int indexer_setup_xids(DB_INDEXER *indexer, ULEHANDLE ule, int i, TXNID xid, XIDS *xids_result);
static int indexer_find_prev(DB_INDEXER *indexer, ULEHANDLE ule, int i);
static int indexer_generate_hot_key_val(DB_INDEXER *indexer, DB *hotdb, ULEHANDLE ule, UXRHANDLE uxr, DBT *hotkey, DBT *hotval);
static int indexer_brt_delete_provisional(DB_INDEXER *indexer, DB *hotdb, DBT *hotkey, XIDS xids);
static int indexer_brt_delete_committed(DB_INDEXER *indexer, DB *hotdb, DBT *hotkey, XIDS xids);
static int indexer_brt_insert_provisional(DB_INDEXER *indexer, DB *hotdb, DBT *hotkey, DBT *hotval, XIDS xids);
static int indexer_brt_insert_committed(DB_INDEXER *indexer, DB *hotdb, DBT *hotkey, DBT *hotval, XIDS xids);
static int indexer_brt_commit(DB_INDEXER *indexer, DB *hotdb, DBT *hotkey, XIDS xids);
static int indexer_maybe_lock_provisional_key(DB_INDEXER *indexer, ULEHANDLE ule, int i, DB *hotdb, DBT *key, XIDS xids);
static int indexer_is_xid_live(DB_INDEXER *indexer, TXNID xid);
int
indexer_undo_do(DB_INDEXER *indexer, DB *hotdb, ULEHANDLE ule) {
int result = 0;
// init the xids to the root xid
XIDS xids = xids_get_root_xids();
// DBTs for the key and val
DBT hotkey; toku_init_dbt(&hotkey); hotkey.flags = DB_DBT_REALLOC;
DBT hotval; toku_init_dbt(&hotval); hotval.flags = DB_DBT_REALLOC;
// DBT for the previous inserted key
DBT previous_insert_key; toku_init_dbt(&previous_insert_key); previous_insert_key.flags = DB_DBT_REALLOC;
// DBTs for the commit keys
DBT delete_key; toku_init_dbt(&delete_key); delete_key.flags = DB_DBT_REALLOC;
DBT insert_key; toku_init_dbt(&insert_key); insert_key.flags = DB_DBT_REALLOC;
// scan the transaction stack from bottom to top
int num_uxrs = ule_get_num_committed(ule) + ule_get_num_provisional(ule);
for (int i = 0; i < num_uxrs; i++) {
// get the ith transaction record
UXRHANDLE uxr = ule_get_uxr(ule, i);
invariant(uxr);
// setup up the xids
result = indexer_setup_xids(indexer, ule, i, uxr_get_txnid(uxr), &xids);
if (result != 0)
break;
// skip placeholders
if (uxr_is_placeholder(uxr)) {
invariant(ule_is_provisional(ule, i));
continue;
}
// init delete and insert commits
BOOL do_delete_commit = FALSE, do_insert_commit = FALSE;
// undo
int previ = indexer_find_prev(indexer, ule, i);
if (previ >= 0) {
UXRHANDLE prevuxr = ule_get_uxr(ule, previ);
if (uxr_is_delete(prevuxr)) {
; // do nothing
} else if (uxr_is_insert(prevuxr)) {
// generate the hot key and val
result = indexer_generate_hot_key_val(indexer, hotdb, ule, prevuxr, &hotkey, &hotval);
if (result == 0) {
// send the delete message
if (ule_is_committed(ule, i) || !indexer_is_xid_live(indexer, xids_get_outermost_xid(xids))) {
result = indexer_brt_delete_committed(indexer, hotdb, &hotkey, xids);
if (result == 0) {
do_delete_commit = TRUE;
result = toku_dbt_set(hotkey.size, hotkey.data, &delete_key, NULL);
}
} else {
result = indexer_brt_delete_provisional(indexer, hotdb, &hotkey, xids);
}
if (result == 0)
result = indexer_maybe_lock_provisional_key(indexer, ule, i, hotdb, &hotkey, xids);
}
} else
invariant(0);
}
if (result != 0)
break;
// do
if (uxr_is_delete(uxr)) {
result = indexer_maybe_lock_provisional_key(indexer, ule, i, hotdb, &previous_insert_key, xids);
} else if (uxr_is_insert(uxr)) {
// generate the hot key and val
result = indexer_generate_hot_key_val(indexer, hotdb, ule, uxr, &hotkey, &hotval);
if (result == 0) {
// send the insert message
if (ule_is_committed(ule, i) || !indexer_is_xid_live(indexer, xids_get_outermost_xid(xids))) {
result = indexer_brt_insert_committed(indexer, hotdb, &hotkey, &hotval, xids);
if (result == 0) {
do_insert_commit = TRUE;
result = toku_dbt_set(hotkey.size, hotkey.data, &insert_key, NULL);
}
} else {
result = indexer_brt_insert_provisional(indexer, hotdb, &hotkey, &hotval, xids);
}
if (result == 0) {
result = indexer_maybe_lock_provisional_key(indexer, ule, i, hotdb, &hotkey, xids);
if (result == 0)
result = toku_dbt_set(hotkey.size, hotkey.data, &previous_insert_key, NULL);
}
}
} else
invariant(0);
// send commit messages if needed
if (result == 0 && do_delete_commit)
result = indexer_brt_commit(indexer, hotdb, &delete_key, xids);
if (result == 0 && do_insert_commit)
result = indexer_brt_commit(indexer, hotdb, &insert_key, xids);
if (result != 0)
break;
}
// cleanup
toku_destroy_dbt(&hotkey);
toku_destroy_dbt(&hotval);
toku_destroy_dbt(&delete_key);
toku_destroy_dbt(&insert_key);
toku_destroy_dbt(&previous_insert_key);
xids_destroy(&xids);
if ( indexer->i->test_only_flags == INDEXER_TEST_ONLY_ERROR_CALLBACK )
result = EINVAL;
return result;
}
static int
indexer_setup_xids(DB_INDEXER *indexer, ULEHANDLE ule, int i, TXNID this_xid, XIDS *xids_result) {
indexer = indexer;
int result = 0;
XIDS old_xids = *xids_result;
XIDS new_xids;
if (i <= ule_get_num_committed(ule) || xids_get_num_xids(old_xids) == 0) {
// setup xids = [ root xid, this xid ]
new_xids = xids_get_root_xids();
if (this_xid > 0) {
XIDS child_xids;
result = xids_create_child(new_xids, &child_xids, this_xid);
xids_destroy(&new_xids);
if (result == 0)
new_xids = child_xids;
}
} else {
// append this_xid to xids
result = xids_create_child(old_xids, &new_xids, this_xid);
}
if (result == 0) {
xids_destroy(&old_xids);
*xids_result = new_xids;
}
return result;
}
static int
indexer_generate_hot_key_val(DB_INDEXER *indexer, DB *hotdb, ULEHANDLE ule, UXRHANDLE uxr, DBT *hotkey, DBT *hotval) {
int result = 0;
// setup the source key
DBT srckey;
toku_fill_dbt(&srckey, ule_get_key(ule), ule_get_keylen(ule));
// setup the source val
DBT srcval;
toku_fill_dbt(&srcval, uxr_get_val(uxr), uxr_get_vallen(uxr));
// generate the secondary row
DB_ENV *env = indexer->i->env;
result = env->i->generate_row_for_put(hotdb, indexer->i->src_db, hotkey, hotval, &srckey, &srcval, indexer->i->generate_extra);
toku_destroy_dbt(&srckey);
toku_destroy_dbt(&srcval);
return result;
}
// looks up the TOKUTXN by TXNID. if it does not exist then the transaction is committed.
// returns TRUE if the xid is committed. otherwise returns FALSE.
static int
indexer_is_xid_live(DB_INDEXER *indexer, TXNID xid) {
int result = 0;
// TEST
if (indexer->i->test_is_xid_live) {
result = indexer->i->test_is_xid_live(indexer, xid);
} else {
DB_ENV *env = indexer->i->env;
TOKUTXN txn = NULL;
int r = toku_txnid2txn(env->i->logger, xid, &txn);
invariant(r == 0);
result = txn != NULL;
}
return result;
}
// Take a write lock on the given key for the outermost xid in the xids list.
static int
indexer_maybe_lock_provisional_key(DB_INDEXER *indexer, ULEHANDLE ule, int i, DB *hotdb, DBT *key, XIDS xids) {
int result = 0;
if (ule_is_provisional(ule, i)) {
TXNID outermost_live_xid = xids_get_outermost_xid(xids);
// TEST
if (indexer->i->test_maybe_lock_provisional_key) {
result = indexer->i->test_maybe_lock_provisional_key(indexer, outermost_live_xid, hotdb, key);
} else {
DB_ENV *env = indexer->i->env;
TOKUTXN txn = NULL;
result = toku_txnid2txn(env->i->logger, outermost_live_xid, &txn);
invariant(result == 0);
if (txn != NULL) {
result = toku_grab_write_lock(hotdb, key, txn);
lazy_assert(result == 0);
}
}
}
return result;
}
static int
indexer_find_prev_committed(ULEHANDLE ule, int i) {
invariant(i < ule_num_uxrs(ule));
int previ = i - 1;
return previ;
}
static int
indexer_find_prev_provisional(ULEHANDLE ule, int i) {
invariant(i < ule_num_uxrs(ule));
int previ = i - 1;
while (previ >= 0) {
UXRHANDLE uxr = ule_get_uxr(ule, previ);
if (!uxr_is_placeholder(uxr))
break;
previ -= 1;
}
return previ;
}
// find the index of the previous transaction record before the ith position in the stack
static int
indexer_find_prev(DB_INDEXER *UU(indexer), ULEHANDLE ule, int i) {
int result;
if (ule_is_committed(ule, i))
result = indexer_find_prev_committed(ule, i);
else
result = indexer_find_prev_provisional(ule, i);
return result;
}
static TOKUTXN
indexer_get_innermost_live_txn(DB_INDEXER *indexer, XIDS xids) {
DB_ENV *env = indexer->i->env;
TOKUTXN txn = NULL;
for (uint8_t num_xids = xids_get_num_xids(xids); txn == NULL && num_xids > 0; num_xids--) {
TXNID xid = xids_get_xid(xids, num_xids-1);
int result = toku_txnid2txn(env->i->logger, xid, &txn);
invariant(result == 0);
}
return txn;
}
// inject "delete" message into brt with logging in recovery and rollback logs,
// and making assocation between txn and brt
static int
indexer_brt_delete_provisional(DB_INDEXER *indexer, DB *hotdb, DBT *hotkey, XIDS xids) {
int result = 0;
// TEST
if (indexer->i->test_delete_provisional) {
result = indexer->i->test_delete_provisional(indexer, hotdb, hotkey, xids);
} else {
result = toku_ydb_check_avail_fs_space(indexer->i->env);
if (result == 0) {
TOKUTXN txn = indexer_get_innermost_live_txn(indexer, xids);
invariant(txn != NULL);
result = toku_brt_maybe_delete (hotdb->i->brt, hotkey, txn, FALSE, ZERO_LSN, TRUE);
}
}
return result;
}
// send a delete message into the tree without rollback or recovery logging
static int
indexer_brt_delete_committed(DB_INDEXER *indexer, DB *hotdb, DBT *hotkey, XIDS xids) {
int result = 0;
// TEST
if (indexer->i->test_delete_committed) {
result = indexer->i->test_delete_committed(indexer, hotdb, hotkey, xids);
} else {
result = toku_ydb_check_avail_fs_space(indexer->i->env);
if (result == 0)
result = toku_brt_send_delete(db_struct_i(hotdb)->brt, hotkey, xids);
}
return result;
}
// inject "insert" message into brt with logging in recovery and rollback logs,
// and making assocation between txn and brt
static int
indexer_brt_insert_provisional(DB_INDEXER *indexer, DB *hotdb, DBT *hotkey, DBT *hotval, XIDS xids) {
int result = 0;
// TEST
if (indexer->i->test_insert_provisional) {
result = indexer->i->test_insert_provisional(indexer, hotdb, hotkey, hotval, xids);
} else {
result = toku_ydb_check_avail_fs_space(indexer->i->env);
if (result == 0) {
TOKUTXN txn = indexer_get_innermost_live_txn(indexer, xids);
invariant(txn != NULL);
result = toku_brt_maybe_insert (hotdb->i->brt, hotkey, hotval, txn, FALSE, ZERO_LSN, TRUE, BRT_INSERT);
}
}
return result;
}
// send an insert message into the tree without rollback or recovery logging
// and without associating the txn and the brt
static int
indexer_brt_insert_committed(DB_INDEXER *indexer, DB *hotdb, DBT *hotkey, DBT *hotval, XIDS xids) {
int result = 0;
// TEST
if (indexer->i->test_insert_committed) {
result = indexer->i->test_insert_committed(indexer, hotdb, hotkey, hotval, xids);
} else {
result = toku_ydb_check_avail_fs_space(indexer->i->env);
if (result == 0)
result = toku_brt_send_insert(db_struct_i(hotdb)->brt, hotkey, hotval, xids, BRT_INSERT);
}
return result;
}
// send a commit message into the tree
// Note: If the xid is zero, then the leafentry will already have a committed transaction
// record and no commit message is needed. (A commit message with xid of zero is
// illegal anyway.)
static int
indexer_brt_commit(DB_INDEXER *indexer, DB *hotdb, DBT *hotkey, XIDS xids) {
int result = 0;
if (xids_get_num_xids(xids) > 0) {// send commit only when not the root xid
// TEST
if (indexer->i->test_commit_any) {
result = indexer->i->test_commit_any(indexer, hotdb, hotkey, xids);
} else {
result = toku_ydb_check_avail_fs_space(indexer->i->env);
if (result == 0)
result = toku_brt_send_commit_any(db_struct_i(hotdb)->brt, hotkey, xids);
}
}
return result;
}
src/indexer.c 0 → 100644
View file @ f31a90e9
/* -*- mode: C; c-basic-offset: 4 -*- */
/*
* Copyright (c) 2010 Tokutek Inc. All rights reserved."
* The technology is licensed by the Massachusetts Institute of Technology,
* Rutgers State University of New Jersey, and the Research Foundation of
* State University of New York at Stony Brook under United States of America
* Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
*/
/*
* The indexer
*/
#include <stdio.h>
#include <string.h>
#include <toku_portability.h>
#include "toku_assert.h"
#include "ydb-internal.h"
#include "le-cursor.h"
#include "indexer.h"
#include "brt-internal.h"
#include "toku_atomic.h"
#include "tokuconst.h"
#include "brt.h"
#include "mempool.h"
#include "leafentry.h"
#include "ule.h"
#include "xids.h"
// for now
static INDEXER_STATUS_S status;
#include "indexer-internal.h"
static int build_index(DB_INDEXER *indexer);
static int close_indexer(DB_INDEXER *indexer);
static int abort_indexer(DB_INDEXER *indexer);
static void free_indexer_resources(DB_INDEXER *indexer);
static void free_indexer(DB_INDEXER *indexer);
static int update_estimated_rows(DB_INDEXER *indexer);
static int maybe_call_poll_func(DB_INDEXER *indexer, uint64_t loop_count);
static int
associate_indexer_with_hot_dbs(DB_INDEXER *indexer, DB *dest_dbs[], int N) {
int result =0;
for (int i = 0; i < N; i++) {
result = toku_db_set_indexer(dest_dbs[i], indexer);
if (result != 0) {
for (int j = 0; j < i; j++) {
int result2 = toku_db_set_indexer(dest_dbs[j], NULL);
lazy_assert(result2 == 0);
}
}
}
return result;
}
static void
disassociate_indexer_from_hot_dbs(DB_INDEXER *indexer) {
for (int i = 0; i < indexer->i->N; i++) {
int result = toku_db_set_indexer(indexer->i->dest_dbs[i], NULL);
lazy_assert(result == 0);
}
}
static void
indexer_add_refs(DB_INDEXER *indexer) {
toku_db_add_ref(indexer->i->src_db);
for (int i = 0; i < indexer->i->N; i++)
toku_db_add_ref(indexer->i->dest_dbs[i]);
}
static void
indexer_release_refs(DB_INDEXER *indexer) {
toku_db_release_ref(indexer->i->src_db);
for (int i = 0; i < indexer->i->N; i++)
toku_db_release_ref(indexer->i->dest_dbs[i]);
}
/*
* free_indexer_resources() frees all of the resources associated with
* struct __toku_indexer_internal
* assumes any previously freed items set the field pointer to NULL
*/
static void
free_indexer_resources(DB_INDEXER *indexer) {
if ( indexer->i ) {
if ( indexer->i->lec ) { le_cursor_close(indexer->i->lec); }
if ( indexer->i->fnums ) { toku_free(indexer->i->fnums); }
indexer_release_refs(indexer);
// indexer->i
toku_free(indexer->i);
indexer->i = NULL;
}
}
static void
free_indexer(DB_INDEXER *indexer) {
if ( indexer ) {
free_indexer_resources(indexer);
toku_free(indexer);
indexer = NULL;
}
}
int
toku_indexer_create_indexer(DB_ENV *env,
DB_TXN *txn,
DB_INDEXER **indexerp,
DB *src_db,
int N,
DB *dest_dbs[N],
uint32_t db_flags[N],
uint32_t indexer_flags)
{
int rval = 0;
DB_INDEXER *indexer = 0; // set later when created
*indexerp = NULL;
rval = toku_grab_read_lock_on_directory (src_db, txn);
if (rval == 0) {
XCALLOC(indexer); // init to all zeroes (thus initializing the error_callback and poll_func)
if ( !indexer ) { rval = ENOMEM; goto create_exit; }
XCALLOC(indexer->i); // init to all zeroes (thus initializing all pointers to NULL)
if ( !indexer->i ) { rval = ENOMEM; goto create_exit; }
indexer->i->env = env;
indexer->i->txn = txn;
indexer->i->src_db = src_db;
indexer->i->N = N;
indexer->i->dest_dbs = dest_dbs;
indexer->i->db_flags = db_flags;
indexer->i->indexer_flags = indexer_flags;
indexer->i->loop_mod = 1000; // call poll_func every 1000 rows
indexer->i->estimated_rows = 0;
indexer->i->generate_extra = NULL; // TODO add a parameter
indexer->i->undo_do = indexer_undo_do; // TEST export the undo do function
XCALLOC_N(N, indexer->i->fnums);
if ( !indexer->i->fnums ) { rval = ENOMEM; goto create_exit; }
for(int i=0;i<indexer->i->N;i++) {
indexer->i->fnums[i] = toku_cachefile_filenum(db_struct_i(dest_dbs[i])->brt->cf);
}
indexer->i->filenums.num = N;
indexer->i->filenums.filenums = indexer->i->fnums;
indexer->i->test_only_flags = 0; // for test use only
indexer->set_error_callback = toku_indexer_set_error_callback;
indexer->set_poll_function = toku_indexer_set_poll_function;
indexer->build = build_index;
indexer->close = close_indexer;
indexer->abort = abort_indexer;
// create and initialize the leafentry cursor
rval = le_cursor_create(&indexer->i->lec, db_struct_i(src_db)->brt, db_txn_struct_i(txn)->tokutxn);
if ( !indexer->i->lec ) { goto create_exit; }
// 2954: add recovery and rollback entries
LSN hot_index_lsn; // not used (yet)
TOKUTXN ttxn = db_txn_struct_i(txn)->tokutxn;
FILENUMS filenums = indexer->i->filenums;
rval = toku_brt_hot_index(NULL, ttxn, filenums, 1, &hot_index_lsn);
}
if (rval == 0)
rval = associate_indexer_with_hot_dbs(indexer, dest_dbs, N);
create_exit:
if ( rval == 0 ) {
indexer_add_refs(indexer);
*indexerp = indexer;
(void) toku_sync_fetch_and_increment_uint64(&status.create);
(void) toku_sync_fetch_and_increment_uint32(&status.current);
if ( status.current > status.max )
status.max = status.current; // not worth a lock to make threadsafe, may be inaccurate
} else {
(void) toku_sync_fetch_and_increment_uint64(&status.create_fail);
free_indexer(indexer);
}
return rval;
}
int
toku_indexer_set_poll_function(DB_INDEXER *indexer,
int (*poll_func)(void *poll_extra,
float progress),
void *poll_extra)
{
invariant(indexer != NULL);
indexer->i->poll_func = poll_func;
indexer->i->poll_extra = poll_extra;
return 0;
}
int
toku_indexer_set_error_callback(DB_INDEXER *indexer,
void (*error_cb)(DB *db, int i, int err,
DBT *key, DBT *val,
void *error_extra),
void *error_extra)
{
invariant(indexer != NULL);
indexer->i->error_callback = error_cb;
indexer->i->error_extra = error_extra;
return 0;
}
int
toku_indexer_is_key_right_of_le_cursor(DB_INDEXER *indexer, DB *db, const DBT *key) {
DB_ENV *env = indexer->i->env;
return is_key_right_of_le_cursor(indexer->i->lec, key, env->i->bt_compare, db);
}
static int
build_index(DB_INDEXER *indexer) {
int result = 0;
(void) toku_sync_fetch_and_increment_uint64(&status.build);
DBT key; toku_init_dbt(&key); key.flags = DB_DBT_REALLOC;
DBT le; toku_init_dbt(&le); le.flags = DB_DBT_REALLOC;
BOOL done = FALSE;
for (uint64_t loop_count = 0; !done; loop_count++) {
toku_ydb_lock();
result = le_cursor_next(indexer->i->lec, &key, &le);
if (result != 0) {
done = TRUE;
if (result == DB_NOTFOUND)
result = 0; // all done, normal way to exit loop successfully
}
else {
ULEHANDLE ule = toku_ule_create(le.data);
for (int which_db = 0; (which_db < indexer->i->N) && (result == 0); which_db++) {
DB *db = indexer->i->dest_dbs[which_db];
result = indexer_undo_do(indexer, db, ule);
if ( (result != 0) && (indexer->i->error_callback != NULL)) {
indexer->i->error_callback(db, which_db, result, &key, NULL, indexer->i->error_extra);
}
}
toku_ule_free(ule);
}
toku_ydb_unlock_and_yield(1000); // if there is lock contention, then sleep for 1 millisecond after the unlock
if (result == 0)
result = maybe_call_poll_func(indexer, loop_count);
if (result != 0)
done = TRUE;
}
toku_destroy_dbt(&key);
toku_destroy_dbt(&le);
// post index creation cleanup
// - optimize?
// - garbage collect?
// - unique checks?
return result;
}
static int
close_indexer(DB_INDEXER *indexer) {
int r = 0;
(void) toku_sync_fetch_and_decrement_uint32(&status.current);
toku_ydb_lock();
/*
Add all created dbs to the transaction’s checkpoint_before_commit list. (This will cause a local checkpoint of created index files, which is necessary because these files are not necessarily on disk and all the operations to create them are not in the recovery log.)
Disassociate the indexer from the hot dbs
*/
disassociate_indexer_from_hot_dbs(indexer);
toku_ydb_unlock();
/*
Destroy the hot cursor
Destroy the indexer object
*/
free_indexer(indexer);
if ( r == 0 ) {
(void) toku_sync_fetch_and_increment_uint64(&status.close);
} else {
(void) toku_sync_fetch_and_increment_uint64(&status.close_fail);
}
return r;
}
static int
abort_indexer(DB_INDEXER *indexer) {
(void) toku_sync_fetch_and_decrement_uint32(&status.current);
(void) toku_sync_fetch_and_increment_uint64(&status.abort);
free_indexer(indexer);
return 0;
}
// derived from ha_tokudb::estimate_num_rows
static int
update_estimated_rows(DB_INDEXER *indexer) {
DBT key; toku_init_dbt(&key);
DBT data; toku_init_dbt(&data);
DBC* crsr;
DB_TXN* txn = NULL;
uint64_t less, equal, greater;
int is_exact;
int error;
DB *db = indexer->i->src_db;
DB_ENV *db_env = indexer->i->env;
error = db_env->txn_begin(db_env, 0, &txn, DB_READ_UNCOMMITTED);
if (error) goto cleanup;
error = db->cursor(db, txn, &crsr, 0);
if (error) { goto cleanup; }
error = crsr->c_get(crsr, &key, &data, DB_FIRST);
if (error == DB_NOTFOUND) {
indexer->i->estimated_rows = 0;
error = 0;
goto cleanup;
}
else if (error) { goto cleanup; }
error = db->key_range64(db, txn, &key,
&less, &equal, &greater,
&is_exact);
if (error) { goto cleanup; }
indexer->i->estimated_rows = equal + greater;
error = 0;
cleanup:
if ( crsr != NULL ) {
int rr = crsr->c_close(crsr);
invariant(rr == 0);
crsr = NULL;
}
txn->commit(txn, 0);
return error;
}
static int
maybe_call_poll_func(DB_INDEXER *indexer, uint64_t loop_count) {
int result = 0;
if ( indexer->i->poll_func != NULL && ( loop_count % indexer->i->loop_mod ) == 0 ) {
int r __attribute__((unused)) = update_estimated_rows(indexer);
// what happens if estimate_rows fails?
// - currently does not modify estimate, which is probably sufficient
float progress;
if ( indexer->i->estimated_rows == 0 ) {
progress = 1.0;
}
else {
progress = (float)loop_count / (float)indexer->i->estimated_rows;
}
result = indexer->i->poll_func(indexer->i->poll_extra, progress);
}
return result;
}
void
toku_indexer_get_status(INDEXER_STATUS s) {
*s = status;
}
// this allows us to force errors under test. Flags are defined in indexer.h
void
toku_indexer_set_test_only_flags(DB_INDEXER *indexer, int flags) {
invariant(indexer != NULL);
indexer->i->test_only_flags = flags;
}
DB *
toku_indexer_get_src_db(DB_INDEXER *indexer) {
return indexer->i->src_db;
}
src/indexer.h 0 → 100644
View file @ f31a90e9
#ifndef TOKU_INDEXER_H
#define TOKU_INDEXER_H
#if defined(__cplusplus)
extern "C" {
#endif
/* Copyright (c) 2007-2010 Tokutek Inc. All rights reserved.
*
* The technology is licensed by the Massachusetts Institute of Technology,
* Rutgers State University of New Jersey, and the Research Foundation of
* State University of New York at Stony Brook under United States of America
* Serial No. 11/760379 and to the patents and/or patent applications resulting from it.
*/
int toku_indexer_create_indexer(DB_ENV *env,
DB_TXN *txn,
DB_INDEXER **indexer,
DB *src_db,
int N,
DB *dest_dbs[N],
uint32_t db_flags[N],
uint32_t indexer_flags) __attribute__((__visibility__("default")));
int toku_indexer_set_poll_function(DB_INDEXER *indexer,
int (*poll_function)(void *poll_extra,
float progress),
void *poll_extra);
int toku_indexer_set_error_callback(DB_INDEXER *indexer,
void (*error_cb)(DB *db, int i, int err,
DBT *key, DBT *val,
void *error_extra),
void *error_extra);
int toku_indexer_is_key_right_of_le_cursor(DB_INDEXER *indexer, DB *db, const DBT *key);
DB *toku_indexer_get_src_db(DB_INDEXER *indexer);
void toku_indexer_set_test_only_flags(DB_INDEXER *indexer, int flags) __attribute__((__visibility__("default")));
#define INDEXER_TEST_ONLY_ERROR_CALLBACK 1
typedef struct indexer_status {
uint64_t create; // number of indexers successfully created
uint64_t create_fail; // number of calls to toku_indexer_create_indexer() that failed
uint64_t build; // number of calls to indexer->build()
uint64_t close; // number of calls to indexer->close()
uint64_t close_fail; // number of calls to indexer->close() that failed
uint64_t abort; // number of calls to indexer->abort()
uint32_t current; // number of indexers currently in existence
uint32_t max; // max number of indexers that ever existed simulataneously
} INDEXER_STATUS_S, *INDEXER_STATUS;
void toku_indexer_get_status(INDEXER_STATUS s);
#if defined(__cplusplus)
}
#endif
#endif // TOKU_INDEXER_H
src/loader.c
View file @ f31a90e9
......@@ -58,6 +58,22 @@ struct __toku_loader_internal {
char **inames_in_env; /* [N] inames of new files to be created */
};
static void
loader_add_refs(DB_LOADER *loader) {
if (loader->i->src_db)
toku_db_add_ref(loader->i->src_db);
for (int i = 0; i < loader->i->N; i++)
toku_db_add_ref(loader->i->dbs[i]);
}
static void
loader_release_refs(DB_LOADER *loader) {
if (loader->i->src_db)
toku_db_release_ref(loader->i->src_db);
for (int i = 0; i < loader->i->N; i++)
toku_db_release_ref(loader->i->dbs[i]);
}
/*
* free_loader_resources() frees all of the resources associated with
* struct __toku_loader_internal
......@@ -67,6 +83,7 @@ struct __toku_loader_internal {
static void free_loader_resources(DB_LOADER *loader)
{
if ( loader->i ) {
loader_release_refs(loader);
for (int i=0; i<loader->i->N; i++) {
if (loader->i->ekeys &&
loader->i->ekeys[i].data &&
......@@ -229,6 +246,7 @@ int toku_loader_create_loader(DB_ENV *env,
}
*blp = loader;
create_exit:
loader_add_refs(loader);
if (rval == 0) {
(void) toku_sync_fetch_and_increment_uint64(&status.create);
(void) toku_sync_fetch_and_increment_uint32(&status.current);
......
src/tests/Makefile
View file @ f31a90e9
......@@ -98,6 +98,18 @@ BDB_DONTRUN_TESTS = \
helgrind1 \
helgrind2 \
helgrind3 \
hotindexer-bw \
hotindexer-db-busy \
hotindexer-error-callback \
hotindexer-insert-committed-optimized \
hotindexer-insert-committed \
hotindexer-insert-provisional \
hotindexer-multiclient \
hotindexer-nested-insert-committed \
hotindexer-put-multiple \
hotindexer-simple-abort \
hotindexer-undo-do-test \
hotindexer-with-queries \
insert-dup-prelock \
isolation \
isolation-read-committed \
......@@ -108,11 +120,13 @@ BDB_DONTRUN_TESTS = \
loader-cleanup-test \
loader-create-abort \
loader-create-close \
loader-db-busy \
loader-dup-test \
loader-no-puts \
loader-reference-test \
loader-stress-test \
loader-tpch-load \
lock-pressure \
manyfiles \
multiprocess \
mvcc-create-table \
......@@ -177,6 +191,7 @@ BDB_DONTRUN_TESTS = \
test_txn_nested3 \
test_txn_nested4 \
test_txn_nested5 \
txn-ignore \
transactional_fileops \
update-multiple-nochange \
update-multiple-key0 \
......@@ -740,6 +755,15 @@ LOADER_USE_DEFAULT_TESTS = create-abort create-close no-puts reference-test stre
loader-tests: $(LOADER_TESTS)
echo $(LOADER_TESTS)
HOTINDEXER_UNDO_TESTS = $(wildcard hotindexer-undo-do-tests/*.test)
CHECK_HOTINDEXER_UNDO_TESTS = $(patsubst %.test,%.run,$(HOTINDEXER_UNDO_TESTS))
hotindexer-undo-do-test.tdbrun: hotindexer-undo-do-test.tdb$(BINSUF) $(CHECK_HOTINDEXER_UNDO_TESTS)
true
$(CHECK_HOTINDEXER_UNDO_TESTS): %.run: %.test
./run-hotindexer-undo-do-tests.bash $< $(SUMMARIZE_CMD)
clean:
rm -f $(ALL_BINS)
rm -rf dir.* *.check.output *.check.valgrind
......
src/tests/hotindexer-bw.c 0 → 100644
View file @ f31a90e9
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2010 Tokutek Inc. All rights reserved."
#include "test.h"
#include "toku_pthread.h"
#include <db.h>
#include <sys/stat.h>
#include "key-val.h"
enum {NUM_INDEXER_INDEXES=1};
static const int NUM_DBS = NUM_INDEXER_INDEXES + 1; // 1 for source DB
static const int NUM_ROWS = 10;
int num_rows;
enum {MAX_CLIENTS=10};
typedef enum {FORWARD = 0, BACKWARD} Direction;
typedef enum {TXN_NONE = 0, TXN_CREATE = 1, TXN_END = 2} TxnWork;
DB_ENV *env;
float last_progress = 0.0;
static int poll_print(void *extra, float progress) {
if ( verbose ) {
if ( last_progress + 0.01 < progress ) {
printf(" progress : %3.0f%%\n", progress * 100.0);
last_progress = progress;
}
}
extra = extra;
return 0;
}
static inline uint32_t key_to_put(int iter, int offset)
{
return (uint32_t)(((iter+1) * MAX_CLIENTS) + offset);
}
static int generate_initial_table(DB *db, DB_TXN *txn, uint32_t rows)
{
struct timeval start, now;
if ( verbose ) {
printf("generate_initial_table\n");
gettimeofday(&start,0);
}
int r = 0;
DBT key, val;
uint32_t k, v, i;
// create keys of stride MAX_CLIENTS
for (i=0; i<rows; i++)
{
k = key_to_put(i, 0);
v = generate_val(k, 0);
dbt_init(&key, &k, sizeof(k));
dbt_init(&val, &v, sizeof(v));
r = db->put(db, txn, &key, &val, 0);
if ( r != 0 ) break;
}
if ( verbose ) {
gettimeofday(&now,0);
int duration = (int)(now.tv_sec - start.tv_sec);
if ( duration > 0 )
printf("generate_initial_table : %u rows in %d sec = %d rows/sec\n", rows, duration, rows/duration);
}
return r;
}
/*
* client() is a routine intended to be run in a separate thread from index creation
* - it takes a client spec which describes work to be done
* - direction : move to ever increasing or decreasing rows
* - txnwork : whether a transaction should be created or closed within the client
* (allows client transaction to start before or during index creation,
* and to close during or after index creation)
*/
typedef struct client_spec {
uint32_t num; // number of rows to write
uint32_t start; // approximate start row
int offset; // offset from stride (= MAX_CLIENTS)
Direction dir;
TxnWork txnwork;
DB_TXN *txn;
DB **dbs;
int client_number;
uint32_t *flags;
} client_spec_t, *client_spec;
int client_count = 0;
static void * client(void *arg)
{
client_spec cs = arg;
client_count++;
if ( verbose ) printf("client[%d]\n", cs->client_number);
assert(cs->client_number < MAX_CLIENTS);
assert(cs->dir == FORWARD || cs->dir == BACKWARD);
int r;
if ( cs->txnwork | TXN_CREATE ) { r = env->txn_begin(env, NULL, &cs->txn, 0); CKERR(r); }
DBT key, val;
DBT dest_keys[NUM_DBS];
DBT dest_vals[NUM_DBS];
uint32_t k, v;
int n = cs->start;
for(int which=0;which<NUM_DBS;which++) {
dbt_init(&dest_keys[which], NULL, 0);
dest_keys[which].flags = DB_DBT_REALLOC;
dbt_init(&dest_vals[which], NULL, 0);
dest_vals[which].flags = DB_DBT_REALLOC;
}
int rr;
for (uint32_t i = 0; i < cs->num; i++ ) {
DB_TXN *txn;
env->txn_begin(env, cs->txn, &txn, 0);
k = key_to_put(n, cs->offset);
v = generate_val(k, 0);
dbt_init(&key, &k, sizeof(k));
dbt_init(&val, &v, sizeof(v));
rr = env->put_multiple(env,
cs->dbs[0],
txn,
&key,
&val,
NUM_DBS,
cs->dbs, // dest dbs
dest_keys,
dest_vals,
cs->flags,
NULL);
if ( rr != 0 ) {
if ( verbose ) printf("client[%u] : put_multiple returns %d, i=%u, n=%u, key=%u\n", cs->client_number, rr, i, n, k);
r = txn->abort(txn); CKERR(r);
break;
}
r = txn->commit(txn, 0); CKERR(r);
n = ( cs->dir == FORWARD ) ? n + 1 : n - 1;
}
if ( cs->txnwork | TXN_END ) { r = cs->txn->commit(cs->txn, DB_TXN_SYNC); CKERR(r); }
if (verbose) printf("client[%d] done\n", cs->client_number);
for (int which=0; which<NUM_DBS; which++) {
toku_free(dest_keys[which].data);
toku_free(dest_vals[which].data);
}
return 0;
}
toku_pthread_t *client_threads;
client_spec_t *client_specs;
static void clients_init(DB **dbs, uint32_t *flags)
{
client_threads = toku_malloc(sizeof(toku_pthread_t) * MAX_CLIENTS);
client_specs = toku_malloc(sizeof(client_spec_t) * MAX_CLIENTS);
client_specs[0].client_number = 0;
// client_specs[0].start = 0;
client_specs[0].start = num_rows - 1;
client_specs[0].num = num_rows;
client_specs[0].offset = -1;
// client_specs[0].dir = FORWARD;
client_specs[0].dir = BACKWARD;
client_specs[0].txnwork = TXN_CREATE | TXN_END;
client_specs[0].txn = NULL;
client_specs[0].dbs = dbs;
client_specs[0].flags = flags;
client_specs[1].client_number = 1;
client_specs[1].start = 0;
client_specs[1].num = num_rows;
client_specs[1].offset = 1;
client_specs[1].dir = FORWARD;
client_specs[1].txnwork = TXN_CREATE | TXN_END;
client_specs[1].txn = NULL;
client_specs[1].dbs = dbs;
client_specs[1].flags = flags;
}
static void clients_cleanup(void)
{
toku_free(client_threads); client_threads = NULL;
toku_free(client_specs); client_specs = NULL;
}
// verify results
// - read the keys in the primary table, then calculate what keys should exist
// in the other DB. Read the other table to verify.
static void check_results(DB *src, DB *db)
{
int r;
int pass = 1;
int clients = client_count;
int max_rows = ( clients + 1 ) * num_rows;
unsigned int *db_keys = (unsigned int *) toku_malloc(max_rows * sizeof (unsigned int));
DBT key, val;
unsigned int k=0, v=0;
dbt_init(&key, &k, sizeof(unsigned int));
dbt_init(&val, &v, sizeof(unsigned int));
DB_TXN *txn;
r = env->txn_begin(env, NULL, &txn, 0); CKERR(r);
DBC *cursor;
r = src->cursor(src, txn, &cursor, 0); CKERR(r);
int which = *(uint32_t*)db->app_private;
// scan the primary table,
// calculate the expected keys in 'db'
int row = 0;
while ( r != DB_NOTFOUND ) {
r = cursor->c_get(cursor, &key, &val, DB_NEXT);
if ( r != DB_NOTFOUND ) {
k = *((uint32_t *)(key.data));
db_keys[row] = twiddle32(k, which);
row++;
}
}
if ( verbose ) printf("primary table scanned, contains %d rows\n", row);
int primary_rows = row;
r = cursor->c_close(cursor); CKERR(r);
// sort the expected keys
qsort(db_keys, primary_rows, sizeof (unsigned int), uint_cmp);
if ( verbose > 1 ) {
for(int i=0;i<primary_rows;i++) {
printf("primary table[%u] = %u\n", i, db_keys[i]);
}
}
// scan the indexer-created DB, comparing keys with expected keys
// - there should be exactly 'primary_rows' in the new index
r = db->cursor(db, txn, &cursor, 0); CKERR(r);
for (int i=0;i<primary_rows;i++) {
r = cursor->c_get(cursor, &key, &val, DB_NEXT);
if ( r == DB_NOTFOUND ) {
printf("scan of index finds last row is %d\n", i);
}
CKERR(r);
k = *((uint32_t *)(key.data));
if ( db_keys[i] != k ) {
if ( verbose ) printf("ERROR expecting key %10u for row %d, found key = %10u\n", db_keys[i],i,k);
pass = 0;
i++;
// goto check_results_error;
}
}
// next cursor op should return DB_NOTFOUND
r = cursor->c_get(cursor, &key, &val, DB_NEXT);
assert(r == DB_NOTFOUND);
// we're done - cleanup and close
//check_results_error:
r = cursor->c_close(cursor); CKERR(r);
toku_free(db_keys);
r = txn->commit(txn, 0); CKERR(r);
if ( pass ) printf("check_results : pass\n");
else printf("check_results : fail\n");
return;
}
static void test_indexer(DB *src, DB **dbs)
{
int r;
DB_TXN *txn;
DB_INDEXER *indexer;
uint32_t db_flags[NUM_DBS];
if ( verbose ) printf("test_indexer\n");
for(int i=0;i<NUM_DBS;i++) {
db_flags[i] = DB_YESOVERWRITE;
}
clients_init(dbs, db_flags);
// create and initialize indexer
r = env->txn_begin(env, NULL, &txn, 0);
CKERR(r);
if ( verbose ) printf("test_indexer create_indexer\n");
r = env->create_indexer(env, txn, &indexer, src, NUM_DBS-1, &dbs[1], db_flags, 0);
CKERR(r);
r = indexer->set_error_callback(indexer, NULL, NULL);
CKERR(r);
r = indexer->set_poll_function(indexer, poll_print, NULL);
CKERR(r);
// start threads doing additional inserts - no lock issues since indexer already created
r = toku_pthread_create(&client_threads[0], 0, client, (void *)&client_specs[0]); CKERR(r);
// r = toku_pthread_create(&client_threads[1], 0, client, (void *)&client_specs[1]); CKERR(r);
struct timeval start, now;
if ( verbose ) {
printf("test_indexer build\n");
gettimeofday(&start,0);
}
r = indexer->build(indexer);
CKERR(r);
if ( verbose ) {
gettimeofday(&now,0);
int duration = (int)(now.tv_sec - start.tv_sec);
if ( duration > 0 )
printf("test_indexer build : sec = %d\n", duration);
}
if ( verbose ) printf("test_indexer close\n");
r = indexer->close(indexer);
CKERR(r);
r = txn->commit(txn, DB_TXN_SYNC);
CKERR(r);
void *t0;
r = toku_pthread_join(client_threads[0], &t0); CKERR(r);
// void *t1;
// r = toku_pthread_join(client_threads[1], &t1); CKERR(r);
clients_cleanup();
if ( verbose ) printf("check_results\n");
check_results(src, dbs[1]);
if ( verbose ) printf("PASS\n");
if ( verbose ) printf("test_indexer done\n");
}
static void run_test(void)
{
int r;
r = system("rm -rf " ENVDIR); CKERR(r);
r = toku_os_mkdir(ENVDIR, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
r = toku_os_mkdir(ENVDIR "/log", S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
r = db_env_create(&env, 0); CKERR(r);
r = env->set_lg_dir(env, "log"); CKERR(r);
r = env->set_default_bt_compare(env, uint_dbt_cmp); CKERR(r);
generate_permute_tables();
r = env->set_generate_row_callback_for_put(env, put_multiple_generate); CKERR(r);
int envflags = DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN | DB_CREATE | DB_PRIVATE | DB_INIT_LOG;
r = env->open(env, ENVDIR, envflags, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
env->set_errfile(env, stderr);
r = env->checkpointing_set_period(env, 0); CKERR(r);
DBT desc;
dbt_init(&desc, "foo", sizeof("foo"));
int ids[MAX_DBS];
DB *dbs[MAX_DBS];
for (int i = 0; i < NUM_DBS; i++) {
ids[i] = i;
r = db_create(&dbs[i], env, 0); CKERR(r);
r = dbs[i]->set_descriptor(dbs[i], 1, &desc); CKERR(r);
dbs[i]->app_private = &ids[i];
char key_name[32];
sprintf(key_name, "key%d", i);
r = dbs[i]->open(dbs[i], NULL, key_name, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); CKERR(r);
}
// generate the src DB (do not use put_multiple)
DB_TXN *txn;
r = env->txn_begin(env, NULL, &txn, 0); CKERR(r);
r = generate_initial_table(dbs[0], txn, num_rows); CKERR(r);
r = txn->commit(txn, DB_TXN_SYNC); CKERR(r);
// -------------------------- //
if (1) test_indexer(dbs[0], dbs);
// -------------------------- //
for(int i=0;i<NUM_DBS;i++) {
r = dbs[i]->close(dbs[i], 0); CKERR(r);
}
r = env->close(env, 0); CKERR(r);
}
// ------------ infrastructure ----------
static inline void
do_args (int argc, char * const argv[]) {
const char *progname=argv[0];
num_rows = NUM_ROWS;
argc--; argv++;
while (argc>0) {
if (strcmp(argv[0],"-v")==0) {
verbose++;
} else if (strcmp(argv[0],"-q")==0) {
verbose=0;
} else if (strcmp(argv[0],"-r")==0) {
argc--; argv++;
num_rows = atoi(argv[0]);
} else {
fprintf(stderr, "Usage:\n %s [-v] [-q] [-r rows]\n", progname);
exit(1);
}
argc--; argv++;
}
}
int test_main(int argc, char * const *argv) {
do_args(argc, argv);
run_test();
return 0;
}
/*
* Please ignore this code - I don't think I'm going to use it, but I don't want to lose it
* I will delete this later - Dave
if ( rr != 0 ) { // possible lock deadlock
if (verbose > 1) {
printf("client[%u] : put_multiple returns %d, i=%u, n=%u, key=%u\n", cs->client_number, rr, i, n, k);
if ( verbose > 2 ) print_engine_status(env);
}
// abort the transaction, freeing up locks associated with previous put_multiples
if ( verbose > 1 ) printf("start txn abort\n");
r = txn->abort(txn); CKERR(r);
if ( verbose > 1 ) printf(" txn aborted\n");
sleep(2 + cs->client_number);
// now retry, waiting until the deadlock resolves itself
r = env->txn_begin(env, cs->txn, &txn, 0); CKERR(r);
if ( verbose > 1 ) printf("txn begin\n");
while ( rr != 0 ) {
rr = env->put_multiple(env,
cs->dbs[0],
txn,
&key,
&val,
NUM_DBS,
cs->dbs, // dest dbs
dest_keys,
dest_vals,
cs->flags,
NULL);
if ( rr != 0 ) {
if ( verbose ) printf("client[%u] : put_multiple returns %d, i=%u, n=%u, key=%u\n", cs->client_number, rr, i, n, k);
if ( verbose ) printf("start txn abort\n");
r = txn->abort(txn); CKERR(r);
if ( verbose ) printf(" txn aborted\n");
sleep(2 + cs->client_number);
r = env->txn_begin(env, cs->txn, &txn, 0); CKERR(r);
if ( verbose ) printf("txn begin\n");
}
}
*/
src/tests/hotindexer-db-busy.c 0 → 100644
View file @ f31a90e9
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2009 Tokutek Inc. All rights reserved."
#ident "$Id: loader-reference-test.c 23787 2010-09-14 17:31:19Z dwells $"
// Verify that the indexer grabs references on the DBs
#include "test.h"
#include "toku_pthread.h"
#include <db.h>
#include <sys/stat.h>
DB_ENV *env;
enum {NUM_DBS=1};
static int put_multiple_generate(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_val, const DBT *src_key, const DBT *src_val, void *extra) {
dest_db = dest_db; src_db = src_db; dest_key = dest_key; dest_val = dest_val; src_key = src_key; src_val = src_val; extra = extra;
assert(0);
return 0;
}
char *src_name="src.db";
static void run_test(void)
{
int r;
r = system("rm -rf " ENVDIR); CKERR(r);
r = toku_os_mkdir(ENVDIR, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
r = toku_os_mkdir(ENVDIR "/log", S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
r = db_env_create(&env, 0); CKERR(r);
r = env->set_lg_dir(env, "log"); CKERR(r);
r = env->set_default_bt_compare(env, int64_dbt_cmp); CKERR(r);
r = env->set_generate_row_callback_for_put(env, put_multiple_generate); CKERR(r);
int envflags = DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN | DB_CREATE | DB_PRIVATE | DB_INIT_LOG;
r = env->open(env, ENVDIR, envflags, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
env->set_errfile(env, stderr);
//Disable auto-checkpointing
r = env->checkpointing_set_period(env, 0); CKERR(r);
DB *src_db = NULL;
r = db_create(&src_db, env, 0); CKERR(r);
r = src_db->open(src_db, NULL, src_name, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); CKERR(r);
DB *dbs[NUM_DBS];
for (int i = 0; i < NUM_DBS; i++) {
r = db_create(&dbs[i], env, 0); CKERR(r);
char key_name[32];
sprintf(key_name, "key%d", i);
r = dbs[i]->open(dbs[i], NULL, key_name, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); CKERR(r);
dbs[i]->app_private = (void *) (intptr_t) i;
}
DB_TXN *hottxn;
r = env->txn_begin(env, NULL, &hottxn, 0); CKERR(r);
DB_INDEXER *indexer;
r = env->create_indexer(env, hottxn, &indexer, src_db, NUM_DBS, dbs, NULL, 0); CKERR(r);
r = src_db->close(src_db, 0);
assert(r == EBUSY); // close the src_db with an active indexer, should not succeed
for(int i = 0; i < NUM_DBS; i++) {
r = dbs[i]->close(dbs[i], 0);
assert(r == EBUSY); // close a dest_db, should not succeed
}
r = indexer->abort(indexer); CKERR(r);
r = src_db->close(src_db, 0); CKERR(r);
r = hottxn->commit(hottxn, DB_TXN_SYNC); CKERR(r);
for(int i = 0;i < NUM_DBS; i++) {
r = dbs[i]->close(dbs[i], 0); CKERR(r);
}
r = env->close(env, 0); CKERR(r);
}
// ------------ infrastructure ----------
static void do_args(int argc, char * const argv[]);
int test_main(int argc, char * const *argv) {
do_args(argc, argv);
run_test();
return 0;
}
static void do_args(int argc, char * const argv[]) {
int resultcode;
char *cmd = argv[0];
argc--; argv++;
while (argc>0) {
if (strcmp(argv[0], "-v")==0) {
verbose++;
} else if (strcmp(argv[0],"-q")==0) {
verbose--;
if (verbose<0) verbose=0;
} else if (strcmp(argv[0], "-h")==0) {
resultcode=0;
do_usage:
fprintf(stderr, "Usage:\n%s\n", cmd);
exit(resultcode);
} else {
fprintf(stderr, "Unknown arg: %s\n", argv[0]);
resultcode=1;
goto do_usage;
}
argc--;
argv++;
}
}
src/tests/hotindexer-error-callback.c 0 → 100644
View file @ f31a90e9
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2010 Tokutek Inc. All rights reserved."
#include "test.h"
#include "toku_pthread.h"
#include <db.h>
#include <sys/stat.h>
#include "key-val.h"
#include "ydb.h"
#include "indexer.h"
enum {NUM_DBS=1};
static const int NUM_ROWS = 10;
enum {MAX_CLIENTS=10};
typedef enum {FORWARD = 0, BACKWARD} Direction;
typedef enum {TXN_NONE = 0, TXN_CREATE = 1, TXN_END = 2} TxnWork;
DB_ENV *env;
float last_progress = 0.0;
int error_cb_count = 0;
static void error_callback(DB *db, int which_db, int err, DBT *key, DBT *val, void *extra)
{
error_cb_count++;
if ( verbose ) {
printf("error_callback (%d) : db_p = %p, which_db = %d, error = %d, key_p = %p, val_p = %p, extra_p = %p\n",
error_cb_count,
db, which_db,
err,
key, val, extra);
}
}
static int poll_print(void *extra, float progress) {
if ( verbose ) {
if ( last_progress + 0.01 < progress ) {
printf(" progress : %3.0f%%\n", progress * 100.0);
last_progress = progress;
}
}
extra = extra;
return 0;
}
static inline uint32_t key_to_put(int iter, int offset)
{
return (uint32_t)(((iter+1) * MAX_CLIENTS) + offset);
}
static int generate_initial_table(DB *db, DB_TXN *txn, uint32_t rows)
{
struct timeval start, now;
if ( verbose ) {
printf("generate_initial_table\n");
gettimeofday(&start,0);
}
int r = 0;
DBT key, val;
uint32_t k, v, i;
// create keys of stride MAX_CLIENTS
for (i=0; i<rows; i++)
{
k = key_to_put(i, 0);
v = generate_val(k, 0);
dbt_init(&key, &k, sizeof(k));
dbt_init(&val, &v, sizeof(v));
r = db->put(db, txn, &key, &val, 0);
if ( r != 0 ) break;
}
if ( verbose ) {
gettimeofday(&now,0);
int duration = (int)(now.tv_sec - start.tv_sec);
if ( duration > 0 )
printf("generate_initial_table : %u rows in %d sec = %d rows/sec\n", rows, duration, rows/duration);
}
return r;
}
static void test_indexer(DB *src, DB **dbs)
{
int r;
DB_TXN *txn;
DB_INDEXER *indexer;
uint32_t db_flags[NUM_DBS];
if ( verbose ) printf("test_indexer\n");
for(int i=0;i<NUM_DBS;i++) {
db_flags[i] = DB_NOOVERWRITE;
}
// create and initialize loader
r = env->txn_begin(env, NULL, &txn, 0);
CKERR(r);
if ( verbose ) printf("test_indexer create_indexer\n");
r = env->create_indexer(env, txn, &indexer, src, NUM_DBS, dbs, db_flags, 0);
CKERR(r);
r = indexer->set_error_callback(indexer, error_callback, NULL);
CKERR(r);
toku_indexer_set_test_only_flags(indexer, INDEXER_TEST_ONLY_ERROR_CALLBACK);
r = indexer->set_poll_function(indexer, poll_print, NULL);
CKERR(r);
r = indexer->build(indexer);
assert(r != 0 ); // build should return an error
assert(error_cb_count == 1); // error callback count should be 1
if ( verbose ) printf("test_indexer close\n");
r = indexer->close(indexer);
CKERR(r);
r = txn->commit(txn, DB_TXN_SYNC);
CKERR(r);
if ( verbose ) printf("PASS\n");
if ( verbose ) printf("test_indexer done\n");
}
static void run_test(void)
{
int r;
r = system("rm -rf " ENVDIR); CKERR(r);
r = toku_os_mkdir(ENVDIR, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
r = toku_os_mkdir(ENVDIR "/log", S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
r = db_env_create(&env, 0); CKERR(r);
r = env->set_lg_dir(env, "log"); CKERR(r);
// r = env->set_default_bt_compare(env, int64_dbt_cmp); CKERR(r);
r = env->set_default_bt_compare(env, int_dbt_cmp); CKERR(r);
generate_permute_tables();
r = env->set_generate_row_callback_for_put(env, put_multiple_generate); CKERR(r);
int envflags = DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN | DB_CREATE | DB_PRIVATE | DB_INIT_LOG;
r = env->open(env, ENVDIR, envflags, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
env->set_errfile(env, stderr);
//Disable auto-checkpointing
r = env->checkpointing_set_period(env, 0); CKERR(r);
DB *src_db = NULL;
char *src_name="src.db";
r = db_create(&src_db, env, 0); CKERR(r);
r = src_db->open(src_db, NULL, src_name, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); CKERR(r);
DB_TXN *txn;
r = env->txn_begin(env, NULL, &txn, 0); CKERR(r);
r = generate_initial_table(src_db, txn, NUM_ROWS); CKERR(r);
r = txn->commit(txn, DB_TXN_SYNC); CKERR(r);
DBT desc;
dbt_init(&desc, "foo", sizeof("foo"));
DB *dbs[NUM_DBS];
int idx[MAX_DBS];
for (int i = 0; i < NUM_DBS; i++) {
idx[i] = i+1;
r = db_create(&dbs[i], env, 0); CKERR(r);
r = dbs[i]->set_descriptor(dbs[i], 1, &desc); CKERR(r);
dbs[i]->app_private = &idx[i];
char key_name[32];
sprintf(key_name, "key%d", i);
r = dbs[i]->open(dbs[i], NULL, key_name, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); CKERR(r);
}
// -------------------------- //
if (1) test_indexer(src_db, dbs);
// -------------------------- //
for(int i=0;i<NUM_DBS;i++) {
r = dbs[i]->close(dbs[i], 0); CKERR(r);
}
r = src_db->close(src_db, 0); CKERR(r);
r = env->close(env, 0); CKERR(r);
}
// ------------ infrastructure ----------
int test_main(int argc, char * const argv[]) {
default_parse_args(argc, argv);
run_test();
return 0;
}
src/tests/hotindexer-insert-committed-optimized.c 0 → 100644
View file @ f31a90e9
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2009 Tokutek Inc. All rights reserved."
#ident "$Id: loader-reference-test.c 23787 2010-09-14 17:31:19Z dwells $"
#include "test.h"
#include "toku_pthread.h"
#include <db.h>
#include <sys/stat.h>
DB_ENV *env;
enum {NUM_DBS=1};
enum {NUM_KV_PAIRS=3};
struct kv_pair {
int64_t key;
int64_t val;
};
struct kv_pair kv_pairs[NUM_KV_PAIRS] = {{1,4},
{2,5},
{3,6}};
static int put_multiple_generate(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_val, const DBT *src_key, const DBT *src_val, void *extra) {
src_db = src_db;
extra = extra;
uint32_t which = (uint32_t) (intptr_t) dest_db->app_private;
assert(which == 0);
// switch the key and val
dbt_init(dest_key, src_val->data, src_val->size);
dbt_init(dest_val, src_key->data, src_key->size);
// printf("dest_key.data = %d\n", *(int*)dest_key->data);
// printf("dest_val.data = %d\n", *(int*)dest_val->data);
return 0;
}
static int poll_print(void *extra, float progress) {
progress = progress;
extra = extra;
if ( verbose ) printf("poll_print %f\n", progress);
return 0;
}
static void test_indexer(DB *src, DB **dbs)
{
int r;
DB_TXN *txn;
DB_INDEXER *indexer;
uint32_t db_flags[NUM_DBS];
if ( verbose ) printf("test_indexer\n");
for(int i=0;i<NUM_DBS;i++) {
db_flags[i] = DB_NOOVERWRITE;
}
// create and initialize loader
r = env->txn_begin(env, NULL, &txn, 0);
CKERR(r);
if ( verbose ) printf("test_indexer create_indexer\n");
r = env->create_indexer(env, txn, &indexer, src, NUM_DBS, dbs, db_flags, 0);
CKERR(r);
r = indexer->set_error_callback(indexer, NULL, NULL);
CKERR(r);
r = indexer->set_poll_function(indexer, poll_print, NULL);
CKERR(r);
if ( verbose ) printf("test_indexer build\n");
r = indexer->build(indexer);
CKERR(r);
if ( verbose ) printf("test_indexer close\n");
r = indexer->close(indexer);
CKERR(r);
r = txn->commit(txn, DB_TXN_SYNC);
CKERR(r);
if ( verbose ) printf("PASS\n");
if ( verbose ) printf("test_indexer done\n");
}
char *src_name="src.db";
static void run_test(void)
{
int r;
r = system("rm -rf " ENVDIR); CKERR(r);
r = toku_os_mkdir(ENVDIR, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
r = toku_os_mkdir(ENVDIR "/log", S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
r = db_env_create(&env, 0); CKERR(r);
r = env->set_lg_dir(env, "log"); CKERR(r);
r = env->set_default_bt_compare(env, int64_dbt_cmp); CKERR(r);
r = env->set_generate_row_callback_for_put(env, put_multiple_generate); CKERR(r);
int envflags = DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN | DB_CREATE | DB_PRIVATE | DB_INIT_LOG;
r = env->open(env, ENVDIR, envflags, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
env->set_errfile(env, stderr);
//Disable auto-checkpointing
r = env->checkpointing_set_period(env, 0); CKERR(r);
DB *src_db = NULL;
r = db_create(&src_db, env, 0); CKERR(r);
r = src_db->open(src_db, NULL, src_name, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); CKERR(r);
DBT key, val;
DB_TXN *txn;
r = env->txn_begin(env, NULL, &txn, 0); CKERR(r);
for(int i=0;i<NUM_KV_PAIRS;i++) {
dbt_init(&key, &kv_pairs[i].key, sizeof(kv_pairs[i].key));
dbt_init(&val, &kv_pairs[i].val, sizeof(kv_pairs[i].val));
r = src_db->put(src_db, txn, &key, &val, 0); CKERR(r);
}
r = txn->commit(txn, DB_TXN_SYNC); CKERR(r);
r = src_db->optimize(src_db); CKERR(r);
DB *dbs[NUM_DBS];
for (int i = 0; i < NUM_DBS; i++) {
r = db_create(&dbs[i], env, 0); CKERR(r);
char key_name[32];
sprintf(key_name, "key%d", i);
r = dbs[i]->open(dbs[i], NULL, key_name, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); CKERR(r);
dbs[i]->app_private = (void *) (intptr_t) i;
}
// -------------------------- //
if (1) test_indexer(src_db, dbs);
// -------------------------- //
for(int i=0;i<NUM_DBS;i++) {
r = dbs[i]->close(dbs[i], 0); CKERR(r);
}
r = src_db->close(src_db, 0); CKERR(r);
r = env->close(env, 0); CKERR(r);
}
// ------------ infrastructure ----------
static void do_args(int argc, char * const argv[]);
int test_main(int argc, char * const *argv) {
do_args(argc, argv);
run_test();
return 0;
}
static void do_args(int argc, char * const argv[]) {
int resultcode;
char *cmd = argv[0];
argc--; argv++;
while (argc>0) {
if (strcmp(argv[0], "-v")==0) {
verbose++;
} else if (strcmp(argv[0],"-q")==0) {
verbose--;
if (verbose<0) verbose=0;
} else if (strcmp(argv[0], "-h")==0) {
resultcode=0;
do_usage:
fprintf(stderr, "Usage:\n%s\n", cmd);
exit(resultcode);
} else {
fprintf(stderr, "Unknown arg: %s\n", argv[0]);
resultcode=1;
goto do_usage;
}
argc--;
argv++;
}
}
src/tests/hotindexer-insert-committed.c 0 → 100644
View file @ f31a90e9
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2009 Tokutek Inc. All rights reserved."
#ident "$Id: loader-reference-test.c 23787 2010-09-14 17:31:19Z dwells $"
#include "test.h"
#include "toku_pthread.h"
#include <db.h>
#include <sys/stat.h>
DB_ENV *env;
enum {NUM_DBS=1};
enum {NUM_KV_PAIRS=3};
struct kv_pair {
int64_t key;
int64_t val;
};
struct kv_pair kv_pairs[NUM_KV_PAIRS] = {{1,4},
{2,5},
{3,6}};
static int put_multiple_generate(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_val, const DBT *src_key, const DBT *src_val, void *extra) {
src_db = src_db;
extra = extra;
uint32_t which = (uint32_t) (intptr_t) dest_db->app_private;
assert(which == 0);
// switch the key and val
dbt_init(dest_key, src_val->data, src_val->size);
dbt_init(dest_val, src_key->data, src_key->size);
// printf("dest_key.data = %d\n", *(int*)dest_key->data);
// printf("dest_val.data = %d\n", *(int*)dest_val->data);
return 0;
}
static int poll_print(void *extra, float progress) {
progress = progress;
extra = extra;
if ( verbose ) printf("poll_print %f\n", progress);
return 0;
}
static void test_indexer(DB *src, DB **dbs)
{
int r;
DB_TXN *txn;
DB_INDEXER *indexer;
uint32_t db_flags[NUM_DBS];
if ( verbose ) printf("test_indexer\n");
for(int i=0;i<NUM_DBS;i++) {
db_flags[i] = DB_NOOVERWRITE;
}
// create and initialize loader
r = env->txn_begin(env, NULL, &txn, 0);
CKERR(r);
if ( verbose ) printf("test_indexer create_indexer\n");
r = env->create_indexer(env, txn, &indexer, src, NUM_DBS, dbs, db_flags, 0);
CKERR(r);
r = indexer->set_error_callback(indexer, NULL, NULL);
CKERR(r);
r = indexer->set_poll_function(indexer, poll_print, NULL);
CKERR(r);
if ( verbose ) printf("test_indexer build\n");
r = indexer->build(indexer);
CKERR(r);
if ( verbose ) printf("test_indexer close\n");
r = indexer->close(indexer);
CKERR(r);
r = txn->commit(txn, DB_TXN_SYNC);
CKERR(r);
if ( verbose ) printf("PASS\n");
if ( verbose ) printf("test_indexer done\n");
}
char *src_name="src.db";
static void run_test(void)
{
int r;
r = system("rm -rf " ENVDIR); CKERR(r);
r = toku_os_mkdir(ENVDIR, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
r = toku_os_mkdir(ENVDIR "/log", S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
r = db_env_create(&env, 0); CKERR(r);
r = env->set_lg_dir(env, "log"); CKERR(r);
r = env->set_default_bt_compare(env, int64_dbt_cmp); CKERR(r);
r = env->set_generate_row_callback_for_put(env, put_multiple_generate); CKERR(r);
int envflags = DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN | DB_CREATE | DB_PRIVATE | DB_INIT_LOG;
r = env->open(env, ENVDIR, envflags, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
env->set_errfile(env, stderr);
//Disable auto-checkpointing
r = env->checkpointing_set_period(env, 0); CKERR(r);
DB *src_db = NULL;
r = db_create(&src_db, env, 0); CKERR(r);
r = src_db->open(src_db, NULL, src_name, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); CKERR(r);
DBT key, val;
DB_TXN *txn;
r = env->txn_begin(env, NULL, &txn, 0); CKERR(r);
for(int i=0;i<NUM_KV_PAIRS;i++) {
dbt_init(&key, &kv_pairs[i].key, sizeof(kv_pairs[i].key));
dbt_init(&val, &kv_pairs[i].val, sizeof(kv_pairs[i].val));
r = src_db->put(src_db, txn, &key, &val, 0); CKERR(r);
}
r = txn->commit(txn, DB_TXN_SYNC); CKERR(r);
DB *dbs[NUM_DBS];
for (int i = 0; i < NUM_DBS; i++) {
r = db_create(&dbs[i], env, 0); CKERR(r);
char key_name[32];
sprintf(key_name, "key%d", i);
r = dbs[i]->open(dbs[i], NULL, key_name, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); CKERR(r);
dbs[i]->app_private = (void *) (intptr_t) i;
}
// -------------------------- //
if (1) test_indexer(src_db, dbs);
// -------------------------- //
for(int i=0;i<NUM_DBS;i++) {
r = dbs[i]->close(dbs[i], 0); CKERR(r);
}
r = src_db->close(src_db, 0); CKERR(r);
r = env->close(env, 0); CKERR(r);
}
// ------------ infrastructure ----------
static void do_args(int argc, char * const argv[]);
int test_main(int argc, char * const *argv) {
do_args(argc, argv);
run_test();
return 0;
}
static void do_args(int argc, char * const argv[]) {
int resultcode;
char *cmd = argv[0];
argc--; argv++;
while (argc>0) {
if (strcmp(argv[0], "-v")==0) {
verbose++;
} else if (strcmp(argv[0],"-q")==0) {
verbose--;
if (verbose<0) verbose=0;
} else if (strcmp(argv[0], "-h")==0) {
resultcode=0;
do_usage:
fprintf(stderr, "Usage:\n%s\n", cmd);
exit(resultcode);
} else {
fprintf(stderr, "Unknown arg: %s\n", argv[0]);
resultcode=1;
goto do_usage;
}
argc--;
argv++;
}
}
src/tests/hotindexer-insert-provisional.c 0 → 100644
View file @ f31a90e9
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2009 Tokutek Inc. All rights reserved."
#ident "$Id: loader-reference-test.c 23787 2010-09-14 17:31:19Z dwells $"
#include "test.h"
#include "toku_pthread.h"
#include <db.h>
#include <sys/stat.h>
DB_ENV *env;
enum {NUM_DBS=1};
enum {NUM_KV_PAIRS=3};
struct kv_pair {
int64_t key;
int64_t val;
};
struct kv_pair kv_pairs[NUM_KV_PAIRS] = {{1,4},
{2,5},
{3,6}};
static int put_multiple_generate(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_val, const DBT *src_key, const DBT *src_val, void *extra) {
src_db = src_db;
extra = extra;
uint32_t which = (uint32_t) (intptr_t) dest_db->app_private;
assert(which == 0);
// switch the key and val
dbt_init(dest_key, src_val->data, src_val->size);
dbt_init(dest_val, src_key->data, src_key->size);
// printf("dest_key.data = %d\n", *(int*)dest_key->data);
// printf("dest_val.data = %d\n", *(int*)dest_val->data);
return 0;
}
static int poll_print(void *extra, float progress) {
progress = progress;
extra = extra;
if ( verbose ) printf("poll_print %f\n", progress);
return 0;
}
static void test_indexer(DB *src, DB **dbs)
{
int r;
DB_TXN *txn;
DB_INDEXER *indexer;
uint32_t db_flags[NUM_DBS];
if ( verbose ) printf("test_indexer\n");
for(int i=0;i<NUM_DBS;i++) {
db_flags[i] = DB_NOOVERWRITE;
}
// create and initialize loader
r = env->txn_begin(env, NULL, &txn, 0);
CKERR(r);
if ( verbose ) printf("test_indexer create_indexer\n");
r = env->create_indexer(env, txn, &indexer, src, NUM_DBS, dbs, db_flags, 0);
CKERR(r);
r = indexer->set_error_callback(indexer, NULL, NULL);
CKERR(r);
r = indexer->set_poll_function(indexer, poll_print, NULL);
CKERR(r);
if ( verbose ) printf("test_indexer build\n");
r = indexer->build(indexer);
CKERR(r);
if ( verbose ) printf("test_indexer close\n");
r = indexer->close(indexer);
CKERR(r);
r = txn->commit(txn, DB_TXN_SYNC);
CKERR(r);
if ( verbose ) printf("PASS\n");
if ( verbose ) printf("test_indexer done\n");
}
char *src_name="src.db";
static void run_test(void)
{
int r;
r = system("rm -rf " ENVDIR); CKERR(r);
r = toku_os_mkdir(ENVDIR, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
r = toku_os_mkdir(ENVDIR "/log", S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
r = db_env_create(&env, 0); CKERR(r);
r = env->set_lg_dir(env, "log"); CKERR(r);
r = env->set_default_bt_compare(env, int64_dbt_cmp); CKERR(r);
r = env->set_generate_row_callback_for_put(env, put_multiple_generate); CKERR(r);
int envflags = DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN | DB_CREATE | DB_PRIVATE | DB_INIT_LOG;
r = env->open(env, ENVDIR, envflags, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
env->set_errfile(env, stderr);
//Disable auto-checkpointing
r = env->checkpointing_set_period(env, 0); CKERR(r);
DB *src_db = NULL;
r = db_create(&src_db, env, 0); CKERR(r);
r = src_db->open(src_db, NULL, src_name, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); CKERR(r);
DBT key, val;
DB_TXN *txn;
r = env->txn_begin(env, NULL, &txn, 0); CKERR(r);
for(int i=0;i<NUM_KV_PAIRS;i++) {
dbt_init(&key, &kv_pairs[i].key, sizeof(kv_pairs[i].key));
dbt_init(&val, &kv_pairs[i].val, sizeof(kv_pairs[i].val));
r = src_db->put(src_db, txn, &key, &val, 0); CKERR(r);
}
DB *dbs[NUM_DBS];
for (int i = 0; i < NUM_DBS; i++) {
r = db_create(&dbs[i], env, 0); CKERR(r);
char key_name[32];
sprintf(key_name, "key%d", i);
r = dbs[i]->open(dbs[i], NULL, key_name, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); CKERR(r);
dbs[i]->app_private = (void *) (intptr_t) i;
}
// -------------------------- //
if (1) test_indexer(src_db, dbs);
// -------------------------- //
r = txn->commit(txn, DB_TXN_SYNC); CKERR(r);
for(int i=0;i<NUM_DBS;i++) {
r = dbs[i]->close(dbs[i], 0); CKERR(r);
}
r = src_db->close(src_db, 0); CKERR(r);
r = env->close(env, 0); CKERR(r);
}
// ------------ infrastructure ----------
static void do_args(int argc, char * const argv[]);
int test_main(int argc, char * const *argv) {
do_args(argc, argv);
run_test();
return 0;
}
static void do_args(int argc, char * const argv[]) {
int resultcode;
char *cmd = argv[0];
argc--; argv++;
while (argc>0) {
if (strcmp(argv[0], "-v")==0) {
verbose++;
} else if (strcmp(argv[0],"-q")==0) {
verbose--;
if (verbose<0) verbose=0;
} else if (strcmp(argv[0], "-h")==0) {
resultcode=0;
do_usage:
fprintf(stderr, "Usage:\n%s\n", cmd);
exit(resultcode);
} else {
fprintf(stderr, "Unknown arg: %s\n", argv[0]);
resultcode=1;
goto do_usage;
}
argc--;
argv++;
}
}
src/tests/hotindexer-multiclient.c 0 → 100644
View file @ f31a90e9
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2010 Tokutek Inc. All rights reserved."
#include "test.h"
#include "toku_pthread.h"
#include <db.h>
#include <sys/stat.h>
#include "key-val.h"
enum {NUM_INDEXER_INDEXES=1};
static const int NUM_DBS = NUM_INDEXER_INDEXES + 1; // 1 for source DB
static const int NUM_ROWS = 10000;
int num_rows;
enum {MAX_CLIENTS=10};
typedef enum {FORWARD = 0, BACKWARD} Direction;
typedef enum {TXN_NONE = 0, TXN_CREATE = 1, TXN_END = 2} TxnWork;
DB_ENV *env;
float last_progress = 0.0;
static int poll_print(void *extra, float progress) {
if ( verbose ) {
if ( last_progress + 0.01 < progress ) {
printf(" progress : %3.0f%%\n", progress * 100.0);
last_progress = progress;
}
}
extra = extra;
return 0;
}
static inline uint32_t key_to_put(int iter, int offset)
{
return (uint32_t)(((iter+1) * MAX_CLIENTS) + offset);
}
static int generate_initial_table(DB *db, DB_TXN *txn, uint32_t rows)
{
struct timeval start, now;
if ( verbose ) {
printf("generate_initial_table\n");
gettimeofday(&start,0);
}
int r = 0;
DBT key, val;
uint32_t k, v, i;
// create keys of stride MAX_CLIENTS
for (i=0; i<rows; i++)
{
k = key_to_put(i, 0);
v = generate_val(k, 0);
dbt_init(&key, &k, sizeof(k));
dbt_init(&val, &v, sizeof(v));
r = db->put(db, txn, &key, &val, 0);
if ( r != 0 ) break;
}
if ( verbose ) {
gettimeofday(&now,0);
int duration = (int)(now.tv_sec - start.tv_sec);
if ( duration > 0 )
printf("generate_initial_table : %u rows in %d sec = %d rows/sec\n", rows, duration, rows/duration);
}
return r;
}
/*
* client() is a routine intended to be run in a separate thread from index creation
* - it takes a client spec which describes work to be done
* - direction : move to ever increasing or decreasing rows
* - txnwork : whether a transaction should be created or closed within the client
* (allows client transaction to start before or during index creation,
* and to close during or after index creation)
*/
typedef struct client_spec {
uint32_t num; // number of rows to write
uint32_t start; // approximate start row
int offset; // offset from stride (= MAX_CLIENTS)
Direction dir;
TxnWork txnwork;
DB_TXN *txn;
uint32_t max_inserts_per_txn; // this is for the parent transaction
DB **dbs;
int client_number;
uint32_t *flags;
} client_spec_t, *client_spec;
int client_count = 0;
static void * client(void *arg)
{
client_spec cs = arg;
client_count++;
if ( verbose ) printf("client[%d]\n", cs->client_number);
assert(cs->client_number < MAX_CLIENTS);
assert(cs->dir == FORWARD || cs->dir == BACKWARD);
int r;
if ( cs->txnwork | TXN_CREATE ) { r = env->txn_begin(env, NULL, &cs->txn, 0); CKERR(r); }
DBT key, val;
DBT dest_keys[NUM_DBS];
DBT dest_vals[NUM_DBS];
uint32_t k, v;
int n = cs->start;
for(int which=0;which<NUM_DBS;which++) {
dbt_init(&dest_keys[which], NULL, 0);
dest_keys[which].flags = DB_DBT_REALLOC;
dbt_init(&dest_vals[which], NULL, 0);
dest_vals[which].flags = DB_DBT_REALLOC;
}
int rr;
uint32_t inserts = 0;
for (uint32_t i = 0; i < cs->num; i++ ) {
DB_TXN *txn;
env->txn_begin(env, cs->txn, &txn, 0);
k = key_to_put(n, cs->offset);
v = generate_val(k, 0);
dbt_init(&key, &k, sizeof(k));
dbt_init(&val, &v, sizeof(v));
rr = env->put_multiple(env,
cs->dbs[0],
txn,
&key,
&val,
NUM_DBS,
cs->dbs, // dest dbs
dest_keys,
dest_vals,
cs->flags,
NULL);
if ( rr != 0 ) {
if ( verbose ) printf("client[%u] : put_multiple returns %d, i=%u, n=%u, key=%u\n", cs->client_number, rr, i, n, k);
r = txn->abort(txn); CKERR(r);
break;
}
r = txn->commit(txn, 0); CKERR(r);
// limit the number of inserts per parent transaction to prevent lock escalation
inserts++;
if ( inserts >= cs->max_inserts_per_txn ) {
r = cs->txn->commit(cs->txn, 0); CKERR(r);
r = env->txn_begin(env, NULL, &cs->txn, 0); CKERR(r);
inserts = 0;
}
n = ( cs->dir == FORWARD ) ? n + 1 : n - 1;
}
if ( cs->txnwork | TXN_END ) { r = cs->txn->commit(cs->txn, DB_TXN_SYNC); CKERR(r); }
if (verbose) printf("client[%d] done\n", cs->client_number);
for (int which=0; which<NUM_DBS; which++) {
toku_free(dest_keys[which].data);
toku_free(dest_vals[which].data);
}
return 0;
}
toku_pthread_t *client_threads;
client_spec_t *client_specs;
static void clients_init(DB **dbs, uint32_t *flags)
{
client_threads = toku_malloc(sizeof(toku_pthread_t) * MAX_CLIENTS);
client_specs = toku_malloc(sizeof(client_spec_t) * MAX_CLIENTS);
client_specs[0].client_number = 0;
client_specs[0].start = 0;
client_specs[0].num = num_rows;
client_specs[0].offset = -1;
client_specs[0].dir = FORWARD;
client_specs[0].txnwork = TXN_CREATE | TXN_END;
client_specs[0].txn = NULL;
client_specs[0].max_inserts_per_txn = 1000;
client_specs[0].dbs = dbs;
client_specs[0].flags = flags;
client_specs[1].client_number = 1;
client_specs[1].start = 0;
client_specs[1].num = num_rows;
client_specs[1].offset = 1;
client_specs[1].dir = FORWARD;
client_specs[1].txnwork = TXN_CREATE | TXN_END;
client_specs[1].txn = NULL;
client_specs[1].max_inserts_per_txn = 100;
client_specs[1].dbs = dbs;
client_specs[1].flags = flags;
client_specs[2].client_number = 2;
client_specs[2].start = num_rows -1;
client_specs[2].num = num_rows;
client_specs[2].offset = -2;
client_specs[2].dir = BACKWARD;
client_specs[2].txnwork = TXN_CREATE | TXN_END;
client_specs[2].txn = NULL;
client_specs[2].max_inserts_per_txn = 1000;
client_specs[2].dbs = dbs;
client_specs[2].flags = flags;
}
static void clients_cleanup(void)
{
toku_free(client_threads); client_threads = NULL;
toku_free(client_specs); client_specs = NULL;
}
// verify results
// - read the keys in the primary table, then calculate what keys should exist
// in the other DB. Read the other table to verify.
static int check_results(DB *src, DB *db)
{
int r;
int fail = 0;
int clients = client_count;
int max_rows = ( clients + 1 ) * num_rows;
unsigned int *db_keys = (unsigned int *) toku_malloc(max_rows * sizeof (unsigned int));
DBT key, val;
unsigned int k=0, v=0;
dbt_init(&key, &k, sizeof(unsigned int));
dbt_init(&val, &v, sizeof(unsigned int));
DB_TXN *txn;
r = env->txn_begin(env, NULL, &txn, 0); CKERR(r);
DBC *cursor;
r = src->cursor(src, txn, &cursor, 0); CKERR(r);
int which = *(uint32_t*)db->app_private;
// scan the primary table,
// calculate the expected keys in 'db'
int row = 0;
while ( r != DB_NOTFOUND ) {
r = cursor->c_get(cursor, &key, &val, DB_NEXT);
if ( r != DB_NOTFOUND ) {
k = *((uint32_t *)(key.data));
db_keys[row] = twiddle32(k, which);
row++;
}
}
if ( verbose ) printf("primary table scanned, contains %d rows\n", row);
int primary_rows = row;
r = cursor->c_close(cursor); CKERR(r);
// sort the expected keys
qsort(db_keys, primary_rows, sizeof (unsigned int), uint_cmp);
if ( verbose > 1 ) {
for(int i=0;i<primary_rows;i++) {
printf("primary table[%u] = %u\n", i, db_keys[i]);
}
}
// scan the indexer-created DB, comparing keys with expected keys
// - there should be exactly 'primary_rows' in the new index
r = db->cursor(db, txn, &cursor, 0); CKERR(r);
for (int i=0;i<primary_rows;i++) {
r = cursor->c_get(cursor, &key, &val, DB_NEXT);
if ( r == DB_NOTFOUND ) {
printf("scan of index finds last row is %d\n", i);
}
CKERR(r);
k = *((uint32_t *)(key.data));
if ( db_keys[i] != k ) {
if ( verbose ) printf("ERROR expecting key %10u for row %d, found key = %10u\n", db_keys[i],i,k);
fail = 1;
goto check_results_error;
}
}
// next cursor op should return DB_NOTFOUND
r = cursor->c_get(cursor, &key, &val, DB_NEXT);
assert(r == DB_NOTFOUND);
// we're done - cleanup and close
check_results_error:
r = cursor->c_close(cursor); CKERR(r);
toku_free(db_keys);
r = txn->commit(txn, 0); CKERR(r);
if ( verbose ) {
if ( fail ) printf("check_results : fail\n");
else printf("check_results : pass\n");
}
return fail;
}
static void test_indexer(DB *src, DB **dbs)
{
int r;
DB_TXN *txn;
DB_INDEXER *indexer;
uint32_t db_flags[NUM_DBS];
if ( verbose ) printf("test_indexer\n");
for(int i=0;i<NUM_DBS;i++) {
db_flags[i] = DB_YESOVERWRITE;
}
clients_init(dbs, db_flags);
// create and initialize indexer
r = env->txn_begin(env, NULL, &txn, 0);
CKERR(r);
if ( verbose ) printf("test_indexer create_indexer\n");
r = env->create_indexer(env, txn, &indexer, src, NUM_DBS-1, &dbs[1], db_flags, 0);
CKERR(r);
r = indexer->set_error_callback(indexer, NULL, NULL);
CKERR(r);
r = indexer->set_poll_function(indexer, poll_print, NULL);
CKERR(r);
// start threads doing additional inserts - no lock issues since indexer already created
r = toku_pthread_create(&client_threads[0], 0, client, (void *)&client_specs[0]); CKERR(r);
r = toku_pthread_create(&client_threads[1], 0, client, (void *)&client_specs[1]); CKERR(r);
// r = toku_pthread_create(&client_threads[2], 0, client, (void *)&client_specs[2]); CKERR(r);
struct timeval start, now;
if ( verbose ) {
printf("test_indexer build\n");
gettimeofday(&start,0);
}
r = indexer->build(indexer);
CKERR(r);
if ( verbose ) {
gettimeofday(&now,0);
int duration = (int)(now.tv_sec - start.tv_sec);
if ( duration > 0 )
printf("test_indexer build : sec = %d\n", duration);
}
void *t0; r = toku_pthread_join(client_threads[0], &t0); CKERR(r);
void *t1; r = toku_pthread_join(client_threads[1], &t1); CKERR(r);
// void *t2; r = toku_pthread_join(client_threads[2], &t2); CKERR(r);
if ( verbose ) printf("test_indexer close\n");
r = indexer->close(indexer);
CKERR(r);
r = txn->commit(txn, DB_TXN_SYNC);
CKERR(r);
clients_cleanup();
if ( verbose ) printf("check_results\n");
r = check_results(src, dbs[1]);
CKERR(r);
if ( verbose && (r == 0)) printf("PASS\n");
if ( verbose && (r == 0)) printf("test_indexer done\n");
}
static void run_test(void)
{
int r;
r = system("rm -rf " ENVDIR); CKERR(r);
r = toku_os_mkdir(ENVDIR, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
r = toku_os_mkdir(ENVDIR "/log", S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
r = db_env_create(&env, 0); CKERR(r);
r = env->set_lg_dir(env, "log"); CKERR(r);
r = env->set_default_bt_compare(env, uint_dbt_cmp); CKERR(r);
generate_permute_tables();
r = env->set_generate_row_callback_for_put(env, put_multiple_generate); CKERR(r);
int envflags = DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN | DB_CREATE | DB_PRIVATE | DB_INIT_LOG;
r = env->open(env, ENVDIR, envflags, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
env->set_errfile(env, stderr);
r = env->checkpointing_set_period(env, 0); CKERR(r);
DBT desc;
dbt_init(&desc, "foo", sizeof("foo"));
int ids[MAX_DBS];
DB *dbs[MAX_DBS];
for (int i = 0; i < NUM_DBS; i++) {
ids[i] = i;
r = db_create(&dbs[i], env, 0); CKERR(r);
r = dbs[i]->set_descriptor(dbs[i], 1, &desc); CKERR(r);
dbs[i]->app_private = &ids[i];
char key_name[32];
sprintf(key_name, "key%d", i);
r = dbs[i]->open(dbs[i], NULL, key_name, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); CKERR(r);
}
// generate the src DB (do not use put_multiple)
DB_TXN *txn;
r = env->txn_begin(env, NULL, &txn, 0); CKERR(r);
r = generate_initial_table(dbs[0], txn, num_rows); CKERR(r);
r = txn->commit(txn, DB_TXN_SYNC); CKERR(r);
// -------------------------- //
if (1) test_indexer(dbs[0], dbs);
// -------------------------- //
for(int i=0;i<NUM_DBS;i++) {
r = dbs[i]->close(dbs[i], 0); CKERR(r);
}
r = env->close(env, 0); CKERR(r);
}
// ------------ infrastructure ----------
static inline void
do_args (int argc, char * const argv[]) {
const char *progname=argv[0];
num_rows = NUM_ROWS;
argc--; argv++;
while (argc>0) {
if (strcmp(argv[0],"-v")==0) {
verbose++;
} else if (strcmp(argv[0],"-q")==0) {
verbose=0;
} else if (strcmp(argv[0],"-r")==0) {
argc--; argv++;
num_rows = atoi(argv[0]);
} else {
fprintf(stderr, "Usage:\n %s [-v] [-q] [-r rows]\n", progname);
exit(1);
}
argc--; argv++;
}
}
int test_main(int argc, char * const *argv) {
do_args(argc, argv);
run_test();
return 0;
}
/*
* Please ignore this code - I don't think I'm going to use it, but I don't want to lose it
* I will delete this later - Dave
if ( rr != 0 ) { // possible lock deadlock
if (verbose > 1) {
printf("client[%u] : put_multiple returns %d, i=%u, n=%u, key=%u\n", cs->client_number, rr, i, n, k);
if ( verbose > 2 ) print_engine_status(env);
}
// abort the transaction, freeing up locks associated with previous put_multiples
if ( verbose > 1 ) printf("start txn abort\n");
r = txn->abort(txn); CKERR(r);
if ( verbose > 1 ) printf(" txn aborted\n");
sleep(2 + cs->client_number);
// now retry, waiting until the deadlock resolves itself
r = env->txn_begin(env, cs->txn, &txn, 0); CKERR(r);
if ( verbose > 1 ) printf("txn begin\n");
while ( rr != 0 ) {
rr = env->put_multiple(env,
cs->dbs[0],
txn,
&key,
&val,
NUM_DBS,
cs->dbs, // dest dbs
dest_keys,
dest_vals,
cs->flags,
NULL);
if ( rr != 0 ) {
if ( verbose ) printf("client[%u] : put_multiple returns %d, i=%u, n=%u, key=%u\n", cs->client_number, rr, i, n, k);
if ( verbose ) printf("start txn abort\n");
r = txn->abort(txn); CKERR(r);
if ( verbose ) printf(" txn aborted\n");
sleep(2 + cs->client_number);
r = env->txn_begin(env, cs->txn, &txn, 0); CKERR(r);
if ( verbose ) printf("txn begin\n");
}
}
*/
src/tests/hotindexer-nested-insert-committed.c 0 → 100644
View file @ f31a90e9
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2009 Tokutek Inc. All rights reserved."
#ident "$Id: loader-reference-test.c 23787 2010-09-14 17:31:19Z dwells $"
#include "test.h"
#include "toku_pthread.h"
#include <db.h>
#include <sys/stat.h>
DB_ENV *env;
enum {NUM_DBS=1};
enum {NUM_KV_PAIRS=3};
struct kv_pair {
int64_t key;
int64_t val;
};
struct kv_pair kv_pairs[NUM_KV_PAIRS] = {{1,4},
{2,5},
{3,6}};
static int put_multiple_generate(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_val, const DBT *src_key, const DBT *src_val, void *extra) {
src_db = src_db;
extra = extra;
uint32_t which = (uint32_t) (intptr_t) dest_db->app_private;
assert(which == 0);
// switch the key and val
dbt_init(dest_key, src_val->data, src_val->size);
dbt_init(dest_val, src_key->data, src_key->size);
// printf("dest_key.data = %d\n", *(int*)dest_key->data);
// printf("dest_val.data = %d\n", *(int*)dest_val->data);
return 0;
}
static int poll_print(void *extra, float progress) {
progress = progress;
extra = extra;
if ( verbose ) printf("poll_print %f\n", progress);
return 0;
}
static void test_indexer(DB *src, DB **dbs)
{
int r;
DB_TXN *txn;
DB_INDEXER *indexer;
uint32_t db_flags[NUM_DBS];
if ( verbose ) printf("test_indexer\n");
for(int i=0;i<NUM_DBS;i++) {
db_flags[i] = DB_NOOVERWRITE;
}
// create and initialize loader
r = env->txn_begin(env, NULL, &txn, 0);
CKERR(r);
if ( verbose ) printf("test_indexer create_indexer\n");
r = env->create_indexer(env, txn, &indexer, src, NUM_DBS, dbs, db_flags, 0);
CKERR(r);
r = indexer->set_error_callback(indexer, NULL, NULL);
CKERR(r);
r = indexer->set_poll_function(indexer, poll_print, NULL);
CKERR(r);
if ( verbose ) printf("test_indexer build\n");
r = indexer->build(indexer);
CKERR(r);
if ( verbose ) printf("test_indexer close\n");
r = indexer->close(indexer);
CKERR(r);
r = txn->commit(txn, DB_TXN_SYNC);
CKERR(r);
if ( verbose ) printf("PASS\n");
if ( verbose ) printf("test_indexer done\n");
}
char *src_name="src.db";
static void run_test(void)
{
int r;
r = system("rm -rf " ENVDIR); CKERR(r);
r = toku_os_mkdir(ENVDIR, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
r = toku_os_mkdir(ENVDIR "/log", S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
r = db_env_create(&env, 0); CKERR(r);
r = env->set_lg_dir(env, "log"); CKERR(r);
r = env->set_default_bt_compare(env, int64_dbt_cmp); CKERR(r);
r = env->set_generate_row_callback_for_put(env, put_multiple_generate); CKERR(r);
int envflags = DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN | DB_CREATE | DB_PRIVATE | DB_INIT_LOG;
r = env->open(env, ENVDIR, envflags, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
env->set_errfile(env, stderr);
//Disable auto-checkpointing
r = env->checkpointing_set_period(env, 0); CKERR(r);
DB *src_db = NULL;
r = db_create(&src_db, env, 0); CKERR(r);
r = src_db->open(src_db, NULL, src_name, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); CKERR(r);
DBT key, val;
DB_TXN *txn0 = NULL;
r = env->txn_begin(env, NULL, &txn0, 0); CKERR(r);
DB_TXN *txn;
r = env->txn_begin(env, txn0, &txn, 0); CKERR(r);
for(int i=0;i<NUM_KV_PAIRS;i++) {
dbt_init(&key, &kv_pairs[i].key, sizeof(kv_pairs[i].key));
dbt_init(&val, &kv_pairs[i].val, sizeof(kv_pairs[i].val));
r = src_db->put(src_db, txn, &key, &val, 0); CKERR(r);
}
r = txn->commit(txn, DB_TXN_SYNC); CKERR(r);
r = txn0->commit(txn0, DB_TXN_SYNC); CKERR(r);
DB *dbs[NUM_DBS];
for (int i = 0; i < NUM_DBS; i++) {
r = db_create(&dbs[i], env, 0); CKERR(r);
char key_name[32];
sprintf(key_name, "key%d", i);
r = dbs[i]->open(dbs[i], NULL, key_name, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); CKERR(r);
dbs[i]->app_private = (void *) (intptr_t) i;
}
// -------------------------- //
if (1) test_indexer(src_db, dbs);
// -------------------------- //
for(int i=0;i<NUM_DBS;i++) {
r = dbs[i]->close(dbs[i], 0); CKERR(r);
}
r = src_db->close(src_db, 0); CKERR(r);
r = env->close(env, 0); CKERR(r);
}
// ------------ infrastructure ----------
static void do_args(int argc, char * const argv[]);
int test_main(int argc, char * const *argv) {
do_args(argc, argv);
run_test();
return 0;
}
static void do_args(int argc, char * const argv[]) {
int resultcode;
char *cmd = argv[0];
argc--; argv++;
while (argc>0) {
if (strcmp(argv[0], "-v")==0) {
verbose++;
} else if (strcmp(argv[0],"-q")==0) {
verbose--;
if (verbose<0) verbose=0;
} else if (strcmp(argv[0], "-h")==0) {
resultcode=0;
do_usage:
fprintf(stderr, "Usage:\n%s\n", cmd);
exit(resultcode);
} else {
fprintf(stderr, "Unknown arg: %s\n", argv[0]);
resultcode=1;
goto do_usage;
}
argc--;
argv++;
}
}
src/tests/hotindexer-put-multiple.c 0 → 100644
View file @ f31a90e9
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2009 Tokutek Inc. All rights reserved."
#ident "$Id: loader-reference-test.c 23787 2010-09-14 17:31:19Z dwells $"
#include "test.h"
#include "toku_pthread.h"
#include <db.h>
#include <sys/stat.h>
DB_ENV *env;
enum {NUM_DBS=1};
enum {NUM_KV_PAIRS=3};
struct kv_pair {
int64_t key;
int64_t val;
};
struct kv_pair kv_pairs[NUM_KV_PAIRS] = {{1,4},
{2,5},
{3,6}};
static int put_multiple_generate(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_val, const DBT *src_key, const DBT *src_val, void *extra) {
src_db = src_db;
extra = extra;
uint32_t which = (uint32_t) (intptr_t) dest_db->app_private;
if (which == NUM_DBS) {
// primary
dbt_init(dest_key, src_key->data, src_key->size);
dbt_init(dest_val, src_val->data, src_val->size);
} else {
// secondaries: switch the key and val
dbt_init(dest_key, src_val->data, src_val->size);
dbt_init(dest_val, src_key->data, src_key->size);
}
// printf("dest_key.data = %d\n", *(int*)dest_key->data);
// printf("dest_val.data = %d\n", *(int*)dest_val->data);
return 0;
}
static int poll_print(void *extra, float progress) {
progress = progress;
extra = extra;
if ( verbose ) printf("poll_print %f\n", progress);
return 0;
}
char *src_name="src.db";
static void run_test(void)
{
int r;
r = system("rm -rf " ENVDIR); CKERR(r);
r = toku_os_mkdir(ENVDIR, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
r = toku_os_mkdir(ENVDIR "/log", S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
r = db_env_create(&env, 0); CKERR(r);
r = env->set_lg_dir(env, "log"); CKERR(r);
r = env->set_default_bt_compare(env, int64_dbt_cmp); CKERR(r);
r = env->set_generate_row_callback_for_put(env, put_multiple_generate); CKERR(r);
int envflags = DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN | DB_CREATE | DB_PRIVATE | DB_INIT_LOG;
r = env->open(env, ENVDIR, envflags, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
env->set_errfile(env, stderr);
//Disable auto-checkpointing
r = env->checkpointing_set_period(env, 0); CKERR(r);
DB *src_db = NULL;
r = db_create(&src_db, env, 0); CKERR(r);
r = src_db->open(src_db, NULL, src_name, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); CKERR(r);
src_db->app_private = (void *) NUM_DBS;
DB_TXN *txn;
r = env->txn_begin(env, NULL, &txn, 0); CKERR(r);
for(int i=0;i<NUM_KV_PAIRS;i++) {
DBT key, val;
dbt_init(&key, &kv_pairs[i].key, sizeof(kv_pairs[i].key));
dbt_init(&val, &kv_pairs[i].val, sizeof(kv_pairs[i].val));
r = src_db->put(src_db, txn, &key, &val, 0); CKERR(r);
}
DB *dbs[NUM_DBS];
for (int i = 0; i < NUM_DBS; i++) {
r = db_create(&dbs[i], env, 0); CKERR(r);
char key_name[32];
sprintf(key_name, "key%d", i);
r = dbs[i]->open(dbs[i], NULL, key_name, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); CKERR(r);
dbs[i]->app_private = (void *) (intptr_t) i;
}
DB_TXN *hottxn;
r = env->txn_begin(env, NULL, &hottxn, 0);
CKERR(r);
DB_INDEXER *indexer;
r = env->create_indexer(env, hottxn, &indexer, src_db, NUM_DBS, dbs, NULL, 0);
CKERR(r);
r = indexer->set_error_callback(indexer, NULL, NULL);
CKERR(r);
r = indexer->set_poll_function(indexer, poll_print, NULL);
CKERR(r);
// setup putm
DB *putm_dbs[NUM_DBS+1];
for (int i = 0; i < NUM_DBS; i++)
putm_dbs[i] = dbs[i];
putm_dbs[NUM_DBS] = src_db;
DBT putm_keys[NUM_DBS+1], putm_vals[NUM_DBS+1];
uint32_t putm_flags[NUM_DBS+1];
for (int i = 0; i < NUM_DBS+1; i++)
putm_flags[i] = 0;
DBT prikey; int64_t pk;
dbt_init(&prikey, &pk, sizeof pk);
DBT prival; int64_t pv;
dbt_init(&prival, &pv, sizeof pv);
// putm (8,9)
pk = 8; pv = 9;
r = env->put_multiple(env, src_db, txn, &prikey, &prival, NUM_DBS+1, putm_dbs, putm_keys, putm_vals, putm_flags, NULL);
CKERR(r);
r = indexer->build(indexer);
CKERR(r);
// putm (9, 10)
pk = 9; pv = 10;
r = env->put_multiple(env, src_db, txn, &prikey, &prival, NUM_DBS+1, putm_dbs, putm_keys, putm_vals, putm_flags, NULL);
CKERR(r);
r = indexer->close(indexer);
CKERR(r);
r = hottxn->commit(hottxn, DB_TXN_SYNC);
CKERR(r);
r = txn->commit(txn, DB_TXN_SYNC); CKERR(r);
for(int i=0;i<NUM_DBS;i++) {
r = dbs[i]->close(dbs[i], 0); CKERR(r);
}
r = src_db->close(src_db, 0); CKERR(r);
r = env->close(env, 0); CKERR(r);
}
// ------------ infrastructure ----------
static void do_args(int argc, char * const argv[]);
int test_main(int argc, char * const *argv) {
do_args(argc, argv);
run_test();
return 0;
}
static void do_args(int argc, char * const argv[]) {
int resultcode;
char *cmd = argv[0];
argc--; argv++;
while (argc>0) {
if (strcmp(argv[0], "-v")==0) {
verbose++;
} else if (strcmp(argv[0],"-q")==0) {
verbose--;
if (verbose<0) verbose=0;
} else if (strcmp(argv[0], "-h")==0) {
resultcode=0;
do_usage:
fprintf(stderr, "Usage:\n%s\n", cmd);
exit(resultcode);
} else {
fprintf(stderr, "Unknown arg: %s\n", argv[0]);
resultcode=1;
goto do_usage;
}
argc--;
argv++;
}
}
src/tests/hotindexer-simple-abort.c 0 → 100644
View file @ f31a90e9
#include "test.h"
static int
put_callback(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_data, const DBT *src_key, const DBT *src_data, void *extra) {
dest_db = dest_db; src_db = src_db; dest_key = dest_key; dest_data = dest_data; src_key = src_key; src_data = src_data;
lazy_assert(src_db != NULL && dest_db != NULL);
lazy_assert(extra == NULL);
dest_key->data = src_data->data;
dest_key->size = src_data->size;
dest_data->size = 0;
return 0;
}
static void
run_test(void) {
int r;
DB_ENV *env = NULL;
r = db_env_create(&env, 0); assert_zero(r);
r = env->set_generate_row_callback_for_put(env, put_callback); assert_zero(r);
r = env->open(env, ENVDIR, DB_INIT_MPOOL|DB_CREATE|DB_THREAD |DB_INIT_LOCK|DB_INIT_LOG|DB_INIT_TXN|DB_PRIVATE, S_IRWXU+S_IRWXG+S_IRWXO); assert_zero(r);
DB *src_db = NULL;
r = db_create(&src_db, env, 0); assert_zero(r);
r = src_db->open(src_db, NULL, "0.tdb", NULL, DB_BTREE, DB_AUTO_COMMIT+DB_CREATE, S_IRWXU+S_IRWXG+S_IRWXO); assert_zero(r);
DB *dest_db = NULL;
r = db_create(&dest_db, env, 0); assert_zero(r);
r = dest_db->open(dest_db, NULL, "1.tdb", NULL, DB_BTREE, DB_AUTO_COMMIT+DB_CREATE, S_IRWXU+S_IRWXG+S_IRWXO); assert_zero(r);
DB_TXN *txn = NULL;
r = env->txn_begin(env, NULL, &txn, 0); assert_zero(r);
DB_INDEXER *indexer = NULL;
r = env->create_indexer(env, txn, &indexer, src_db, 1, &dest_db, NULL, 0); assert_zero(r);
r = indexer->abort(indexer); assert_zero(r);
r = txn->abort(txn); assert_zero(r);
r = src_db->close(src_db, 0); assert_zero(r);
r = dest_db->close(dest_db, 0); assert_zero(r);
r = env->close(env, 0); assert_zero(r);
}
int
test_main(int argc, char * const argv[]) {
int r;
// parse_args(argc, argv);
for (int i = 1; i < argc; i++) {
char * const arg = argv[i];
if (strcmp(arg, "-v") == 0) {
verbose++;
continue;
}
if (strcmp(arg, "-q") == 0) {
verbose = 0;
continue;
}
}
r = system("rm -rf " ENVDIR); assert_zero(r);
r = toku_os_mkdir(ENVDIR, S_IRWXU+S_IRWXG+S_IRWXO); assert_zero(r);
run_test();
return 0;
}
src/tests/hotindexer-undo-do-test.c 0 → 100644
View file @ f31a90e9
// test the hotindexer undo do function
// read a description of the live transactions and a leafentry from a test file, run the undo do function,
// and print out the actions taken by the undo do function while processing the leafentry
#include "test.h"
#include <stdbool.h>
#include "tokuconst.h"
#include "brttypes.h"
#include "omt.h"
#include "mempool.h"
#include "leafentry.h"
#include "ule.h"
#include "ule-internal.h"
#include "le-cursor.h"
#include "indexer-internal.h"
#include "xids-internal.h"
struct live {
int n;
int o;
TXNID *xids;
};
static void
live_init(struct live *live) {
live->n = live->o = 0;
live->xids = NULL;
}
static void
live_destroy(struct live *live) {
toku_free(live->xids);
}
static void
live_add(struct live *live, TXNID xid) {
if (live->o >= live->n) {
int newn = live->n == 0 ? 1 : live->n * 2;
live->xids = (TXNID *) toku_realloc(live->xids, newn * sizeof (TXNID));
resource_assert(live->xids);
live->n = newn;
}
live->xids[live->o++] = xid;
}
static int
is_live(struct live *live, TXNID xid) {
int r = 0;
for (int i = 0; i < live->o; i++) {
if (live->xids[i] == xid) {
r = 1;
break;
}
}
return r;
}
// live transaction ID set
struct live live_xids;
static void
uxr_init(UXR uxr, uint8_t type, void *val, uint32_t vallen, TXNID xid) {
uxr->type = type;
uxr->valp = toku_malloc(vallen); resource_assert(uxr->valp);
memcpy(uxr->valp, val, vallen);
uxr->vallen = vallen;
uxr->xid = xid;
}
static void
uxr_destroy(UXR uxr) {
toku_free(uxr->valp);
uxr->valp = NULL;
}
static ULE
ule_init(ULE ule) {
ule->num_puxrs = 0;
ule->num_cuxrs = 0;
ule->keyp = NULL;
ule->keylen = 0;
ule->uxrs = ule->uxrs_static;
return ule;
}
static void
ule_set_key(ULE ule, void *key, uint32_t keylen) {
ule->keyp = toku_realloc(ule->keyp, keylen);
memcpy(ule->keyp, key, keylen);
ule->keylen = keylen;
}
static void
ule_destroy(ULE ule) {
for (unsigned int i = 0; i < ule->num_cuxrs + ule->num_puxrs; i++)
uxr_destroy(&ule->uxrs[i]);
toku_free(ule->keyp);
ule->keyp = NULL;
}
static void
ule_add_provisional(ULE ule, UXR uxr) {
invariant(ule->num_cuxrs + ule->num_puxrs + 1 <= MAX_TRANSACTION_RECORDS*2);
ule->uxrs[ule->num_cuxrs + ule->num_puxrs] = *uxr;
ule->num_puxrs++;
}
static void
ule_add_committed(ULE ule, UXR uxr) {
lazy_assert(ule->num_puxrs == 0);
invariant(ule->num_cuxrs + 1 <= MAX_TRANSACTION_RECORDS*2);
ule->uxrs[ule->num_cuxrs] = *uxr;
ule->num_cuxrs++;
}
static ULE
ule_create(void) {
ULE ule = (ULE) toku_calloc(1, sizeof (ULE_S)); resource_assert(ule);
if (ule)
ule_init(ule);
return ule;
}
static void
ule_free(ULE ule) {
ule_destroy(ule);
toku_free(ule);
}
static void
print_xids(XIDS xids) {
printf("[");
if (xids->num_xids == 0)
printf("0");
else {
for (int i = 0; i < xids->num_xids; i++) {
printf("%lu", xids->ids[i]);
if (i+1 < xids->num_xids)
printf(",");
}
}
printf("] ");
}
static void
print_dbt(DBT *dbt) {
printf("%.*s ", dbt->size, (char *) dbt->data);
}
static int
put_callback(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_data, const DBT *src_key, const DBT *src_data, void *extra) {
dest_db = dest_db; src_db = src_db; dest_key = dest_key; dest_data = dest_data; src_key = src_key; src_data = src_data;
lazy_assert(src_db != NULL && dest_db != NULL);
lazy_assert(extra == NULL);
switch (dest_key->flags) {
case 0:
dest_key->data = src_data->data;
dest_key->size = src_data->size;
break;
case DB_DBT_REALLOC:
dest_key->data = toku_realloc(dest_key->data, src_data->size);
memcpy(dest_key->data, src_data->data, src_data->size);
dest_key->size = src_data->size;
break;
default:
lazy_assert(0);
}
switch (dest_data->flags) {
case 0:
lazy_assert(0);
break;
case DB_DBT_REALLOC:
dest_data->data = toku_realloc(dest_data->data, src_key->size);
memcpy(dest_data->data, src_key->data, src_key->size);
dest_data->size = src_key->size;
break;
default:
lazy_assert(0);
}
return 0;
}
static int
test_is_xid_live(DB_INDEXER *indexer, TXNID xid) {
indexer = indexer;
int r = is_live(&live_xids, xid);
return r;
}
static int
test_maybe_lock_provisional_key(DB_INDEXER *indexer, TXNID xid, DB *hotdb, DBT *key) {
hotdb = hotdb;
if (test_is_xid_live(indexer, xid)) {
printf("lock [%lu] ", xid);
print_dbt(key);
printf("\n");
}
return 0;
}
static int
test_delete_provisional(DB_INDEXER *indexer, DB *hotdb, DBT *hotkey, XIDS xids) {
indexer = indexer; hotdb = hotdb;
printf("delete_provisional ");
print_xids(xids);
print_dbt(hotkey);
printf("\n");
return 0;
}
static int
test_delete_committed(DB_INDEXER *indexer, DB *hotdb, DBT *hotkey, XIDS xids) {
indexer = indexer; hotdb = hotdb;
printf("delete_committed ");
print_xids(xids);
print_dbt(hotkey);
printf("\n");
return 0;
}
static int
test_insert_provisional(DB_INDEXER *indexer, DB *hotdb, DBT *hotkey, DBT *hotval, XIDS xids) {
indexer = indexer; hotdb = hotdb;
printf("insert_provisional ");
print_xids(xids);
print_dbt(hotkey);
print_dbt(hotval);
printf("\n");
return 0;
}
static int
test_insert_committed(DB_INDEXER *indexer, DB *hotdb, DBT *hotkey, DBT *hotval, XIDS xids) {
indexer = indexer; hotdb = hotdb;
printf("insert_committed ");
print_xids(xids);
print_dbt(hotkey);
print_dbt(hotval);
printf("\n");
return 0;
}
static int
test_commit_any(DB_INDEXER *indexer, DB *hotdb, DBT *hotkey, XIDS xids) {
indexer = indexer; hotdb = hotdb;
printf("commit_any ");
print_xids(xids);
print_dbt(hotkey);
printf("\n");
return 0;
}
static int
split_fields(char *line, char *fields[], int maxfields) {
int i;
for (i = 0; i < maxfields; i++, line = NULL) {
fields[i] = strtok(line, " ");
if (fields[i] == NULL)
break;
}
return i;
}
static int
read_line(char **line_ptr, size_t *len_ptr, FILE *f) {
char *line = *line_ptr;
size_t len = 0;
bool in_comment = false;
while (1) {
int c = fgetc(f);
if (c == EOF)
break;
else if (c == '\n') {
in_comment = false;
if (len > 0)
break;
} else {
if (c == '#')
in_comment = true;
if (!in_comment) {
line = toku_realloc(line, len+1);
line[len++] = c;
}
}
}
if (len > 0) {
line = toku_realloc(line, len+1);
line[len] = '\0';
}
*line_ptr = line;
*len_ptr = len;
return len == 0 ? -1 : 0;
}
static int
read_test(char *testname, ULE ule) {
int r = 0;
FILE *f = fopen(testname, "r");
if (f) {
char *line = NULL;
size_t len = 0;
while (read_line(&line, &len, f) != -1) {
// printf("%s", line);
const int maxfields = 8;
char *fields[maxfields];
int nfields = split_fields(line, fields, maxfields);
// for (int i = 0; i < nfields; i++); printf("%s ", fields[i]); printf("\n");
if (nfields < 1)
continue;
// live xid...
if (strcmp(fields[0], "live") == 0) {
for (int i = 1; i < nfields; i++)
live_add(&live_xids, atoll(fields[i]));
continue;
}
// key KEY
if (strcmp(fields[0], "key") == 0 && nfields == 2) {
ule_set_key(ule, fields[1], strlen(fields[1]));
continue;
}
// insert committed|provisional XID DATA
if (strcmp(fields[0], "insert") == 0 && nfields == 4) {
UXR_S uxr_s;
uxr_init(&uxr_s, XR_INSERT, fields[3], strlen(fields[3]), atoll(fields[2]));
if (fields[1][0] == 'p')
ule_add_provisional(ule, &uxr_s);
if (fields[1][0] == 'c')
ule_add_committed(ule, &uxr_s);
continue;
}
// delete committed|provisional XID
if (strcmp(fields[0], "delete") == 0 && nfields == 3) {
UXR_S uxr_s;
uxr_init(&uxr_s, XR_DELETE, NULL, 0, atoll(fields[2]));
if (fields[1][0] == 'p')
ule_add_provisional(ule, &uxr_s);
if (fields[1][0] == 'c')
ule_add_committed(ule, &uxr_s);
continue;
}
// placeholder XID
if (strcmp(fields[0], "placeholder") == 0 && nfields == 2) {
UXR_S uxr_s;
uxr_init(&uxr_s, XR_PLACEHOLDER, NULL, 0, atoll(fields[1]));
ule_add_provisional(ule, &uxr_s);
continue;
}
printf("%s???\n", line);
r = EINVAL;
}
toku_free(line);
fclose(f);
} else {
r = errno;
printf("fopen %s errno=%d\n", testname, errno);
}
return r;
}
static void
run_test(char *envdir, char *testname) {
if (verbose)
printf("%s\n", testname);
live_init(&live_xids);
int r;
DB_ENV *env = NULL;
r = db_env_create(&env, 0); assert_zero(r);
r = env->set_generate_row_callback_for_put(env, put_callback); assert_zero(r);
r = env->open(env, envdir, DB_INIT_MPOOL|DB_CREATE|DB_THREAD |DB_INIT_LOCK|DB_INIT_LOG|DB_INIT_TXN|DB_PRIVATE, S_IRWXU+S_IRWXG+S_IRWXO); assert_zero(r);
DB *src_db = NULL;
r = db_create(&src_db, env, 0); assert_zero(r);
r = src_db->open(src_db, NULL, "0.tdb", NULL, DB_BTREE, DB_AUTO_COMMIT+DB_CREATE, S_IRWXU+S_IRWXG+S_IRWXO); assert_zero(r);
DB *dest_db = NULL;
r = db_create(&dest_db, env, 0); assert_zero(r);
r = dest_db->open(dest_db, NULL, "1.tdb", NULL, DB_BTREE, DB_AUTO_COMMIT+DB_CREATE, S_IRWXU+S_IRWXG+S_IRWXO); assert_zero(r);
DB_TXN *txn = NULL;
r = env->txn_begin(env, NULL, &txn, 0); assert_zero(r);
DB_INDEXER *indexer = NULL;
r = env->create_indexer(env, txn, &indexer, src_db, 1, &dest_db, NULL, 0); assert_zero(r);
// set test callbacks
indexer->i->test_is_xid_live = test_is_xid_live;
indexer->i->test_maybe_lock_provisional_key = test_maybe_lock_provisional_key;
indexer->i->test_delete_provisional = test_delete_provisional;
indexer->i->test_delete_committed = test_delete_committed;
indexer->i->test_insert_provisional = test_insert_provisional;
indexer->i->test_insert_committed = test_insert_committed;
indexer->i->test_commit_any = test_commit_any;
// create a ule
ULE ule = ule_create();
ule_init(ule);
// read the test
r = read_test(testname, ule); assert_zero(r);
r = indexer->i->undo_do(indexer, dest_db, ule); assert_zero(r);
ule_free(ule);
r = indexer->close(indexer); assert_zero(r);
r = txn->abort(txn); assert_zero(r);
r = src_db->close(src_db, 0); assert_zero(r);
r = dest_db->close(dest_db, 0); assert_zero(r);
r = env->close(env, 0); assert_zero(r);
live_destroy(&live_xids);
}
int
test_main(int argc, char * const argv[]) {
int r;
// parse_args(argc, argv);
int i;
for (i = 1; i < argc; i++) {
char * const arg = argv[i];
if (strcmp(arg, "-v") == 0) {
verbose++;
continue;
}
if (strcmp(arg, "-q") == 0) {
verbose = 0;
continue;
}
break;
}
for ( ; i < argc; i++) {
char *testname = argv[i];
char envdir[strlen(ENVDIR) + 1 + 32 + 1];
sprintf(envdir, "%s.%d", ENVDIR, toku_os_getpid());
char syscmd[32 + strlen(envdir)];
sprintf(syscmd, "rm -rf %s", envdir);
r = system(syscmd); assert_zero(r);
r = toku_os_mkdir(envdir, S_IRWXU+S_IRWXG+S_IRWXO); assert_zero(r);
run_test(envdir, testname);
}
return 0;
}
src/tests/hotindexer-undo-do-tests/README 0 → 100644
View file @ f31a90e9
= Hotindexer undo do testing =
The hotindexer undo do function is tested by feeding it a leafentry,
capturing the actions taken on the hot dictionary,
and comparing the actions with the precomputed expected actions.
The test passes if the actions match the expected actions.
Each test is described by a .test file and a .result file.
The .test file describes the set of transactions that are live
when the undo do function is called as well as the transaction records
that comprise a leaf entry.
The .result file describes the actions taken by the undo do function
when processing the leaf entry.
= Contents of a *.test =
Comments begin with '#'.
An <XIDLIST> is a list of transaction id's separated by a SPACE.
<XIDS> is a stack of transaction id's separated by a COMMA.
An <XID> is a 64 bit number.
A <KEY> is a string.
A <VALUE> is a string.
The field separator is a single SPACE.
== set the leaf entry key ==
key <KEY>
== add transaction IDs to the live transaction set ==
live <XIDLIST>
the live transaction set is initially empty
== push a delete transaction record onto the leaf entry stack ==
delete <committed|provisional> <XID>
== push an insert transaction records onto the leaf entry stack ==
insert <committed|provisional> <XID> <VALUE>
== push a placeholder onto the leaf entry stack ==
placeholder <XID>
= Contents of a *.result =
== insert committed ==
insert_committed [<XIDS>] <KEY> <VALUE>
commit_any [<XIDS>] <KEY>
== delete committed ==
delete_committed [<XIDS>] <KEY>
== insert provisional ==
insert_provisional [<XIDS>] <KEY> <VALUE>
== delete provisional ==
delete_provisional [<XIDS>] <KEY>
== lock ==
lock [<XIDS>] <KEY>
src/tests/hotindexer-undo-do-tests/delete.0.test 0 → 100644
View file @ f31a90e9
# this test runs the undo do function on a leaf entry that consists of a single committed delete
key 1
delete committed 0
src/tests/hotindexer-undo-do-tests/insert.0.test 0 → 100644
View file @ f31a90e9
# commited insert
key 1
insert committed 0 100
src/tests/hotindexer-undo-do-tests/insert.100.result 0 → 100644
View file @ f31a90e9
insert_committed [100] 100 1
commit_any [100] 100
src/tests/hotindexer-undo-do-tests/insert.100.test 0 → 100644
View file @ f31a90e9
key 1
delete committed 0
insert committed 100 100
src/tests/hotindexer-undo-do-tests/insert.200.result 0 → 100644
View file @ f31a90e9
insert_committed [100] 100 1
commit_any [100] 100
delete_committed [200] 100
insert_committed [200] 200 1
commit_any [200] 100
commit_any [200] 200
src/tests/hotindexer-undo-do-tests/insert.200.test 0 → 100644
View file @ f31a90e9
key 1
delete committed 0
insert committed 100 100
insert committed 200 200
src/tests/hotindexer-undo-do-tests/placeholder.0.live.result 0 → 100644
View file @ f31a90e9
insert_provisional [300,301,302] 10 1
lock [300] 10
src/tests/hotindexer-undo-do-tests/placeholder.0.live.test 0 → 100644
View file @ f31a90e9
live 300 301 302
key 1
delete committed 0
placeholder 300
placeholder 301
insert provisional 302 10
src/tests/hotindexer-undo-do-tests/placeholder.0.result 0 → 100644
View file @ f31a90e9
insert_committed [300,301,302] 10 1
commit_any [300,301,302] 10
src/tests/hotindexer-undo-do-tests/placeholder.0.test 0 → 100644
View file @ f31a90e9
key 1
delete committed 0
placeholder 300
placeholder 301
insert provisional 302 10
src/tests/hotindexer-undo-do-tests/placeholder.1.live.result 0 → 100644
View file @ f31a90e9
insert_provisional [300,301,302] 10 1
lock [300] 10
src/tests/hotindexer-undo-do-tests/placeholder.1.live.test 0 → 100644
View file @ f31a90e9
live 300 301
key 1
delete committed 0
placeholder 300
placeholder 301
insert provisional 302 10
src/tests/hotindexer-undo-do-tests/placeholder.2.live.result 0 → 100644
View file @ f31a90e9
insert_committed [100] 10 1
commit_any [100] 10
delete_provisional [300,301,302] 10
lock [300] 10
insert_provisional [300,301,302] 20 1
lock [300] 20
src/tests/hotindexer-undo-do-tests/placeholder.2.live.test 0 → 100644
View file @ f31a90e9
live 300 301 302
key 1
delete committed 0
insert committed 100 10
placeholder 300
placeholder 301
insert provisional 302 20
src/tests/hotindexer-undo-do-tests/placeholder.2.result 0 → 100644
View file @ f31a90e9
insert_committed [100] 10 1
commit_any [100] 10
delete_committed [300,301,302] 10
insert_committed [300,301,302] 20 1
commit_any [300,301,302] 10
commit_any [300,301,302] 20
src/tests/hotindexer-undo-do-tests/placeholder.2.test 0 → 100644
View file @ f31a90e9
key 1
delete committed 0
insert committed 100 10
placeholder 300
placeholder 301
insert provisional 302 20
src/tests/hotindexer-undo-do-tests/prov.1.live.result 0 → 100644
View file @ f31a90e9
insert_committed [0] 10 1
delete_committed [100] 10
insert_committed [100] 20 1
commit_any [100] 10
commit_any [100] 20
delete_committed [200] 20
insert_committed [200] 10 1
commit_any [200] 20
commit_any [200] 10
delete_provisional [300,301] 10
lock [300] 10
insert_provisional [300,301] 30 1
lock [300] 30
delete_provisional [300,301,302] 30
lock [300] 30
lock [300] 30
src/tests/hotindexer-undo-do-tests/prov.1.live.test 0 → 100644
View file @ f31a90e9
live 300 301 302
key 1
insert committed 0 10
insert committed 100 20
insert committed 200 10
placeholder 300
insert provisional 301 30
delete provisional 302
src/tests/hotindexer-undo-do-tests/prov.1.result 0 → 100644
View file @ f31a90e9
insert_committed [0] 10 1
delete_committed [100] 10
insert_committed [100] 20 1
commit_any [100] 10
commit_any [100] 20
delete_committed [200] 20
insert_committed [200] 10 1
commit_any [200] 20
commit_any [200] 10
delete_committed [300,301] 10
insert_committed [300,301] 30 1
commit_any [300,301] 10
commit_any [300,301] 30
delete_committed [300,301,302] 30
commit_any [300,301,302] 30
src/tests/hotindexer-undo-do-tests/prov.1.test 0 → 100644
View file @ f31a90e9
key 1
insert committed 0 10
insert committed 100 20
insert committed 200 10
placeholder 300
insert provisional 301 30
delete provisional 302
src/tests/hotindexer-undo-do-tests/prov.insert.100.live.result 0 → 100644
View file @ f31a90e9
insert_provisional [100] 100 1
lock [100] 100
src/tests/hotindexer-undo-do-tests/prov.insert.100.live.test 0 → 100644
View file @ f31a90e9
live 100
key 1
delete committed 0
insert provisional 100 100
src/tests/hotindexer-undo-do-tests/prov.insert.100.result 0 → 100644
View file @ f31a90e9
insert_committed [100] 100 1
commit_any [100] 100
src/tests/hotindexer-undo-do-tests/prov.insert.100.test 0 → 100644
View file @ f31a90e9
key 1
delete committed 0
insert provisional 100 100
src/tests/hotindexer-with-queries.c 0 → 100644
View file @ f31a90e9
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2010 Tokutek Inc. All rights reserved."
#include "test.h"
#include "toku_pthread.h"
#include <db.h>
#include <sys/stat.h>
#include "key-val.h"
enum {NUM_INDEXER_INDEXES=1};
static const int NUM_DBS = NUM_INDEXER_INDEXES + 1; // 1 for source DB
static const int NUM_ROWS = 1000000;
int num_rows;
enum {MAX_CLIENTS=10};
typedef enum {FORWARD = 0, BACKWARD} Direction;
typedef enum {TXN_NONE = 0, TXN_CREATE = 1, TXN_END = 2} TxnWork;
DB_ENV *env;
float last_progress = 0.0;
static int UU() poll_print(void *extra, float progress) {
if ( verbose > 1 ) {
if ( last_progress + 0.01 < progress ) {
printf(" progress : %3.0f%%\n", progress * 100.0);
last_progress = progress;
}
}
extra = extra;
return 0;
}
static inline uint32_t key_to_put(int iter, int offset)
{
return (uint32_t)(((iter+1) * MAX_CLIENTS) + offset);
}
static int generate_initial_table(DB *db, DB_TXN *txn, uint32_t rows)
{
struct timeval start, now;
if ( verbose ) {
printf("generate_initial_table\n");
gettimeofday(&start,0);
}
int r = 0;
DBT key, val;
uint32_t k, v, i;
// create keys of stride MAX_CLIENTS
for (i=0; i<rows; i++)
{
k = key_to_put(i, 0);
v = generate_val(k, 0);
dbt_init(&key, &k, sizeof(k));
dbt_init(&val, &v, sizeof(v));
r = db->put(db, txn, &key, &val, 0);
if ( r != 0 ) break;
}
if ( verbose ) {
gettimeofday(&now,0);
int duration = (int)(now.tv_sec - start.tv_sec);
if ( duration > 0 )
printf("generate_initial_table : %u rows in %d sec = %d rows/sec\n", rows, duration, rows/duration);
}
return r;
}
/*
* client scans the primary table (like a range query)
*/
static void * client(void *arg)
{
DB *src = (DB *)arg;
if ( verbose ) printf("client start\n");
int r;
struct timeval start, now;
DB_TXN *txn;
DBT key, val;
uint32_t k, v;
dbt_init(&key, &k, sizeof(unsigned int));
dbt_init(&val, &v, sizeof(unsigned int));
r = env->txn_begin(env, NULL, &txn, 0); CKERR(r);
DBC *cursor;
r = src->cursor(src, txn, &cursor, 0); CKERR(r);
int row = 0;
gettimeofday(&start,0);
while ( r != DB_NOTFOUND ) {
r = cursor->c_get(cursor, &key, &val, DB_NEXT);
if ( r != DB_NOTFOUND ) {
row++;
}
}
gettimeofday(&now, 0);
if ( verbose ) printf("client : primary table scanned in %d sec, contains %d rows\n",
(int)(now.tv_sec - start.tv_sec),
row);
r = cursor->c_close(cursor); CKERR(r);
r = txn->commit(txn, 0); CKERR(r);
if ( verbose ) printf("client done\n");
return NULL;
}
toku_pthread_t *client_thread;
static void client_init(void)
{
client_thread = (toku_pthread_t *)toku_malloc(sizeof(toku_pthread_t));
}
static void client_cleanup(void)
{
toku_free(client_thread); client_thread = NULL;
}
static void query_only(DB *src)
{
int r;
void *t0;
client_init();
// start thread doing query
r = toku_pthread_create(client_thread, 0, client, (void *)src);
CKERR(r);
r = toku_pthread_join(*client_thread, &t0);
CKERR(r);
client_cleanup();
}
static void test_indexer(DB *src, DB **dbs)
{
int r;
DB_TXN *txn;
DB_INDEXER *indexer;
uint32_t db_flags[NUM_DBS];
if ( verbose ) printf("test_indexer\n");
for(int i=0;i<NUM_DBS;i++) {
db_flags[i] = DB_YESOVERWRITE;
}
client_init();
// create and initialize indexer
r = env->txn_begin(env, NULL, &txn, 0);
CKERR(r);
if ( verbose ) printf("test_indexer create_indexer\n");
r = env->create_indexer(env, txn, &indexer, src, NUM_DBS-1, &dbs[1], db_flags, 0);
CKERR(r);
r = indexer->set_error_callback(indexer, NULL, NULL);
CKERR(r);
r = indexer->set_poll_function(indexer, poll_print, NULL);
CKERR(r);
// start thread doing query
r = toku_pthread_create(client_thread, 0, client, (void *)src); CKERR(r);
struct timeval start, now;
if ( verbose ) {
printf("test_indexer build\n");
gettimeofday(&start,0);
}
r = indexer->build(indexer);
CKERR(r);
if ( verbose ) {
gettimeofday(&now,0);
int duration = (int)(now.tv_sec - start.tv_sec);
if ( duration > 0 )
printf("test_indexer build : sec = %d\n", duration);
}
void *t0;
r = toku_pthread_join(*client_thread, &t0); CKERR(r);
if ( verbose ) printf("test_indexer close\n");
r = indexer->close(indexer);
CKERR(r);
r = txn->commit(txn, DB_TXN_SYNC);
CKERR(r);
client_cleanup();
}
static void run_test(void)
{
int r;
r = system("rm -rf " ENVDIR); CKERR(r);
r = toku_os_mkdir(ENVDIR, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
r = toku_os_mkdir(ENVDIR "/log", S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
r = db_env_create(&env, 0); CKERR(r);
r = env->set_lg_dir(env, "log"); CKERR(r);
r = env->set_default_bt_compare(env, uint_dbt_cmp); CKERR(r);
generate_permute_tables();
r = env->set_generate_row_callback_for_put(env, put_multiple_generate); CKERR(r);
int envflags = DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN | DB_CREATE | DB_PRIVATE | DB_INIT_LOG;
r = env->open(env, ENVDIR, envflags, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
env->set_errfile(env, stderr);
r = env->checkpointing_set_period(env, 0); CKERR(r);
DBT desc;
dbt_init(&desc, "foo", sizeof("foo"));
int ids[MAX_DBS];
DB *dbs[MAX_DBS];
for (int i = 0; i < NUM_DBS; i++) {
ids[i] = i;
r = db_create(&dbs[i], env, 0); CKERR(r);
r = dbs[i]->set_descriptor(dbs[i], 1, &desc); CKERR(r);
dbs[i]->app_private = &ids[i];
char key_name[32];
sprintf(key_name, "key%d", i);
r = dbs[i]->open(dbs[i], NULL, key_name, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); CKERR(r);
}
// generate the src DB (do not use put_multiple)
DB_TXN *txn;
r = env->txn_begin(env, NULL, &txn, 0); CKERR(r);
r = generate_initial_table(dbs[0], txn, num_rows); CKERR(r);
r = txn->commit(txn, DB_TXN_SYNC); CKERR(r);
// scan the whole table twice to reduce possible flattening effects
// -------------------------- //
query_only(dbs[0]);
query_only(dbs[0]);
// -------------------------- //
// scan the whole table while running the indexer
// -------------------------- //
test_indexer(dbs[0], dbs);
// -------------------------- //
// scan the whole table again to confirm performance
// -------------------------- //
query_only(dbs[0]);
// -------------------------- //
for(int i=0;i<NUM_DBS;i++) {
r = dbs[i]->close(dbs[i], 0); CKERR(r);
}
r = env->close(env, 0); CKERR(r);
if ( verbose && (r == 0)) printf("PASS\n");
}
// ------------ infrastructure ----------
#include <sched.h>
static inline void
do_args (int argc, char * const argv[]) {
const char *progname=argv[0];
num_rows = NUM_ROWS;
int num_cpus = 0;
argc--; argv++;
while (argc>0) {
if (strcmp(argv[0],"-v")==0) {
verbose++;
} else if (strcmp(argv[0],"-q")==0) {
verbose=0;
} else if (strcmp(argv[0],"-r")==0) {
argc--; argv++;
num_rows = atoi(argv[0]);
} else if (strcmp(argv[0], "--ncpus") == 0 && argc+1 > 0) {
argc--; argv++;
num_cpus = atoi(argv[0]);
} else {
fprintf(stderr, "Usage:\n %s [-v] [-q] [-r rows]\n", progname);
exit(1);
}
argc--; argv++;
}
if (num_cpus > 0) {
cpu_set_t cpuset;
CPU_ZERO(&cpuset);
for (int i = 0; i < num_cpus; i++)
CPU_SET(i, &cpuset);
int r;
r = sched_setaffinity(toku_os_getpid(), sizeof cpuset, &cpuset);
assert(r == 0);
cpu_set_t use_cpuset;
CPU_ZERO(&use_cpuset);
r = sched_getaffinity(toku_os_getpid(), sizeof use_cpuset, &use_cpuset);
assert(r == 0);
assert(memcmp(&cpuset, &use_cpuset, sizeof cpuset) == 0);
}
}
int test_main(int argc, char * const *argv) {
do_args(argc, argv);
run_test();
return 0;
}
src/tests/key-val.h 0 → 100644
View file @ f31a90e9
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2010 Tokutek Inc. All rights reserved."
#ifndef KEY_VAL_H
#define KEY_VAL_H
//
// Functions to create unique key/value pairs, row generators, checkers, ... for each of NUM_DBS
//
// To use, during initialization:
// generate_permute_tables();
// r = env->set_generate_row_callback_for_put(env, put_multiple_generate);
//
#if defined(__cilkplusplus) || defined (__cplusplus)
extern "C" {
#endif
enum {MAX_DBS=32};
enum {MAGIC=311};
// a is the bit-wise permute table. For DB[i], permute bits as described in a[i] using 'twiddle32'
// inv is the inverse bit-wise permute of a[]. To get the original value from a twiddled value, twiddle32 (again) with inv[]
int a[MAX_DBS][32];
int inv[MAX_DBS][32];
// rotate right and left functions
static inline uint32_t UU() rotr32(const uint32_t x, const uint32_t num) {
const uint32_t n = num % 32;
return (x >> n) | ( x << (32 - n));
}
static inline uint64_t UU() rotr64(const uint64_t x, const uint64_t num) {
const uint64_t n = num % 64;
return ( x >> n ) | ( x << (64 - n));
}
static inline uint32_t UU() rotl32(const uint32_t x, const uint32_t num) {
const uint32_t n = num % 32;
return (x << n) | ( x >> (32 - n));
}
static inline uint64_t UU() rotl64(const uint64_t x, const uint64_t num) {
const uint64_t n = num % 64;
return ( x << n ) | ( x >> (64 - n));
}
static void UU() generate_permute_tables(void) {
int i, j, tmp;
for(int db=0;db<MAX_DBS;db++) {
for(i=0;i<32;i++) {
a[db][i] = i;
}
for(i=0;i<32;i++) {
j = random() % (i + 1);
tmp = a[db][j];
a[db][j] = a[db][i];
a[db][i] = tmp;
}
// if(db < NUM_DBS){ printf("a[%d] = ", db); for(i=0;i<32;i++) { printf("%2d ", a[db][i]); } printf("\n");}
for(i=0;i<32;i++) {
inv[db][a[db][i]] = i;
}
}
}
// permute bits of x based on permute table bitmap
static uint32_t UU() twiddle32(uint32_t x, int db)
{
uint32_t b = 0;
for(int i=0;i<32;i++) {
b |= (( x >> i ) & 1) << a[db][i];
}
return b;
}
// permute bits of x based on inverse permute table bitmap
static uint32_t UU() inv_twiddle32(uint32_t x, int db)
{
uint32_t b = 0;
for(int i=0;i<32;i++) {
b |= (( x >> i ) & 1) << inv[db][i];
}
return b;
}
// generate val from key, index
static uint32_t UU() generate_val(int key, int i) {
return rotl32((key + MAGIC), i);
}
static uint32_t UU() pkey_for_val(int key, int i) {
return rotr32(key, i) - MAGIC;
}
// There is no handlerton in this test, so this function is a local replacement
// for the handlerton's generate_row_for_put().
static int UU() put_multiple_generate(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_val, const DBT *src_key, const DBT *src_val, void *extra) {
src_db = src_db;
extra = extra;
uint32_t which = *(uint32_t*)dest_db->app_private;
if ( which == 0 ) {
if (dest_key->flags==DB_DBT_REALLOC) {
if (dest_key->data) toku_free(dest_key->data);
dest_key->flags = 0;
dest_key->ulen = 0;
}
if (dest_val->flags==DB_DBT_REALLOC) {
if (dest_val->data) toku_free(dest_val->data);
dest_val->flags = 0;
dest_val->ulen = 0;
}
dbt_init(dest_key, src_key->data, src_key->size);
dbt_init(dest_val, src_val->data, src_val->size);
}
else {
assert(dest_key->flags==DB_DBT_REALLOC);
if (dest_key->ulen < sizeof(uint32_t)) {
dest_key->data = toku_xrealloc(dest_key->data, sizeof(uint32_t));
dest_key->ulen = sizeof(uint32_t);
}
assert(dest_val->flags==DB_DBT_REALLOC);
if (dest_val->ulen < sizeof(uint32_t)) {
dest_val->data = toku_xrealloc(dest_val->data, sizeof(uint32_t));
dest_val->ulen = sizeof(uint32_t);
}
uint32_t *new_key = (uint32_t *)dest_key->data;
uint32_t *new_val = (uint32_t *)dest_val->data;
*new_key = twiddle32(*(uint32_t*)src_key->data, which);
*new_val = generate_val(*(uint32_t*)src_key->data, which);
dest_key->size = sizeof(uint32_t);
dest_val->size = sizeof(uint32_t);
//data is already set above
}
// printf("pmg : dest_key.data = %u, dest_val.data = %u \n", *(unsigned int*)dest_key->data, *(unsigned int*)dest_val->data);
return 0;
}
static int UU() uint_cmp(const void *ap, const void *bp) {
unsigned int an = *(unsigned int *)ap;
unsigned int bn = *(unsigned int *)bp;
if (an < bn)
return -1;
if (an > bn)
return +1;
return 0;
}
#if defined(__cilkplusplus) || defined(__cplusplus)
} // extern "C"
#endif
#endif // KEY_VAL_H
src/tests/loader-create-close.c
View file @ f31a90e9
......@@ -44,7 +44,7 @@ static void loader_open_abort(int ndb) {
r = env->txn_begin(env, NULL, &txn, 0); CKERR(r);
DB_LOADER *loader;
r = env->create_loader(env, txn, &loader, dbs[0], ndb, dbs, db_flags, dbt_flags, loader_flags); CKERR(r);
r = env->create_loader(env, txn, &loader, ndb > 0 ? dbs[0] : NULL, ndb, dbs, db_flags, dbt_flags, loader_flags); CKERR(r);
r = loader->close(loader); CKERR(r);
......
src/tests/loader-db-busy.c 0 → 100644
View file @ f31a90e9
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2010 Tokutek Inc. All rights reserved."
#ident "$Id: loader-create-close.c 20421 2010-05-19 19:24:57Z bkuszmaul $"
// Verify that the loader grabs references on the DB's
#include "test.h"
#include <db.h>
static int loader_flags = 0;
static int put_multiple_generate(DB *UU(dest_db), DB *UU(src_db), DBT *UU(dest_key), DBT *UU(dest_val), const DBT *UU(src_key), const DBT *UU(src_val), void *UU(extra)) {
return ENOMEM;
}
static void loader_open_abort(int ndb) {
int r;
r = system("rm -rf " ENVDIR); CKERR(r);
r = toku_os_mkdir(ENVDIR, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
DB_ENV *env;
r = db_env_create(&env, 0); CKERR(r);
r = env->set_generate_row_callback_for_put(env, put_multiple_generate);
CKERR(r);
int envflags = DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN | DB_CREATE | DB_PRIVATE;
r = env->open(env, ENVDIR, envflags, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
env->set_errfile(env, stderr);
DB *dbs[ndb];
uint32_t db_flags[ndb];
uint32_t dbt_flags[ndb];
for (int i = 0; i < ndb; i++) {
db_flags[i] = DB_NOOVERWRITE;
dbt_flags[i] = 0;
r = db_create(&dbs[i], env, 0); CKERR(r);
char name[32];
sprintf(name, "db%d", i);
r = dbs[i]->open(dbs[i], NULL, name, NULL, DB_BTREE, DB_CREATE, 0666); CKERR(r);
}
DB_TXN *txn;
r = env->txn_begin(env, NULL, &txn, 0); CKERR(r);
DB_LOADER *loader;
r = env->create_loader(env, txn, &loader, ndb > 0 ? dbs[0] : NULL, ndb, dbs, db_flags, dbt_flags, loader_flags); CKERR(r);
for (int i = 0; i < ndb; i++) {
r = dbs[i]->close(dbs[i], 0);
assert(r == EBUSY);
}
r = loader->close(loader); CKERR(r);
r = txn->commit(txn, 0); CKERR(r);
for (int i = 0; i < ndb; i++) {
r = dbs[i]->close(dbs[i], 0); CKERR(r);
}
r = env->close(env, 0); CKERR(r);
}
static void do_args(int argc, char * const argv[]) {
int resultcode;
char *cmd = argv[0];
argc--; argv++;
while (argc>0) {
if (strcmp(argv[0], "-h")==0) {
resultcode=0;
do_usage:
fprintf(stderr, "Usage: %s -h -v -q -p\n", cmd);
exit(resultcode);
} else if (strcmp(argv[0], "-v")==0) {
verbose++;
} else if (strcmp(argv[0],"-q")==0) {
verbose--;
if (verbose<0) verbose=0;
} else if (strcmp(argv[0], "-p") == 0) {
loader_flags = LOADER_USE_PUTS;
} else {
fprintf(stderr, "Unknown arg: %s\n", argv[0]);
resultcode=1;
goto do_usage;
}
argc--;
argv++;
}
}
int test_main(int argc, char * const *argv) {
do_args(argc, argv);
loader_open_abort(0);
loader_open_abort(1);
loader_open_abort(2);
return 0;
}
src/tests/lock-pressure.c 0 → 100644
View file @ f31a90e9
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2010 Tokutek Inc. All rights reserved."
#include "test.h"
#include "toku_pthread.h"
#include <db.h>
#include <sys/stat.h>
#include "key-val.h"
enum {NUM_INDEXER_INDEXES=1};
static const int NUM_DBS = NUM_INDEXER_INDEXES + 1; // 1 for source DB
static const int NUM_ROWS = 10000;
int num_rows;
enum {MAX_CLIENTS=10};
typedef enum {FORWARD = 0, BACKWARD} Direction;
typedef enum {TXN_NONE = 0, TXN_CREATE = 1, TXN_END = 2} TxnWork;
DB_ENV *env;
float last_progress = 0.0;
static int UU() poll_print(void *extra, float progress) {
if ( verbose ) {
if ( last_progress + 0.01 < progress ) {
printf(" progress : %3.0f%%\n", progress * 100.0);
last_progress = progress;
}
}
extra = extra;
return 0;
}
static inline uint32_t key_to_put(int iter, int offset)
{
return (uint32_t)(((iter+1) * MAX_CLIENTS) + offset);
}
static int generate_initial_table(DB *db, DB_TXN *txn, uint32_t rows)
{
struct timeval start, now;
if ( verbose ) {
printf("generate_initial_table\n");
gettimeofday(&start,0);
}
int r = 0;
DBT key, val;
uint32_t k, v, i;
// create keys of stride MAX_CLIENTS
for (i=0; i<rows; i++)
{
k = key_to_put(i, 0);
v = generate_val(k, 0);
dbt_init(&key, &k, sizeof(k));
dbt_init(&val, &v, sizeof(v));
r = db->put(db, txn, &key, &val, 0);
if ( r != 0 ) break;
}
if ( verbose ) {
gettimeofday(&now,0);
int duration = (int)(now.tv_sec - start.tv_sec);
if ( duration > 0 )
printf("generate_initial_table : %u rows in %d sec = %d rows/sec\n", rows, duration, rows/duration);
}
return r;
}
/*
* client() is a routine intended to be run in a separate thread from index creation
* - it takes a client spec which describes work to be done
* - direction : move to ever increasing or decreasing rows
* - txnwork : whether a transaction should be created or closed within the client
* (allows client transaction to start before or during index creation,
* and to close during or after index creation)
*/
typedef struct client_spec {
uint32_t num; // number of rows to write
uint32_t start; // approximate start row
int offset; // offset from stride (= MAX_CLIENTS)
Direction dir;
TxnWork txnwork;
DB_TXN *txn;
DB **dbs;
int client_number;
uint32_t *flags;
} client_spec_t, *client_spec;
int client_count = 0;
static void * client(void *arg)
{
client_spec cs = arg;
client_count++;
if ( verbose ) printf("client[%d]\n", cs->client_number);
assert(cs->client_number < MAX_CLIENTS);
assert(cs->dir == FORWARD || cs->dir == BACKWARD);
int r;
if ( cs->txnwork | TXN_CREATE ) { r = env->txn_begin(env, NULL, &cs->txn, 0); CKERR(r); }
DBT key, val;
DBT dest_keys[NUM_DBS];
DBT dest_vals[NUM_DBS];
uint32_t k, v;
int n = cs->start;
for(int which=0;which<NUM_DBS;which++) {
dbt_init(&dest_keys[which], NULL, 0);
dest_keys[which].flags = DB_DBT_REALLOC;
dbt_init(&dest_vals[which], NULL, 0);
dest_vals[which].flags = DB_DBT_REALLOC;
}
int rr;
for (uint32_t i = 0; i < cs->num; i++ ) {
DB_TXN *txn;
env->txn_begin(env, cs->txn, &txn, 0);
k = key_to_put(n, cs->offset);
v = generate_val(k, 0);
dbt_init(&key, &k, sizeof(k));
dbt_init(&val, &v, sizeof(v));
rr = env->put_multiple(env,
cs->dbs[0],
txn,
&key,
&val,
NUM_DBS,
cs->dbs, // dest dbs
dest_keys,
dest_vals,
cs->flags,
NULL);
if ( rr != 0 ) {
if ( verbose ) printf("client[%u] : put_multiple returns %d, i=%u, n=%u, key=%u\n", cs->client_number, rr, i, n, k);
r = txn->abort(txn); CKERR(r);
break;
}
r = txn->commit(txn, 0); CKERR(r);
n = ( cs->dir == FORWARD ) ? n + 1 : n - 1;
}
if ( cs->txnwork | TXN_END ) { r = cs->txn->commit(cs->txn, DB_TXN_SYNC); CKERR(r); }
if (verbose) printf("client[%d] done\n", cs->client_number);
return 0;
}
toku_pthread_t *client_threads;
client_spec_t *client_specs;
static void clients_init(DB **dbs, uint32_t *flags)
{
client_threads = toku_malloc(sizeof(toku_pthread_t) * MAX_CLIENTS);
client_specs = toku_malloc(sizeof(client_spec_t) * MAX_CLIENTS);
client_specs[0].client_number = 0;
client_specs[0].start = 0;
// client_specs[0].start = num_rows - 1;
client_specs[0].num = num_rows;
client_specs[0].offset = -1;
client_specs[0].dir = FORWARD;
// client_specs[0].dir = BACKWARD;
client_specs[0].txnwork = TXN_CREATE | TXN_END;
client_specs[0].txn = NULL;
client_specs[0].dbs = dbs;
client_specs[0].flags = flags;
client_specs[1].client_number = 1;
client_specs[1].start = 0;
client_specs[1].num = num_rows;
client_specs[1].offset = 1;
client_specs[1].dir = FORWARD;
client_specs[1].txnwork = TXN_CREATE | TXN_END;
client_specs[1].txn = NULL;
client_specs[1].dbs = dbs;
client_specs[1].flags = flags;
}
static void clients_cleanup(void)
{
toku_free(client_threads); client_threads = NULL;
toku_free(client_specs); client_specs = NULL;
}
// verify results
// - read the keys in the primary table, then calculate what keys should exist
// in the other DB. Read the other table to verify.
static int UU() check_results(DB *src, DB *db)
{
int r;
int fail = 0;
int clients = client_count;
int max_rows = ( clients + 1 ) * num_rows;
unsigned int *db_keys = (unsigned int *) toku_malloc(max_rows * sizeof (unsigned int));
DBT key, val;
unsigned int k=0, v=0;
dbt_init(&key, &k, sizeof(unsigned int));
dbt_init(&val, &v, sizeof(unsigned int));
DB_TXN *txn;
r = env->txn_begin(env, NULL, &txn, 0); CKERR(r);
DBC *cursor;
r = src->cursor(src, txn, &cursor, 0); CKERR(r);
int which = *(uint32_t*)db->app_private;
// scan the primary table,
// calculate the expected keys in 'db'
int row = 0;
while ( r != DB_NOTFOUND ) {
r = cursor->c_get(cursor, &key, &val, DB_NEXT);
if ( r != DB_NOTFOUND ) {
k = *((uint32_t *)(key.data));
db_keys[row] = twiddle32(k, which);
row++;
}
}
if ( verbose ) printf("primary table scanned, contains %d rows\n", row);
int primary_rows = row;
r = cursor->c_close(cursor); CKERR(r);
// sort the expected keys
qsort(db_keys, primary_rows, sizeof (unsigned int), uint_cmp);
if ( verbose > 1 ) {
for(int i=0;i<primary_rows;i++) {
printf("primary table[%u] = %u\n", i, db_keys[i]);
}
}
// scan the indexer-created DB, comparing keys with expected keys
// - there should be exactly 'primary_rows' in the new index
r = db->cursor(db, txn, &cursor, 0); CKERR(r);
for (int i=0;i<primary_rows;i++) {
r = cursor->c_get(cursor, &key, &val, DB_NEXT);
if ( r == DB_NOTFOUND ) {
printf("scan of index finds last row is %d\n", i);
}
CKERR(r);
k = *((uint32_t *)(key.data));
if ( db_keys[i] != k ) {
if ( verbose ) printf("ERROR expecting key %10u for row %d, found key = %10u\n", db_keys[i],i,k);
fail = 1;
goto check_results_error;
}
}
// next cursor op should return DB_NOTFOUND
r = cursor->c_get(cursor, &key, &val, DB_NEXT);
assert(r == DB_NOTFOUND);
// we're done - cleanup and close
check_results_error:
r = cursor->c_close(cursor); CKERR(r);
toku_free(db_keys);
r = txn->commit(txn, 0); CKERR(r);
if ( verbose ) {
if ( fail ) printf("check_results : fail\n");
else printf("check_results : pass\n");
}
return fail;
}
static void test_indexer(DB *src, DB **dbs)
{
src = src; dbs = dbs;
int r;
DB_TXN *txn;
#if 0
DB_INDEXER *indexer;
#endif
uint32_t db_flags[NUM_DBS];
if ( verbose ) printf("test_indexer\n");
for(int i=0;i<NUM_DBS;i++) {
db_flags[i] = DB_YESOVERWRITE;
}
clients_init(dbs, db_flags);
// create and initialize indexer
r = env->txn_begin(env, NULL, &txn, 0);
CKERR(r);
#if 0
if ( verbose ) printf("test_indexer create_indexer\n");
r = env->create_indexer(env, txn, &indexer, src, NUM_DBS-1, &dbs[1], db_flags, 0);
CKERR(r);
r = indexer->set_error_callback(indexer, NULL, NULL);
CKERR(r);
r = indexer->set_poll_function(indexer, poll_print, NULL);
CKERR(r);
#endif
// start threads doing additional inserts - no lock issues since indexer already created
r = toku_pthread_create(&client_threads[0], 0, client, (void *)&client_specs[0]); CKERR(r);
r = toku_pthread_create(&client_threads[1], 0, client, (void *)&client_specs[1]); CKERR(r);
struct timeval start, now;
if ( verbose ) {
printf("test_indexer build\n");
gettimeofday(&start,0);
}
#if 0
r = indexer->build(indexer);
CKERR(r);
#endif
if ( verbose ) {
gettimeofday(&now,0);
int duration = (int)(now.tv_sec - start.tv_sec);
if ( duration > 0 )
printf("test_indexer build : sec = %d\n", duration);
}
void *t0;
r = toku_pthread_join(client_threads[0], &t0); CKERR(r);
void *t1;
r = toku_pthread_join(client_threads[1], &t1); CKERR(r);
#if 0
if ( verbose ) printf("test_indexer close\n");
r = indexer->close(indexer);
CKERR(r);
#endif
r = txn->commit(txn, DB_TXN_SYNC);
CKERR(r);
clients_cleanup();
#if 0
if ( verbose ) printf("check_results\n");
r = check_results(src, dbs[1]);
CKERR(r);
#endif
if ( verbose && (r == 0)) printf("PASS\n");
if ( verbose && (r == 0)) printf("test_indexer done\n");
}
static void run_test(void)
{
int r;
r = system("rm -rf " ENVDIR); CKERR(r);
r = toku_os_mkdir(ENVDIR, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
r = toku_os_mkdir(ENVDIR "/log", S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
r = db_env_create(&env, 0); CKERR(r);
r = env->set_lg_dir(env, "log"); CKERR(r);
r = env->set_default_bt_compare(env, uint_dbt_cmp); CKERR(r);
r = env->set_lk_max_locks(env, ~0); CKERR(r);
generate_permute_tables();
r = env->set_generate_row_callback_for_put(env, put_multiple_generate); CKERR(r);
int envflags = DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN | DB_CREATE | DB_PRIVATE | DB_INIT_LOG;
r = env->open(env, ENVDIR, envflags, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
env->set_errfile(env, stderr);
r = env->checkpointing_set_period(env, 0); CKERR(r);
DBT desc;
dbt_init(&desc, "foo", sizeof("foo"));
int ids[MAX_DBS];
DB *dbs[MAX_DBS];
for (int i = 0; i < NUM_DBS; i++) {
ids[i] = i;
r = db_create(&dbs[i], env, 0); CKERR(r);
r = dbs[i]->set_descriptor(dbs[i], 1, &desc); CKERR(r);
dbs[i]->app_private = &ids[i];
char key_name[32];
sprintf(key_name, "key%d", i);
r = dbs[i]->open(dbs[i], NULL, key_name, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); CKERR(r);
}
// generate the src DB (do not use put_multiple)
DB_TXN *txn;
r = env->txn_begin(env, NULL, &txn, 0); CKERR(r);
r = generate_initial_table(dbs[0], txn, num_rows); CKERR(r);
r = txn->commit(txn, DB_TXN_SYNC); CKERR(r);
// -------------------------- //
if (1) test_indexer(dbs[0], dbs);
// -------------------------- //
for(int i=0;i<NUM_DBS;i++) {
r = dbs[i]->close(dbs[i], 0); CKERR(r);
}
r = env->close(env, 0); CKERR(r);
}
// ------------ infrastructure ----------
static inline void
do_args (int argc, char * const argv[]) {
const char *progname=argv[0];
num_rows = NUM_ROWS;
argc--; argv++;
while (argc>0) {
if (strcmp(argv[0],"-v")==0) {
verbose++;
} else if (strcmp(argv[0],"-q")==0) {
verbose=0;
} else if (strcmp(argv[0],"-r")==0) {
argc--; argv++;
num_rows = atoi(argv[0]);
} else {
fprintf(stderr, "Usage:\n %s [-v] [-q] [-r rows]\n", progname);
exit(1);
}
argc--; argv++;
}
}
int test_main(int argc, char * const *argv) {
do_args(argc, argv);
run_test();
return 0;
}
/*
* Please ignore this code - I don't think I'm going to use it, but I don't want to lose it
* I will delete this later - Dave
if ( rr != 0 ) { // possible lock deadlock
if (verbose > 1) {
printf("client[%u] : put_multiple returns %d, i=%u, n=%u, key=%u\n", cs->client_number, rr, i, n, k);
if ( verbose > 2 ) print_engine_status(env);
}
// abort the transaction, freeing up locks associated with previous put_multiples
if ( verbose > 1 ) printf("start txn abort\n");
r = txn->abort(txn); CKERR(r);
if ( verbose > 1 ) printf(" txn aborted\n");
sleep(2 + cs->client_number);
// now retry, waiting until the deadlock resolves itself
r = env->txn_begin(env, cs->txn, &txn, 0); CKERR(r);
if ( verbose > 1 ) printf("txn begin\n");
while ( rr != 0 ) {
rr = env->put_multiple(env,
cs->dbs[0],
txn,
&key,
&val,
NUM_DBS,
cs->dbs, // dest dbs
dest_keys,
dest_vals,
cs->flags,
NULL);
if ( rr != 0 ) {
if ( verbose ) printf("client[%u] : put_multiple returns %d, i=%u, n=%u, key=%u\n", cs->client_number, rr, i, n, k);
if ( verbose ) printf("start txn abort\n");
r = txn->abort(txn); CKERR(r);
if ( verbose ) printf(" txn aborted\n");
sleep(2 + cs->client_number);
r = env->txn_begin(env, cs->txn, &txn, 0); CKERR(r);
if ( verbose ) printf("txn begin\n");
}
}
*/
src/tests/run-hotindexer-undo-do-tests.bash 0 → 100644
View file @ f31a90e9
#!/usr/bin/env bash
# run a sequence of hotindexer undo tests.
tests=""
verbose=0
for arg in $* ; do
if [ $arg = "--verbose" ] ; then
verbose=1
else
tests="$tests $arg"
fi
done
for t in $tests ; do
testdir=`dirname $t`
testfile=`basename $t`
testname=""
resultfile=""
if [[ $testfile =~ "(.*)\.test$" ]] ; then
testname=${BASH_REMATCH[1]}
resultfile=$testname.result
else
exit 1
fi
if [ $verbose != 0 ] ; then echo $testdir $testname $testfile $resultfile; fi
./hotindexer-undo-do-test.tdb $testdir/$testfile >$testdir/$testname.run
if [ -f $testdir/$resultfile ] ; then
diff -q $testdir/$testname.run $testdir/$resultfile >/dev/null 2>&1
exitcode=$?
else
exitcode=1
fi
if [ $verbose != 0 ] ; then
echo $testname $exitcode
else
rm $testdir/$testname.run
fi
if [ $exitcode != 0 ] ; then break; fi
done
exit $exitcode
\ No newline at end of file
src/tests/test.h
View file @ f31a90e9
......@@ -304,6 +304,24 @@ multiply_locks_for_n_dbs(DB_ENV *env, int num_dbs) {
#endif
}
static inline void
default_parse_args (int argc, char * const argv[]) {
const char *progname=argv[0];
argc--; argv++;
while (argc>0) {
if (strcmp(argv[0],"-v")==0) {
verbose=1;
} else if (strcmp(argv[0],"-q")==0) {
verbose=0;
} else {
fprintf(stderr, "Usage:\n %s [-v] [-q]\n", progname);
exit(1);
}
argc--; argv++;
}
}
#if defined(__cilkplusplus) || defined(__cplusplus)
}
#endif
......
src/ydb-internal.h
View file @ f31a90e9
......@@ -2,7 +2,7 @@
#ifndef YDB_INTERNAL_H
#define YDB_INTERNAL_H
#ident "Copyright (c) 2007 Tokutek Inc. All rights reserved."
#ident "Copyright (c) 2007-2010 Tokutek Inc. All rights reserved."
#include <db.h>
#include "../newbrt/brttypes.h"
......@@ -31,8 +31,16 @@ struct __toku_db_internal {
char *dname; // dname is constant for this handle (handle must be closed before file is renamed)
BOOL is_zombie; // True if DB->close has been called on this DB
struct toku_list dbs_that_must_close_before_abort;
DB_INDEXER *indexer;
int refs; // reference count including indexers and loaders
};
int toku_db_set_indexer(DB *db, DB_INDEXER *indexer);
DB_INDEXER *toku_db_get_indexer(DB *db);
void toku_db_add_ref(DB *db);
void toku_db_release_ref(DB *db);
#if DB_VERSION_MAJOR == 4 && DB_VERSION_MINOR == 1
typedef void (*toku_env_errcall_t)(const char *, char *);
#elif DB_VERSION_MAJOR == 4 && DB_VERSION_MINOR >= 3
......@@ -119,9 +127,12 @@ int toku_ydb_lock_init(void);
int toku_ydb_lock_destroy(void);
void toku_ydb_lock(void);
void toku_ydb_unlock(void);
void toku_ydb_unlock_and_yield(unsigned long useconds);
void toku_ydb_lock_get_status(SCHEDULE_STATUS statp);
int toku_ydb_check_avail_fs_space(DB_ENV *env);
/* *********************************************************
......@@ -215,6 +226,10 @@ struct __toku_dbc_internal {
int toku_db_pre_acquire_table_lock(DB *db, DB_TXN *txn, BOOL just_lock);
int toku_grab_write_lock (DB* db, DBT* key, TOKUTXN tokutxn);
int toku_grab_read_lock_on_directory (DB* db, DB_TXN * txn);
#if defined(__cplusplus)
}
#endif
......
src/ydb-le-cursor.h
View file @ f31a90e9
#include "le_cursor.h"
#include "le-cursor.h"
static inline int le_cursor_create_db_txn(LE_CURSOR *le_cursor_result, DB *db, DB_TXN *txn) {
return le_cursor_create(le_cursor_result, db_struct_i(db)->brt, db_txn_struct_i(txn)->tokutxn);
......
src/ydb.c
View file @ f31a90e9
......@@ -30,6 +30,7 @@ const char *toku_copyright_string = "Copyright (c) 2007-2009 Tokutek Inc. All r
#include "checkpoint.h"
#include "key.h"
#include "loader.h"
#include "indexer.h"
#include "ydb_load.h"
#include "brtloader.h"
......@@ -337,6 +338,12 @@ env_check_avail_fs_space(DB_ENV *env) {
return r;
}
int
toku_ydb_check_avail_fs_space(DB_ENV *env) {
int rval = env_check_avail_fs_space(env);
return rval;
}
/* db methods */
static inline int db_opened(DB *db) {
return db->i->opened != 0;
......@@ -2078,6 +2085,7 @@ toku_env_create(DB_ENV ** envp, u_int32_t flags) {
SENV(set_redzone);
#undef SENV
result->create_loader = toku_loader_create_loader;
result->create_indexer = toku_indexer_create_indexer;
MALLOC(result->i);
if (result->i == 0) { r = ENOMEM; goto cleanup; }
......@@ -2128,7 +2136,10 @@ cleanup:
int
DB_ENV_CREATE_FUN (DB_ENV ** envp, u_int32_t flags) {
toku_ydb_lock(); int r = toku_env_create(envp, flags); toku_ydb_unlock(); return r;
toku_ydb_lock();
int r = toku_env_create(envp, flags);
toku_ydb_unlock();
return r;
}
static int
......@@ -2244,7 +2255,7 @@ toku_txn_commit(DB_TXN * txn, u_int32_t flags,
#if !TOKUDB_NATIVE_H
toku_free(db_txn_struct_i(txn));
#endif
toku_free(txn);
toku_free(txn); txn = NULL;
if (flags!=0) return EINVAL;
return r;
}
......@@ -2497,15 +2508,15 @@ toku_txn_begin(DB_ENV *env, DB_TXN * stxn, DB_TXN ** txn, u_int32_t flags, int i
break;
}
}
r = toku_txn_begin_txn(
r = toku_txn_begin_txn(result,
stxn ? db_txn_struct_i(stxn)->tokutxn : 0,
&db_txn_struct_i(result)->tokutxn,
env->i->logger,
snapshot_type
);
if (r != 0)
return r;
//Add to the list of children for the parent.
if (result->parent) {
assert(!db_txn_struct_i(result->parent)->child);
......@@ -2718,9 +2729,24 @@ env_get_zombie_db_with_dname(DB_ENV *env, const char *dname) {
return rval;
}
void
toku_db_add_ref(DB *db) {
db->i->refs++;
}
void
toku_db_release_ref(DB *db){
db->i->refs--;
}
//DB->close()
static int
toku_db_close(DB * db, u_int32_t flags) {
int r = 0;
if (db->i->refs != 1) {
r = EBUSY;
} else {
db->i->refs = 0;
if (db_opened(db) && db->i->dname) {
// internal (non-user) dictionary has no dname
env_note_db_closed(db->dbenv, db); // tell env that this db is no longer in use by the user of this api (user-closed, may still be in use by fractal tree internals)
......@@ -2731,7 +2757,8 @@ toku_db_close(DB * db, u_int32_t flags) {
if (!toku_list_empty(&db->i->dbs_that_must_close_before_abort))
toku_list_remove(&db->i->dbs_that_must_close_before_abort);
int r = toku_brt_db_delay_closed(db->i->brt, db, db_close_before_brt, flags);
r = toku_brt_db_delay_closed(db->i->brt, db, db_close_before_brt, flags);
}
return r;
}
......@@ -3027,7 +3054,7 @@ grab_range_lock(RANGE_LOCK_REQUEST request) {
return r;
}
static int
int
toku_grab_read_lock_on_directory (DB* db, DB_TXN * txn) {
RANGE_LOCK_REQUEST_S request;
char * dname = db->i->dname;
......@@ -3787,14 +3814,37 @@ log_del_multiple(DB_TXN *txn, DB *src_db, const DBT *key, const DBT *val, uint32
return r;
}
static uint32_t
lookup_src_db(uint32_t num_dbs, DB *db_array[], DB *src_db) {
uint32_t which_db;
for (which_db = 0; which_db < num_dbs; which_db++)
if (db_array[which_db] == src_db)
break;
return which_db;
}
static int
do_del_multiple(DB_TXN *txn, uint32_t num_dbs, DB *db_array[], DBT keys[]) {
int r = 0;
TOKUTXN ttxn = db_txn_struct_i(txn)->tokutxn;
for (uint32_t which_db = 0; r == 0 && which_db < num_dbs; which_db++) {
DB *db = db_array[which_db];
int do_delete = TRUE;
DB_INDEXER *indexer = toku_db_get_indexer(db);
if (indexer) {
DB *src_db = toku_indexer_get_src_db(indexer);
invariant(src_db != NULL);
uint32_t which_src_db = lookup_src_db(num_dbs, db_array, src_db);
if (which_src_db >= num_dbs)
r = EINVAL;
else
do_delete = !toku_indexer_is_key_right_of_le_cursor(indexer, src_db, &keys[which_src_db]);
}
if (r == 0 && do_delete) {
num_deletes++;
r = toku_brt_maybe_delete(db->i->brt, &keys[which_db], ttxn, FALSE, ZERO_LSN, FALSE);
}
}
return r;
}
......@@ -3886,7 +3936,6 @@ env_del_multiple(
if (r == 0)
r = do_del_multiple(txn, num_dbs, db_array, del_keys);
}
cleanup:
......@@ -4488,9 +4537,22 @@ do_put_multiple(DB_TXN *txn, uint32_t num_dbs, DB *db_array[], DBT keys[], DBT v
TOKUTXN ttxn = db_txn_struct_i(txn)->tokutxn;
for (uint32_t which_db = 0; r == 0 && which_db < num_dbs; which_db++) {
DB *db = db_array[which_db];
int do_put = TRUE;
DB_INDEXER *indexer = toku_db_get_indexer(db);
if (indexer) {
DB *src_db = toku_indexer_get_src_db(indexer);
invariant(src_db != NULL);
uint32_t which_src_db = lookup_src_db(num_dbs, db_array, src_db);
if (which_src_db >= num_dbs)
r = EINVAL;
else
do_put = !toku_indexer_is_key_right_of_le_cursor(indexer, src_db, &keys[which_src_db]);
}
if (r == 0 && do_put) {
num_inserts++;
r = toku_brt_maybe_insert(db->i->brt, &keys[which_db], &vals[which_db], ttxn, FALSE, ZERO_LSN, FALSE, BRT_INSERT);
}
}
return r;
}
......@@ -5304,7 +5366,10 @@ toku_db_destruct_autotxn(DB_TXN *txn, int r, BOOL changed) {
static int
locked_db_close(DB * db, u_int32_t flags) {
toku_ydb_lock(); int r = toku_db_close(db, flags); toku_ydb_unlock(); return r;
toku_ydb_lock();
int r = toku_db_close(db, flags);
toku_ydb_unlock();
return r;
}
static inline int
......@@ -5607,6 +5672,27 @@ locked_db_get_fragmentation(DB * db, TOKU_DB_FRAGMENTATION report) {
return r;
}
int
toku_db_set_indexer(DB *db, DB_INDEXER * indexer) {
db->i->indexer = indexer;
return 0;
}
static int
locked_db_set_indexer(DB *db, DB_INDEXER *indexer) {
toku_ydb_lock(); int r = toku_db_set_indexer(db, indexer); toku_ydb_unlock(); return r;
}
DB_INDEXER *
toku_db_get_indexer(DB *db) {
return db->i->indexer;
}
static void
locked_db_get_indexer(DB *db, DB_INDEXER **indexer_ptr) {
toku_ydb_lock(); *indexer_ptr = toku_db_get_indexer(db); toku_ydb_unlock();
}
static int
toku_db_create(DB ** db, DB_ENV * env, u_int32_t flags) {
int r;
......@@ -5649,6 +5735,8 @@ toku_db_create(DB ** db, DB_ENV * env, u_int32_t flags) {
SDB(flatten);
SDB(optimize);
SDB(get_fragmentation);
SDB(set_indexer);
SDB(get_indexer);
#undef SDB
result->dbt_pos_infty = toku_db_dbt_pos_infty;
result->dbt_neg_infty = toku_db_dbt_neg_infty;
......@@ -5663,6 +5751,8 @@ toku_db_create(DB ** db, DB_ENV * env, u_int32_t flags) {
result->i->open_flags = 0;
result->i->open_mode = 0;
result->i->brt = 0;
result->i->indexer = NULL;
result->i->refs = 1;
toku_list_init(&result->i->dbs_that_must_close_before_abort);
r = toku_brt_create(&result->i->brt);
if (r != 0) {
......@@ -5676,7 +5766,10 @@ toku_db_create(DB ** db, DB_ENV * env, u_int32_t flags) {
int
DB_CREATE_FUN (DB ** db, DB_ENV * env, u_int32_t flags) {
toku_ydb_lock(); int r = toku_db_create(db, env, flags); toku_ydb_unlock(); return r;
toku_ydb_lock();
int r = toku_db_create(db, env, flags);
toku_ydb_unlock();
return r;
}
/* need db_strerror_r for multiple threads */
......@@ -6000,3 +6093,15 @@ toku_test_get_checkpointing_user_data_status (void) {
return toku_cachetable_get_checkpointing_user_data_status();
}
int
toku_grab_write_lock (DB* db, DBT* key, TOKUTXN tokutxn) {
RANGE_LOCK_REQUEST_S request;
DB_TXN * txn = toku_txn_get_container_db_txn(tokutxn);
//Left end of range == right end of range (point lock)
write_lock_request_init(&request, txn, db,
key,
key);
int r = grab_range_lock(&request);
return r;
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7