Skip to content

M
mariadb
Commit f31a90e9 authored by Dave Wells's avatar Dave Wells Committed by Yoni Fogel
Browse files
Options

merge indexer to main refs #2843 [t:2843] 

git-svn-id: file:///svn/toku/tokudb@25593 c7de825b-a66e-492c-adef-691d508d4ae1
parent ff2b488e
Expand all Show whitespace changes
Inline Side-by-side
Showing with 5796 additions and 179 deletions
buildheader/db.h_4_1
View file @ f31a90e9
...@@ -54,6 +54,16 @@ struct __toku_loader { ...@@ -54,6 +54,16 @@ struct __toku_loader {
int (*close)(DB_LOADER *loader); /* finish loading, free memory */ int (*close)(DB_LOADER *loader); /* finish loading, free memory */
int (*abort)(DB_LOADER *loader); /* abort loading, free memory */ int (*abort)(DB_LOADER *loader); /* abort loading, free memory */
}; };
typedef struct __toku_indexer DB_INDEXER;
struct __toku_indexer_internal;
struct __toku_indexer {
struct __toku_indexer_internal *i;
int (*set_error_callback)(DB_INDEXER *indexer, void (*error_cb)(DB *db, int i, int err, DBT *key, DBT *val, void *error_extra), void *error_extra); /* set the error callback */
int (*set_poll_function)(DB_INDEXER *indexer, int (*poll_func)(void *extra, float progress), void *poll_extra); /* set the polling function */
int (*build)(DB_INDEXER *indexer); /* build the indexes */
int (*close)(DB_INDEXER *indexer); /* finish indexing, free memory */
int (*abort)(DB_INDEXER *indexer); /* abort indexing, free memory */
};
typedef struct __toku_engine_status { typedef struct __toku_engine_status {
char creationtime[26]; /* time of environment creation */ char creationtime[26]; /* time of environment creation */
char startuptime[26]; /* time of engine startup */ char startuptime[26]; /* time of engine startup */
...@@ -270,6 +280,7 @@ struct __toku_db_env { ...@@ -270,6 +280,7 @@ struct __toku_db_env {
int (*get_engine_status_text) (DB_ENV*, char*, int) /* Fill in status text */; int (*get_engine_status_text) (DB_ENV*, char*, int) /* Fill in status text */;
int (*get_iname) (DB_ENV* env, DBT* dname_dbt, DBT* iname_dbt) /* FOR TEST ONLY: lookup existing iname */; int (*get_iname) (DB_ENV* env, DBT* dname_dbt, DBT* iname_dbt) /* FOR TEST ONLY: lookup existing iname */;
int (*create_loader) (DB_ENV *env, DB_TXN *txn, DB_LOADER **blp, DB *src_db, int N, DB *dbs[/*N*/], uint32_t db_flags[/*N*/], uint32_t dbt_flags[/*N*/], uint32_t loader_flags); int (*create_loader) (DB_ENV *env, DB_TXN *txn, DB_LOADER **blp, DB *src_db, int N, DB *dbs[/*N*/], uint32_t db_flags[/*N*/], uint32_t dbt_flags[/*N*/], uint32_t loader_flags);
int (*create_indexer) (DB_ENV *env, DB_TXN *txn, DB_INDEXER **idxrp, DB *src_db, int N, DB *dbs[/*N*/], uint32_t db_flags[/*N*/], uint32_t indexer_flags);
int (*put_multiple) (DB_ENV *env, DB *src_db, DB_TXN *txn, int (*put_multiple) (DB_ENV *env, DB *src_db, DB_TXN *txn,
const DBT *key, const DBT *val, const DBT *key, const DBT *val,
uint32_t num_dbs, DB **db_array, DBT *keys, DBT *vals, uint32_t *flags_array, uint32_t num_dbs, DB **db_array, DBT *keys, DBT *vals, uint32_t *flags_array,
...@@ -299,7 +310,7 @@ struct __toku_db_env { ...@@ -299,7 +310,7 @@ struct __toku_db_env {
int (*set_redzone) (DB_ENV *env, int redzone) /* set the redzone limit in percent of total space */; int (*set_redzone) (DB_ENV *env, int redzone) /* set the redzone limit in percent of total space */;
int (*set_lk_max_memory) (DB_ENV *env, uint64_t max); int (*set_lk_max_memory) (DB_ENV *env, uint64_t max);
int (*get_lk_max_memory) (DB_ENV *env, uint64_t *max); int (*get_lk_max_memory) (DB_ENV *env, uint64_t *max);
void* __toku_dummy0[15]; void* __toku_dummy0[14];
char __toku_dummy1[64]; char __toku_dummy1[64];
void *api1_internal; /* 32-bit offset=212 size=4, 64=bit offset=360 size=8 */ void *api1_internal; /* 32-bit offset=212 size=4, 64=bit offset=360 size=8 */
void* __toku_dummy2[7]; void* __toku_dummy2[7];
...@@ -394,7 +405,9 @@ struct __toku_db { ...@@ -394,7 +405,9 @@ struct __toku_db {
int (*flatten)(DB*, DB_TXN*) /* Flatten a dictionary, similar to (but faster than) a table scan */; int (*flatten)(DB*, DB_TXN*) /* Flatten a dictionary, similar to (but faster than) a table scan */;
int (*optimize)(DB*) /* Run garbage collecion and promote all transactions older than oldest. Amortized (happens during flattening) */; int (*optimize)(DB*) /* Run garbage collecion and promote all transactions older than oldest. Amortized (happens during flattening) */;
int (*get_fragmentation)(DB*,TOKU_DB_FRAGMENTATION); int (*get_fragmentation)(DB*,TOKU_DB_FRAGMENTATION);
void* __toku_dummy0[19]; int (*set_indexer)(DB*, DB_INDEXER*);
void (*get_indexer)(DB*, DB_INDEXER**);
void* __toku_dummy0[17];
char __toku_dummy1[96]; char __toku_dummy1[96];
void *api_internal; /* 32-bit offset=236 size=4, 64=bit offset=376 size=8 */ void *api_internal; /* 32-bit offset=236 size=4, 64=bit offset=376 size=8 */
void* __toku_dummy2[5]; void* __toku_dummy2[5];
......
buildheader/db.h_4_3
View file @ f31a90e9
...@@ -54,6 +54,16 @@ struct __toku_loader { ...@@ -54,6 +54,16 @@ struct __toku_loader {
int (*close)(DB_LOADER *loader); /* finish loading, free memory */ int (*close)(DB_LOADER *loader); /* finish loading, free memory */
int (*abort)(DB_LOADER *loader); /* abort loading, free memory */ int (*abort)(DB_LOADER *loader); /* abort loading, free memory */
}; };
typedef struct __toku_indexer DB_INDEXER;
struct __toku_indexer_internal;
struct __toku_indexer {
struct __toku_indexer_internal *i;
int (*set_error_callback)(DB_INDEXER *indexer, void (*error_cb)(DB *db, int i, int err, DBT *key, DBT *val, void *error_extra), void *error_extra); /* set the error callback */
int (*set_poll_function)(DB_INDEXER *indexer, int (*poll_func)(void *extra, float progress), void *poll_extra); /* set the polling function */
int (*build)(DB_INDEXER *indexer); /* build the indexes */
int (*close)(DB_INDEXER *indexer); /* finish indexing, free memory */
int (*abort)(DB_INDEXER *indexer); /* abort indexing, free memory */
};
typedef struct __toku_engine_status { typedef struct __toku_engine_status {
char creationtime[26]; /* time of environment creation */ char creationtime[26]; /* time of environment creation */
char startuptime[26]; /* time of engine startup */ char startuptime[26]; /* time of engine startup */
...@@ -272,6 +282,7 @@ struct __toku_db_env { ...@@ -272,6 +282,7 @@ struct __toku_db_env {
int (*get_iname) (DB_ENV* env, DBT* dname_dbt, DBT* iname_dbt) /* FOR TEST ONLY: lookup existing iname */; int (*get_iname) (DB_ENV* env, DBT* dname_dbt, DBT* iname_dbt) /* FOR TEST ONLY: lookup existing iname */;
void *app_private; /* 32-bit offset=44 size=4, 64=bit offset=88 size=8 */ void *app_private; /* 32-bit offset=44 size=4, 64=bit offset=88 size=8 */
int (*create_loader) (DB_ENV *env, DB_TXN *txn, DB_LOADER **blp, DB *src_db, int N, DB *dbs[/*N*/], uint32_t db_flags[/*N*/], uint32_t dbt_flags[/*N*/], uint32_t loader_flags); int (*create_loader) (DB_ENV *env, DB_TXN *txn, DB_LOADER **blp, DB *src_db, int N, DB *dbs[/*N*/], uint32_t db_flags[/*N*/], uint32_t dbt_flags[/*N*/], uint32_t loader_flags);
int (*create_indexer) (DB_ENV *env, DB_TXN *txn, DB_INDEXER **idxrp, DB *src_db, int N, DB *dbs[/*N*/], uint32_t db_flags[/*N*/], uint32_t indexer_flags);
int (*put_multiple) (DB_ENV *env, DB *src_db, DB_TXN *txn, int (*put_multiple) (DB_ENV *env, DB *src_db, DB_TXN *txn,
const DBT *key, const DBT *val, const DBT *key, const DBT *val,
uint32_t num_dbs, DB **db_array, DBT *keys, DBT *vals, uint32_t *flags_array, uint32_t num_dbs, DB **db_array, DBT *keys, DBT *vals, uint32_t *flags_array,
...@@ -301,7 +312,7 @@ struct __toku_db_env { ...@@ -301,7 +312,7 @@ struct __toku_db_env {
int (*set_redzone) (DB_ENV *env, int redzone) /* set the redzone limit in percent of total space */; int (*set_redzone) (DB_ENV *env, int redzone) /* set the redzone limit in percent of total space */;
int (*set_lk_max_memory) (DB_ENV *env, uint64_t max); int (*set_lk_max_memory) (DB_ENV *env, uint64_t max);
int (*get_lk_max_memory) (DB_ENV *env, uint64_t *max); int (*get_lk_max_memory) (DB_ENV *env, uint64_t *max);
void* __toku_dummy0[15]; void* __toku_dummy0[14];
char __toku_dummy1[96]; char __toku_dummy1[96];
void *api1_internal; /* 32-bit offset=244 size=4, 64=bit offset=392 size=8 */ void *api1_internal; /* 32-bit offset=244 size=4, 64=bit offset=392 size=8 */
void* __toku_dummy2[7]; void* __toku_dummy2[7];
...@@ -404,7 +415,9 @@ struct __toku_db { ...@@ -404,7 +415,9 @@ struct __toku_db {
int (*flatten)(DB*, DB_TXN*) /* Flatten a dictionary, similar to (but faster than) a table scan */; int (*flatten)(DB*, DB_TXN*) /* Flatten a dictionary, similar to (but faster than) a table scan */;
int (*optimize)(DB*) /* Run garbage collecion and promote all transactions older than oldest. Amortized (happens during flattening) */; int (*optimize)(DB*) /* Run garbage collecion and promote all transactions older than oldest. Amortized (happens during flattening) */;
int (*get_fragmentation)(DB*,TOKU_DB_FRAGMENTATION); int (*get_fragmentation)(DB*,TOKU_DB_FRAGMENTATION);
void* __toku_dummy0[22]; int (*set_indexer)(DB*, DB_INDEXER*);
void (*get_indexer)(DB*, DB_INDEXER**);
void* __toku_dummy0[20];
char __toku_dummy1[96]; char __toku_dummy1[96];
void *api_internal; /* 32-bit offset=248 size=4, 64=bit offset=400 size=8 */ void *api_internal; /* 32-bit offset=248 size=4, 64=bit offset=400 size=8 */
void* __toku_dummy2[5]; void* __toku_dummy2[5];
......
buildheader/db.h_4_4
View file @ f31a90e9
...@@ -54,6 +54,16 @@ struct __toku_loader { ...@@ -54,6 +54,16 @@ struct __toku_loader {
int (*close)(DB_LOADER *loader); /* finish loading, free memory */ int (*close)(DB_LOADER *loader); /* finish loading, free memory */
int (*abort)(DB_LOADER *loader); /* abort loading, free memory */ int (*abort)(DB_LOADER *loader); /* abort loading, free memory */
}; };
typedef struct __toku_indexer DB_INDEXER;
struct __toku_indexer_internal;
struct __toku_indexer {
struct __toku_indexer_internal *i;
int (*set_error_callback)(DB_INDEXER *indexer, void (*error_cb)(DB *db, int i, int err, DBT *key, DBT *val, void *error_extra), void *error_extra); /* set the error callback */
int (*set_poll_function)(DB_INDEXER *indexer, int (*poll_func)(void *extra, float progress), void *poll_extra); /* set the polling function */
int (*build)(DB_INDEXER *indexer); /* build the indexes */
int (*close)(DB_INDEXER *indexer); /* finish indexing, free memory */
int (*abort)(DB_INDEXER *indexer); /* abort indexing, free memory */
};
typedef struct __toku_engine_status { typedef struct __toku_engine_status {
char creationtime[26]; /* time of environment creation */ char creationtime[26]; /* time of environment creation */
char startuptime[26]; /* time of engine startup */ char startuptime[26]; /* time of engine startup */
...@@ -272,6 +282,7 @@ struct __toku_db_env { ...@@ -272,6 +282,7 @@ struct __toku_db_env {
int (*get_iname) (DB_ENV* env, DBT* dname_dbt, DBT* iname_dbt) /* FOR TEST ONLY: lookup existing iname */; int (*get_iname) (DB_ENV* env, DBT* dname_dbt, DBT* iname_dbt) /* FOR TEST ONLY: lookup existing iname */;
void *app_private; /* 32-bit offset=44 size=4, 64=bit offset=88 size=8 */ void *app_private; /* 32-bit offset=44 size=4, 64=bit offset=88 size=8 */
int (*create_loader) (DB_ENV *env, DB_TXN *txn, DB_LOADER **blp, DB *src_db, int N, DB *dbs[/*N*/], uint32_t db_flags[/*N*/], uint32_t dbt_flags[/*N*/], uint32_t loader_flags); int (*create_loader) (DB_ENV *env, DB_TXN *txn, DB_LOADER **blp, DB *src_db, int N, DB *dbs[/*N*/], uint32_t db_flags[/*N*/], uint32_t dbt_flags[/*N*/], uint32_t loader_flags);
int (*create_indexer) (DB_ENV *env, DB_TXN *txn, DB_INDEXER **idxrp, DB *src_db, int N, DB *dbs[/*N*/], uint32_t db_flags[/*N*/], uint32_t indexer_flags);
int (*put_multiple) (DB_ENV *env, DB *src_db, DB_TXN *txn, int (*put_multiple) (DB_ENV *env, DB *src_db, DB_TXN *txn,
const DBT *key, const DBT *val, const DBT *key, const DBT *val,
uint32_t num_dbs, DB **db_array, DBT *keys, DBT *vals, uint32_t *flags_array, uint32_t num_dbs, DB **db_array, DBT *keys, DBT *vals, uint32_t *flags_array,
...@@ -301,7 +312,7 @@ struct __toku_db_env { ...@@ -301,7 +312,7 @@ struct __toku_db_env {
int (*set_redzone) (DB_ENV *env, int redzone) /* set the redzone limit in percent of total space */; int (*set_redzone) (DB_ENV *env, int redzone) /* set the redzone limit in percent of total space */;
int (*set_lk_max_memory) (DB_ENV *env, uint64_t max); int (*set_lk_max_memory) (DB_ENV *env, uint64_t max);
int (*get_lk_max_memory) (DB_ENV *env, uint64_t *max); int (*get_lk_max_memory) (DB_ENV *env, uint64_t *max);
void* __toku_dummy0[30]; void* __toku_dummy0[29];
char __toku_dummy1[128]; char __toku_dummy1[128];
void *api1_internal; /* 32-bit offset=336 size=4, 64=bit offset=544 size=8 */ void *api1_internal; /* 32-bit offset=336 size=4, 64=bit offset=544 size=8 */
void* __toku_dummy2[7]; void* __toku_dummy2[7];
...@@ -406,7 +417,9 @@ struct __toku_db { ...@@ -406,7 +417,9 @@ struct __toku_db {
int (*flatten)(DB*, DB_TXN*) /* Flatten a dictionary, similar to (but faster than) a table scan */; int (*flatten)(DB*, DB_TXN*) /* Flatten a dictionary, similar to (but faster than) a table scan */;
int (*optimize)(DB*) /* Run garbage collecion and promote all transactions older than oldest. Amortized (happens during flattening) */; int (*optimize)(DB*) /* Run garbage collecion and promote all transactions older than oldest. Amortized (happens during flattening) */;
int (*get_fragmentation)(DB*,TOKU_DB_FRAGMENTATION); int (*get_fragmentation)(DB*,TOKU_DB_FRAGMENTATION);
void* __toku_dummy0[24]; int (*set_indexer)(DB*, DB_INDEXER*);
void (*get_indexer)(DB*, DB_INDEXER**);
void* __toku_dummy0[22];
char __toku_dummy1[96]; char __toku_dummy1[96];
void *api_internal; /* 32-bit offset=256 size=4, 64=bit offset=416 size=8 */ void *api_internal; /* 32-bit offset=256 size=4, 64=bit offset=416 size=8 */
void* __toku_dummy2[5]; void* __toku_dummy2[5];
......
buildheader/db.h_4_5
View file @ f31a90e9
...@@ -54,6 +54,16 @@ struct __toku_loader { ...@@ -54,6 +54,16 @@ struct __toku_loader {
int (*close)(DB_LOADER *loader); /* finish loading, free memory */ int (*close)(DB_LOADER *loader); /* finish loading, free memory */
int (*abort)(DB_LOADER *loader); /* abort loading, free memory */ int (*abort)(DB_LOADER *loader); /* abort loading, free memory */
}; };
typedef struct __toku_indexer DB_INDEXER;
struct __toku_indexer_internal;
struct __toku_indexer {
struct __toku_indexer_internal *i;
int (*set_error_callback)(DB_INDEXER *indexer, void (*error_cb)(DB *db, int i, int err, DBT *key, DBT *val, void *error_extra), void *error_extra); /* set the error callback */
int (*set_poll_function)(DB_INDEXER *indexer, int (*poll_func)(void *extra, float progress), void *poll_extra); /* set the polling function */
int (*build)(DB_INDEXER *indexer); /* build the indexes */
int (*close)(DB_INDEXER *indexer); /* finish indexing, free memory */
int (*abort)(DB_INDEXER *indexer); /* abort indexing, free memory */
};
typedef struct __toku_engine_status { typedef struct __toku_engine_status {
char creationtime[26]; /* time of environment creation */ char creationtime[26]; /* time of environment creation */
char startuptime[26]; /* time of engine startup */ char startuptime[26]; /* time of engine startup */
...@@ -271,11 +281,12 @@ struct __toku_db_env { ...@@ -271,11 +281,12 @@ struct __toku_db_env {
int (*get_engine_status_text) (DB_ENV*, char*, int) /* Fill in status text */; int (*get_engine_status_text) (DB_ENV*, char*, int) /* Fill in status text */;
int (*get_iname) (DB_ENV* env, DBT* dname_dbt, DBT* iname_dbt) /* FOR TEST ONLY: lookup existing iname */; int (*get_iname) (DB_ENV* env, DBT* dname_dbt, DBT* iname_dbt) /* FOR TEST ONLY: lookup existing iname */;
int (*create_loader) (DB_ENV *env, DB_TXN *txn, DB_LOADER **blp, DB *src_db, int N, DB *dbs[/*N*/], uint32_t db_flags[/*N*/], uint32_t dbt_flags[/*N*/], uint32_t loader_flags); int (*create_loader) (DB_ENV *env, DB_TXN *txn, DB_LOADER **blp, DB *src_db, int N, DB *dbs[/*N*/], uint32_t db_flags[/*N*/], uint32_t dbt_flags[/*N*/], uint32_t loader_flags);
int (*create_indexer) (DB_ENV *env, DB_TXN *txn, DB_INDEXER **idxrp, DB *src_db, int N, DB *dbs[/*N*/], uint32_t db_flags[/*N*/], uint32_t indexer_flags);
void *app_private; /* 32-bit offset=52 size=4, 64=bit offset=104 size=8 */
int (*put_multiple) (DB_ENV *env, DB *src_db, DB_TXN *txn, int (*put_multiple) (DB_ENV *env, DB *src_db, DB_TXN *txn,
const DBT *key, const DBT *val, const DBT *key, const DBT *val,
uint32_t num_dbs, DB **db_array, DBT *keys, DBT *vals, uint32_t *flags_array, uint32_t num_dbs, DB **db_array, DBT *keys, DBT *vals, uint32_t *flags_array,
void *extra) /* insert into multiple DBs */; void *extra) /* insert into multiple DBs */;
void *app_private; /* 32-bit offset=52 size=4, 64=bit offset=104 size=8 */
int (*set_generate_row_callback_for_put) (DB_ENV *env, int (*set_generate_row_callback_for_put) (DB_ENV *env,
int (*generate_row_for_put)(DB *dest_db, DB *src_db, int (*generate_row_for_put)(DB *dest_db, DB *src_db,
DBT *dest_key, DBT *dest_val, DBT *dest_key, DBT *dest_val,
...@@ -301,7 +312,7 @@ struct __toku_db_env { ...@@ -301,7 +312,7 @@ struct __toku_db_env {
int (*set_redzone) (DB_ENV *env, int redzone) /* set the redzone limit in percent of total space */; int (*set_redzone) (DB_ENV *env, int redzone) /* set the redzone limit in percent of total space */;
int (*set_lk_max_memory) (DB_ENV *env, uint64_t max); int (*set_lk_max_memory) (DB_ENV *env, uint64_t max);
int (*get_lk_max_memory) (DB_ENV *env, uint64_t *max); int (*get_lk_max_memory) (DB_ENV *env, uint64_t *max);
void* __toku_dummy0[30]; void* __toku_dummy0[29];
char __toku_dummy1[128]; char __toku_dummy1[128];
void *api1_internal; /* 32-bit offset=336 size=4, 64=bit offset=544 size=8 */ void *api1_internal; /* 32-bit offset=336 size=4, 64=bit offset=544 size=8 */
void* __toku_dummy2[8]; void* __toku_dummy2[8];
...@@ -406,7 +417,9 @@ struct __toku_db { ...@@ -406,7 +417,9 @@ struct __toku_db {
int (*flatten)(DB*, DB_TXN*) /* Flatten a dictionary, similar to (but faster than) a table scan */; int (*flatten)(DB*, DB_TXN*) /* Flatten a dictionary, similar to (but faster than) a table scan */;
int (*optimize)(DB*) /* Run garbage collecion and promote all transactions older than oldest. Amortized (happens during flattening) */; int (*optimize)(DB*) /* Run garbage collecion and promote all transactions older than oldest. Amortized (happens during flattening) */;
int (*get_fragmentation)(DB*,TOKU_DB_FRAGMENTATION); int (*get_fragmentation)(DB*,TOKU_DB_FRAGMENTATION);
void* __toku_dummy0[27]; int (*set_indexer)(DB*, DB_INDEXER*);
void (*get_indexer)(DB*, DB_INDEXER**);
void* __toku_dummy0[25];
char __toku_dummy1[96]; char __toku_dummy1[96];
void *api_internal; /* 32-bit offset=268 size=4, 64=bit offset=440 size=8 */ void *api_internal; /* 32-bit offset=268 size=4, 64=bit offset=440 size=8 */
void* __toku_dummy2[5]; void* __toku_dummy2[5];
......
buildheader/db.h_4_6
View file @ f31a90e9
...@@ -54,6 +54,16 @@ struct __toku_loader { ...@@ -54,6 +54,16 @@ struct __toku_loader {
int (*close)(DB_LOADER *loader); /* finish loading, free memory */ int (*close)(DB_LOADER *loader); /* finish loading, free memory */
int (*abort)(DB_LOADER *loader); /* abort loading, free memory */ int (*abort)(DB_LOADER *loader); /* abort loading, free memory */
}; };
typedef struct __toku_indexer DB_INDEXER;
struct __toku_indexer_internal;
struct __toku_indexer {
struct __toku_indexer_internal *i;
int (*set_error_callback)(DB_INDEXER *indexer, void (*error_cb)(DB *db, int i, int err, DBT *key, DBT *val, void *error_extra), void *error_extra); /* set the error callback */
int (*set_poll_function)(DB_INDEXER *indexer, int (*poll_func)(void *extra, float progress), void *poll_extra); /* set the polling function */
int (*build)(DB_INDEXER *indexer); /* build the indexes */
int (*close)(DB_INDEXER *indexer); /* finish indexing, free memory */
int (*abort)(DB_INDEXER *indexer); /* abort indexing, free memory */
};
typedef struct __toku_engine_status { typedef struct __toku_engine_status {
char creationtime[26]; /* time of environment creation */ char creationtime[26]; /* time of environment creation */
char startuptime[26]; /* time of engine startup */ char startuptime[26]; /* time of engine startup */
...@@ -272,11 +282,12 @@ struct __toku_db_env { ...@@ -272,11 +282,12 @@ struct __toku_db_env {
int (*get_engine_status_text) (DB_ENV*, char*, int) /* Fill in status text */; int (*get_engine_status_text) (DB_ENV*, char*, int) /* Fill in status text */;
int (*get_iname) (DB_ENV* env, DBT* dname_dbt, DBT* iname_dbt) /* FOR TEST ONLY: lookup existing iname */; int (*get_iname) (DB_ENV* env, DBT* dname_dbt, DBT* iname_dbt) /* FOR TEST ONLY: lookup existing iname */;
int (*create_loader) (DB_ENV *env, DB_TXN *txn, DB_LOADER **blp, DB *src_db, int N, DB *dbs[/*N*/], uint32_t db_flags[/*N*/], uint32_t dbt_flags[/*N*/], uint32_t loader_flags); int (*create_loader) (DB_ENV *env, DB_TXN *txn, DB_LOADER **blp, DB *src_db, int N, DB *dbs[/*N*/], uint32_t db_flags[/*N*/], uint32_t dbt_flags[/*N*/], uint32_t loader_flags);
int (*create_indexer) (DB_ENV *env, DB_TXN *txn, DB_INDEXER **idxrp, DB *src_db, int N, DB *dbs[/*N*/], uint32_t db_flags[/*N*/], uint32_t indexer_flags);
void *app_private; /* 32-bit offset=52 size=4, 64=bit offset=104 size=8 */
int (*put_multiple) (DB_ENV *env, DB *src_db, DB_TXN *txn, int (*put_multiple) (DB_ENV *env, DB *src_db, DB_TXN *txn,
const DBT *key, const DBT *val, const DBT *key, const DBT *val,
uint32_t num_dbs, DB **db_array, DBT *keys, DBT *vals, uint32_t *flags_array, uint32_t num_dbs, DB **db_array, DBT *keys, DBT *vals, uint32_t *flags_array,
void *extra) /* insert into multiple DBs */; void *extra) /* insert into multiple DBs */;
void *app_private; /* 32-bit offset=52 size=4, 64=bit offset=104 size=8 */
int (*set_generate_row_callback_for_put) (DB_ENV *env, int (*set_generate_row_callback_for_put) (DB_ENV *env,
int (*generate_row_for_put)(DB *dest_db, DB *src_db, int (*generate_row_for_put)(DB *dest_db, DB *src_db,
DBT *dest_key, DBT *dest_val, DBT *dest_key, DBT *dest_val,
...@@ -302,7 +313,7 @@ struct __toku_db_env { ...@@ -302,7 +313,7 @@ struct __toku_db_env {
int (*set_redzone) (DB_ENV *env, int redzone) /* set the redzone limit in percent of total space */; int (*set_redzone) (DB_ENV *env, int redzone) /* set the redzone limit in percent of total space */;
int (*set_lk_max_memory) (DB_ENV *env, uint64_t max); int (*set_lk_max_memory) (DB_ENV *env, uint64_t max);
int (*get_lk_max_memory) (DB_ENV *env, uint64_t *max); int (*get_lk_max_memory) (DB_ENV *env, uint64_t *max);
void* __toku_dummy0[31]; void* __toku_dummy0[30];
char __toku_dummy1[144]; char __toku_dummy1[144];
void *api1_internal; /* 32-bit offset=356 size=4, 64=bit offset=568 size=8 */ void *api1_internal; /* 32-bit offset=356 size=4, 64=bit offset=568 size=8 */
void* __toku_dummy2[8]; void* __toku_dummy2[8];
...@@ -409,7 +420,9 @@ struct __toku_db { ...@@ -409,7 +420,9 @@ struct __toku_db {
int (*flatten)(DB*, DB_TXN*) /* Flatten a dictionary, similar to (but faster than) a table scan */; int (*flatten)(DB*, DB_TXN*) /* Flatten a dictionary, similar to (but faster than) a table scan */;
int (*optimize)(DB*) /* Run garbage collecion and promote all transactions older than oldest. Amortized (happens during flattening) */; int (*optimize)(DB*) /* Run garbage collecion and promote all transactions older than oldest. Amortized (happens during flattening) */;
int (*get_fragmentation)(DB*,TOKU_DB_FRAGMENTATION); int (*get_fragmentation)(DB*,TOKU_DB_FRAGMENTATION);
void* __toku_dummy1[31]; int (*set_indexer)(DB*, DB_INDEXER*);
void (*get_indexer)(DB*, DB_INDEXER**);
void* __toku_dummy1[29];
char __toku_dummy2[80]; char __toku_dummy2[80];
void *api_internal; /* 32-bit offset=276 size=4, 64=bit offset=464 size=8 */ void *api_internal; /* 32-bit offset=276 size=4, 64=bit offset=464 size=8 */
void* __toku_dummy3[5]; void* __toku_dummy3[5];
......
buildheader/make_db_h.c
View file @ f31a90e9
...@@ -437,6 +437,18 @@ int main (int argc __attribute__((__unused__)), char *const argv[] __attribute__ ...@@ -437,6 +437,18 @@ int main (int argc __attribute__((__unused__)), char *const argv[] __attribute__
printf(" int (*abort)(DB_LOADER *loader); /* abort loading, free memory */\n"); printf(" int (*abort)(DB_LOADER *loader); /* abort loading, free memory */\n");
printf("};\n"); printf("};\n");
//indexer
printf("typedef struct __toku_indexer DB_INDEXER;\n");
printf("struct __toku_indexer_internal;\n");
printf("struct __toku_indexer {\n");
printf(" struct __toku_indexer_internal *i;\n");
printf(" int (*set_error_callback)(DB_INDEXER *indexer, void (*error_cb)(DB *db, int i, int err, DBT *key, DBT *val, void *error_extra), void *error_extra); /* set the error callback */\n");
printf(" int (*set_poll_function)(DB_INDEXER *indexer, int (*poll_func)(void *extra, float progress), void *poll_extra); /* set the polling function */\n");
printf(" int (*build)(DB_INDEXER *indexer); /* build the indexes */\n");
printf(" int (*close)(DB_INDEXER *indexer); /* finish indexing, free memory */\n");
printf(" int (*abort)(DB_INDEXER *indexer); /* abort indexing, free memory */\n");
printf("};\n");
//engine status info //engine status info
printf("typedef struct __toku_engine_status {\n"); printf("typedef struct __toku_engine_status {\n");
printf(" char creationtime[26]; /* time of environment creation */ \n"); printf(" char creationtime[26]; /* time of environment creation */ \n");
...@@ -571,6 +583,7 @@ int main (int argc __attribute__((__unused__)), char *const argv[] __attribute__ ...@@ -571,6 +583,7 @@ int main (int argc __attribute__((__unused__)), char *const argv[] __attribute__
"int (*get_engine_status_text) (DB_ENV*, char*, int) /* Fill in status text */", "int (*get_engine_status_text) (DB_ENV*, char*, int) /* Fill in status text */",
"int (*get_iname) (DB_ENV* env, DBT* dname_dbt, DBT* iname_dbt) /* FOR TEST ONLY: lookup existing iname */", "int (*get_iname) (DB_ENV* env, DBT* dname_dbt, DBT* iname_dbt) /* FOR TEST ONLY: lookup existing iname */",
"int (*create_loader) (DB_ENV *env, DB_TXN *txn, DB_LOADER **blp, DB *src_db, int N, DB *dbs[/*N*/], uint32_t db_flags[/*N*/], uint32_t dbt_flags[/*N*/], uint32_t loader_flags)", "int (*create_loader) (DB_ENV *env, DB_TXN *txn, DB_LOADER **blp, DB *src_db, int N, DB *dbs[/*N*/], uint32_t db_flags[/*N*/], uint32_t dbt_flags[/*N*/], uint32_t loader_flags)",
"int (*create_indexer) (DB_ENV *env, DB_TXN *txn, DB_INDEXER **idxrp, DB *src_db, int N, DB *dbs[/*N*/], uint32_t db_flags[/*N*/], uint32_t indexer_flags)",
"int (*put_multiple) (DB_ENV *env, DB *src_db, DB_TXN *txn,\n" "int (*put_multiple) (DB_ENV *env, DB *src_db, DB_TXN *txn,\n"
" const DBT *key, const DBT *val,\n" " const DBT *key, const DBT *val,\n"
" uint32_t num_dbs, DB **db_array, DBT *keys, DBT *vals, uint32_t *flags_array,\n" " uint32_t num_dbs, DB **db_array, DBT *keys, DBT *vals, uint32_t *flags_array,\n"
...@@ -652,6 +665,8 @@ int main (int argc __attribute__((__unused__)), char *const argv[] __attribute__ ...@@ -652,6 +665,8 @@ int main (int argc __attribute__((__unused__)), char *const argv[] __attribute__
"int (*flatten)(DB*, DB_TXN*) /* Flatten a dictionary, similar to (but faster than) a table scan */", "int (*flatten)(DB*, DB_TXN*) /* Flatten a dictionary, similar to (but faster than) a table scan */",
"int (*optimize)(DB*) /* Run garbage collecion and promote all transactions older than oldest. Amortized (happens during flattening) */", "int (*optimize)(DB*) /* Run garbage collecion and promote all transactions older than oldest. Amortized (happens during flattening) */",
"int (*get_fragmentation)(DB*,TOKU_DB_FRAGMENTATION)", "int (*get_fragmentation)(DB*,TOKU_DB_FRAGMENTATION)",
"int (*set_indexer)(DB*, DB_INDEXER*)",
"void (*get_indexer)(DB*, DB_INDEXER**)",
NULL}; NULL};
print_struct("db", 1, db_fields32, db_fields64, sizeof(db_fields32)/sizeof(db_fields32[0]), extra); print_struct("db", 1, db_fields32, db_fields64, sizeof(db_fields32)/sizeof(db_fields32[0]), extra);
} }
......
buildheader/tdb.h
View file @ f31a90e9
...@@ -54,6 +54,16 @@ struct __toku_loader { ...@@ -54,6 +54,16 @@ struct __toku_loader {
int (*close)(DB_LOADER *loader); /* finish loading, free memory */ int (*close)(DB_LOADER *loader); /* finish loading, free memory */
int (*abort)(DB_LOADER *loader); /* abort loading, free memory */ int (*abort)(DB_LOADER *loader); /* abort loading, free memory */
}; };
typedef struct __toku_indexer DB_INDEXER;
struct __toku_indexer_internal;
struct __toku_indexer {
struct __toku_indexer_internal *i;
int (*set_error_callback)(DB_INDEXER *indexer, void (*error_cb)(DB *db, int i, int err, DBT *key, DBT *val, void *error_extra), void *error_extra); /* set the error callback */
int (*set_poll_function)(DB_INDEXER *indexer, int (*poll_func)(void *extra, float progress), void *poll_extra); /* set the polling function */
int (*build)(DB_INDEXER *indexer); /* build the indexes */
int (*close)(DB_INDEXER *indexer); /* finish indexing, free memory */
int (*abort)(DB_INDEXER *indexer); /* abort indexing, free memory */
};
typedef struct __toku_engine_status { typedef struct __toku_engine_status {
char creationtime[26]; /* time of environment creation */ char creationtime[26]; /* time of environment creation */
char startuptime[26]; /* time of engine startup */ char startuptime[26]; /* time of engine startup */
...@@ -272,11 +282,12 @@ struct __toku_db_env { ...@@ -272,11 +282,12 @@ struct __toku_db_env {
int (*get_engine_status_text) (DB_ENV*, char*, int) /* Fill in status text */; int (*get_engine_status_text) (DB_ENV*, char*, int) /* Fill in status text */;
int (*get_iname) (DB_ENV* env, DBT* dname_dbt, DBT* iname_dbt) /* FOR TEST ONLY: lookup existing iname */; int (*get_iname) (DB_ENV* env, DBT* dname_dbt, DBT* iname_dbt) /* FOR TEST ONLY: lookup existing iname */;
int (*create_loader) (DB_ENV *env, DB_TXN *txn, DB_LOADER **blp, DB *src_db, int N, DB *dbs[/*N*/], uint32_t db_flags[/*N*/], uint32_t dbt_flags[/*N*/], uint32_t loader_flags); int (*create_loader) (DB_ENV *env, DB_TXN *txn, DB_LOADER **blp, DB *src_db, int N, DB *dbs[/*N*/], uint32_t db_flags[/*N*/], uint32_t dbt_flags[/*N*/], uint32_t loader_flags);
int (*create_indexer) (DB_ENV *env, DB_TXN *txn, DB_INDEXER **idxrp, DB *src_db, int N, DB *dbs[/*N*/], uint32_t db_flags[/*N*/], uint32_t indexer_flags);
void *app_private;
int (*put_multiple) (DB_ENV *env, DB *src_db, DB_TXN *txn, int (*put_multiple) (DB_ENV *env, DB *src_db, DB_TXN *txn,
const DBT *key, const DBT *val, const DBT *key, const DBT *val,
uint32_t num_dbs, DB **db_array, DBT *keys, DBT *vals, uint32_t *flags_array, uint32_t num_dbs, DB **db_array, DBT *keys, DBT *vals, uint32_t *flags_array,
void *extra) /* insert into multiple DBs */; void *extra) /* insert into multiple DBs */;
void *app_private;
int (*set_generate_row_callback_for_put) (DB_ENV *env, int (*set_generate_row_callback_for_put) (DB_ENV *env,
int (*generate_row_for_put)(DB *dest_db, DB *src_db, int (*generate_row_for_put)(DB *dest_db, DB *src_db,
DBT *dest_key, DBT *dest_val, DBT *dest_key, DBT *dest_val,
...@@ -378,6 +389,8 @@ struct __toku_db { ...@@ -378,6 +389,8 @@ struct __toku_db {
int (*flatten)(DB*, DB_TXN*) /* Flatten a dictionary, similar to (but faster than) a table scan */; int (*flatten)(DB*, DB_TXN*) /* Flatten a dictionary, similar to (but faster than) a table scan */;
int (*optimize)(DB*) /* Run garbage collecion and promote all transactions older than oldest. Amortized (happens during flattening) */; int (*optimize)(DB*) /* Run garbage collecion and promote all transactions older than oldest. Amortized (happens during flattening) */;
int (*get_fragmentation)(DB*,TOKU_DB_FRAGMENTATION); int (*get_fragmentation)(DB*,TOKU_DB_FRAGMENTATION);
int (*set_indexer)(DB*, DB_INDEXER*);
void (*get_indexer)(DB*, DB_INDEXER**);
void *api_internal; void *api_internal;
int (*close) (DB*, u_int32_t); int (*close) (DB*, u_int32_t);
int (*cursor) (DB *, DB_TXN *, DBC **, u_int32_t); int (*cursor) (DB *, DB_TXN *, DBC **, u_int32_t);
......
include/db.h
View file @ f31a90e9
...@@ -54,6 +54,16 @@ struct __toku_loader { ...@@ -54,6 +54,16 @@ struct __toku_loader {
int (*close)(DB_LOADER *loader); /* finish loading, free memory */ int (*close)(DB_LOADER *loader); /* finish loading, free memory */
int (*abort)(DB_LOADER *loader); /* abort loading, free memory */ int (*abort)(DB_LOADER *loader); /* abort loading, free memory */
}; };
typedef struct __toku_indexer DB_INDEXER;
struct __toku_indexer_internal;
struct __toku_indexer {
struct __toku_indexer_internal *i;
int (*set_error_callback)(DB_INDEXER *indexer, void (*error_cb)(DB *db, int i, int err, DBT *key, DBT *val, void *error_extra), void *error_extra); /* set the error callback */
int (*set_poll_function)(DB_INDEXER *indexer, int (*poll_func)(void *extra, float progress), void *poll_extra); /* set the polling function */
int (*build)(DB_INDEXER *indexer); /* build the indexes */
int (*close)(DB_INDEXER *indexer); /* finish indexing, free memory */
int (*abort)(DB_INDEXER *indexer); /* abort indexing, free memory */
};
typedef struct __toku_engine_status { typedef struct __toku_engine_status {
char creationtime[26]; /* time of environment creation */ char creationtime[26]; /* time of environment creation */
char startuptime[26]; /* time of engine startup */ char startuptime[26]; /* time of engine startup */
...@@ -272,11 +282,12 @@ struct __toku_db_env { ...@@ -272,11 +282,12 @@ struct __toku_db_env {
int (*get_engine_status_text) (DB_ENV*, char*, int) /* Fill in status text */; int (*get_engine_status_text) (DB_ENV*, char*, int) /* Fill in status text */;
int (*get_iname) (DB_ENV* env, DBT* dname_dbt, DBT* iname_dbt) /* FOR TEST ONLY: lookup existing iname */; int (*get_iname) (DB_ENV* env, DBT* dname_dbt, DBT* iname_dbt) /* FOR TEST ONLY: lookup existing iname */;
int (*create_loader) (DB_ENV *env, DB_TXN *txn, DB_LOADER **blp, DB *src_db, int N, DB *dbs[/*N*/], uint32_t db_flags[/*N*/], uint32_t dbt_flags[/*N*/], uint32_t loader_flags); int (*create_loader) (DB_ENV *env, DB_TXN *txn, DB_LOADER **blp, DB *src_db, int N, DB *dbs[/*N*/], uint32_t db_flags[/*N*/], uint32_t dbt_flags[/*N*/], uint32_t loader_flags);
int (*create_indexer) (DB_ENV *env, DB_TXN *txn, DB_INDEXER **idxrp, DB *src_db, int N, DB *dbs[/*N*/], uint32_t db_flags[/*N*/], uint32_t indexer_flags);
void *app_private;
int (*put_multiple) (DB_ENV *env, DB *src_db, DB_TXN *txn, int (*put_multiple) (DB_ENV *env, DB *src_db, DB_TXN *txn,
const DBT *key, const DBT *val, const DBT *key, const DBT *val,
uint32_t num_dbs, DB **db_array, DBT *keys, DBT *vals, uint32_t *flags_array, uint32_t num_dbs, DB **db_array, DBT *keys, DBT *vals, uint32_t *flags_array,
void *extra) /* insert into multiple DBs */; void *extra) /* insert into multiple DBs */;
void *app_private;
int (*set_generate_row_callback_for_put) (DB_ENV *env, int (*set_generate_row_callback_for_put) (DB_ENV *env,
int (*generate_row_for_put)(DB *dest_db, DB *src_db, int (*generate_row_for_put)(DB *dest_db, DB *src_db,
DBT *dest_key, DBT *dest_val, DBT *dest_key, DBT *dest_val,
...@@ -378,6 +389,8 @@ struct __toku_db { ...@@ -378,6 +389,8 @@ struct __toku_db {
int (*flatten)(DB*, DB_TXN*) /* Flatten a dictionary, similar to (but faster than) a table scan */; int (*flatten)(DB*, DB_TXN*) /* Flatten a dictionary, similar to (but faster than) a table scan */;
int (*optimize)(DB*) /* Run garbage collecion and promote all transactions older than oldest. Amortized (happens during flattening) */; int (*optimize)(DB*) /* Run garbage collecion and promote all transactions older than oldest. Amortized (happens during flattening) */;
int (*get_fragmentation)(DB*,TOKU_DB_FRAGMENTATION); int (*get_fragmentation)(DB*,TOKU_DB_FRAGMENTATION);
int (*set_indexer)(DB*, DB_INDEXER*);
void (*get_indexer)(DB*, DB_INDEXER**);
void *api_internal; void *api_internal;
int (*close) (DB*, u_int32_t); int (*close) (DB*, u_int32_t);
int (*cursor) (DB *, DB_TXN *, DBC **, u_int32_t); int (*cursor) (DB *, DB_TXN *, DBC **, u_int32_t);
......
newbrt/brt.c
View file @ f31a90e9
...@@ -2619,6 +2619,26 @@ toku_brt_load_recovery(TOKUTXN txn, char const * old_iname, char const * new_ina ...@@ -2619,6 +2619,26 @@ toku_brt_load_recovery(TOKUTXN txn, char const * old_iname, char const * new_ina
return r; return r;
} }
// 2954
// this function handles the tasks needed to be recoverable
// - write to rollback log
// - write to recovery log
int
toku_brt_hot_index_recovery(TOKUTXN txn, FILENUMS filenums, int do_fsync, int do_log, LSN *hot_index_lsn)
{
int r = 0;
assert(txn);
TOKULOGGER logger = toku_txn_logger(txn);
// write to the rollback log
r = toku_logger_save_rollback_hot_index(txn, &filenums);
if ( r==0 && do_log && logger) {
TXNID xid = toku_txn_get_txnid(txn);
// write to the recovery log
r = toku_log_hot_index(logger, hot_index_lsn, do_fsync, xid, filenums);
}
return r;
}
static int brt_optimize (BRT brt, BOOL upgrade); static int brt_optimize (BRT brt, BOOL upgrade);
...@@ -2683,6 +2703,16 @@ toku_brt_load(BRT brt, TOKUTXN txn, char const * new_iname, int do_fsync, LSN *l ...@@ -2683,6 +2703,16 @@ toku_brt_load(BRT brt, TOKUTXN txn, char const * new_iname, int do_fsync, LSN *l
return r; return r;
} }
// 2954
// brt actions for logging hot index filenums
int
toku_brt_hot_index(BRT brt __attribute__ ((unused)), TOKUTXN txn, FILENUMS filenums, int do_fsync, LSN *lsn) {
int r = 0;
int do_log = 1;
r = toku_brt_hot_index_recovery(txn, filenums, do_fsync, do_log, lsn);
return r;
}
int int
toku_brt_log_put (TOKUTXN txn, BRT brt, const DBT *key, const DBT *val) { toku_brt_log_put (TOKUTXN txn, BRT brt, const DBT *key, const DBT *val) {
int r = 0; int r = 0;
...@@ -2728,7 +2758,7 @@ toku_brt_log_put_multiple (TOKUTXN txn, BRT src_brt, BRT *brts, int num_brts, co ...@@ -2728,7 +2758,7 @@ toku_brt_log_put_multiple (TOKUTXN txn, BRT src_brt, BRT *brts, int num_brts, co
} }
int int
toku_brt_maybe_insert (BRT brt, DBT *key, DBT *val, TOKUTXN txn, BOOL oplsn_valid, LSN oplsn, int do_logging, enum brt_msg_type type) { toku_brt_maybe_insert (BRT brt, DBT *key, DBT *val, TOKUTXN txn, BOOL oplsn_valid, LSN oplsn, BOOL do_logging, enum brt_msg_type type) {
lazy_assert(type==BRT_INSERT || type==BRT_INSERT_NO_OVERWRITE); lazy_assert(type==BRT_INSERT || type==BRT_INSERT_NO_OVERWRITE);
int r = 0; int r = 0;
XIDS message_xids = xids_get_root_xids(); //By default use committed messages XIDS message_xids = xids_get_root_xids(); //By default use committed messages
...@@ -2766,12 +2796,26 @@ toku_brt_maybe_insert (BRT brt, DBT *key, DBT *val, TOKUTXN txn, BOOL oplsn_vali ...@@ -2766,12 +2796,26 @@ toku_brt_maybe_insert (BRT brt, DBT *key, DBT *val, TOKUTXN txn, BOOL oplsn_vali
if (oplsn_valid && oplsn.lsn <= (treelsn = toku_brt_checkpoint_lsn(brt)).lsn) { if (oplsn_valid && oplsn.lsn <= (treelsn = toku_brt_checkpoint_lsn(brt)).lsn) {
r = 0; r = 0;
} else { } else {
BRT_MSG_S brtcmd = { type, message_xids, .u.id={key,val}}; r = toku_brt_send_insert(brt, key, val, message_xids, type);
r = toku_brt_root_put_cmd(brt, &brtcmd);
} }
return r; return r;
} }
int
toku_brt_send_insert(BRT brt, DBT *key, DBT *val, XIDS xids, enum brt_msg_type type) {
BRT_MSG_S brtcmd = { type, xids, .u.id = { key, val }};
int r = toku_brt_root_put_cmd(brt, &brtcmd);
return r;
}
int
toku_brt_send_commit_any(BRT brt, DBT *key, XIDS xids) {
DBT val;
BRT_MSG_S brtcmd = { BRT_COMMIT_ANY, xids, .u.id = { key, toku_init_dbt(&val) }};
int r = toku_brt_root_put_cmd(brt, &brtcmd);
return r;
}
int toku_brt_delete(BRT brt, DBT *key, TOKUTXN txn) { int toku_brt_delete(BRT brt, DBT *key, TOKUTXN txn) {
return toku_brt_maybe_delete(brt, key, txn, FALSE, ZERO_LSN, TRUE); return toku_brt_maybe_delete(brt, key, txn, FALSE, ZERO_LSN, TRUE);
} }
...@@ -2817,7 +2861,7 @@ toku_brt_log_del_multiple (TOKUTXN txn, BRT src_brt, BRT *brts, int num_brts, co ...@@ -2817,7 +2861,7 @@ toku_brt_log_del_multiple (TOKUTXN txn, BRT src_brt, BRT *brts, int num_brts, co
} }
int int
toku_brt_maybe_delete(BRT brt, DBT *key, TOKUTXN txn, BOOL oplsn_valid, LSN oplsn, int do_logging) { toku_brt_maybe_delete(BRT brt, DBT *key, TOKUTXN txn, BOOL oplsn_valid, LSN oplsn, BOOL do_logging) {
int r; int r;
XIDS message_xids = xids_get_root_xids(); //By default use committed messages XIDS message_xids = xids_get_root_xids(); //By default use committed messages
TXNID xid = toku_txn_get_txnid(txn); TXNID xid = toku_txn_get_txnid(txn);
...@@ -2848,13 +2892,19 @@ toku_brt_maybe_delete(BRT brt, DBT *key, TOKUTXN txn, BOOL oplsn_valid, LSN opls ...@@ -2848,13 +2892,19 @@ toku_brt_maybe_delete(BRT brt, DBT *key, TOKUTXN txn, BOOL oplsn_valid, LSN opls
if (oplsn_valid && oplsn.lsn <= (treelsn = toku_brt_checkpoint_lsn(brt)).lsn) { if (oplsn_valid && oplsn.lsn <= (treelsn = toku_brt_checkpoint_lsn(brt)).lsn) {
r = 0; r = 0;
} else { } else {
DBT val; r = toku_brt_send_delete(brt, key, message_xids);
BRT_MSG_S brtcmd = { BRT_DELETE_ANY, message_xids, .u.id={key, toku_init_dbt(&val)}};
r = toku_brt_root_put_cmd(brt, &brtcmd);
} }
return r; return r;
} }
int
toku_brt_send_delete(BRT brt, DBT *key, XIDS xids) {
DBT val; toku_init_dbt(&val);
BRT_MSG_S brtcmd = { BRT_DELETE_ANY, xids, .u.id = { key, &val }};
int result = toku_brt_root_put_cmd(brt, &brtcmd);
return result;
}
struct omt_compressor_state { struct omt_compressor_state {
struct mempool *new_kvspace; struct mempool *new_kvspace;
OMT omt; OMT omt;
......
newbrt/brt.h
View file @ f31a90e9
...@@ -72,9 +72,15 @@ int toku_brt_optimize_for_upgrade (BRT brt) __attribute__ ((warn_unused_result) ...@@ -72,9 +72,15 @@ int toku_brt_optimize_for_upgrade (BRT brt) __attribute__ ((warn_unused_result)
// Effect: Insert a key and data pair into a brt if the oplsn is newer than the brt lsn. This function is called during recovery. // Effect: Insert a key and data pair into a brt if the oplsn is newer than the brt lsn. This function is called during recovery.
// Returns 0 if successful // Returns 0 if successful
int toku_brt_maybe_insert (BRT brt, DBT *k, DBT *v, TOKUTXN txn, BOOL oplsn_valid, LSN oplsn, int do_logging, enum brt_msg_type type) __attribute__ ((warn_unused_result)); int toku_brt_maybe_insert (BRT brt, DBT *k, DBT *v, TOKUTXN txn, BOOL oplsn_valid, LSN oplsn, BOOL do_logging, enum brt_msg_type type) __attribute__ ((warn_unused_result));
int toku_brt_load_recovery(TOKUTXN txn, char const * old_iname, char const * new_iname, int do_fsync, int do_log, LSN *load_lsn) __attribute__ ((warn_unused_result)); int toku_brt_load_recovery(TOKUTXN txn, char const * old_iname, char const * new_iname, int do_fsync, int do_log, LSN *load_lsn) __attribute__ ((warn_unused_result));
int toku_brt_load(BRT brt, TOKUTXN txn, char const * new_iname, int do_fsync, LSN *get_lsn) __attribute__ ((warn_unused_result)); int toku_brt_load(BRT brt, TOKUTXN txn, char const * new_iname, int do_fsync, LSN *get_lsn) __attribute__ ((warn_unused_result));
// 2954
int toku_brt_hot_index_recovery(TOKUTXN txn, FILENUMS filenums, int do_fsync, int do_log, LSN *hot_index_lsn);
int toku_brt_hot_index(BRT brt, TOKUTXN txn, FILENUMS filenums, int do_fsync, LSN *lsn) __attribute__ ((warn_unused_result));
int toku_brt_log_put_multiple (TOKUTXN txn, BRT src_brt, BRT *brts, int num_brts, const DBT *key, const DBT *val) __attribute__ ((warn_unused_result)); int toku_brt_log_put_multiple (TOKUTXN txn, BRT src_brt, BRT *brts, int num_brts, const DBT *key, const DBT *val) __attribute__ ((warn_unused_result));
int toku_brt_log_put (TOKUTXN txn, BRT brt, const DBT *key, const DBT *val) __attribute__ ((warn_unused_result)); int toku_brt_log_put (TOKUTXN txn, BRT brt, const DBT *key, const DBT *val) __attribute__ ((warn_unused_result));
int toku_brt_log_del_multiple (TOKUTXN txn, BRT src_brt, BRT *brts, int num_brts, const DBT *key, const DBT *val) __attribute__ ((warn_unused_result)); int toku_brt_log_del_multiple (TOKUTXN txn, BRT src_brt, BRT *brts, int num_brts, const DBT *key, const DBT *val) __attribute__ ((warn_unused_result));
...@@ -86,16 +92,11 @@ int toku_brt_delete (BRT brt, DBT *k, TOKUTXN txn) __attribute__ ((warn_unused_ ...@@ -86,16 +92,11 @@ int toku_brt_delete (BRT brt, DBT *k, TOKUTXN txn) __attribute__ ((warn_unused_
// Effect: Delete a key from a brt if the oplsn is newer than the brt lsn. This function is called during recovery. // Effect: Delete a key from a brt if the oplsn is newer than the brt lsn. This function is called during recovery.
// Returns 0 if successful // Returns 0 if successful
int toku_brt_maybe_delete (BRT brt, DBT *k, TOKUTXN txn, BOOL oplsn_valid, LSN oplsn, int do_logging) __attribute__ ((warn_unused_result)); int toku_brt_maybe_delete (BRT brt, DBT *k, TOKUTXN txn, BOOL oplsn_valid, LSN oplsn, BOOL do_logging) __attribute__ ((warn_unused_result));
// Effect: Delete a pair only if both k and v are equal according to the comparison function. int toku_brt_send_insert(BRT brt, DBT *key, DBT *val, XIDS xids, enum brt_msg_type type) __attribute__ ((warn_unused_result));
// Returns 0 if successful int toku_brt_send_delete(BRT brt, DBT *key, XIDS xids) __attribute__ ((warn_unused_result));
int toku_brt_delete_both (BRT brt, DBT *k, DBT *v, TOKUTXN txn) __attribute__ ((warn_unused_result)); int toku_brt_send_commit_any(BRT brt, DBT *key, XIDS xids) __attribute__ ((warn_unused_result));
// Effect: Delete a pair only if both k and v are equal according to the comparison function and the
// oplsn is newer than the brt lsn. This function is called by recovery.
// Returns 0 if successful
int toku_brt_maybe_delete_both (BRT brt, DBT *k, DBT *v, TOKUTXN txn, BOOL oplsn_valid, LSN oplsn) __attribute__ ((warn_unused_result));
int toku_brt_db_delay_closed (BRT brt, DB* db, int (*close_db)(DB*, u_int32_t), u_int32_t close_flags) __attribute__ ((warn_unused_result)); int toku_brt_db_delay_closed (BRT brt, DB* db, int (*close_db)(DB*, u_int32_t), u_int32_t close_flags) __attribute__ ((warn_unused_result));
int toku_close_brt (BRT, char **error_string) __attribute__ ((warn_unused_result)); int toku_close_brt (BRT, char **error_string) __attribute__ ((warn_unused_result));
...@@ -155,7 +156,6 @@ int toku_brt_cursor_set_range_reverse(BRT_CURSOR cursor, DBT *key, BRT_GET_CALLB ...@@ -155,7 +156,6 @@ int toku_brt_cursor_set_range_reverse(BRT_CURSOR cursor, DBT *key, BRT_GET_CALLB
int toku_brt_cursor_get_both_range(BRT_CURSOR cursor, DBT *key, DBT *val, BRT_GET_CALLBACK_FUNCTION getf, void *getf_v) __attribute__ ((warn_unused_result)); int toku_brt_cursor_get_both_range(BRT_CURSOR cursor, DBT *key, DBT *val, BRT_GET_CALLBACK_FUNCTION getf, void *getf_v) __attribute__ ((warn_unused_result));
int toku_brt_cursor_get_both_range_reverse(BRT_CURSOR cursor, DBT *key, DBT *val, BRT_GET_CALLBACK_FUNCTION getf, void *getf_v) __attribute__ ((warn_unused_result)); int toku_brt_cursor_get_both_range_reverse(BRT_CURSOR cursor, DBT *key, DBT *val, BRT_GET_CALLBACK_FUNCTION getf, void *getf_v) __attribute__ ((warn_unused_result));
int toku_brt_cursor_delete(BRT_CURSOR cursor, int flags, TOKUTXN) __attribute__ ((warn_unused_result)); int toku_brt_cursor_delete(BRT_CURSOR cursor, int flags, TOKUTXN) __attribute__ ((warn_unused_result));
int toku_brt_cursor_close (BRT_CURSOR curs) __attribute__ ((warn_unused_result)); int toku_brt_cursor_close (BRT_CURSOR curs) __attribute__ ((warn_unused_result));
BOOL toku_brt_cursor_uninitialized(BRT_CURSOR c) __attribute__ ((warn_unused_result)); BOOL toku_brt_cursor_uninitialized(BRT_CURSOR c) __attribute__ ((warn_unused_result));
...@@ -228,6 +228,12 @@ BOOL toku_brt_is_empty (BRT brt, BOOL *try_again) __attribute__ ((warn_unused_re ...@@ -228,6 +228,12 @@ BOOL toku_brt_is_empty (BRT brt, BOOL *try_again) __attribute__ ((warn_unused_re
// Effect: Return TRUE iff the tree is empty. (However if *try_again is set to TRUE by toku_brt_is_empty, then the answer is inconclusive, and the function should // Effect: Return TRUE iff the tree is empty. (However if *try_again is set to TRUE by toku_brt_is_empty, then the answer is inconclusive, and the function should
// be tried again. It's a good idea to release the big ydb lock in this case. // be tried again. It's a good idea to release the big ydb lock in this case.
BOOL toku_brt_is_empty_fast (BRT brt);
// Effect: Return TRUE if there are no messages or leaf entries in the tree. If so, it's empty. If there are messages or leaf entries, we say it's not empty
// even though if we were to optimize the tree it might turn out that they are empty.
double get_tdiff(void) __attribute__((__visibility__("default")));
BOOL toku_brt_is_empty_fast (BRT brt) __attribute__ ((warn_unused_result)); BOOL toku_brt_is_empty_fast (BRT brt) __attribute__ ((warn_unused_result));
// Effect: Return TRUE if there are no messages or leaf entries in the tree. If so, it's empty. If there are messages or leaf entries, we say it's not empty // Effect: Return TRUE if there are no messages or leaf entries in the tree. If so, it's empty. If there are messages or leaf entries, we say it's not empty
// even though if we were to optimize the tree it might turn out that they are empty. // even though if we were to optimize the tree it might turn out that they are empty.
......
newbrt/le-cursor.c
View file @ f31a90e9
...@@ -72,7 +72,7 @@ le_cursor_next(LE_CURSOR le_cursor, DBT *key, DBT *val) { ...@@ -72,7 +72,7 @@ le_cursor_next(LE_CURSOR le_cursor, DBT *key, DBT *val) {
} }
int int
is_key_right_of_le_cursor(LE_CURSOR le_cursor, DBT *key, int (*keycompare)(DB *, const DBT *, const DBT *), DB *db) { is_key_right_of_le_cursor(LE_CURSOR le_cursor, const DBT *key, int (*keycompare)(DB *, const DBT *, const DBT *), DB *db) {
int result; int result;
if (le_cursor->neg_infinity) if (le_cursor->neg_infinity)
result = TRUE; result = TRUE;
......
newbrt/le-cursor.h
View file @ f31a90e9
...@@ -2,6 +2,9 @@ ...@@ -2,6 +2,9 @@
#ident "Copyright (c) 2010 Tokutek Inc. All rights reserved." #ident "Copyright (c) 2010 Tokutek Inc. All rights reserved."
#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it." #ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
#ifndef LE_CURSOR_H
#define LE_CURSOR_H
// A leaf entry cursor (LE_CURSOR) is a special type of BRT_CURSOR that visits all of the leaf entries in a tree // A leaf entry cursor (LE_CURSOR) is a special type of BRT_CURSOR that visits all of the leaf entries in a tree
// and returns the leaf entry to the caller. It maintains a copy of the key that it was last positioned over to // and returns the leaf entry to the caller. It maintains a copy of the key that it was last positioned over to
// speed up key comparisions with a given key. For example, the hot indexing could use the _key_right_of_cursor // speed up key comparisions with a given key. For example, the hot indexing could use the _key_right_of_cursor
...@@ -32,5 +35,6 @@ int le_cursor_next(LE_CURSOR le_cursor, DBT *key, DBT *val); ...@@ -32,5 +35,6 @@ int le_cursor_next(LE_CURSOR le_cursor, DBT *key, DBT *val);
// The LE_CURSOR position is intialized to -infinity. Any key comparision with -infinity returns TRUE. // The LE_CURSOR position is intialized to -infinity. Any key comparision with -infinity returns TRUE.
// When the cursor runs off the right edge of the tree, the LE_CURSOR position is set to +infinity. Any key comparision with +infinity // When the cursor runs off the right edge of the tree, the LE_CURSOR position is set to +infinity. Any key comparision with +infinity
// returns FALSE. // returns FALSE.
int is_key_right_of_le_cursor(LE_CURSOR le_cursor, DBT *key, int (*keycompare)(DB *extra, const DBT *, const DBT *), DB *extra); int is_key_right_of_le_cursor(LE_CURSOR le_cursor, const DBT *key, int (*keycompare)(DB *extra, const DBT *, const DBT *), DB *extra);
#endif
newbrt/log-internal.h
View file @ f31a90e9
...@@ -11,6 +11,7 @@ ...@@ -11,6 +11,7 @@
#include "toku_list.h" #include "toku_list.h"
#include "memarena.h" #include "memarena.h"
#include "logfilemgr.h" #include "logfilemgr.h"
#include "txn.h"
#include <stdio.h> #include <stdio.h>
#include <toku_pthread.h> #include <toku_pthread.h>
#include <sys/types.h> #include <sys/types.h>
...@@ -129,6 +130,7 @@ struct tokutxn { ...@@ -129,6 +130,7 @@ struct tokutxn {
u_int64_t snapshot_txnid64; /* this is the lsn of the snapshot */ u_int64_t snapshot_txnid64; /* this is the lsn of the snapshot */
TOKULOGGER logger; TOKULOGGER logger;
TOKUTXN parent; TOKUTXN parent;
DB_TXN* container_db_txn; // reference to DB_TXN that contains this tokutxn
u_int64_t rollentry_raw_count; // the total count of every byte in the transaction and all its children. u_int64_t rollentry_raw_count; // the total count of every byte in the transaction and all its children.
OMT open_brts; // a collection of the brts that we touched. Indexed by filenum. OMT open_brts; // a collection of the brts that we touched. Indexed by filenum.
...@@ -150,6 +152,7 @@ struct tokutxn { ...@@ -150,6 +152,7 @@ struct tokutxn {
BOOL recovered_from_checkpoint; BOOL recovered_from_checkpoint;
ROLLBACK_LOG_NODE pinned_inprogress_rollback_log; ROLLBACK_LOG_NODE pinned_inprogress_rollback_log;
struct toku_list checkpoint_before_commit; struct toku_list checkpoint_before_commit;
TXN_IGNORE ignore_errors; // 2954
}; };
struct txninfo { struct txninfo {
......
newbrt/logformat.c
View file @ f31a90e9
...@@ -70,6 +70,9 @@ const struct logtype rollbacks[] = { ...@@ -70,6 +70,9 @@ const struct logtype rollbacks[] = {
{"load", 'l', FA{{"BYTESTRING", "old_iname", 0}, {"load", 'l', FA{{"BYTESTRING", "old_iname", 0},
{"BYTESTRING", "new_iname", 0}, {"BYTESTRING", "new_iname", 0},
NULLFIELD}}, NULLFIELD}},
// #2954
{"hot_index", 'h', FA{{"FILENUMS", "hot_index_filenums", 0},
NULLFIELD}},
{"dictionary_redirect", 'R', FA{{"FILENUM", "old_filenum", 0}, {"dictionary_redirect", 'R', FA{{"FILENUM", "old_filenum", 0},
{"FILENUM", "new_filenum", 0}, {"FILENUM", "new_filenum", 0},
NULLFIELD}}, NULLFIELD}},
...@@ -172,6 +175,10 @@ const struct logtype logtypes[] = { ...@@ -172,6 +175,10 @@ const struct logtype logtypes[] = {
{"BYTESTRING", "old_iname", 0}, {"BYTESTRING", "old_iname", 0},
{"BYTESTRING", "new_iname", 0}, {"BYTESTRING", "new_iname", 0},
NULLFIELD}}, NULLFIELD}},
// #2954
{"hot_index", 'h', FA{{"TXNID", "xid", 0},
{"FILENUMS", "hot_index_filenums", 0},
NULLFIELD}},
{0,0,FA{NULLFIELD}} {0,0,FA{NULLFIELD}}
}; };
...@@ -528,6 +535,9 @@ generate_rollbacks (void) { ...@@ -528,6 +535,9 @@ generate_rollbacks (void) {
if ( strcmp(ft->type, "BYTESTRING") == 0 ) { if ( strcmp(ft->type, "BYTESTRING") == 0 ) {
fprintf2(cf, hf, ", BYTESTRING *%s_ptr", ft->name); fprintf2(cf, hf, ", BYTESTRING *%s_ptr", ft->name);
} }
else if ( strcmp(ft->type, "FILENUMS") == 0 ) {
fprintf2(cf, hf, ", FILENUMS *%s_ptr", ft->name);
}
else { else {
fprintf2(cf, hf, ", %s %s", ft->type, ft->name); fprintf2(cf, hf, ", %s %s", ft->type, ft->name);
} }
...@@ -548,6 +558,13 @@ generate_rollbacks (void) { ...@@ -548,6 +558,13 @@ generate_rollbacks (void) {
" };\n", " };\n",
ft->name, ft->name, ft->name, ft->name); ft->name, ft->name, ft->name, ft->name);
} }
if ( strcmp(ft->type, "FILENUMS") == 0 ) {
fprintf(cf, " FILENUMS %s = {\n"
" .num = %s_ptr->num,\n"
" .filenums = toku_memdup_in_rollback(log, %s_ptr->filenums, %s_ptr->num * (sizeof (FILENUM)))\n"
" };\n",
ft->name, ft->name, ft->name, ft->name);
}
}); });
{ {
int count=0; int count=0;
......
newbrt/rbuf.h
View file @ f31a90e9
...@@ -131,6 +131,26 @@ static inline void rbuf_ma_FILENUM (struct rbuf *r, MEMARENA ma __attribute__((_ ...@@ -131,6 +131,26 @@ static inline void rbuf_ma_FILENUM (struct rbuf *r, MEMARENA ma __attribute__((_
rbuf_FILENUM(r, filenum); rbuf_FILENUM(r, filenum);
} }
// 2954
// Don't try to use the same space, malloc it
static inline void rbuf_FILENUMS (struct rbuf *r, FILENUMS *filenums) {
filenums->num = rbuf_int(r);
u_int32_t newndone = r->ndone + ( filenums->num * sizeof(FILENUM) );
assert(newndone <= r->size);
filenums->filenums = (FILENUM *) toku_memdup(&r->buf[r->ndone], (size_t)filenums->num * sizeof(FILENUM));
assert(filenums->filenums);
r->ndone = newndone;
}
// 2954
static inline void rbuf_ma_FILENUMS (struct rbuf *r, MEMARENA ma __attribute__((__unused__)), FILENUMS *filenums) {
filenums->num = rbuf_int(r);
u_int32_t newndone = r->ndone + ( filenums->num * sizeof(FILENUM) );
assert(newndone <= r->size);
filenums->filenums = (FILENUM *) memarena_memdup(ma, &r->buf[r->ndone], (size_t)filenums->num * sizeof(FILENUM));
assert(filenums->filenums);
r->ndone = newndone;
}
// Don't try to use the same space, malloc it // Don't try to use the same space, malloc it
static inline void rbuf_BYTESTRING (struct rbuf *r, BYTESTRING *bs) { static inline void rbuf_BYTESTRING (struct rbuf *r, BYTESTRING *bs) {
bs->len = rbuf_int(r); bs->len = rbuf_int(r);
......
newbrt/recover.c
View file @ f31a90e9
...@@ -977,6 +977,26 @@ static int toku_recover_backward_load(struct logtype_load *UU(l), RECOVER_ENV UU ...@@ -977,6 +977,26 @@ static int toku_recover_backward_load(struct logtype_load *UU(l), RECOVER_ENV UU
return 0; return 0;
} }
// #2954
static int toku_recover_hot_index(struct logtype_hot_index *UU(l), RECOVER_ENV UU(renv)) {
int r;
TOKUTXN txn = NULL;
r = toku_txnid2txn(renv->logger, l->xid, &txn);
assert(r == 0);
assert(txn!=NULL);
// just make an entry in the rollback log
// - set do_log = 0 -> don't write to recovery log
r = toku_brt_hot_index_recovery(txn, l->hot_index_filenums, 0, 0, (LSN*)NULL);
assert(r == 0);
return 0;
}
// #2954
static int toku_recover_backward_hot_index(struct logtype_hot_index *UU(l), RECOVER_ENV UU(renv)) {
// nothing
return 0;
}
// Effects: If there are no log files, or if there is a "clean" checkpoint at the end of the log, // Effects: If there are no log files, or if there is a "clean" checkpoint at the end of the log,
// then we don't need recovery to run. Skip the shutdown log entry if there is one. // then we don't need recovery to run. Skip the shutdown log entry if there is one.
// Returns: TRUE if we need recovery, otherwise FALSE. // Returns: TRUE if we need recovery, otherwise FALSE.
......
newbrt/roll.c
View file @ f31a90e9
...@@ -129,8 +129,11 @@ static int find_brt_from_filenum (OMTVALUE v, void *filenumvp) { ...@@ -129,8 +129,11 @@ static int find_brt_from_filenum (OMTVALUE v, void *filenumvp) {
static int do_insertion (enum brt_msg_type type, FILENUM filenum, BYTESTRING key, BYTESTRING *data, TOKUTXN txn, LSN oplsn) { static int do_insertion (enum brt_msg_type type, FILENUM filenum, BYTESTRING key, BYTESTRING *data, TOKUTXN txn, LSN oplsn) {
CACHEFILE cf; CACHEFILE cf;
// 2954 - ignore messages for aborted hot-index
int r = toku_txn_ignore_contains(txn, filenum);
if ( r != ENOENT ) goto done; // ENOENT => filenum not in ignore list
//printf("%s:%d committing insert %s %s\n", __FILE__, __LINE__, key.data, data.data); //printf("%s:%d committing insert %s %s\n", __FILE__, __LINE__, key.data, data.data);
int r = toku_cachefile_of_filenum(txn->logger->ct, filenum, &cf); r = toku_cachefile_of_filenum(txn->logger->ct, filenum, &cf);
if (r==ENOENT) { //Missing file on recovered transaction is not an error if (r==ENOENT) { //Missing file on recovered transaction is not an error
assert(txn->recovered_from_checkpoint); assert(txn->recovered_from_checkpoint);
r = 0; r = 0;
...@@ -378,6 +381,44 @@ toku_rollback_load (BYTESTRING UU(old_iname), ...@@ -378,6 +381,44 @@ toku_rollback_load (BYTESTRING UU(old_iname),
return 0; return 0;
} }
//2954
int
toku_commit_hot_index (FILENUMS UU(hot_index_filenums),
TOKUTXN UU(txn),
YIELDF UU(yield),
void * UU(yield_v),
LSN UU(oplsn))
{
// nothing
return 0;
}
//2954
// function called by toku_omt_iterate to add hot_index filenums to
// each live txn's ignore list when a hot index is aborted
static int
live_txn_ignore(OMTVALUE vtxn, u_int32_t UU(idx) , void *vfn) {
TOKUTXN txn = vtxn;
FILENUMS *hot_index_filenums = vfn;
int r;
for (uint32_t i=0; i<hot_index_filenums->num;i++) {
r = toku_txn_ignore_add(txn, hot_index_filenums->filenums[i]);
if ( r != 0 ) return r;
}
return 0;
}
int
toku_rollback_hot_index (FILENUMS UU(hot_index_filenums),
TOKUTXN UU(txn),
YIELDF UU(yield),
void * UU(yield_v),
LSN UU(oplsn))
{
int r = toku_omt_iterate(txn->logger->live_txns, live_txn_ignore, &hot_index_filenums);
assert(r == 0);
return r;
}
int int
toku_commit_dictionary_redirect (FILENUM UU(old_filenum), toku_commit_dictionary_redirect (FILENUM UU(old_filenum),
......
newbrt/rollback.c
View file @ f31a90e9
...@@ -289,6 +289,7 @@ void toku_rollback_txn_close (TOKUTXN txn) { ...@@ -289,6 +289,7 @@ void toku_rollback_txn_close (TOKUTXN txn) {
note_txn_closing(txn); note_txn_closing(txn);
xids_destroy(&txn->xids); xids_destroy(&txn->xids);
toku_txn_ignore_free(txn); // 2954
toku_free(txn); toku_free(txn);
return; return;
} }
......
newbrt/tests/is_empty.c
View file @ f31a90e9
...@@ -37,7 +37,7 @@ static void test_it (int N) { ...@@ -37,7 +37,7 @@ static void test_it (int N) {
r = toku_logger_open_rollback(logger, ct, TRUE); CKERR(r); r = toku_logger_open_rollback(logger, ct, TRUE); CKERR(r);
TOKUTXN txn; TOKUTXN txn;
r = toku_txn_begin_txn((TOKUTXN)0, &txn, logger, TXN_SNAPSHOT_ROOT); CKERR(r); r = toku_txn_begin_txn((DB_TXN*)NULL, (TOKUTXN)0, &txn, logger, TXN_SNAPSHOT_ROOT); CKERR(r);
r = toku_open_brt(FILENAME, 1, &brt, 1024, ct, txn, toku_builtin_compare_fun, NULL); CKERR(r); r = toku_open_brt(FILENAME, 1, &brt, 1024, ct, txn, toku_builtin_compare_fun, NULL); CKERR(r);
...@@ -49,12 +49,12 @@ static void test_it (int N) { ...@@ -49,12 +49,12 @@ static void test_it (int N) {
unsigned int rands[N]; unsigned int rands[N];
for (int i=0; i<N; i++) { for (int i=0; i<N; i++) {
r = toku_txn_begin_txn((TOKUTXN)0, &txn, logger, TXN_SNAPSHOT_ROOT); CKERR(r); r = toku_txn_begin_txn((DB_TXN*)NULL, (TOKUTXN)0, &txn, logger, TXN_SNAPSHOT_ROOT); CKERR(r);
r = toku_open_brt(FILENAME, 0, &brt, 1024, ct, txn, toku_builtin_compare_fun, NULL); CKERR(r); r = toku_open_brt(FILENAME, 0, &brt, 1024, ct, txn, toku_builtin_compare_fun, NULL); CKERR(r);
r = toku_txn_commit_txn(txn, FALSE, do_yield, NULL, NULL, NULL); CKERR(r); r = toku_txn_commit_txn(txn, FALSE, do_yield, NULL, NULL, NULL); CKERR(r);
toku_txn_close_txn(txn); toku_txn_close_txn(txn);
r = toku_txn_begin_txn((TOKUTXN)0, &txn, logger, TXN_SNAPSHOT_ROOT); CKERR(r); r = toku_txn_begin_txn((DB_TXN*)NULL, (TOKUTXN)0, &txn, logger, TXN_SNAPSHOT_ROOT); CKERR(r);
char key[100],val[300]; char key[100],val[300];
DBT k, v; DBT k, v;
rands[i] = random(); rands[i] = random();
...@@ -72,12 +72,12 @@ static void test_it (int N) { ...@@ -72,12 +72,12 @@ static void test_it (int N) {
if (verbose) printf("i=%d\n", i); if (verbose) printf("i=%d\n", i);
} }
for (int i=0; i<N; i++) { for (int i=0; i<N; i++) {
r = toku_txn_begin_txn((TOKUTXN)0, &txn, logger, TXN_SNAPSHOT_ROOT); CKERR(r); r = toku_txn_begin_txn((DB_TXN*)NULL, (TOKUTXN)0, &txn, logger, TXN_SNAPSHOT_ROOT); CKERR(r);
r = toku_open_brt(FILENAME, 0, &brt, 1024, ct, txn, toku_builtin_compare_fun, NULL); CKERR(r); r = toku_open_brt(FILENAME, 0, &brt, 1024, ct, txn, toku_builtin_compare_fun, NULL); CKERR(r);
r = toku_txn_commit_txn(txn, FALSE, do_yield, NULL, NULL, NULL); CKERR(r); r = toku_txn_commit_txn(txn, FALSE, do_yield, NULL, NULL, NULL); CKERR(r);
toku_txn_close_txn(txn); toku_txn_close_txn(txn);
r = toku_txn_begin_txn((TOKUTXN)0, &txn, logger, TXN_SNAPSHOT_ROOT); CKERR(r); r = toku_txn_begin_txn((DB_TXN*)NULL, (TOKUTXN)0, &txn, logger, TXN_SNAPSHOT_ROOT); CKERR(r);
char key[100]; char key[100];
DBT k; DBT k;
snprintf(key, sizeof(key), "key%x.%x", rands[i], i); snprintf(key, sizeof(key), "key%x.%x", rands[i], i);
...@@ -102,7 +102,7 @@ static void test_it (int N) { ...@@ -102,7 +102,7 @@ static void test_it (int N) {
if (verbose) printf("d=%d\n", i); if (verbose) printf("d=%d\n", i);
} }
r = toku_txn_begin_txn((TOKUTXN)0, &txn, logger, TXN_SNAPSHOT_ROOT); CKERR(r); r = toku_txn_begin_txn((DB_TXN*)NULL, (TOKUTXN)0, &txn, logger, TXN_SNAPSHOT_ROOT); CKERR(r);
r = toku_open_brt(FILENAME, 0, &brt, 1024, ct, txn, toku_builtin_compare_fun, NULL); CKERR(r); r = toku_open_brt(FILENAME, 0, &brt, 1024, ct, txn, toku_builtin_compare_fun, NULL); CKERR(r);
r = toku_txn_commit_txn(txn, FALSE, do_yield, NULL, NULL, NULL); CKERR(r); r = toku_txn_commit_txn(txn, FALSE, do_yield, NULL, NULL, NULL); CKERR(r);
toku_txn_close_txn(txn); toku_txn_close_txn(txn);
......
newbrt/tests/le-cursor-provdel.c
View file @ f31a90e9
...@@ -41,7 +41,7 @@ create_populate_tree(const char *logdir, const char *fname, int n) { ...@@ -41,7 +41,7 @@ create_populate_tree(const char *logdir, const char *fname, int n) {
assert(error == 0); assert(error == 0);
TOKUTXN txn = NULL; TOKUTXN txn = NULL;
error = toku_txn_begin_txn(NULL, &txn, logger, TXN_SNAPSHOT_NONE); error = toku_txn_begin_txn(NULL, NULL, &txn, logger, TXN_SNAPSHOT_NONE);
assert(error == 0); assert(error == 0);
BRT brt = NULL; BRT brt = NULL;
...@@ -53,7 +53,7 @@ create_populate_tree(const char *logdir, const char *fname, int n) { ...@@ -53,7 +53,7 @@ create_populate_tree(const char *logdir, const char *fname, int n) {
toku_txn_close_txn(txn); toku_txn_close_txn(txn);
txn = NULL; txn = NULL;
error = toku_txn_begin_txn(NULL, &txn, logger, TXN_SNAPSHOT_NONE); error = toku_txn_begin_txn(NULL, NULL, &txn, logger, TXN_SNAPSHOT_NONE);
assert(error == 0); assert(error == 0);
// insert keys 0, 1, 2, .. (n-1) // insert keys 0, 1, 2, .. (n-1)
...@@ -107,7 +107,7 @@ test_provdel(const char *logdir, const char *fname, int n) { ...@@ -107,7 +107,7 @@ test_provdel(const char *logdir, const char *fname, int n) {
assert(error == 0); assert(error == 0);
TOKUTXN txn = NULL; TOKUTXN txn = NULL;
error = toku_txn_begin_txn(NULL, &txn, logger, TXN_SNAPSHOT_NONE); error = toku_txn_begin_txn(NULL, NULL, &txn, logger, TXN_SNAPSHOT_NONE);
assert(error == 0); assert(error == 0);
BRT brt = NULL; BRT brt = NULL;
...@@ -119,7 +119,7 @@ test_provdel(const char *logdir, const char *fname, int n) { ...@@ -119,7 +119,7 @@ test_provdel(const char *logdir, const char *fname, int n) {
toku_txn_close_txn(txn); toku_txn_close_txn(txn);
txn = NULL; txn = NULL;
error = toku_txn_begin_txn(NULL, &txn, logger, TXN_SNAPSHOT_NONE); error = toku_txn_begin_txn(NULL, NULL, &txn, logger, TXN_SNAPSHOT_NONE);
assert(error == 0); assert(error == 0);
// del keys 0, 2, 4, ... // del keys 0, 2, 4, ...
...@@ -132,7 +132,7 @@ test_provdel(const char *logdir, const char *fname, int n) { ...@@ -132,7 +132,7 @@ test_provdel(const char *logdir, const char *fname, int n) {
} }
TOKUTXN cursortxn = NULL; TOKUTXN cursortxn = NULL;
error = toku_txn_begin_txn(NULL, &cursortxn, logger, TXN_SNAPSHOT_NONE); error = toku_txn_begin_txn(NULL, NULL, &cursortxn, logger, TXN_SNAPSHOT_NONE);
assert(error == 0); assert(error == 0);
LE_CURSOR cursor = NULL; LE_CURSOR cursor = NULL;
......
newbrt/tests/le-cursor-right.c
View file @ f31a90e9
...@@ -44,7 +44,7 @@ create_populate_tree(const char *logdir, const char *fname, int n) { ...@@ -44,7 +44,7 @@ create_populate_tree(const char *logdir, const char *fname, int n) {
assert(error == 0); assert(error == 0);
TOKUTXN txn = NULL; TOKUTXN txn = NULL;
error = toku_txn_begin_txn(NULL, &txn, logger, TXN_SNAPSHOT_NONE); error = toku_txn_begin_txn(NULL, NULL, &txn, logger, TXN_SNAPSHOT_NONE);
assert(error == 0); assert(error == 0);
BRT brt = NULL; BRT brt = NULL;
...@@ -56,7 +56,7 @@ create_populate_tree(const char *logdir, const char *fname, int n) { ...@@ -56,7 +56,7 @@ create_populate_tree(const char *logdir, const char *fname, int n) {
toku_txn_close_txn(txn); toku_txn_close_txn(txn);
txn = NULL; txn = NULL;
error = toku_txn_begin_txn(NULL, &txn, logger, TXN_SNAPSHOT_NONE); error = toku_txn_begin_txn(NULL, NULL, &txn, logger, TXN_SNAPSHOT_NONE);
assert(error == 0); assert(error == 0);
// insert keys 0, 1, 2, .. (n-1) // insert keys 0, 1, 2, .. (n-1)
......
newbrt/tests/le-cursor-walk.c
View file @ f31a90e9
...@@ -41,7 +41,7 @@ create_populate_tree(const char *logdir, const char *fname, int n) { ...@@ -41,7 +41,7 @@ create_populate_tree(const char *logdir, const char *fname, int n) {
assert(error == 0); assert(error == 0);
TOKUTXN txn = NULL; TOKUTXN txn = NULL;
error = toku_txn_begin_txn(NULL, &txn, logger, TXN_SNAPSHOT_NONE); error = toku_txn_begin_txn(NULL, NULL, &txn, logger, TXN_SNAPSHOT_NONE);
assert(error == 0); assert(error == 0);
BRT brt = NULL; BRT brt = NULL;
...@@ -53,7 +53,7 @@ create_populate_tree(const char *logdir, const char *fname, int n) { ...@@ -53,7 +53,7 @@ create_populate_tree(const char *logdir, const char *fname, int n) {
toku_txn_close_txn(txn); toku_txn_close_txn(txn);
txn = NULL; txn = NULL;
error = toku_txn_begin_txn(NULL, &txn, logger, TXN_SNAPSHOT_NONE); error = toku_txn_begin_txn(NULL, NULL, &txn, logger, TXN_SNAPSHOT_NONE);
assert(error == 0); assert(error == 0);
// insert keys 0, 1, 2, .. (n-1) // insert keys 0, 1, 2, .. (n-1)
......
newbrt/tests/test-leafentry-nested.c
View file @ f31a90e9
...@@ -5,6 +5,7 @@ ...@@ -5,6 +5,7 @@
#include "brttypes.h" #include "brttypes.h"
#include "includes.h" #include "includes.h"
#include "ule.h" #include "ule.h"
#include "ule-internal.h"
enum {MAX_SIZE = 256}; enum {MAX_SIZE = 256};
static XIDS nested_xids[MAX_TRANSACTION_RECORDS]; static XIDS nested_xids[MAX_TRANSACTION_RECORDS];
......
newbrt/tests/txn-ignore.c 0 → 100644
View file @ f31a90e9
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2010 Tokutek Inc. All rights reserved."
#include "test.h"
#include "includes.h"
#include "../brttypes.h"
#include "../txn.h"
/*
* a test of the txn filenums to ignore utilities:
* - toku_txn_ignore_create()
* - toku_txn_ignore_free()
* - toku_txn_ignore_add()
* - toku_txn_ignore_delete()
* - toku_txn_ignore_contains()
*/
int
test_main(int argc, const char *argv[]) {
default_parse_args(argc, argv);
TOKUTXN txn = (TOKUTXN) toku_malloc(sizeof(struct tokutxn));
int r;
r = toku_txn_ignore_init(txn); CKERR(r);
FILENUM f1 = {1};
FILENUM f2 = {2};
FILENUM f3 = {3};
FILENUM f4 = {4};
FILENUM f5 = {5};
FILENUM f6 = {6};
FILENUM f7 = {7};
FILENUM f8 = {8};
FILENUM f9 = {9};
r = toku_txn_ignore_add(txn, f1); CKERR(r);
r = toku_txn_ignore_add(txn, f3); CKERR(r);
r = toku_txn_ignore_add(txn, f5); CKERR(r);
r = toku_txn_ignore_add(txn, f7); CKERR(r);
r = toku_txn_ignore_add(txn, f9); CKERR(r);
r = toku_txn_ignore_remove(txn, f3); CKERR(r);
r = toku_txn_ignore_remove(txn, f2); assert( r == ENOENT );
r = toku_txn_ignore_contains(txn, f1); CKERR(r);
r = toku_txn_ignore_contains(txn, f2); assert( r == ENOENT );
r = toku_txn_ignore_contains(txn, f3); assert( r == ENOENT );
r = toku_txn_ignore_contains(txn, f4); assert( r == ENOENT );
r = toku_txn_ignore_contains(txn, f5); CKERR(r);
r = toku_txn_ignore_contains(txn, f6); assert( r == ENOENT );
r = toku_txn_ignore_contains(txn, f7); CKERR(r);
r = toku_txn_ignore_contains(txn, f8); assert( r == ENOENT );
r = toku_txn_ignore_contains(txn, f9); CKERR(r);
assert(txn->ignore_errors->fns_allocated == 8);
assert(txn->ignore_errors->filenums.num == 4);
r = toku_txn_ignore_add(txn, f2); CKERR(r);
r = toku_txn_ignore_add(txn, f3); CKERR(r);
r = toku_txn_ignore_add(txn, f4); CKERR(r);
r = toku_txn_ignore_add(txn, f6); CKERR(r);
r = toku_txn_ignore_add(txn, f8); CKERR(r);
assert(txn->ignore_errors->fns_allocated == 16);
assert(txn->ignore_errors->filenums.num == 9);
// check that dups are ignored
for (int i=0;i<10;i++) {
r = toku_txn_ignore_add(txn, f2); CKERR(r);
}
assert(txn->ignore_errors->fns_allocated == 16);
assert(txn->ignore_errors->filenums.num == 9);
toku_txn_ignore_free(txn);
toku_free(txn);
return 0;
}
newbrt/txn.c
View file @ f31a90e9
...@@ -26,19 +26,34 @@ toku_txn_get_status(TXN_STATUS s) { ...@@ -26,19 +26,34 @@ toku_txn_get_status(TXN_STATUS s) {
*s = status; *s = status;
} }
int toku_txn_begin_txn (
int
toku_txn_begin_txn (
DB_TXN *container_db_txn,
TOKUTXN parent_tokutxn, TOKUTXN parent_tokutxn,
TOKUTXN *tokutxn, TOKUTXN *tokutxn,
TOKULOGGER logger, TOKULOGGER logger,
TXN_SNAPSHOT_TYPE snapshot_type TXN_SNAPSHOT_TYPE snapshot_type
) )
{ {
return toku_txn_begin_with_xid( int r;
parent_tokutxn, r = toku_txn_begin_with_xid(parent_tokutxn,
tokutxn, logger, tokutxn, logger,
0, 0,
snapshot_type snapshot_type
); );
if (r == 0) {
// container_db_txn set here, not in helper function toku_txn_begin_with_xid()
// because helper function is used by recovery, which does not have DB_TXN
(*tokutxn)->container_db_txn = container_db_txn; // internal struct points to container
}
return r;
}
DB_TXN *
toku_txn_get_container_db_txn (TOKUTXN tokutxn) {
DB_TXN * container = tokutxn->container_db_txn;
return container;
} }
static int static int
...@@ -221,6 +236,10 @@ int toku_txn_begin_with_xid ( ...@@ -221,6 +236,10 @@ int toku_txn_begin_with_xid (
result->force_fsync_on_commit = FALSE; result->force_fsync_on_commit = FALSE;
result->recovered_from_checkpoint = FALSE; result->recovered_from_checkpoint = FALSE;
toku_list_init(&result->checkpoint_before_commit); toku_list_init(&result->checkpoint_before_commit);
// 2954
r = toku_txn_ignore_init(result);
if (r != 0) goto died;
*tokutxn = result; *tokutxn = result;
status.begin++; status.begin++;
if (garbage_collection_debug) { if (garbage_collection_debug) {
...@@ -549,3 +568,106 @@ verify_snapshot_system(TOKULOGGER logger) { ...@@ -549,3 +568,106 @@ verify_snapshot_system(TOKULOGGER logger) {
} }
} }
// routines for checking if rollback errors should be ignored because a hot index create was aborted
// 2954
// returns
// 0 on success
// ENOMEM if can't alloc memory
// EINVAL if txn = NULL
// -1 on other errors
int toku_txn_ignore_init(TOKUTXN txn)
{
if ( !txn ) return EINVAL;
TXN_IGNORE txni = toku_malloc(sizeof(TXN_IGNORE_S));
if ( txni == NULL ) return ENOMEM;
txni->fns_allocated = 0;
txni->filenums.num = 0;
txni->filenums.filenums = NULL;
txn->ignore_errors = txni;
return 0;
}
void toku_txn_ignore_free(TOKUTXN txn)
{
TXN_IGNORE txni = txn->ignore_errors;
toku_free(txni->filenums.filenums);
toku_free(txni);
txni = NULL;
}
// returns
// 0 on success
// ENOMEM if can't alloc memory
// EINVAL if txn = NULL
// -1 on other errors
int toku_txn_ignore_add(TOKUTXN txn, FILENUM filenum)
{
if ( !txn ) return EINVAL;
// check for dups
if ( toku_txn_ignore_contains(txn, filenum) == 0 ) return 0;
// alloc more space if needed
const int N = 2;
TXN_IGNORE txni = txn->ignore_errors;
if ( txni->filenums.num == txni->fns_allocated ) {
if ( txni->fns_allocated == 0 ) {
CALLOC_N(N, txni->filenums.filenums);
if ( txni->filenums.filenums == NULL ) return ENOMEM;
txni->fns_allocated = N;
}
else {
XREALLOC_N(txni->fns_allocated * N, txni->filenums.filenums);
if ( txni->filenums.filenums == NULL ) return ENOMEM;
txni->fns_allocated = txni->fns_allocated * N;
}
}
txni->filenums.num++;
txni->filenums.filenums[txni->filenums.num - 1].fileid = filenum.fileid;
return 0;
}
// returns
// 0 on success
// ENOENT if not found
// EINVAL if txn = NULL
// -1 on other errors
int toku_txn_ignore_remove(TOKUTXN txn, FILENUM filenum)
{
if ( !txn ) return EINVAL;
TXN_IGNORE txni = txn->ignore_errors;
int found_fn = 0;
if ( txni->filenums.num == 0 ) return ENOENT;
for(uint32_t i=0; i<txni->filenums.num; i++) {
if ( !found_fn ) {
if ( txni->filenums.filenums[i].fileid == filenum.fileid ) {
found_fn = 1;
}
}
else { // remove bubble in array
txni->filenums.filenums[i-1].fileid = txni->filenums.filenums[i].fileid;
}
}
if ( !found_fn ) return ENOENT;
txni->filenums.num--;
return 0;
}
// returns
// 0 on success
// ENOENT if not found
// EINVAL if txn = NULL
// -1 on other errors
int toku_txn_ignore_contains(TOKUTXN txn, FILENUM filenum)
{
if ( !txn ) return EINVAL;
TXN_IGNORE txni = txn->ignore_errors;
for(uint32_t i=0; i<txni->filenums.num; i++) {
if ( txni->filenums.filenums[i].fileid == filenum.fileid ) {
return 0;
}
}
return ENOENT;
}
newbrt/txn.h
View file @ f31a90e9
...@@ -10,12 +10,16 @@ extern "C" { ...@@ -10,12 +10,16 @@ extern "C" {
#endif #endif
int toku_txn_begin_txn ( int toku_txn_begin_txn (
DB_TXN *container_db_txn,
TOKUTXN parent_tokutxn, TOKUTXN parent_tokutxn,
TOKUTXN *tokutxn, TOKUTXN *tokutxn,
TOKULOGGER logger, TOKULOGGER logger,
TXN_SNAPSHOT_TYPE snapshot_type TXN_SNAPSHOT_TYPE snapshot_type
); );
DB_TXN * toku_txn_get_container_db_txn (TOKUTXN tokutxn);
// toku_txn_begin_with_xid is called from recovery and has no containing DB_TXN
int toku_txn_begin_with_xid ( int toku_txn_begin_with_xid (
TOKUTXN parent_tokutxn, TOKUTXN parent_tokutxn,
TOKUTXN *tokutxn, TOKUTXN *tokutxn,
...@@ -23,6 +27,7 @@ int toku_txn_begin_with_xid ( ...@@ -23,6 +27,7 @@ int toku_txn_begin_with_xid (
TXNID xid, TXNID xid,
TXN_SNAPSHOT_TYPE snapshot_type TXN_SNAPSHOT_TYPE snapshot_type
); );
int toku_txn_load_txninfo (TOKUTXN txn, TXNINFO info); int toku_txn_load_txninfo (TOKUTXN txn, TXNINFO info);
int toku_txn_commit_txn (TOKUTXN txn, int nosync, YIELDF yield, void *yieldv, int toku_txn_commit_txn (TOKUTXN txn, int nosync, YIELDF yield, void *yieldv,
...@@ -69,6 +74,18 @@ typedef struct { ...@@ -69,6 +74,18 @@ typedef struct {
TXNID xid2; TXNID xid2;
} XID_PAIR_S, *XID_PAIR; } XID_PAIR_S, *XID_PAIR;
// 2954
typedef struct tokutxn_filenum_ignore_errors {
uint32_t fns_allocated;
FILENUMS filenums;
} TXN_IGNORE_S, *TXN_IGNORE;
int toku_txn_ignore_init(TOKUTXN txn);
void toku_txn_ignore_free(TOKUTXN txn);
int toku_txn_ignore_add(TOKUTXN txn, FILENUM filenum);
int toku_txn_ignore_remove(TOKUTXN txn, FILENUM filenum);
int toku_txn_ignore_contains(TOKUTXN txn, FILENUM filenum);
#if defined(__cplusplus) || defined(__cilkplusplus) #if defined(__cplusplus) || defined(__cilkplusplus)
}; };
#endif #endif
......
newbrt/ule-internal.h 0 → 100644
View file @ f31a90e9
/* -*- mode: C; c-basic-offset: 4 -*- */
/* Purpose of this file is to provide the test programs with internal
* ule mechanisms that do not belong in the public interface.
*/
#ifndef TOKU_ULE_INTERNAL_H
#define TOKU_ULE_INTERNAL_H
#ident "$Id: ule.h 24600 2010-10-15 15:22:18Z dwells $"
#ident "Copyright (c) 2007-2010 Tokutek Inc. All rights reserved."
#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
#if defined(__cplusplus) || defined(__cilkplusplus)
extern "C" {
#endif
//1 does much slower debugging
#define ULE_DEBUG 0
/////////////////////////////////////////////////////////////////////////////////
// Following data structures are the unpacked format of a leafentry.
// * ule is the unpacked leaf entry, that contains an array of unpacked
// transaction records
// * uxr is the unpacked transaction record
//
//Types of transaction records.
enum {XR_INSERT = 1,
XR_DELETE = 2,
XR_PLACEHOLDER = 3};
typedef struct uxr { // unpacked transaction record
uint8_t type; // delete/insert/placeholder
uint32_t vallen; // number of bytes in value
void * valp; // pointer to value (Where is value really stored?)
TXNID xid; // transaction id
// Note: when packing ule into a new leafentry, will need
// to copy actual data from valp to new leafentry
} UXR_S, *UXR;
// Unpacked Leaf Entry is of fixed size because it's just on the
// stack and we care about ease of access more than the memory footprint.
typedef struct ule { // unpacked leaf entry
uint32_t num_puxrs; // how many of uxrs[] are provisional
uint32_t num_cuxrs; // how many of uxrs[] are committed
uint32_t keylen;
void * keyp;
UXR_S uxrs_static[MAX_TRANSACTION_RECORDS*2]; // uxrs[0] is oldest committed (txn commit time, not txn start time), uxrs[num_cuxrs] is outermost provisional value (if any exist/num_puxrs > 0)
UXR uxrs; //If num_cuxrs < MAX_TRANSACTION_RECORDS then &uxrs_static[0].
//Otherwise we use a dynamically allocated array of size num_cuxrs + 1 + MAX_TRANSATION_RECORD.
} ULE_S, *ULE;
void test_msg_modify_ule(ULE ule, BRT_MSG msg);
//////////////////////////////////////////////////////////////////////////////////////
//Functions exported for test purposes only (used internally for non-test purposes).
void le_unpack(ULE ule, LEAFENTRY le);
int le_pack(ULE ule, // data to be packed into new leafentry
size_t *new_leafentry_memorysize,
size_t *new_leafentry_disksize,
LEAFENTRY * const new_leafentry_p, // this is what this function creates
OMT omt,
struct mempool *mp,
void **maybe_free);
size_t le_memsize_from_ule (ULE ule);
void ule_cleanup(ULE ule);
#if defined(__cplusplus) || defined(__cilkplusplus)
}
#endif
#endif // TOKU_ULE_H
newbrt/ule.c
View file @ f31a90e9
...@@ -29,6 +29,31 @@ ...@@ -29,6 +29,31 @@
#include "xids.h" #include "xids.h"
#include "brt_msg.h" #include "brt_msg.h"
#include "ule.h" #include "ule.h"
#include "ule-internal.h"
#define ULE_DEBUG 0
///////////////////////////////////////////////////////////////////////////////////
// Accessor functions used by outside world (e.g. indexer)
//
ULEHANDLE
toku_ule_create(void * le_p) {
ULE ule_p = toku_malloc(sizeof(ULE_S));
le_unpack(ule_p, le_p);
return (ULEHANDLE) ule_p;
}
void toku_ule_free(ULEHANDLE ule_p) {
ule_cleanup((ULE) ule_p);
toku_free(ule_p);
}
/////////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////////////
// //
...@@ -84,9 +109,6 @@ static void ule_optimize(ULE ule, XIDS xids); ...@@ -84,9 +109,6 @@ static void ule_optimize(ULE ule, XIDS xids);
static inline BOOL uxr_type_is_insert(u_int8_t type); static inline BOOL uxr_type_is_insert(u_int8_t type);
static inline BOOL uxr_type_is_delete(u_int8_t type); static inline BOOL uxr_type_is_delete(u_int8_t type);
static inline BOOL uxr_type_is_placeholder(u_int8_t type); static inline BOOL uxr_type_is_placeholder(u_int8_t type);
static inline BOOL uxr_is_insert(UXR uxr);
static inline BOOL uxr_is_delete(UXR uxr);
static inline BOOL uxr_is_placeholder(UXR uxr);
static inline size_t uxr_pack_txnid(UXR uxr, uint8_t *p); static inline size_t uxr_pack_txnid(UXR uxr, uint8_t *p);
static inline size_t uxr_pack_type_and_length(UXR uxr, uint8_t *p); static inline size_t uxr_pack_type_and_length(UXR uxr, uint8_t *p);
static inline size_t uxr_pack_length_and_bit(UXR uxr, uint8_t *p); static inline size_t uxr_pack_length_and_bit(UXR uxr, uint8_t *p);
...@@ -1650,6 +1672,49 @@ ule_add_placeholders(ULE ule, XIDS xids) { ...@@ -1650,6 +1672,49 @@ ule_add_placeholders(ULE ule, XIDS xids) {
} }
} }
int
ule_num_uxrs(ULE ule) {
return ule->num_cuxrs + ule->num_puxrs;
}
UXR
ule_get_uxr(ULE ule, int ith) {
invariant(0 <= ith && ith < ule_num_uxrs(ule));
return &ule->uxrs[ith];
}
int
ule_get_num_committed(ULE ule) {
return ule->num_cuxrs;
}
int
ule_get_num_provisional(ULE ule) {
return ule->num_puxrs;
}
int
ule_is_committed(ULE ule, int ith) {
invariant(0 <= ith && ith < ule_num_uxrs(ule));
return ith < (int) ule->num_cuxrs;
}
int
ule_is_provisional(ULE ule, int ith) {
invariant(0 <= ith && ith < ule_num_uxrs(ule));
return ith >= (int) ule->num_cuxrs;
}
void *
ule_get_key(ULE ule) {
return ule->keyp;
}
uint32_t
ule_get_keylen(ULE ule) {
return ule->keylen;
}
///////////////////////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////////////////////
// This layer of abstraction (uxr_xxx) understands uxr and nothing else. // This layer of abstraction (uxr_xxx) understands uxr and nothing else.
// //
...@@ -1660,7 +1725,7 @@ uxr_type_is_insert(u_int8_t type) { ...@@ -1660,7 +1725,7 @@ uxr_type_is_insert(u_int8_t type) {
return rval; return rval;
} }
static inline BOOL BOOL
uxr_is_insert(UXR uxr) { uxr_is_insert(UXR uxr) {
return uxr_type_is_insert(uxr->type); return uxr_type_is_insert(uxr->type);
} }
...@@ -1671,7 +1736,7 @@ uxr_type_is_delete(u_int8_t type) { ...@@ -1671,7 +1736,7 @@ uxr_type_is_delete(u_int8_t type) {
return rval; return rval;
} }
static inline BOOL BOOL
uxr_is_delete(UXR uxr) { uxr_is_delete(UXR uxr) {
return uxr_type_is_delete(uxr->type); return uxr_type_is_delete(uxr->type);
} }
...@@ -1682,11 +1747,27 @@ uxr_type_is_placeholder(u_int8_t type) { ...@@ -1682,11 +1747,27 @@ uxr_type_is_placeholder(u_int8_t type) {
return rval; return rval;
} }
static inline BOOL BOOL
uxr_is_placeholder(UXR uxr) { uxr_is_placeholder(UXR uxr) {
return uxr_type_is_placeholder(uxr->type); return uxr_type_is_placeholder(uxr->type);
} }
void *
uxr_get_val(UXR uxr) {
return uxr->valp;
}
uint32_t
uxr_get_vallen(UXR uxr) {
return uxr->vallen;
}
TXNID
uxr_get_txnid(UXR uxr) {
return uxr->xid;
}
static int static int
le_iterate_get_accepted_index(TXNID *xids, uint32_t *index, uint32_t num_xids, LE_ITERATE_CALLBACK f, TOKUTXN context) { le_iterate_get_accepted_index(TXNID *xids, uint32_t *index, uint32_t num_xids, LE_ITERATE_CALLBACK f, TOKUTXN context) {
uint32_t i; uint32_t i;
......
newbrt/ule.h
View file @ f31a90e9
...@@ -16,43 +16,32 @@ ...@@ -16,43 +16,32 @@
extern "C" { extern "C" {
#endif #endif
// opaque handles used by outside world (i.e. indexer)
typedef struct ule *ULEHANDLE;
typedef struct uxr *UXRHANDLE;
ULEHANDLE toku_ule_create(void * le_p);
void toku_ule_free(ULEHANDLE ule_p);
int ule_num_uxrs(ULEHANDLE ule);
int ule_get_num_committed(ULEHANDLE ule);
int ule_get_num_provisional(ULEHANDLE ule);
UXRHANDLE ule_get_uxr(ULEHANDLE ule, int ith);
int ule_is_committed(ULEHANDLE ule, int ith);
int ule_is_provisional(ULEHANDLE ule, int ith);
void *ule_get_key(ULEHANDLE ule);
uint32_t ule_get_keylen(ULEHANDLE ule);
BOOL uxr_is_insert(UXRHANDLE uxr);
BOOL uxr_is_delete(UXRHANDLE uxr);
BOOL uxr_is_placeholder(UXRHANDLE uxr);
void *uxr_get_val(UXRHANDLE uxr);
uint32_t uxr_get_vallen(UXRHANDLE uxr);
TXNID uxr_get_txnid(UXRHANDLE uxr);
//1 does much slower debugging //1 does much slower debugging
#define ULE_DEBUG 0 #define GARBAGE_COLLECTION_DEBUG 0
/////////////////////////////////////////////////////////////////////////////////
// Following data structures are the unpacked format of a leafentry.
// * ule is the unpacked leaf entry, that contains an array of unpacked
// transaction records
// * uxr is the unpacked transaction record
//
//Types of transaction records.
enum {XR_INSERT = 1,
XR_DELETE = 2,
XR_PLACEHOLDER = 3};
typedef struct { // unpacked transaction record
uint8_t type; // delete/insert/placeholder
uint32_t vallen; // number of bytes in value
void * valp; // pointer to value (Where is value really stored?)
TXNID xid; // transaction id
// Note: when packing ule into a new leafentry, will need
// to copy actual data from valp to new leafentry
} UXR_S, *UXR;
// Unpacked Leaf Entry is of fixed size because it's just on the
// stack and we care about ease of access more than the memory footprint.
typedef struct { // unpacked leaf entry
uint32_t num_puxrs; // how many of uxrs[] are provisional
uint32_t num_cuxrs; // how many of uxrs[] are committed
uint32_t keylen;
void * keyp;
UXR_S uxrs_static[MAX_TRANSACTION_RECORDS*2]; // uxrs[0] is oldest committed (txn commit time, not txn start time), uxrs[num_cuxrs] is outermost provisional value (if any exist/num_puxrs > 0)
UXR uxrs; //If num_cuxrs < MAX_TRANSACTION_RECORDS then uxrs = &uxrs_static[0].
//Otherwise we use a dynamically allocated array of size num_cuxrs + 1 + MAX_TRANSATION_RECORD.
//Felt that MAX_TRANSACTION_RECORD was good upper bound for number of committed UXRs we would be willing to allocate off stack.
} ULE_S, *ULE;
int apply_msg_to_leafentry(BRT_MSG msg, int apply_msg_to_leafentry(BRT_MSG msg,
LEAFENTRY old_leafentry, // NULL if there was no stored data. LEAFENTRY old_leafentry, // NULL if there was no stored data.
...@@ -65,28 +54,11 @@ int apply_msg_to_leafentry(BRT_MSG msg, ...@@ -65,28 +54,11 @@ int apply_msg_to_leafentry(BRT_MSG msg,
OMT snapshot_xids, OMT snapshot_xids,
OMT live_list_reverse); OMT live_list_reverse);
void
test_msg_modify_ule(ULE ule, BRT_MSG msg);
//////////////////////////////////////////////////////////////////////////////////////
//Functions exported for test purposes only (used internally for non-test purposes).
void le_unpack(ULE ule, LEAFENTRY le);
int le_pack(ULE ule, // data to be packed into new leafentry
size_t *new_leafentry_memorysize,
size_t *new_leafentry_disksize,
LEAFENTRY * const new_leafentry_p, // this is what this function creates
OMT omt,
struct mempool *mp,
void **maybe_free);
size_t le_memsize_from_ule (ULE ule);
void ule_cleanup(ULE ule);
TXNID toku_get_youngest_live_list_txnid_for(TXNID xc, OMT live_list_reverse); TXNID toku_get_youngest_live_list_txnid_for(TXNID xc, OMT live_list_reverse);
#if defined(__cplusplus) || defined(__cilkplusplus) #if defined(__cplusplus) || defined(__cilkplusplus)
}; }
#endif #endif
#endif // TOKU_ULE_H #endif // TOKU_ULE_H
......
newbrt/xids.c
View file @ f31a90e9
...@@ -158,6 +158,14 @@ xids_get_innermost_xid(XIDS xids) { ...@@ -158,6 +158,14 @@ xids_get_innermost_xid(XIDS xids) {
return rval; return rval;
} }
TXNID
xids_get_outermost_xid(XIDS xids) {
TXNID rval = TXNID_NONE;
if (xids_get_num_xids(xids))
rval = xids_get_xid(xids, 0);
return rval;
}
void void
xids_cpy(XIDS target, XIDS source) { xids_cpy(XIDS target, XIDS source) {
size_t size = xids_get_size(source); size_t size = xids_get_size(source);
......
newbrt/xids.h
View file @ f31a90e9
...@@ -46,6 +46,7 @@ u_int8_t xids_find_index_of_xid(XIDS xids, TXNID target_xid); ...@@ -46,6 +46,7 @@ u_int8_t xids_find_index_of_xid(XIDS xids, TXNID target_xid);
u_int8_t xids_get_num_xids(XIDS xids); u_int8_t xids_get_num_xids(XIDS xids);
TXNID xids_get_innermost_xid(XIDS xids); TXNID xids_get_innermost_xid(XIDS xids);
TXNID xids_get_outermost_xid(XIDS xids);
// return size in bytes // return size in bytes
u_int32_t xids_get_size(XIDS xids); u_int32_t xids_get_size(XIDS xids);
......
src/Makefile
View file @ f31a90e9
...@@ -36,6 +36,8 @@ OBJS_RAW = \ ...@@ -36,6 +36,8 @@ OBJS_RAW = \
errors \ errors \
dlmalloc \ dlmalloc \
loader \ loader \
indexer \
indexer-undo-do \
elocks \ elocks \
#\end #\end
#OBJS automatically defined. #OBJS automatically defined.
......
src/elocks.c
View file @ f31a90e9
...@@ -63,9 +63,9 @@ ydbtime_destr(void *a) { ...@@ -63,9 +63,9 @@ ydbtime_destr(void *a) {
struct ydbtime *ydbtime = (struct ydbtime *) a; struct ydbtime *ydbtime = (struct ydbtime *) a;
int r; int r;
// printf("%s %p\n", __FUNCTION__, a); // printf("%s %p\n", __FUNCTION__, a);
r = toku_pthread_mutex_lock(&ydb_big_lock.lock); assert(r == 0); r = toku_pthread_mutex_lock(&ydb_big_lock.lock); resource_assert_zero(r);
toku_list_remove(&ydbtime->all_ydbtimes); toku_list_remove(&ydbtime->all_ydbtimes);
r = toku_pthread_mutex_unlock(&ydb_big_lock.lock); assert(r == 0); r = toku_pthread_mutex_unlock(&ydb_big_lock.lock); resource_assert_zero(r);
toku_free(ydbtime); toku_free(ydbtime);
} }
...@@ -74,7 +74,7 @@ static uint64_t ...@@ -74,7 +74,7 @@ static uint64_t
get_tnow(void) { get_tnow(void) {
struct timeval tv; struct timeval tv;
int r = gettimeofday(&tv, NULL); int r = gettimeofday(&tv, NULL);
assert(r == 0); resource_assert_zero(r);
return tv.tv_sec * 1000000ULL + tv.tv_usec; return tv.tv_sec * 1000000ULL + tv.tv_usec;
} }
#endif #endif
...@@ -83,7 +83,7 @@ static void ...@@ -83,7 +83,7 @@ static void
init_status(void) { init_status(void) {
uint64_t cpuhz = 0; uint64_t cpuhz = 0;
#if YDB_LOCK_MISS_TIME #if YDB_LOCK_MISS_TIME
int r = toku_os_get_processor_frequency(&cpuhz); assert(r == 0); int r = toku_os_get_processor_frequency(&cpuhz); resource_assert_zero(r);
#endif #endif
status.ydb_lock_ctr = 0; status.ydb_lock_ctr = 0;
status.max_possible_sleep = MAX_SLEEP; status.max_possible_sleep = MAX_SLEEP;
...@@ -108,10 +108,10 @@ toku_ydb_lock_get_status(SCHEDULE_STATUS statp) { ...@@ -108,10 +108,10 @@ toku_ydb_lock_get_status(SCHEDULE_STATUS statp) {
int int
toku_ydb_lock_init(void) { toku_ydb_lock_init(void) {
int r; int r;
r = toku_pthread_mutex_init(&ydb_big_lock.lock, NULL); assert(r == 0); r = toku_pthread_mutex_init(&ydb_big_lock.lock, NULL); resource_assert_zero(r);
#if YDB_LOCK_MISS_TIME #if YDB_LOCK_MISS_TIME
ydb_big_lock.waiters = 0; ydb_big_lock.waiters = 0;
r = toku_pthread_key_create(&ydb_big_lock.time_key, ydbtime_destr); assert(r == 0); r = toku_pthread_key_create(&ydb_big_lock.time_key, ydbtime_destr); resource_assert_zero(r);
toku_list_init(&ydb_big_lock.all_ydbtimes); toku_list_init(&ydb_big_lock.all_ydbtimes);
#endif #endif
init_status(); init_status();
...@@ -122,21 +122,21 @@ int ...@@ -122,21 +122,21 @@ int
toku_ydb_lock_destroy(void) { toku_ydb_lock_destroy(void) {
int r; int r;
#if YDB_LOCK_MISS_TIME #if YDB_LOCK_MISS_TIME
r = toku_pthread_key_delete(ydb_big_lock.time_key); assert(r == 0); r = toku_pthread_key_delete(ydb_big_lock.time_key); resource_assert_zero(r);
while (!toku_list_empty(&ydb_big_lock.all_ydbtimes)) { while (!toku_list_empty(&ydb_big_lock.all_ydbtimes)) {
struct toku_list *list = toku_list_pop(&ydb_big_lock.all_ydbtimes); struct toku_list *list = toku_list_pop(&ydb_big_lock.all_ydbtimes);
struct ydbtime *ydbtime = toku_list_struct(list, struct ydbtime, all_ydbtimes); struct ydbtime *ydbtime = toku_list_struct(list, struct ydbtime, all_ydbtimes);
ydbtime_destr(ydbtime); ydbtime_destr(ydbtime);
} }
#endif #endif
r = toku_pthread_mutex_destroy(&ydb_big_lock.lock); assert(r == 0); r = toku_pthread_mutex_destroy(&ydb_big_lock.lock); resource_assert_zero(r);
return r; return r;
} }
void void
toku_ydb_lock(void) { toku_ydb_lock(void) {
#if !YDB_LOCK_MISS_TIME #if !YDB_LOCK_MISS_TIME
int r = toku_pthread_mutex_lock(&ydb_big_lock.lock); assert(r == 0); int r = toku_pthread_mutex_lock(&ydb_big_lock.lock); resource_assert_zero(r);
#endif #endif
#if YDB_LOCK_MISS_TIME #if YDB_LOCK_MISS_TIME
...@@ -147,10 +147,10 @@ toku_ydb_lock(void) { ...@@ -147,10 +147,10 @@ toku_ydb_lock(void) {
if (!ydbtime) { // allocate the per thread timestamp if not yet allocated if (!ydbtime) { // allocate the per thread timestamp if not yet allocated
new_ydbtime = TRUE; new_ydbtime = TRUE;
ydbtime = toku_malloc(sizeof (struct ydbtime)); ydbtime = toku_malloc(sizeof (struct ydbtime));
assert(ydbtime); resource_assert(ydbtime);
memset(ydbtime, 0, sizeof (struct ydbtime)); memset(ydbtime, 0, sizeof (struct ydbtime));
r = toku_pthread_setspecific(ydb_big_lock.time_key, ydbtime); r = toku_pthread_setspecific(ydb_big_lock.time_key, ydbtime);
assert(r == 0); resource_assert_zero(r);
(void) toku_sync_fetch_and_increment_uint64(&status.total_clients); (void) toku_sync_fetch_and_increment_uint64(&status.total_clients);
} }
if (ydbtime->tacquire) { // delay the thread if the lock acquire time is set and is less than the current time if (ydbtime->tacquire) { // delay the thread if the lock acquire time is set and is less than the current time
...@@ -177,7 +177,7 @@ toku_ydb_lock(void) { ...@@ -177,7 +177,7 @@ toku_ydb_lock(void) {
(void) toku_sync_fetch_and_increment_int32(&ydb_big_lock.waiters); (void) toku_sync_fetch_and_increment_int32(&ydb_big_lock.waiters);
(void) toku_sync_fetch_and_increment_uint64(&status.total_waiters); (void) toku_sync_fetch_and_increment_uint64(&status.total_waiters);
r = toku_pthread_mutex_lock(&ydb_big_lock.lock); r = toku_pthread_mutex_lock(&ydb_big_lock.lock);
assert(r == 0); resource_assert_zero(r);
(void) toku_sync_fetch_and_decrement_int32(&ydb_big_lock.waiters); (void) toku_sync_fetch_and_decrement_int32(&ydb_big_lock.waiters);
} }
status.max_requested_sleep = u64max(status.max_requested_sleep, requested_sleep); status.max_requested_sleep = u64max(status.max_requested_sleep, requested_sleep);
...@@ -188,25 +188,27 @@ toku_ydb_lock(void) { ...@@ -188,25 +188,27 @@ toku_ydb_lock(void) {
#endif #endif
status.ydb_lock_ctr++; status.ydb_lock_ctr++;
assert((status.ydb_lock_ctr & 0x01) == 1); invariant((status.ydb_lock_ctr & 0x01) == 1);
} }
void // low_priority causes the routine to sleep if there are others waiting for the ydb_lock
toku_ydb_unlock(void) { static void
ydb_unlock_internal(unsigned long useconds) {
status.ydb_lock_ctr++; status.ydb_lock_ctr++;
assert((status.ydb_lock_ctr & 0x01) == 0); invariant((status.ydb_lock_ctr & 0x01) == 0);
#if !YDB_LOCK_MISS_TIME #if !YDB_LOCK_MISS_TIME
int r = toku_pthread_mutex_unlock(&ydb_big_lock.lock); assert(r == 0); int r = toku_pthread_mutex_unlock(&ydb_big_lock.lock); resource_assert_zero(r);
#endif #endif
#if YDB_LOCK_MISS_TIME #if YDB_LOCK_MISS_TIME
struct ydbtime *ydbtime = toku_pthread_getspecific(ydb_big_lock.time_key); struct ydbtime *ydbtime = toku_pthread_getspecific(ydb_big_lock.time_key);
assert(ydbtime); resource_assert(ydbtime);
int r; int r;
uint64_t theld; uint64_t theld;
int waiters = ydb_big_lock.waiters; // get the number of lock waiters (used to compute the lock acquisition time) int waiters = ydb_big_lock.waiters; // get the number of lock waiters (used to compute the lock acquisition time)
int do_sleep = 0;
if (waiters == 0) { if (waiters == 0) {
theld = 0; theld = 0;
} else { } else {
...@@ -219,7 +221,7 @@ toku_ydb_unlock(void) { ...@@ -219,7 +221,7 @@ toku_ydb_unlock(void) {
disk_count += fsync_count - ydb_big_lock.start_fsync_count; // time waiting for fsyncs to complete disk_count += fsync_count - ydb_big_lock.start_fsync_count; // time waiting for fsyncs to complete
disk_time += fsync_time - ydb_big_lock.start_fsync_time; disk_time += fsync_time - ydb_big_lock.start_fsync_time;
if (disk_count == 0) { if (disk_count == 0) {
theld = 0; theld = 0; do_sleep = 1;
} else { } else {
theld = disk_time ? disk_time : disk_count * 20000ULL; // if we decide not to compile in miss_time, then backoff to 20 milliseconds per cache miss theld = disk_time ? disk_time : disk_count * 20000ULL; // if we decide not to compile in miss_time, then backoff to 20 milliseconds per cache miss
...@@ -235,7 +237,10 @@ toku_ydb_unlock(void) { ...@@ -235,7 +237,10 @@ toku_ydb_unlock(void) {
} }
} }
r = toku_pthread_mutex_unlock(&ydb_big_lock.lock); assert(r == 0); r = toku_pthread_mutex_unlock(&ydb_big_lock.lock); resource_assert_zero(r);
if (do_sleep && useconds > 0)
usleep(useconds);
// we use a lower bound of 100 microseconds on the sleep time to avoid system call overhead for short sleeps // we use a lower bound of 100 microseconds on the sleep time to avoid system call overhead for short sleeps
if (waiters == 0 || theld <= 100ULL) if (waiters == 0 || theld <= 100ULL)
...@@ -246,3 +251,12 @@ toku_ydb_unlock(void) { ...@@ -246,3 +251,12 @@ toku_ydb_unlock(void) {
} }
void
toku_ydb_unlock(void) {
ydb_unlock_internal(0);
}
void
toku_ydb_unlock_and_yield(unsigned long useconds) {
ydb_unlock_internal(useconds);
}
src/export.map
View file @ f31a90e9
...@@ -37,6 +37,7 @@ ...@@ -37,6 +37,7 @@
toku_free; toku_free;
toku_malloc; toku_malloc;
toku_calloc;
toku_xmemdup; toku_xmemdup;
toku_xrealloc; toku_xrealloc;
toku_os_get_file_size; toku_os_get_file_size;
...@@ -77,6 +78,7 @@ ...@@ -77,6 +78,7 @@
toku_test_db_redirect_dictionary; toku_test_db_redirect_dictionary;
toku_test_get_latest_lsn; toku_test_get_latest_lsn;
toku_test_get_checkpointing_user_data_status; toku_test_get_checkpointing_user_data_status;
toku_indexer_set_test_only_flags;
local: *; local: *;
}; };
......
src/indexer-internal.h 0 → 100644
View file @ f31a90e9
/* -*- mode: C; c-basic-offset: 4 -*- */
/*
* Copyright (c) 2010 Tokutek Inc. All rights reserved."
* The technology is licensed by the Massachusetts Institute of Technology,
* Rutgers State University of New Jersey, and the Research Foundation of
* State University of New York at Stony Brook under United States of America
* Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
*/
#ifndef TOKU_INDEXER_INTERNAL_H
#define TOKU_INDEXER_INTERNAL_H
struct __toku_indexer_internal {
DB_ENV *env;
DB_TXN *txn;
DB *src_db;
int N;
DB **dest_dbs; /* [N] */
uint32_t *db_flags;
uint32_t indexer_flags;
void (*error_callback)(DB *db, int i, int err, DBT *key, DBT *val, void *error_extra);
void *error_extra;
int (*poll_func)(void *poll_extra, float progress);
void *poll_extra;
uint64_t estimated_rows; // current estimate of table size
uint64_t loop_mod; // how often to call poll_func
LE_CURSOR lec;
void *generate_extra;
FILENUM *fnums; /* [N] */
FILENUMS filenums;
// test functions
int (*undo_do)(DB_INDEXER *indexer, DB *hotdb, ULEHANDLE ule);
int (*test_is_xid_live)(DB_INDEXER *indexer, TXNID xid);
int (*test_maybe_lock_provisional_key)(DB_INDEXER *indexer, TXNID xid, DB *hotdb, DBT *key);
int (*test_delete_provisional)(DB_INDEXER *indexer, DB *hotdb, DBT *hotkey, XIDS xids);
int (*test_delete_committed)(DB_INDEXER *indexer, DB *hotdb, DBT *hotkey, XIDS xids);
int (*test_insert_provisional)(DB_INDEXER *indexer, DB *hotdb, DBT *hotkey, DBT *hotval, XIDS xids);
int (*test_insert_committed)(DB_INDEXER *indexer, DB *hotdb, DBT *hotkey, DBT *hotval, XIDS xids);
int (*test_commit_any)(DB_INDEXER *indexer, DB *db, DBT *key, XIDS xids);
// test flags
int test_only_flags;
};
int indexer_undo_do(DB_INDEXER *indexer, DB *hotdb, ULEHANDLE ule);
#endif
src/indexer-undo-do.c 0 → 100644
View file @ f31a90e9
This diff is collapsed.
src/indexer.c 0 → 100644
View file @ f31a90e9
This diff is collapsed.
src/indexer.h 0 → 100644
View file @ f31a90e9
#ifndef TOKU_INDEXER_H
#define TOKU_INDEXER_H
#if defined(__cplusplus)
extern "C" {
#endif
/* Copyright (c) 2007-2010 Tokutek Inc. All rights reserved.
*
* The technology is licensed by the Massachusetts Institute of Technology,
* Rutgers State University of New Jersey, and the Research Foundation of
* State University of New York at Stony Brook under United States of America
* Serial No. 11/760379 and to the patents and/or patent applications resulting from it.
*/
int toku_indexer_create_indexer(DB_ENV *env,
DB_TXN *txn,
DB_INDEXER **indexer,
DB *src_db,
int N,
DB *dest_dbs[N],
uint32_t db_flags[N],
uint32_t indexer_flags) __attribute__((__visibility__("default")));
int toku_indexer_set_poll_function(DB_INDEXER *indexer,
int (*poll_function)(void *poll_extra,
float progress),
void *poll_extra);
int toku_indexer_set_error_callback(DB_INDEXER *indexer,
void (*error_cb)(DB *db, int i, int err,
DBT *key, DBT *val,
void *error_extra),
void *error_extra);
int toku_indexer_is_key_right_of_le_cursor(DB_INDEXER *indexer, DB *db, const DBT *key);
DB *toku_indexer_get_src_db(DB_INDEXER *indexer);
void toku_indexer_set_test_only_flags(DB_INDEXER *indexer, int flags) __attribute__((__visibility__("default")));
#define INDEXER_TEST_ONLY_ERROR_CALLBACK 1
typedef struct indexer_status {
uint64_t create; // number of indexers successfully created
uint64_t create_fail; // number of calls to toku_indexer_create_indexer() that failed
uint64_t build; // number of calls to indexer->build()
uint64_t close; // number of calls to indexer->close()
uint64_t close_fail; // number of calls to indexer->close() that failed
uint64_t abort; // number of calls to indexer->abort()
uint32_t current; // number of indexers currently in existence
uint32_t max; // max number of indexers that ever existed simulataneously
} INDEXER_STATUS_S, *INDEXER_STATUS;
void toku_indexer_get_status(INDEXER_STATUS s);
#if defined(__cplusplus)
}
#endif
#endif // TOKU_INDEXER_H
src/loader.c
View file @ f31a90e9
...@@ -58,6 +58,22 @@ struct __toku_loader_internal { ...@@ -58,6 +58,22 @@ struct __toku_loader_internal {
char **inames_in_env; /* [N] inames of new files to be created */ char **inames_in_env; /* [N] inames of new files to be created */
}; };
static void
loader_add_refs(DB_LOADER *loader) {
if (loader->i->src_db)
toku_db_add_ref(loader->i->src_db);
for (int i = 0; i < loader->i->N; i++)
toku_db_add_ref(loader->i->dbs[i]);
}
static void
loader_release_refs(DB_LOADER *loader) {
if (loader->i->src_db)
toku_db_release_ref(loader->i->src_db);
for (int i = 0; i < loader->i->N; i++)
toku_db_release_ref(loader->i->dbs[i]);
}
/* /*
* free_loader_resources() frees all of the resources associated with * free_loader_resources() frees all of the resources associated with
* struct __toku_loader_internal * struct __toku_loader_internal
...@@ -67,6 +83,7 @@ struct __toku_loader_internal { ...@@ -67,6 +83,7 @@ struct __toku_loader_internal {
static void free_loader_resources(DB_LOADER *loader) static void free_loader_resources(DB_LOADER *loader)
{ {
if ( loader->i ) { if ( loader->i ) {
loader_release_refs(loader);
for (int i=0; i<loader->i->N; i++) { for (int i=0; i<loader->i->N; i++) {
if (loader->i->ekeys && if (loader->i->ekeys &&
loader->i->ekeys[i].data && loader->i->ekeys[i].data &&
...@@ -229,6 +246,7 @@ int toku_loader_create_loader(DB_ENV *env, ...@@ -229,6 +246,7 @@ int toku_loader_create_loader(DB_ENV *env,
} }
*blp = loader; *blp = loader;
create_exit: create_exit:
loader_add_refs(loader);
if (rval == 0) { if (rval == 0) {
(void) toku_sync_fetch_and_increment_uint64(&status.create); (void) toku_sync_fetch_and_increment_uint64(&status.create);
(void) toku_sync_fetch_and_increment_uint32(&status.current); (void) toku_sync_fetch_and_increment_uint32(&status.current);
......
src/tests/Makefile
View file @ f31a90e9
...@@ -98,6 +98,18 @@ BDB_DONTRUN_TESTS = \ ...@@ -98,6 +98,18 @@ BDB_DONTRUN_TESTS = \
helgrind1 \ helgrind1 \
helgrind2 \ helgrind2 \
helgrind3 \ helgrind3 \
hotindexer-bw \
hotindexer-db-busy \
hotindexer-error-callback \
hotindexer-insert-committed-optimized \
hotindexer-insert-committed \
hotindexer-insert-provisional \
hotindexer-multiclient \
hotindexer-nested-insert-committed \
hotindexer-put-multiple \
hotindexer-simple-abort \
hotindexer-undo-do-test \
hotindexer-with-queries \
insert-dup-prelock \ insert-dup-prelock \
isolation \ isolation \
isolation-read-committed \ isolation-read-committed \
...@@ -108,11 +120,13 @@ BDB_DONTRUN_TESTS = \ ...@@ -108,11 +120,13 @@ BDB_DONTRUN_TESTS = \
loader-cleanup-test \ loader-cleanup-test \
loader-create-abort \ loader-create-abort \
loader-create-close \ loader-create-close \
loader-db-busy \
loader-dup-test \ loader-dup-test \
loader-no-puts \ loader-no-puts \
loader-reference-test \ loader-reference-test \
loader-stress-test \ loader-stress-test \
loader-tpch-load \ loader-tpch-load \
lock-pressure \
manyfiles \ manyfiles \
multiprocess \ multiprocess \
mvcc-create-table \ mvcc-create-table \
...@@ -177,6 +191,7 @@ BDB_DONTRUN_TESTS = \ ...@@ -177,6 +191,7 @@ BDB_DONTRUN_TESTS = \
test_txn_nested3 \ test_txn_nested3 \
test_txn_nested4 \ test_txn_nested4 \
test_txn_nested5 \ test_txn_nested5 \
txn-ignore \
transactional_fileops \ transactional_fileops \
update-multiple-nochange \ update-multiple-nochange \
update-multiple-key0 \ update-multiple-key0 \
...@@ -740,6 +755,15 @@ LOADER_USE_DEFAULT_TESTS = create-abort create-close no-puts reference-test stre ...@@ -740,6 +755,15 @@ LOADER_USE_DEFAULT_TESTS = create-abort create-close no-puts reference-test stre
loader-tests: $(LOADER_TESTS) loader-tests: $(LOADER_TESTS)
echo $(LOADER_TESTS) echo $(LOADER_TESTS)
HOTINDEXER_UNDO_TESTS = $(wildcard hotindexer-undo-do-tests/*.test)
CHECK_HOTINDEXER_UNDO_TESTS = $(patsubst %.test,%.run,$(HOTINDEXER_UNDO_TESTS))
hotindexer-undo-do-test.tdbrun: hotindexer-undo-do-test.tdb$(BINSUF) $(CHECK_HOTINDEXER_UNDO_TESTS)
true
$(CHECK_HOTINDEXER_UNDO_TESTS): %.run: %.test
./run-hotindexer-undo-do-tests.bash $< $(SUMMARIZE_CMD)
clean: clean:
rm -f $(ALL_BINS) rm -f $(ALL_BINS)
rm -rf dir.* *.check.output *.check.valgrind rm -rf dir.* *.check.output *.check.valgrind
......
src/tests/hotindexer-bw.c 0 → 100644
View file @ f31a90e9
This diff is collapsed.
src/tests/hotindexer-db-busy.c 0 → 100644
View file @ f31a90e9
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2009 Tokutek Inc. All rights reserved."
#ident "$Id: loader-reference-test.c 23787 2010-09-14 17:31:19Z dwells $"
// Verify that the indexer grabs references on the DBs
#include "test.h"
#include "toku_pthread.h"
#include <db.h>
#include <sys/stat.h>
DB_ENV *env;
enum {NUM_DBS=1};
static int put_multiple_generate(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_val, const DBT *src_key, const DBT *src_val, void *extra) {
dest_db = dest_db; src_db = src_db; dest_key = dest_key; dest_val = dest_val; src_key = src_key; src_val = src_val; extra = extra;
assert(0);
return 0;
}
char *src_name="src.db";
static void run_test(void)
{
int r;
r = system("rm -rf " ENVDIR); CKERR(r);
r = toku_os_mkdir(ENVDIR, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
r = toku_os_mkdir(ENVDIR "/log", S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
r = db_env_create(&env, 0); CKERR(r);
r = env->set_lg_dir(env, "log"); CKERR(r);
r = env->set_default_bt_compare(env, int64_dbt_cmp); CKERR(r);
r = env->set_generate_row_callback_for_put(env, put_multiple_generate); CKERR(r);
int envflags = DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN | DB_CREATE | DB_PRIVATE | DB_INIT_LOG;
r = env->open(env, ENVDIR, envflags, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
env->set_errfile(env, stderr);
//Disable auto-checkpointing
r = env->checkpointing_set_period(env, 0); CKERR(r);
DB *src_db = NULL;
r = db_create(&src_db, env, 0); CKERR(r);
r = src_db->open(src_db, NULL, src_name, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); CKERR(r);
DB *dbs[NUM_DBS];
for (int i = 0; i < NUM_DBS; i++) {
r = db_create(&dbs[i], env, 0); CKERR(r);
char key_name[32];
sprintf(key_name, "key%d", i);
r = dbs[i]->open(dbs[i], NULL, key_name, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); CKERR(r);
dbs[i]->app_private = (void *) (intptr_t) i;
}
DB_TXN *hottxn;
r = env->txn_begin(env, NULL, &hottxn, 0); CKERR(r);
DB_INDEXER *indexer;
r = env->create_indexer(env, hottxn, &indexer, src_db, NUM_DBS, dbs, NULL, 0); CKERR(r);
r = src_db->close(src_db, 0);
assert(r == EBUSY); // close the src_db with an active indexer, should not succeed
for(int i = 0; i < NUM_DBS; i++) {
r = dbs[i]->close(dbs[i], 0);
assert(r == EBUSY); // close a dest_db, should not succeed
}
r = indexer->abort(indexer); CKERR(r);
r = src_db->close(src_db, 0); CKERR(r);
r = hottxn->commit(hottxn, DB_TXN_SYNC); CKERR(r);
for(int i = 0;i < NUM_DBS; i++) {
r = dbs[i]->close(dbs[i], 0); CKERR(r);
}
r = env->close(env, 0); CKERR(r);
}
// ------------ infrastructure ----------
static void do_args(int argc, char * const argv[]);
int test_main(int argc, char * const *argv) {
do_args(argc, argv);
run_test();
return 0;
}
static void do_args(int argc, char * const argv[]) {
int resultcode;
char *cmd = argv[0];
argc--; argv++;
while (argc>0) {
if (strcmp(argv[0], "-v")==0) {
verbose++;
} else if (strcmp(argv[0],"-q")==0) {
verbose--;
if (verbose<0) verbose=0;
} else if (strcmp(argv[0], "-h")==0) {
resultcode=0;
do_usage:
fprintf(stderr, "Usage:\n%s\n", cmd);
exit(resultcode);
} else {
fprintf(stderr, "Unknown arg: %s\n", argv[0]);
resultcode=1;
goto do_usage;
}
argc--;
argv++;
}
}
src/tests/hotindexer-error-callback.c 0 → 100644
View file @ f31a90e9
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2010 Tokutek Inc. All rights reserved."
#include "test.h"
#include "toku_pthread.h"
#include <db.h>
#include <sys/stat.h>
#include "key-val.h"
#include "ydb.h"
#include "indexer.h"
enum {NUM_DBS=1};
static const int NUM_ROWS = 10;
enum {MAX_CLIENTS=10};
typedef enum {FORWARD = 0, BACKWARD} Direction;
typedef enum {TXN_NONE = 0, TXN_CREATE = 1, TXN_END = 2} TxnWork;
DB_ENV *env;
float last_progress = 0.0;
int error_cb_count = 0;
static void error_callback(DB *db, int which_db, int err, DBT *key, DBT *val, void *extra)
{
error_cb_count++;
if ( verbose ) {
printf("error_callback (%d) : db_p = %p, which_db = %d, error = %d, key_p = %p, val_p = %p, extra_p = %p\n",
error_cb_count,
db, which_db,
err,
key, val, extra);
}
}
static int poll_print(void *extra, float progress) {
if ( verbose ) {
if ( last_progress + 0.01 < progress ) {
printf(" progress : %3.0f%%\n", progress * 100.0);
last_progress = progress;
}
}
extra = extra;
return 0;
}
static inline uint32_t key_to_put(int iter, int offset)
{
return (uint32_t)(((iter+1) * MAX_CLIENTS) + offset);
}
static int generate_initial_table(DB *db, DB_TXN *txn, uint32_t rows)
{
struct timeval start, now;
if ( verbose ) {
printf("generate_initial_table\n");
gettimeofday(&start,0);
}
int r = 0;
DBT key, val;
uint32_t k, v, i;
// create keys of stride MAX_CLIENTS
for (i=0; i<rows; i++)
{
k = key_to_put(i, 0);
v = generate_val(k, 0);
dbt_init(&key, &k, sizeof(k));
dbt_init(&val, &v, sizeof(v));
r = db->put(db, txn, &key, &val, 0);
if ( r != 0 ) break;
}
if ( verbose ) {
gettimeofday(&now,0);
int duration = (int)(now.tv_sec - start.tv_sec);
if ( duration > 0 )
printf("generate_initial_table : %u rows in %d sec = %d rows/sec\n", rows, duration, rows/duration);
}
return r;
}
static void test_indexer(DB *src, DB **dbs)
{
int r;
DB_TXN *txn;
DB_INDEXER *indexer;
uint32_t db_flags[NUM_DBS];
if ( verbose ) printf("test_indexer\n");
for(int i=0;i<NUM_DBS;i++) {
db_flags[i] = DB_NOOVERWRITE;
}
// create and initialize loader
r = env->txn_begin(env, NULL, &txn, 0);
CKERR(r);
if ( verbose ) printf("test_indexer create_indexer\n");
r = env->create_indexer(env, txn, &indexer, src, NUM_DBS, dbs, db_flags, 0);
CKERR(r);
r = indexer->set_error_callback(indexer, error_callback, NULL);
CKERR(r);
toku_indexer_set_test_only_flags(indexer, INDEXER_TEST_ONLY_ERROR_CALLBACK);
r = indexer->set_poll_function(indexer, poll_print, NULL);
CKERR(r);
r = indexer->build(indexer);
assert(r != 0 ); // build should return an error
assert(error_cb_count == 1); // error callback count should be 1
if ( verbose ) printf("test_indexer close\n");
r = indexer->close(indexer);
CKERR(r);
r = txn->commit(txn, DB_TXN_SYNC);
CKERR(r);
if ( verbose ) printf("PASS\n");
if ( verbose ) printf("test_indexer done\n");
}
static void run_test(void)
{
int r;
r = system("rm -rf " ENVDIR); CKERR(r);
r = toku_os_mkdir(ENVDIR, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
r = toku_os_mkdir(ENVDIR "/log", S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
r = db_env_create(&env, 0); CKERR(r);
r = env->set_lg_dir(env, "log"); CKERR(r);
// r = env->set_default_bt_compare(env, int64_dbt_cmp); CKERR(r);
r = env->set_default_bt_compare(env, int_dbt_cmp); CKERR(r);
generate_permute_tables();
r = env->set_generate_row_callback_for_put(env, put_multiple_generate); CKERR(r);
int envflags = DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN | DB_CREATE | DB_PRIVATE | DB_INIT_LOG;
r = env->open(env, ENVDIR, envflags, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
env->set_errfile(env, stderr);
//Disable auto-checkpointing
r = env->checkpointing_set_period(env, 0); CKERR(r);
DB *src_db = NULL;
char *src_name="src.db";
r = db_create(&src_db, env, 0); CKERR(r);
r = src_db->open(src_db, NULL, src_name, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); CKERR(r);
DB_TXN *txn;
r = env->txn_begin(env, NULL, &txn, 0); CKERR(r);
r = generate_initial_table(src_db, txn, NUM_ROWS); CKERR(r);
r = txn->commit(txn, DB_TXN_SYNC); CKERR(r);
DBT desc;
dbt_init(&desc, "foo", sizeof("foo"));
DB *dbs[NUM_DBS];
int idx[MAX_DBS];
for (int i = 0; i < NUM_DBS; i++) {
idx[i] = i+1;
r = db_create(&dbs[i], env, 0); CKERR(r);
r = dbs[i]->set_descriptor(dbs[i], 1, &desc); CKERR(r);
dbs[i]->app_private = &idx[i];
char key_name[32];
sprintf(key_name, "key%d", i);
r = dbs[i]->open(dbs[i], NULL, key_name, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); CKERR(r);
}
// -------------------------- //
if (1) test_indexer(src_db, dbs);
// -------------------------- //
for(int i=0;i<NUM_DBS;i++) {
r = dbs[i]->close(dbs[i], 0); CKERR(r);
}
r = src_db->close(src_db, 0); CKERR(r);
r = env->close(env, 0); CKERR(r);
}
// ------------ infrastructure ----------
int test_main(int argc, char * const argv[]) {
default_parse_args(argc, argv);
run_test();
return 0;
}
src/tests/hotindexer-insert-committed-optimized.c 0 → 100644
View file @ f31a90e9
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2009 Tokutek Inc. All rights reserved."
#ident "$Id: loader-reference-test.c 23787 2010-09-14 17:31:19Z dwells $"
#include "test.h"
#include "toku_pthread.h"
#include <db.h>
#include <sys/stat.h>
DB_ENV *env;
enum {NUM_DBS=1};
enum {NUM_KV_PAIRS=3};
struct kv_pair {
int64_t key;
int64_t val;
};
struct kv_pair kv_pairs[NUM_KV_PAIRS] = {{1,4},
{2,5},
{3,6}};
static int put_multiple_generate(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_val, const DBT *src_key, const DBT *src_val, void *extra) {
src_db = src_db;
extra = extra;
uint32_t which = (uint32_t) (intptr_t) dest_db->app_private;
assert(which == 0);
// switch the key and val
dbt_init(dest_key, src_val->data, src_val->size);
dbt_init(dest_val, src_key->data, src_key->size);
// printf("dest_key.data = %d\n", *(int*)dest_key->data);
// printf("dest_val.data = %d\n", *(int*)dest_val->data);
return 0;
}
static int poll_print(void *extra, float progress) {
progress = progress;
extra = extra;
if ( verbose ) printf("poll_print %f\n", progress);
return 0;
}
static void test_indexer(DB *src, DB **dbs)
{
int r;
DB_TXN *txn;
DB_INDEXER *indexer;
uint32_t db_flags[NUM_DBS];
if ( verbose ) printf("test_indexer\n");
for(int i=0;i<NUM_DBS;i++) {
db_flags[i] = DB_NOOVERWRITE;
}
// create and initialize loader
r = env->txn_begin(env, NULL, &txn, 0);
CKERR(r);
if ( verbose ) printf("test_indexer create_indexer\n");
r = env->create_indexer(env, txn, &indexer, src, NUM_DBS, dbs, db_flags, 0);
CKERR(r);
r = indexer->set_error_callback(indexer, NULL, NULL);
CKERR(r);
r = indexer->set_poll_function(indexer, poll_print, NULL);
CKERR(r);
if ( verbose ) printf("test_indexer build\n");
r = indexer->build(indexer);
CKERR(r);
if ( verbose ) printf("test_indexer close\n");
r = indexer->close(indexer);
CKERR(r);
r = txn->commit(txn, DB_TXN_SYNC);
CKERR(r);
if ( verbose ) printf("PASS\n");
if ( verbose ) printf("test_indexer done\n");
}
char *src_name="src.db";
static void run_test(void)
{
int r;
r = system("rm -rf " ENVDIR); CKERR(r);
r = toku_os_mkdir(ENVDIR, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
r = toku_os_mkdir(ENVDIR "/log", S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
r = db_env_create(&env, 0); CKERR(r);
r = env->set_lg_dir(env, "log"); CKERR(r);
r = env->set_default_bt_compare(env, int64_dbt_cmp); CKERR(r);
r = env->set_generate_row_callback_for_put(env, put_multiple_generate); CKERR(r);
int envflags = DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN | DB_CREATE | DB_PRIVATE | DB_INIT_LOG;
r = env->open(env, ENVDIR, envflags, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
env->set_errfile(env, stderr);
//Disable auto-checkpointing
r = env->checkpointing_set_period(env, 0); CKERR(r);
DB *src_db = NULL;
r = db_create(&src_db, env, 0); CKERR(r);
r = src_db->open(src_db, NULL, src_name, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); CKERR(r);
DBT key, val;
DB_TXN *txn;
r = env->txn_begin(env, NULL, &txn, 0); CKERR(r);
for(int i=0;i<NUM_KV_PAIRS;i++) {
dbt_init(&key, &kv_pairs[i].key, sizeof(kv_pairs[i].key));
dbt_init(&val, &kv_pairs[i].val, sizeof(kv_pairs[i].val));
r = src_db->put(src_db, txn, &key, &val, 0); CKERR(r);
}
r = txn->commit(txn, DB_TXN_SYNC); CKERR(r);
r = src_db->optimize(src_db); CKERR(r);
DB *dbs[NUM_DBS];
for (int i = 0; i < NUM_DBS; i++) {
r = db_create(&dbs[i], env, 0); CKERR(r);
char key_name[32];
sprintf(key_name, "key%d", i);
r = dbs[i]->open(dbs[i], NULL, key_name, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); CKERR(r);
dbs[i]->app_private = (void *) (intptr_t) i;
}
// -------------------------- //
if (1) test_indexer(src_db, dbs);
// -------------------------- //
for(int i=0;i<NUM_DBS;i++) {
r = dbs[i]->close(dbs[i], 0); CKERR(r);
}
r = src_db->close(src_db, 0); CKERR(r);
r = env->close(env, 0); CKERR(r);
}
// ------------ infrastructure ----------
static void do_args(int argc, char * const argv[]);
int test_main(int argc, char * const *argv) {
do_args(argc, argv);
run_test();
return 0;
}
static void do_args(int argc, char * const argv[]) {
int resultcode;
char *cmd = argv[0];
argc--; argv++;
while (argc>0) {
if (strcmp(argv[0], "-v")==0) {
verbose++;
} else if (strcmp(argv[0],"-q")==0) {
verbose--;
if (verbose<0) verbose=0;
} else if (strcmp(argv[0], "-h")==0) {
resultcode=0;
do_usage:
fprintf(stderr, "Usage:\n%s\n", cmd);
exit(resultcode);
} else {
fprintf(stderr, "Unknown arg: %s\n", argv[0]);
resultcode=1;
goto do_usage;
}
argc--;
argv++;
}
}
src/tests/hotindexer-insert-committed.c 0 → 100644
View file @ f31a90e9
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2009 Tokutek Inc. All rights reserved."
#ident "$Id: loader-reference-test.c 23787 2010-09-14 17:31:19Z dwells $"
#include "test.h"
#include "toku_pthread.h"
#include <db.h>
#include <sys/stat.h>
DB_ENV *env;
enum {NUM_DBS=1};
enum {NUM_KV_PAIRS=3};
struct kv_pair {
int64_t key;
int64_t val;
};
struct kv_pair kv_pairs[NUM_KV_PAIRS] = {{1,4},
{2,5},
{3,6}};
static int put_multiple_generate(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_val, const DBT *src_key, const DBT *src_val, void *extra) {
src_db = src_db;
extra = extra;
uint32_t which = (uint32_t) (intptr_t) dest_db->app_private;
assert(which == 0);
// switch the key and val
dbt_init(dest_key, src_val->data, src_val->size);
dbt_init(dest_val, src_key->data, src_key->size);
// printf("dest_key.data = %d\n", *(int*)dest_key->data);
// printf("dest_val.data = %d\n", *(int*)dest_val->data);
return 0;
}
static int poll_print(void *extra, float progress) {
progress = progress;
extra = extra;
if ( verbose ) printf("poll_print %f\n", progress);
return 0;
}
static void test_indexer(DB *src, DB **dbs)
{
int r;
DB_TXN *txn;
DB_INDEXER *indexer;
uint32_t db_flags[NUM_DBS];
if ( verbose ) printf("test_indexer\n");
for(int i=0;i<NUM_DBS;i++) {
db_flags[i] = DB_NOOVERWRITE;
}
// create and initialize loader
r = env->txn_begin(env, NULL, &txn, 0);
CKERR(r);
if ( verbose ) printf("test_indexer create_indexer\n");
r = env->create_indexer(env, txn, &indexer, src, NUM_DBS, dbs, db_flags, 0);
CKERR(r);
r = indexer->set_error_callback(indexer, NULL, NULL);
CKERR(r);
r = indexer->set_poll_function(indexer, poll_print, NULL);
CKERR(r);
if ( verbose ) printf("test_indexer build\n");
r = indexer->build(indexer);
CKERR(r);
if ( verbose ) printf("test_indexer close\n");
r = indexer->close(indexer);
CKERR(r);
r = txn->commit(txn, DB_TXN_SYNC);
CKERR(r);
if ( verbose ) printf("PASS\n");
if ( verbose ) printf("test_indexer done\n");
}
char *src_name="src.db";
static void run_test(void)
{
int r;
r = system("rm -rf " ENVDIR); CKERR(r);
r = toku_os_mkdir(ENVDIR, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
r = toku_os_mkdir(ENVDIR "/log", S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
r = db_env_create(&env, 0); CKERR(r);
r = env->set_lg_dir(env, "log"); CKERR(r);
r = env->set_default_bt_compare(env, int64_dbt_cmp); CKERR(r);
r = env->set_generate_row_callback_for_put(env, put_multiple_generate); CKERR(r);
int envflags = DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN | DB_CREATE | DB_PRIVATE | DB_INIT_LOG;
r = env->open(env, ENVDIR, envflags, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
env->set_errfile(env, stderr);
//Disable auto-checkpointing
r = env->checkpointing_set_period(env, 0); CKERR(r);
DB *src_db = NULL;
r = db_create(&src_db, env, 0); CKERR(r);
r = src_db->open(src_db, NULL, src_name, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); CKERR(r);
DBT key, val;
DB_TXN *txn;
r = env->txn_begin(env, NULL, &txn, 0); CKERR(r);
for(int i=0;i<NUM_KV_PAIRS;i++) {
dbt_init(&key, &kv_pairs[i].key, sizeof(kv_pairs[i].key));
dbt_init(&val, &kv_pairs[i].val, sizeof(kv_pairs[i].val));
r = src_db->put(src_db, txn, &key, &val, 0); CKERR(r);
}
r = txn->commit(txn, DB_TXN_SYNC); CKERR(r);
DB *dbs[NUM_DBS];
for (int i = 0; i < NUM_DBS; i++) {
r = db_create(&dbs[i], env, 0); CKERR(r);
char key_name[32];
sprintf(key_name, "key%d", i);
r = dbs[i]->open(dbs[i], NULL, key_name, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); CKERR(r);
dbs[i]->app_private = (void *) (intptr_t) i;
}
// -------------------------- //
if (1) test_indexer(src_db, dbs);
// -------------------------- //
for(int i=0;i<NUM_DBS;i++) {
r = dbs[i]->close(dbs[i], 0); CKERR(r);
}
r = src_db->close(src_db, 0); CKERR(r);
r = env->close(env, 0); CKERR(r);
}
// ------------ infrastructure ----------
static void do_args(int argc, char * const argv[]);
int test_main(int argc, char * const *argv) {
do_args(argc, argv);
run_test();
return 0;
}
static void do_args(int argc, char * const argv[]) {
int resultcode;
char *cmd = argv[0];
argc--; argv++;
while (argc>0) {
if (strcmp(argv[0], "-v")==0) {
verbose++;
} else if (strcmp(argv[0],"-q")==0) {
verbose--;
if (verbose<0) verbose=0;
} else if (strcmp(argv[0], "-h")==0) {
resultcode=0;
do_usage:
fprintf(stderr, "Usage:\n%s\n", cmd);
exit(resultcode);
} else {
fprintf(stderr, "Unknown arg: %s\n", argv[0]);
resultcode=1;
goto do_usage;
}
argc--;
argv++;
}
}
src/tests/hotindexer-insert-provisional.c 0 → 100644
View file @ f31a90e9
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2009 Tokutek Inc. All rights reserved."
#ident "$Id: loader-reference-test.c 23787 2010-09-14 17:31:19Z dwells $"
#include "test.h"
#include "toku_pthread.h"
#include <db.h>
#include <sys/stat.h>
DB_ENV *env;
enum {NUM_DBS=1};
enum {NUM_KV_PAIRS=3};
struct kv_pair {
int64_t key;
int64_t val;
};
struct kv_pair kv_pairs[NUM_KV_PAIRS] = {{1,4},
{2,5},
{3,6}};
static int put_multiple_generate(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_val, const DBT *src_key, const DBT *src_val, void *extra) {
src_db = src_db;
extra = extra;
uint32_t which = (uint32_t) (intptr_t) dest_db->app_private;
assert(which == 0);
// switch the key and val
dbt_init(dest_key, src_val->data, src_val->size);
dbt_init(dest_val, src_key->data, src_key->size);
// printf("dest_key.data = %d\n", *(int*)dest_key->data);
// printf("dest_val.data = %d\n", *(int*)dest_val->data);
return 0;
}
static int poll_print(void *extra, float progress) {
progress = progress;
extra = extra;
if ( verbose ) printf("poll_print %f\n", progress);
return 0;
}
static void test_indexer(DB *src, DB **dbs)
{
int r;
DB_TXN *txn;
DB_INDEXER *indexer;
uint32_t db_flags[NUM_DBS];
if ( verbose ) printf("test_indexer\n");
for(int i=0;i<NUM_DBS;i++) {
db_flags[i] = DB_NOOVERWRITE;
}
// create and initialize loader
r = env->txn_begin(env, NULL, &txn, 0);
CKERR(r);
if ( verbose ) printf("test_indexer create_indexer\n");
r = env->create_indexer(env, txn, &indexer, src, NUM_DBS, dbs, db_flags, 0);
CKERR(r);
r = indexer->set_error_callback(indexer, NULL, NULL);
CKERR(r);
r = indexer->set_poll_function(indexer, poll_print, NULL);
CKERR(r);
if ( verbose ) printf("test_indexer build\n");
r = indexer->build(indexer);
CKERR(r);
if ( verbose ) printf("test_indexer close\n");
r = indexer->close(indexer);
CKERR(r);
r = txn->commit(txn, DB_TXN_SYNC);
CKERR(r);
if ( verbose ) printf("PASS\n");
if ( verbose ) printf("test_indexer done\n");
}
char *src_name="src.db";
static void run_test(void)
{
int r;
r = system("rm -rf " ENVDIR); CKERR(r);
r = toku_os_mkdir(ENVDIR, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
r = toku_os_mkdir(ENVDIR "/log", S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
r = db_env_create(&env, 0); CKERR(r);
r = env->set_lg_dir(env, "log"); CKERR(r);
r = env->set_default_bt_compare(env, int64_dbt_cmp); CKERR(r);
r = env->set_generate_row_callback_for_put(env, put_multiple_generate); CKERR(r);
int envflags = DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN | DB_CREATE | DB_PRIVATE | DB_INIT_LOG;
r = env->open(env, ENVDIR, envflags, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
env->set_errfile(env, stderr);
//Disable auto-checkpointing
r = env->checkpointing_set_period(env, 0); CKERR(r);
DB *src_db = NULL;
r = db_create(&src_db, env, 0); CKERR(r);
r = src_db->open(src_db, NULL, src_name, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); CKERR(r);
DBT key, val;
DB_TXN *txn;
r = env->txn_begin(env, NULL, &txn, 0); CKERR(r);
for(int i=0;i<NUM_KV_PAIRS;i++) {
dbt_init(&key, &kv_pairs[i].key, sizeof(kv_pairs[i].key));
dbt_init(&val, &kv_pairs[i].val, sizeof(kv_pairs[i].val));
r = src_db->put(src_db, txn, &key, &val, 0); CKERR(r);
}
DB *dbs[NUM_DBS];
for (int i = 0; i < NUM_DBS; i++) {
r = db_create(&dbs[i], env, 0); CKERR(r);
char key_name[32];
sprintf(key_name, "key%d", i);
r = dbs[i]->open(dbs[i], NULL, key_name, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); CKERR(r);
dbs[i]->app_private = (void *) (intptr_t) i;
}
// -------------------------- //
if (1) test_indexer(src_db, dbs);
// -------------------------- //
r = txn->commit(txn, DB_TXN_SYNC); CKERR(r);
for(int i=0;i<NUM_DBS;i++) {
r = dbs[i]->close(dbs[i], 0); CKERR(r);
}
r = src_db->close(src_db, 0); CKERR(r);
r = env->close(env, 0); CKERR(r);
}
// ------------ infrastructure ----------
static void do_args(int argc, char * const argv[]);
int test_main(int argc, char * const *argv) {
do_args(argc, argv);
run_test();
return 0;
}
static void do_args(int argc, char * const argv[]) {
int resultcode;
char *cmd = argv[0];
argc--; argv++;
while (argc>0) {
if (strcmp(argv[0], "-v")==0) {
verbose++;
} else if (strcmp(argv[0],"-q")==0) {
verbose--;
if (verbose<0) verbose=0;
} else if (strcmp(argv[0], "-h")==0) {
resultcode=0;
do_usage:
fprintf(stderr, "Usage:\n%s\n", cmd);
exit(resultcode);
} else {
fprintf(stderr, "Unknown arg: %s\n", argv[0]);
resultcode=1;
goto do_usage;
}
argc--;
argv++;
}
}
src/tests/hotindexer-multiclient.c 0 → 100644
View file @ f31a90e9
This diff is collapsed.
src/tests/hotindexer-nested-insert-committed.c 0 → 100644
View file @ f31a90e9
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2009 Tokutek Inc. All rights reserved."
#ident "$Id: loader-reference-test.c 23787 2010-09-14 17:31:19Z dwells $"
#include "test.h"
#include "toku_pthread.h"
#include <db.h>
#include <sys/stat.h>
DB_ENV *env;
enum {NUM_DBS=1};
enum {NUM_KV_PAIRS=3};
struct kv_pair {
int64_t key;
int64_t val;
};
struct kv_pair kv_pairs[NUM_KV_PAIRS] = {{1,4},
{2,5},
{3,6}};
static int put_multiple_generate(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_val, const DBT *src_key, const DBT *src_val, void *extra) {
src_db = src_db;
extra = extra;
uint32_t which = (uint32_t) (intptr_t) dest_db->app_private;
assert(which == 0);
// switch the key and val
dbt_init(dest_key, src_val->data, src_val->size);
dbt_init(dest_val, src_key->data, src_key->size);
// printf("dest_key.data = %d\n", *(int*)dest_key->data);
// printf("dest_val.data = %d\n", *(int*)dest_val->data);
return 0;
}
static int poll_print(void *extra, float progress) {
progress = progress;
extra = extra;
if ( verbose ) printf("poll_print %f\n", progress);
return 0;
}
static void test_indexer(DB *src, DB **dbs)
{
int r;
DB_TXN *txn;
DB_INDEXER *indexer;
uint32_t db_flags[NUM_DBS];
if ( verbose ) printf("test_indexer\n");
for(int i=0;i<NUM_DBS;i++) {
db_flags[i] = DB_NOOVERWRITE;
}
// create and initialize loader
r = env->txn_begin(env, NULL, &txn, 0);
CKERR(r);
if ( verbose ) printf("test_indexer create_indexer\n");
r = env->create_indexer(env, txn, &indexer, src, NUM_DBS, dbs, db_flags, 0);
CKERR(r);
r = indexer->set_error_callback(indexer, NULL, NULL);
CKERR(r);
r = indexer->set_poll_function(indexer, poll_print, NULL);
CKERR(r);
if ( verbose ) printf("test_indexer build\n");
r = indexer->build(indexer);
CKERR(r);
if ( verbose ) printf("test_indexer close\n");
r = indexer->close(indexer);
CKERR(r);
r = txn->commit(txn, DB_TXN_SYNC);
CKERR(r);
if ( verbose ) printf("PASS\n");
if ( verbose ) printf("test_indexer done\n");
}
char *src_name="src.db";
static void run_test(void)
{
int r;
r = system("rm -rf " ENVDIR); CKERR(r);
r = toku_os_mkdir(ENVDIR, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
r = toku_os_mkdir(ENVDIR "/log", S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
r = db_env_create(&env, 0); CKERR(r);
r = env->set_lg_dir(env, "log"); CKERR(r);
r = env->set_default_bt_compare(env, int64_dbt_cmp); CKERR(r);
r = env->set_generate_row_callback_for_put(env, put_multiple_generate); CKERR(r);
int envflags = DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN | DB_CREATE | DB_PRIVATE | DB_INIT_LOG;
r = env->open(env, ENVDIR, envflags, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
env->set_errfile(env, stderr);
//Disable auto-checkpointing
r = env->checkpointing_set_period(env, 0); CKERR(r);
DB *src_db = NULL;
r = db_create(&src_db, env, 0); CKERR(r);
r = src_db->open(src_db, NULL, src_name, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); CKERR(r);
DBT key, val;
DB_TXN *txn0 = NULL;
r = env->txn_begin(env, NULL, &txn0, 0); CKERR(r);
DB_TXN *txn;
r = env->txn_begin(env, txn0, &txn, 0); CKERR(r);
for(int i=0;i<NUM_KV_PAIRS;i++) {
dbt_init(&key, &kv_pairs[i].key, sizeof(kv_pairs[i].key));
dbt_init(&val, &kv_pairs[i].val, sizeof(kv_pairs[i].val));
r = src_db->put(src_db, txn, &key, &val, 0); CKERR(r);
}
r = txn->commit(txn, DB_TXN_SYNC); CKERR(r);
r = txn0->commit(txn0, DB_TXN_SYNC); CKERR(r);
DB *dbs[NUM_DBS];
for (int i = 0; i < NUM_DBS; i++) {
r = db_create(&dbs[i], env, 0); CKERR(r);
char key_name[32];
sprintf(key_name, "key%d", i);
r = dbs[i]->open(dbs[i], NULL, key_name, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); CKERR(r);
dbs[i]->app_private = (void *) (intptr_t) i;
}
// -------------------------- //
if (1) test_indexer(src_db, dbs);
// -------------------------- //
for(int i=0;i<NUM_DBS;i++) {
r = dbs[i]->close(dbs[i], 0); CKERR(r);
}
r = src_db->close(src_db, 0); CKERR(r);
r = env->close(env, 0); CKERR(r);
}
// ------------ infrastructure ----------
static void do_args(int argc, char * const argv[]);
int test_main(int argc, char * const *argv) {
do_args(argc, argv);
run_test();
return 0;
}
static void do_args(int argc, char * const argv[]) {
int resultcode;
char *cmd = argv[0];
argc--; argv++;
while (argc>0) {
if (strcmp(argv[0], "-v")==0) {
verbose++;
} else if (strcmp(argv[0],"-q")==0) {
verbose--;
if (verbose<0) verbose=0;
} else if (strcmp(argv[0], "-h")==0) {
resultcode=0;
do_usage:
fprintf(stderr, "Usage:\n%s\n", cmd);
exit(resultcode);
} else {
fprintf(stderr, "Unknown arg: %s\n", argv[0]);
resultcode=1;
goto do_usage;
}
argc--;
argv++;
}
}
src/tests/hotindexer-put-multiple.c 0 → 100644
View file @ f31a90e9
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2009 Tokutek Inc. All rights reserved."
#ident "$Id: loader-reference-test.c 23787 2010-09-14 17:31:19Z dwells $"
#include "test.h"
#include "toku_pthread.h"
#include <db.h>
#include <sys/stat.h>
DB_ENV *env;
enum {NUM_DBS=1};
enum {NUM_KV_PAIRS=3};
struct kv_pair {
int64_t key;
int64_t val;
};
struct kv_pair kv_pairs[NUM_KV_PAIRS] = {{1,4},
{2,5},
{3,6}};
static int put_multiple_generate(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_val, const DBT *src_key, const DBT *src_val, void *extra) {
src_db = src_db;
extra = extra;
uint32_t which = (uint32_t) (intptr_t) dest_db->app_private;
if (which == NUM_DBS) {
// primary
dbt_init(dest_key, src_key->data, src_key->size);
dbt_init(dest_val, src_val->data, src_val->size);
} else {
// secondaries: switch the key and val
dbt_init(dest_key, src_val->data, src_val->size);
dbt_init(dest_val, src_key->data, src_key->size);
}
// printf("dest_key.data = %d\n", *(int*)dest_key->data);
// printf("dest_val.data = %d\n", *(int*)dest_val->data);
return 0;
}
static int poll_print(void *extra, float progress) {
progress = progress;
extra = extra;
if ( verbose ) printf("poll_print %f\n", progress);
return 0;
}
char *src_name="src.db";
static void run_test(void)
{
int r;
r = system("rm -rf " ENVDIR); CKERR(r);
r = toku_os_mkdir(ENVDIR, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
r = toku_os_mkdir(ENVDIR "/log", S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
r = db_env_create(&env, 0); CKERR(r);
r = env->set_lg_dir(env, "log"); CKERR(r);
r = env->set_default_bt_compare(env, int64_dbt_cmp); CKERR(r);
r = env->set_generate_row_callback_for_put(env, put_multiple_generate); CKERR(r);
int envflags = DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN | DB_CREATE | DB_PRIVATE | DB_INIT_LOG;
r = env->open(env, ENVDIR, envflags, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
env->set_errfile(env, stderr);
//Disable auto-checkpointing
r = env->checkpointing_set_period(env, 0); CKERR(r);
DB *src_db = NULL;
r = db_create(&src_db, env, 0); CKERR(r);
r = src_db->open(src_db, NULL, src_name, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); CKERR(r);
src_db->app_private = (void *) NUM_DBS;
DB_TXN *txn;
r = env->txn_begin(env, NULL, &txn, 0); CKERR(r);
for(int i=0;i<NUM_KV_PAIRS;i++) {
DBT key, val;
dbt_init(&key, &kv_pairs[i].key, sizeof(kv_pairs[i].key));
dbt_init(&val, &kv_pairs[i].val, sizeof(kv_pairs[i].val));
r = src_db->put(src_db, txn, &key, &val, 0); CKERR(r);
}
DB *dbs[NUM_DBS];
for (int i = 0; i < NUM_DBS; i++) {
r = db_create(&dbs[i], env, 0); CKERR(r);
char key_name[32];
sprintf(key_name, "key%d", i);
r = dbs[i]->open(dbs[i], NULL, key_name, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); CKERR(r);
dbs[i]->app_private = (void *) (intptr_t) i;
}
DB_TXN *hottxn;
r = env->txn_begin(env, NULL, &hottxn, 0);
CKERR(r);
DB_INDEXER *indexer;
r = env->create_indexer(env, hottxn, &indexer, src_db, NUM_DBS, dbs, NULL, 0);
CKERR(r);
r = indexer->set_error_callback(indexer, NULL, NULL);
CKERR(r);
r = indexer->set_poll_function(indexer, poll_print, NULL);
CKERR(r);
// setup putm
DB *putm_dbs[NUM_DBS+1];
for (int i = 0; i < NUM_DBS; i++)
putm_dbs[i] = dbs[i];
putm_dbs[NUM_DBS] = src_db;
DBT putm_keys[NUM_DBS+1], putm_vals[NUM_DBS+1];
uint32_t putm_flags[NUM_DBS+1];
for (int i = 0; i < NUM_DBS+1; i++)
putm_flags[i] = 0;
DBT prikey; int64_t pk;
dbt_init(&prikey, &pk, sizeof pk);
DBT prival; int64_t pv;
dbt_init(&prival, &pv, sizeof pv);
// putm (8,9)
pk = 8; pv = 9;
r = env->put_multiple(env, src_db, txn, &prikey, &prival, NUM_DBS+1, putm_dbs, putm_keys, putm_vals, putm_flags, NULL);
CKERR(r);
r = indexer->build(indexer);
CKERR(r);
// putm (9, 10)
pk = 9; pv = 10;
r = env->put_multiple(env, src_db, txn, &prikey, &prival, NUM_DBS+1, putm_dbs, putm_keys, putm_vals, putm_flags, NULL);
CKERR(r);
r = indexer->close(indexer);
CKERR(r);
r = hottxn->commit(hottxn, DB_TXN_SYNC);
CKERR(r);
r = txn->commit(txn, DB_TXN_SYNC); CKERR(r);
for(int i=0;i<NUM_DBS;i++) {
r = dbs[i]->close(dbs[i], 0); CKERR(r);
}
r = src_db->close(src_db, 0); CKERR(r);
r = env->close(env, 0); CKERR(r);
}
// ------------ infrastructure ----------
static void do_args(int argc, char * const argv[]);
int test_main(int argc, char * const *argv) {
do_args(argc, argv);
run_test();
return 0;
}
static void do_args(int argc, char * const argv[]) {
int resultcode;
char *cmd = argv[0];
argc--; argv++;
while (argc>0) {
if (strcmp(argv[0], "-v")==0) {
verbose++;
} else if (strcmp(argv[0],"-q")==0) {
verbose--;
if (verbose<0) verbose=0;
} else if (strcmp(argv[0], "-h")==0) {
resultcode=0;
do_usage:
fprintf(stderr, "Usage:\n%s\n", cmd);
exit(resultcode);
} else {
fprintf(stderr, "Unknown arg: %s\n", argv[0]);
resultcode=1;
goto do_usage;
}
argc--;
argv++;
}
}
src/tests/hotindexer-simple-abort.c 0 → 100644
View file @ f31a90e9
#include "test.h"
static int
put_callback(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_data, const DBT *src_key, const DBT *src_data, void *extra) {
dest_db = dest_db; src_db = src_db; dest_key = dest_key; dest_data = dest_data; src_key = src_key; src_data = src_data;
lazy_assert(src_db != NULL && dest_db != NULL);
lazy_assert(extra == NULL);
dest_key->data = src_data->data;
dest_key->size = src_data->size;
dest_data->size = 0;
return 0;
}
static void
run_test(void) {
int r;
DB_ENV *env = NULL;
r = db_env_create(&env, 0); assert_zero(r);
r = env->set_generate_row_callback_for_put(env, put_callback); assert_zero(r);
r = env->open(env, ENVDIR, DB_INIT_MPOOL|DB_CREATE|DB_THREAD |DB_INIT_LOCK|DB_INIT_LOG|DB_INIT_TXN|DB_PRIVATE, S_IRWXU+S_IRWXG+S_IRWXO); assert_zero(r);
DB *src_db = NULL;
r = db_create(&src_db, env, 0); assert_zero(r);
r = src_db->open(src_db, NULL, "0.tdb", NULL, DB_BTREE, DB_AUTO_COMMIT+DB_CREATE, S_IRWXU+S_IRWXG+S_IRWXO); assert_zero(r);
DB *dest_db = NULL;
r = db_create(&dest_db, env, 0); assert_zero(r);
r = dest_db->open(dest_db, NULL, "1.tdb", NULL, DB_BTREE, DB_AUTO_COMMIT+DB_CREATE, S_IRWXU+S_IRWXG+S_IRWXO); assert_zero(r);
DB_TXN *txn = NULL;
r = env->txn_begin(env, NULL, &txn, 0); assert_zero(r);
DB_INDEXER *indexer = NULL;
r = env->create_indexer(env, txn, &indexer, src_db, 1, &dest_db, NULL, 0); assert_zero(r);
r = indexer->abort(indexer); assert_zero(r);
r = txn->abort(txn); assert_zero(r);
r = src_db->close(src_db, 0); assert_zero(r);
r = dest_db->close(dest_db, 0); assert_zero(r);
r = env->close(env, 0); assert_zero(r);
}
int
test_main(int argc, char * const argv[]) {
int r;
// parse_args(argc, argv);
for (int i = 1; i < argc; i++) {
char * const arg = argv[i];
if (strcmp(arg, "-v") == 0) {
verbose++;
continue;
}
if (strcmp(arg, "-q") == 0) {
verbose = 0;
continue;
}
}
r = system("rm -rf " ENVDIR); assert_zero(r);
r = toku_os_mkdir(ENVDIR, S_IRWXU+S_IRWXG+S_IRWXO); assert_zero(r);
run_test();
return 0;
}
src/tests/hotindexer-undo-do-test.c 0 → 100644
View file @ f31a90e9
This diff is collapsed.
src/tests/hotindexer-undo-do-tests/README 0 → 100644
View file @ f31a90e9
= Hotindexer undo do testing =
The hotindexer undo do function is tested by feeding it a leafentry,
capturing the actions taken on the hot dictionary,
and comparing the actions with the precomputed expected actions.
The test passes if the actions match the expected actions.
Each test is described by a .test file and a .result file.
The .test file describes the set of transactions that are live
when the undo do function is called as well as the transaction records
that comprise a leaf entry.
The .result file describes the actions taken by the undo do function
when processing the leaf entry.
= Contents of a *.test =
Comments begin with '#'.
An <XIDLIST> is a list of transaction id's separated by a SPACE.
<XIDS> is a stack of transaction id's separated by a COMMA.
An <XID> is a 64 bit number.
A <KEY> is a string.
A <VALUE> is a string.
The field separator is a single SPACE.
== set the leaf entry key ==
key <KEY>
== add transaction IDs to the live transaction set ==
live <XIDLIST>
the live transaction set is initially empty
== push a delete transaction record onto the leaf entry stack ==
delete <committed|provisional> <XID>
== push an insert transaction records onto the leaf entry stack ==
insert <committed|provisional> <XID> <VALUE>
== push a placeholder onto the leaf entry stack ==
placeholder <XID>
= Contents of a *.result =
== insert committed ==
insert_committed [<XIDS>] <KEY> <VALUE>
commit_any [<XIDS>] <KEY>
== delete committed ==
delete_committed [<XIDS>] <KEY>
== insert provisional ==
insert_provisional [<XIDS>] <KEY> <VALUE>
== delete provisional ==
delete_provisional [<XIDS>] <KEY>
== lock ==
lock [<XIDS>] <KEY>
src/tests/hotindexer-undo-do-tests/delete.0.test 0 → 100644
View file @ f31a90e9
# this test runs the undo do function on a leaf entry that consists of a single committed delete
key 1
delete committed 0
src/tests/hotindexer-undo-do-tests/insert.0.test 0 → 100644
View file @ f31a90e9
# commited insert
key 1
insert committed 0 100
src/tests/hotindexer-undo-do-tests/insert.100.result 0 → 100644
View file @ f31a90e9
insert_committed [100] 100 1
commit_any [100] 100
src/tests/hotindexer-undo-do-tests/insert.100.test 0 → 100644
View file @ f31a90e9
key 1
delete committed 0
insert committed 100 100
src/tests/hotindexer-undo-do-tests/insert.200.result 0 → 100644
View file @ f31a90e9
insert_committed [100] 100 1
commit_any [100] 100
delete_committed [200] 100
insert_committed [200] 200 1
commit_any [200] 100
commit_any [200] 200
src/tests/hotindexer-undo-do-tests/insert.200.test 0 → 100644
View file @ f31a90e9
key 1
delete committed 0
insert committed 100 100
insert committed 200 200
src/tests/hotindexer-undo-do-tests/placeholder.0.live.result 0 → 100644
View file @ f31a90e9
insert_provisional [300,301,302] 10 1
lock [300] 10
src/tests/hotindexer-undo-do-tests/placeholder.0.live.test 0 → 100644
View file @ f31a90e9
live 300 301 302
key 1
delete committed 0
placeholder 300
placeholder 301
insert provisional 302 10
src/tests/hotindexer-undo-do-tests/placeholder.0.result 0 → 100644
View file @ f31a90e9
insert_committed [300,301,302] 10 1
commit_any [300,301,302] 10
src/tests/hotindexer-undo-do-tests/placeholder.0.test 0 → 100644
View file @ f31a90e9
key 1
delete committed 0
placeholder 300
placeholder 301
insert provisional 302 10
src/tests/hotindexer-undo-do-tests/placeholder.1.live.result 0 → 100644
View file @ f31a90e9
insert_provisional [300,301,302] 10 1
lock [300] 10
src/tests/hotindexer-undo-do-tests/placeholder.1.live.test 0 → 100644
View file @ f31a90e9
live 300 301
key 1
delete committed 0
placeholder 300
placeholder 301
insert provisional 302 10
src/tests/hotindexer-undo-do-tests/placeholder.2.live.result 0 → 100644
View file @ f31a90e9
insert_committed [100] 10 1
commit_any [100] 10
delete_provisional [300,301,302] 10
lock [300] 10
insert_provisional [300,301,302] 20 1
lock [300] 20
src/tests/hotindexer-undo-do-tests/placeholder.2.live.test 0 → 100644
View file @ f31a90e9
live 300 301 302
key 1
delete committed 0
insert committed 100 10
placeholder 300
placeholder 301
insert provisional 302 20
src/tests/hotindexer-undo-do-tests/placeholder.2.result 0 → 100644
View file @ f31a90e9
insert_committed [100] 10 1
commit_any [100] 10
delete_committed [300,301,302] 10
insert_committed [300,301,302] 20 1
commit_any [300,301,302] 10
commit_any [300,301,302] 20
src/tests/hotindexer-undo-do-tests/placeholder.2.test 0 → 100644
View file @ f31a90e9
key 1
delete committed 0
insert committed 100 10
placeholder 300
placeholder 301
insert provisional 302 20
src/tests/hotindexer-undo-do-tests/prov.1.live.result 0 → 100644
View file @ f31a90e9
insert_committed [0] 10 1
delete_committed [100] 10
insert_committed [100] 20 1
commit_any [100] 10
commit_any [100] 20
delete_committed [200] 20
insert_committed [200] 10 1
commit_any [200] 20
commit_any [200] 10
delete_provisional [300,301] 10
lock [300] 10
insert_provisional [300,301] 30 1
lock [300] 30
delete_provisional [300,301,302] 30
lock [300] 30
lock [300] 30
src/tests/hotindexer-undo-do-tests/prov.1.live.test 0 → 100644
View file @ f31a90e9
live 300 301 302
key 1
insert committed 0 10
insert committed 100 20
insert committed 200 10
placeholder 300
insert provisional 301 30
delete provisional 302
src/tests/hotindexer-undo-do-tests/prov.1.result 0 → 100644
View file @ f31a90e9
insert_committed [0] 10 1
delete_committed [100] 10
insert_committed [100] 20 1
commit_any [100] 10
commit_any [100] 20
delete_committed [200] 20
insert_committed [200] 10 1
commit_any [200] 20
commit_any [200] 10
delete_committed [300,301] 10
insert_committed [300,301] 30 1
commit_any [300,301] 10
commit_any [300,301] 30
delete_committed [300,301,302] 30
commit_any [300,301,302] 30
src/tests/hotindexer-undo-do-tests/prov.1.test 0 → 100644
View file @ f31a90e9
key 1
insert committed 0 10
insert committed 100 20
insert committed 200 10
placeholder 300
insert provisional 301 30
delete provisional 302
src/tests/hotindexer-undo-do-tests/prov.insert.100.live.result 0 → 100644
View file @ f31a90e9
insert_provisional [100] 100 1
lock [100] 100
src/tests/hotindexer-undo-do-tests/prov.insert.100.live.test 0 → 100644
View file @ f31a90e9
live 100
key 1
delete committed 0
insert provisional 100 100
src/tests/hotindexer-undo-do-tests/prov.insert.100.result 0 → 100644
View file @ f31a90e9
insert_committed [100] 100 1
commit_any [100] 100
src/tests/hotindexer-undo-do-tests/prov.insert.100.test 0 → 100644
View file @ f31a90e9
key 1
delete committed 0
insert provisional 100 100
src/tests/hotindexer-with-queries.c 0 → 100644
View file @ f31a90e9
This diff is collapsed.
src/tests/key-val.h 0 → 100644
View file @ f31a90e9
This diff is collapsed.
src/tests/loader-create-close.c
View file @ f31a90e9
...@@ -44,7 +44,7 @@ static void loader_open_abort(int ndb) { ...@@ -44,7 +44,7 @@ static void loader_open_abort(int ndb) {
r = env->txn_begin(env, NULL, &txn, 0); CKERR(r); r = env->txn_begin(env, NULL, &txn, 0); CKERR(r);
DB_LOADER *loader; DB_LOADER *loader;
r = env->create_loader(env, txn, &loader, dbs[0], ndb, dbs, db_flags, dbt_flags, loader_flags); CKERR(r); r = env->create_loader(env, txn, &loader, ndb > 0 ? dbs[0] : NULL, ndb, dbs, db_flags, dbt_flags, loader_flags); CKERR(r);
r = loader->close(loader); CKERR(r); r = loader->close(loader); CKERR(r);
......
src/tests/loader-db-busy.c 0 → 100644
View file @ f31a90e9
This diff is collapsed.
src/tests/lock-pressure.c 0 → 100644
View file @ f31a90e9
This diff is collapsed.
src/tests/run-hotindexer-undo-do-tests.bash 0 → 100644
View file @ f31a90e9
#!/usr/bin/env bash
# run a sequence of hotindexer undo tests.
tests=""
verbose=0
for arg in $* ; do
if [ $arg = "--verbose" ] ; then
verbose=1
else
tests="$tests $arg"
fi
done
for t in $tests ; do
testdir=`dirname $t`
testfile=`basename $t`
testname=""
resultfile=""
if [[ $testfile =~ "(.*)\.test$" ]] ; then
testname=${BASH_REMATCH[1]}
resultfile=$testname.result
else
exit 1
fi
if [ $verbose != 0 ] ; then echo $testdir $testname $testfile $resultfile; fi
./hotindexer-undo-do-test.tdb $testdir/$testfile >$testdir/$testname.run
if [ -f $testdir/$resultfile ] ; then
diff -q $testdir/$testname.run $testdir/$resultfile >/dev/null 2>&1
exitcode=$?
else
exitcode=1
fi
if [ $verbose != 0 ] ; then
echo $testname $exitcode
else
rm $testdir/$testname.run
fi
if [ $exitcode != 0 ] ; then break; fi
done
exit $exitcode
\ No newline at end of file
src/tests/test.h
View file @ f31a90e9
This diff is collapsed.
src/ydb-internal.h
View file @ f31a90e9
This diff is collapsed.
src/ydb-le-cursor.h
View file @ f31a90e9
#include "le_cursor.h" #include "le-cursor.h"
static inline int le_cursor_create_db_txn(LE_CURSOR *le_cursor_result, DB *db, DB_TXN *txn) { static inline int le_cursor_create_db_txn(LE_CURSOR *le_cursor_result, DB *db, DB_TXN *txn) {
return le_cursor_create(le_cursor_result, db_struct_i(db)->brt, db_txn_struct_i(txn)->tokutxn); return le_cursor_create(le_cursor_result, db_struct_i(db)->brt, db_txn_struct_i(txn)->tokutxn);
......
src/ydb.c
View file @ f31a90e9
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7