[t:2499] Allow test program to set loader's scale_factor. Improve...

[t:2499] Allow test program to set loader's scale_factor. Improve loader-cleanup-test to exercise with different combinations of size_factor, ENOSPC from toku_os_write(), and abort_on_poll. Command line options are used to exercise known bugs.

git-svn-id: file:///svn/toku/tokudb@20109 c7de825b-a66e-492c-adef-691d508d4ae1

buildheader/db.h_4_1

@@ -492,6 +492,7 @@ void db_env_set_checkpoint_callback (void (*)(void*), void*) __attribute__((__vi
 void db_env_set_checkpoint_callback2 (void (*)(void*), void*) __attribute__((__visibility__("default")));
 void db_env_set_recover_callback (void (*)(void*), void*) __attribute__((__visibility__("default")));
 void db_env_set_recover_callback2 (void (*)(void*), void*) __attribute__((__visibility__("default")));
+void db_env_set_loader_size_factor (uint32_t) __attribute__((__visibility__("default")));
 #if defined(__cplusplus)
 }
 #endif

buildheader/db.h_4_3

@@ -508,6 +508,7 @@ void db_env_set_checkpoint_callback (void (*)(void*), void*) __attribute__((__vi
 void db_env_set_checkpoint_callback2 (void (*)(void*), void*) __attribute__((__visibility__("default")));
 void db_env_set_recover_callback (void (*)(void*), void*) __attribute__((__visibility__("default")));
 void db_env_set_recover_callback2 (void (*)(void*), void*) __attribute__((__visibility__("default")));
+void db_env_set_loader_size_factor (uint32_t) __attribute__((__visibility__("default")));
 #if defined(__cplusplus)
 }
 #endif

buildheader/db.h_4_6

@@ -13,9 +13,9 @@ extern "C" {
 #define TOKUDB_NATIVE_H 0
 #define DB_VERSION_MAJOR 4
 #define DB_VERSION_MINOR 6
-#define DB_VERSION_PATCH 21
+#define DB_VERSION_PATCH 19
 #ifndef _TOKUDB_WRAP_H
-#define DB_VERSION_STRING "Tokutek: TokuDB 4.6.21"
+#define DB_VERSION_STRING "Tokutek: TokuDB 4.6.19"
 #else
 #define DB_VERSION_STRING_ydb "Tokutek: TokuDB (wrapped bdb)"
 #endif
@@ -521,6 +521,7 @@ void db_env_set_checkpoint_callback (void (*)(void*), void*) __attribute__((__vi
 void db_env_set_checkpoint_callback2 (void (*)(void*), void*) __attribute__((__visibility__("default")));
 void db_env_set_recover_callback (void (*)(void*), void*) __attribute__((__visibility__("default")));
 void db_env_set_recover_callback2 (void (*)(void*), void*) __attribute__((__visibility__("default")));
+void db_env_set_loader_size_factor (uint32_t) __attribute__((__visibility__("default")));
 #if defined(__cplusplus)
 }
 #endif

buildheader/make_db_h.c

@@ -677,6 +677,7 @@ int main (int argc __attribute__((__unused__)), char *const argv[] __attribute__
     printf("void db_env_set_checkpoint_callback2 (void (*)(void*), void*) %s;\n", VISIBLE);
     printf("void db_env_set_recover_callback (void (*)(void*), void*) %s;\n", VISIBLE);
     printf("void db_env_set_recover_callback2 (void (*)(void*), void*) %s;\n", VISIBLE);
+    printf("void db_env_set_loader_size_factor (uint32_t) %s;\n", VISIBLE);
     printf("#if defined(__cplusplus)\n}\n#endif\n");
     printf("#endif\n");
     return 0;

buildheader/tdb.h

@@ -453,6 +453,7 @@ void db_env_set_checkpoint_callback (void (*)(void*), void*) __attribute__((__vi
 void db_env_set_checkpoint_callback2 (void (*)(void*), void*) __attribute__((__visibility__("default")));
 void db_env_set_recover_callback (void (*)(void*), void*) __attribute__((__visibility__("default")));
 void db_env_set_recover_callback2 (void (*)(void*), void*) __attribute__((__visibility__("default")));
+void db_env_set_loader_size_factor (uint32_t) __attribute__((__visibility__("default")));
 #if defined(__cplusplus)
 }
 #endif

include/db.h

@@ -453,6 +453,7 @@ void db_env_set_checkpoint_callback (void (*)(void*), void*) __attribute__((__vi
 void db_env_set_checkpoint_callback2 (void (*)(void*), void*) __attribute__((__visibility__("default")));
 void db_env_set_recover_callback (void (*)(void*), void*) __attribute__((__visibility__("default")));
 void db_env_set_recover_callback2 (void (*)(void*), void*) __attribute__((__visibility__("default")));
+void db_env_set_loader_size_factor (uint32_t) __attribute__((__visibility__("default")));
 #if defined(__cplusplus)
 }
 #endif

newbrt/brtloader.c

@@ -68,6 +68,24 @@ static size_t do_fwrite (const void *ptr, size_t size, size_t nmemb, FILE *strea
     }
 }
 
+
+// 1024 is the right size_factor for production.  
+// Different values for these sizes may be used for testing.
+static uint32_t size_factor = 1024;
+static size_t   data_buffer_limit = 1024*1024*64; 
+static int      nodesize = (1<<22);
+
+
+
+void
+toku_brtloader_set_size_factor(uint32_t factor) {
+// For test purposes only
+    size_factor = factor;
+    data_buffer_limit = 1024*size_factor*64;
+    nodesize = (size_factor==1) ? (1<<15) : (1<<22);
+}
+
+
 static void add_big_buffer(struct file_info *file) {
     if (file->buffer == NULL)
         file->buffer = toku_malloc(file->buffer_size);
@@ -480,10 +498,6 @@ int loader_read_row (FILE *f, DBT *key, DBT *val, BRTLOADER bl)
 }
 
 
-// 1024 is the right number for production.
-//#define SIZE_FACTOR 1
-#define SIZE_FACTOR 1024
-
 int init_rowset (struct rowset *rows)
 /* Effect: Initialize a collection of rows to be empty. */
 {
@@ -494,7 +508,7 @@ int init_rowset (struct rowset *rows)
     rows->n_rows_limit = 100;
     MALLOC_N(rows->n_rows_limit, rows->rows);
     rows->n_bytes = 0;
-    rows->n_bytes_limit = 1024*SIZE_FACTOR*16;
+    rows->n_bytes_limit = 1024*size_factor*16;
     rows->data = (char *) toku_malloc(rows->n_bytes_limit);
     if (rows->rows==NULL || rows->data==NULL) {
 	int r = errno;
@@ -518,7 +532,7 @@ void destroy_rowset (struct rowset *rows) {
     zero_rowset(rows);
 }
 
-const size_t data_buffer_limit = 1024*SIZE_FACTOR*64;
+
 
 static int row_wont_fit (struct rowset *rows, size_t size)
 /* Effect: Return nonzero if adding a row of size SIZE would be too big (bigger than the buffer limit) */ 
@@ -1307,7 +1321,7 @@ static int merge_some_files (const BOOL to_q, FIDX dest_data, QUEUE q, int n_sou
         }
             
         n_rows_done++;
-	const u_int64_t rows_per_report = SIZE_FACTOR*1024;
+	const u_int64_t rows_per_report = size_factor*1024;
 	if (n_rows_done%rows_per_report==0) {
 	    // need to update the progress.
 	    double fraction_of_remaining_we_just_did = (double)rows_per_report / (double)(n_rows - n_rows_done + rows_per_report);
@@ -1363,8 +1377,8 @@ int merge_files (struct merge_fileset *fs,
  */
 {
     //printf(" merge_files use %d progress=%d fin at %d\n", progress_allocation, bl->progress, bl->progress+progress_allocation);
-    const int mergelimit = (SIZE_FACTOR == 1) ? 4 : 256;
-//    const int mergelimit = (SIZE_FACTOR == 1) ? 4 : 16;
+    const int mergelimit = (size_factor == 1) ? 4 : 256;
+//    const int mergelimit = (size_factor == 1) ? 4 : 16;
     int n_passes_left = (fs->n_temp_files==1) ? 1 : n_passes(fs->n_temp_files, mergelimit);
     //printf("%d files, %d per pass, %d passes\n", fs->n_temp_files, mergelimit, n_passes_left);
     int result = 0;
@@ -1514,7 +1528,6 @@ struct leaf_buf {
     int nkeys_p, ndata_p, dsize_p, partitions_p, n_in_buf_p;
 };
 
-const int nodesize = (SIZE_FACTOR==1) ? (1<<15) : (1<<22);
 
 struct translation {
     int64_t off, size;

newbrt/brtloader.h

@@ -39,6 +39,10 @@ int toku_brt_loader_abort(BRTLOADER bl,
 
 void brtloader_set_os_fwrite (size_t (*fwrite_fun)(const void*,size_t,size_t,FILE*));
 
+// For test purposes only
+void toku_brtloader_set_size_factor (uint32_t factor);
+
+
 #if defined(__cplusplus) || defined(__cilkplusplus)
 };
 #endif

src/export.map

@@ -16,6 +16,7 @@
    db_env_set_checkpoint_callback2;
    db_env_set_recover_callback;
    db_env_set_recover_callback2;
+   db_env_set_loader_size_factor;
 
    toku_os_get_max_rss;
 

src/tests/loader-cleanup-test.c

@@ -6,7 +6,13 @@
 /* TODO:
  *
  * When loader is fixed to not crash on failed call to toku_os_write(),
- * get rid of INDUCE_ENOSPC and -e argument, just make regular part of test.
+ * get rid of INDUCE_BAD_WRITE and -e argument.
+ *
+ * When loader is fixed to not crash when poll function returns non-zero
+ * value with size_factor=1, get rid of INDUCE_ABORT and -a argument.
+ *
+ * When loader is fixed to not crash on failed call to do_fwrite() with small size factor,
+ * get rid of SIZE_FACTOR and -s argument.
  */
 
 
@@ -64,7 +70,9 @@ int NUM_ROWS=100000;
 //static int NUM_ROWS=50000000;
 int CHECK_RESULTS=0;
 int USE_PUTS=0;
-int INDUCE_ENOSPC=0;
+int INDUCE_BAD_WRITE=0;
+int INDUCE_ABORT = 0;
+int SIZE_FACTOR = 0;
 enum {MAGIC=311};
 
 
@@ -77,6 +85,7 @@ static void get_inames(DBT* inames, DB** dbs);
 static int verify_file(char * dirname, char * filename);
 static void assert_inames_missing(DBT* inames);
 static ssize_t bad_write(int, const void *, size_t);
+static void run_all_tests(void);
 
 int fwrite_count = 0;
 int fwrite_enospc = 0;
@@ -580,20 +589,23 @@ static void run_test(enum test_type t, int trigger)
 // ------------ infrastructure ----------
 static void do_args(int argc, char * const argv[]);
 
-int test_main(int argc, char * const *argv) {
+static void run_all_tests(void) {
     int trigger;
-    do_args(argc, argv);
+
     if (verbose) printf("\n\nTesting loader with close and commit (normal)\n");
     run_test(commit, 0);
+
     if (verbose) printf("\n\nTesting loader with loader abort and txn abort\n");
     run_test(abort_loader, 0);
-    if (!USE_PUTS) {
+    if (!USE_PUTS && INDUCE_ABORT) {
 	if (verbose) printf("\n\nTesting loader with loader abort_via_poll and txn abort\n");
 	run_test(abort_via_poll, 0);
     }
+
     if (verbose) printf("\n\nTesting loader with loader close and txn abort\n");
     run_test(abort_txn, 0);
-    if (INDUCE_ENOSPC) {
+
+    if (INDUCE_BAD_WRITE) {
 	int i;
 	for (i = 1; i < 5; i++) {
 	    trigger = write_count_nominal / i;
@@ -604,25 +616,37 @@ int test_main(int argc, char * const *argv) {
     {
 	int i;
 	// induce write error at beginning of process
-	for (i = 1; i < 5 * NUM_DBS; i++) {
+	for (i = 1; i < 5 * NUM_DBS && i < fwrite_count_nominal; i++) {
 	    trigger = i;
 	    if (verbose) printf("\n\nTesting loader with enospc induced at fwrite count %d\n", trigger);
 	    run_test(enospc_f, trigger);
 	}
 	// induce write error sprinkled through process
-	for (i = 2; i < 5; i++) {
+	for (i = 2; i < 5 && i < fwrite_count_nominal; i++) {
 	    trigger = fwrite_count_nominal / i;
 	    if (verbose) printf("\n\nTesting loader with enospc induced at fwrite count %d\n", trigger);
 	    run_test(enospc_f, trigger);
 	}
 	// induce write error at end of process
-	for (i = 1; i < 5 * NUM_DBS; i++) {
+	for (i = 0; i < 5 * NUM_DBS && i < fwrite_count_nominal; i++) {
 	    trigger =  fwrite_count_nominal - i;
             assert(trigger > 0);
 	    if (verbose) printf("\n\nTesting loader with enospc induced at fwrite count %d\n", trigger);
 	    run_test(enospc_f, trigger);
 	}
     }
+}
+
+int test_main(int argc, char * const *argv) {
+    do_args(argc, argv);
+
+    if (verbose) printf("\nTesting loader with default size_factor\n");
+    run_all_tests();
+    if (SIZE_FACTOR) {
+	if (verbose) printf("\nTesting loader with size_factor=1\n");
+	db_env_set_loader_size_factor(1);
+	run_all_tests();
+    }
     return 0;
 }
 
@@ -658,8 +682,14 @@ static void do_args(int argc, char * const argv[]) {
             USE_PUTS = LOADER_USE_PUTS;
 	    printf("Using puts\n");
         } else if (strcmp(argv[0], "-e")==0) {
-	    INDUCE_ENOSPC = 1;
+	    INDUCE_BAD_WRITE = 1;
 	    printf("Using enospc\n");
+	} else if (strcmp(argv[0], "-a")==0) {
+	    INDUCE_ABORT = 1;
+	    printf ("Testing abort via poll with smaller size factor\n");
+	} else if (strcmp(argv[0], "-s")==0) {
+	    SIZE_FACTOR = 1;
+	    printf ("Testing abort via poll with smaller size factor\n");
 	} else {
 	    fprintf(stderr, "Unknown arg: %s\n", argv[0]);
 	    resultcode=1;

src/ydb.c

@@ -5636,6 +5636,7 @@ int db_env_set_func_realloc (void *(*f)(void*, size_t)) {
 int db_env_set_func_free (void (*f)(void*)) {
     return toku_set_func_free(f);
 }
+
 // Got to call dlmalloc, or else it won't get included.
 void setup_dlmalloc (void) {
     db_env_set_func_malloc(dlmalloc);
@@ -5668,6 +5669,14 @@ void db_env_set_recover_callback2 (void (*callback_f)(void*), void* extra) {
     toku_recover_set_callback2(callback_f, extra);
 }
 
+void db_env_set_loader_size_factor (uint32_t factor) {
+    toku_brtloader_set_size_factor(factor);
+}
+
+
+
+
+
 // HACK: To ensure toku_pthread_yield gets included in the .so
 // non-static would require a prototype in a header
 // static (since unused) would give a warning