Merge the finer-grained progress report for the loader. Refs #2462. [t:2462]

{{{
svn merge -r 18775:18791 https://svn.tokutek.com/tokudb/toku/tokudb.2462
}}}
.


git-svn-id: file:///svn/toku/tokudb@18792 c7de825b-a66e-492c-adef-691d508d4ae1

newbrt/brtloader-internal.h

@@ -10,6 +10,7 @@ struct file_info {
     BOOL is_extant; // if true, the file must be unlinked.
     char *fname;
     FILE *file;
+    u_int64_t n_rows; // how many rows were written into that file
 };
 struct file_infos {
     int n_files;
@@ -36,6 +37,8 @@ struct brtloader_s {
     const struct descriptor **descriptors; // N of these
     const char **new_fnames_in_env; // the file names that the final data will be written to (relative to env).
 
+    u_int64_t n_rows; // how many rows have been put?
+
     const char *temp_file_template;
     FIDX fprimary_rows; // the file index (in the file_infos) for the data
     FIDX fprimary_idx;  // the file index for the index
@@ -43,6 +46,16 @@ struct brtloader_s {
     CACHETABLE cachetable;
     /* To make it easier to recover from errors, we don't use FILE*, instead we use an index into the file_infos. */
     struct file_infos file_infos;
+
+#define PROGRESS_MAX (1<<16)
+    int progress;       // Progress runs from 0 to PROGRESS_MAX.  When we call the poll function we convert to a float from 0.0 to 1.0
+    // We use an integer so that we can add to the progress using a fetch-and-add instruction.
+
+    // These two are set in the close function, and used while running close
+    int (*poll_function)(void *extra, float progress);
+    void *poll_extra;
+
+    int user_said_stop; // 0 if the poll_function always returned zero.  If it ever returns nonzero, then store that value here.
 };
 
 /* These data structures are used for manipulating a collection of rows in main memory. */
@@ -87,9 +100,9 @@ void init_merge_fileset (struct merge_fileset *fs);
 void destroy_merge_fileset (struct merge_fileset *fs);
 
 int sort_and_write_rows (struct rowset *rows, struct merge_fileset *fs, BRTLOADER bl, DB *dest_db, brt_compare_func,
-			 struct error_callback_s *error_callback);
-int merge_files (struct merge_fileset *fs, BRTLOADER bl, DB *dest_db, brt_compare_func, struct error_callback_s *);
-int write_file_to_dbfile (int outfile, FIDX infile, BRTLOADER bl, const struct descriptor *descriptor);
+			 struct error_callback_s *error_callback, int progress_allocation);
+int merge_files (struct merge_fileset *fs, BRTLOADER bl, DB *dest_db, brt_compare_func, struct error_callback_s *, int progress_allocation);
+int write_file_to_dbfile (int outfile, FIDX infile, BRTLOADER bl, const struct descriptor *descriptor, int progress_allocation);
 
 int brtloader_init_file_infos (struct file_infos *fi);
 void brtloader_fi_destroy (struct file_infos *fi, BOOL is_error);

newbrt/tests/brtloader-test.c

@@ -259,14 +259,14 @@ static void test_merge_files (char *template) {
 
     struct error_callback_s cb;
     cb.error_callback = err_cb;
-    r = sort_and_write_rows(&aset, &fs, &bl, dest_db, compare_ints, &cb);  CKERR(r);
-    r = sort_and_write_rows(&bset, &fs, &bl, dest_db, compare_ints, &cb);  CKERR(r);
+    r = sort_and_write_rows(&aset, &fs, &bl, dest_db, compare_ints, &cb, 0);  CKERR(r);
+    r = sort_and_write_rows(&bset, &fs, &bl, dest_db, compare_ints, &cb, 0);  CKERR(r);
     assert(fs.n_temp_files==2 && fs.n_temp_files_limit >= fs.n_temp_files);
     destroy_rowset(&aset);
     destroy_rowset(&bset);
     for (int i=0; i<2; i++) assert(fs.data_fidxs[i].idx != -1 && fs.idx_fidxs[i].idx != -1);
 
-    r = merge_files(&fs, &bl, dest_db, compare_ints, &cb); CKERR(r);
+    r = merge_files(&fs, &bl, dest_db, compare_ints, &cb, 0); CKERR(r);
 
     assert(fs.n_temp_files==1);
 
@@ -278,7 +278,7 @@ static void test_merge_files (char *template) {
     fprintf(stderr, "Final data in %s\n", name);
     assert(r>=0);
     struct descriptor desc = {.version = 1, .dbt = (DBT){.size = 4, .data="abcd"}};
-    r = write_file_to_dbfile(fd, inf, &bl, &desc);
+    r = write_file_to_dbfile(fd, inf, &bl, &desc, 1);
     CKERR(r);
     r = brtloader_fi_close(&bl.file_infos, inf);
     CKERR(r);

src/tests/loader-stress-test.c

@@ -180,9 +180,19 @@ static void check_results(DB **dbs)
 static void *expect_poll_void = &expect_poll_void;
 static int poll_count=0;
 static int poll_function (void *extra, float progress) {
-    //printf("progress: %5.1f%%\n", progress*100);
+    if (0) {
+	static int did_one=0;
+	static struct timeval start;
+	struct timeval now;
+	gettimeofday(&now, 0);
+	if (!did_one) {
+	    start=now;
+	    did_one=1;
+	}
+	printf("%6.6f %5.1f%%\n", now.tv_sec - start.tv_sec + 1e-6*(now.tv_usec - start.tv_usec), progress*100);
+    }
     assert(extra==expect_poll_void);
-    assert(0.0<=progress && progress<1.0);
+    assert(0.0<=progress && progress<=1.0);
     poll_count++;
     return 0;
 }