#2868 refs[t:2868] fix the presort speedup in the loader

git-svn-id: file:///svn/toku/tokudb@22868 c7de825b-a66e-492c-adef-691d508d4ae1

newbrt/brtloader.c

@@ -1536,22 +1536,28 @@ int sort_and_write_rows (struct rowset rows, struct merge_fileset *fs, BRTLOADER
         //bl_time_t after_sort = bl_time_now();
 
         if (result == 0) {
-            DBT akey = make_dbt(rows.data+rows.rows[0].off,  rows.rows[0].klen);
-            if (fs->have_sorted_output && compare(dest_db, &fs->prev_key, &akey)<0) {
-                // write everything to the same output.
+            DBT min_rowset_key = make_dbt(rows.data+rows.rows[0].off, rows.rows[0].klen);
+            if (fs->have_sorted_output && compare(dest_db, &fs->prev_key, &min_rowset_key) < 0) {
+                // write everything to the same output if the max key in the temp file (prev_key) is < min of the sorted rowset
                 result = write_rowset_to_file(bl, fs->sorted_output, rows);
+                if (result == 0) {
+                    // set the max key in the temp file to the max key in the sorted rowset
+                    result = toku_dbt_set(rows.rows[rows.n_rows-1].klen, rows.data + rows.rows[rows.n_rows-1].off, &fs->prev_key, NULL);
+                }
             } else {
-                FIDX sfile = FIDX_NULL;
-                result = extend_fileset(bl, fs, &sfile);
+                // write the sorted rowset into a new temp file
+                if (fs->have_sorted_output) {
+                    fs->have_sorted_output = FALSE;
+                    result = brtloader_fi_close(&bl->file_infos, fs->sorted_output);
+                }
                 if (result == 0) {
-                    if (fs->have_sorted_output) {
-                        fs->have_sorted_output = FALSE;
-                        result = brtloader_fi_close(&bl->file_infos, fs->sorted_output);
-                    }
+                    FIDX sfile = FIDX_NULL;
+                    result = extend_fileset(bl, fs, &sfile);
                     if (result == 0) {
                         result = write_rowset_to_file(bl, sfile, rows);
                         if (result == 0) {
                             fs->have_sorted_output = TRUE; fs->sorted_output = sfile;
+                            // set the max key in the temp file to the max key in the sorted rowset
                             result = toku_dbt_set(rows.rows[rows.n_rows-1].klen, rows.data + rows.rows[rows.n_rows-1].off, &fs->prev_key, NULL);
                         }
                     }

newbrt/tests/Makefile

@@ -98,9 +98,9 @@ check_brtloader-test$(BINSUF): EXTRA_ARGS=dir.$@
 
 check_brtloader-test-bad-generate$(BINSUF): EXTRA_ARGS=dir.$@
 
-check_brtloader-test-extractor$(BINSUF): $(patsubst %,check_brtloader-test-extractor-%, 1 2 3 1a 2a 3a)
+check_brtloader-test-extractor$(BINSUF): $(patsubst %,check_brtloader-test-extractor-%, 1 2 3 1a 2a 3a 4a 5a)
 	true $(SUMMARIZE_CMD)
-# the 1,2,3 tests don't use valgrind, the 1a,2a,3a tests use valgrind.
+# the 1,2,3 tests don't use valgrind, the 1a,2a,3a,4a,5a tests use valgrind.
 check_brtloader-test-extractor-1: brtloader-test-extractor$(BINSUF)
 	./$< $(VERBVERBOSE) -s -r 1000 --rowsets 1000 --asc dir.$@ $(SUMMARIZE_CMD)
 check_brtloader-test-extractor-2: brtloader-test-extractor$(BINSUF)
@@ -113,6 +113,10 @@ check_brtloader-test-extractor-2a: brtloader-test-extractor$(BINSUF)
 	$(VGRIND) ./$< $(VERBVERBOSE) -s -r 1000 --rowsets 1000 --dsc dir.$@ $(SUMMARIZE_CMD)
 check_brtloader-test-extractor-3a: brtloader-test-extractor$(BINSUF)
 	$(VGRIND) ./$< $(VERBVERBOSE) -s -r 1000 --rowsets 1000 --random dir.$@ $(SUMMARIZE_CMD)
+check_brtloader-test-extractor-4a: brtloader-test-extractor$(BINSUF)
+	$(VGRIND) ./$< $(VERBVERBOSE) -s -r 1000 --rowsets 3 --asc dir.$@ $(SUMMARIZE_CMD)
+check_brtloader-test-extractor-5a: brtloader-test-extractor$(BINSUF)
+	$(VGRIND) ./$< $(VERBVERBOSE) -s -r 1000 --rowsets 3 --asc --asc-poison dir.$@ $(SUMMARIZE_CMD)
 
 check_brtloader-test-extractor-errors$(BINSUF): $(patsubst %,check_brtloader-test-extractor-errors-%, 1 2)
 	true $(SUMMARIZE_CMD)

newbrt/tests/brtloader-test-extractor.c

@@ -277,6 +277,7 @@ static void shuffle(int a[], int n) {
 }
 
 static int ascending_keys = 0;
+static int ascending_keys_poison = 0;
 static int descending_keys = 0;
 static int random_keys = 0;
 
@@ -288,7 +289,12 @@ static void test_extractor(int nrows, int nrowsets, const char *testdir) {
     int nkeys = nrows * nrowsets;
     int *keys = toku_calloc(nkeys, sizeof (int)); assert(keys);
     for (int i = 0; i < nkeys; i++)
-        keys[i] = ascending_keys ? i : nkeys - i;
+        keys[i] = ascending_keys ? 2*i : nkeys - i;
+    if (ascending_keys_poison) {
+        if (verbose)
+            printf("poison %d %d %d\n", nrows*(nrowsets-1), keys[nrows*(nrowsets-1)], keys[nrows-1] -1);
+        keys[nrows*(nrowsets-1)] = keys[nrows-1] - 1;
+    }
     if (random_keys)
         shuffle(keys, nkeys);
 
@@ -384,6 +390,9 @@ int test_main (int argc, const char *argv[]) {
             descending_keys = 1;
         } else if (strcmp(argv[0],"--random") == 0) {
             random_keys = 1;
+        } else if (strcmp(argv[0], "--asc-poison") == 0) {
+            ascending_keys = 1;
+            ascending_keys_poison = 1;
 	} else if (argc!=1) {
             return usage(progname);
 	    exit(1);