Commit 2309e70b authored by Rich Prohaska's avatar Rich Prohaska Committed by Yoni Fogel

#2868 refs[t:2868] fix the presort speedup in the loader

git-svn-id: file:///svn/toku/tokudb@22868 c7de825b-a66e-492c-adef-691d508d4ae1
parent a66a05e0
...@@ -1536,22 +1536,28 @@ int sort_and_write_rows (struct rowset rows, struct merge_fileset *fs, BRTLOADER ...@@ -1536,22 +1536,28 @@ int sort_and_write_rows (struct rowset rows, struct merge_fileset *fs, BRTLOADER
//bl_time_t after_sort = bl_time_now(); //bl_time_t after_sort = bl_time_now();
if (result == 0) { if (result == 0) {
DBT akey = make_dbt(rows.data+rows.rows[0].off, rows.rows[0].klen); DBT min_rowset_key = make_dbt(rows.data+rows.rows[0].off, rows.rows[0].klen);
if (fs->have_sorted_output && compare(dest_db, &fs->prev_key, &akey)<0) { if (fs->have_sorted_output && compare(dest_db, &fs->prev_key, &min_rowset_key) < 0) {
// write everything to the same output. // write everything to the same output if the max key in the temp file (prev_key) is < min of the sorted rowset
result = write_rowset_to_file(bl, fs->sorted_output, rows); result = write_rowset_to_file(bl, fs->sorted_output, rows);
if (result == 0) {
// set the max key in the temp file to the max key in the sorted rowset
result = toku_dbt_set(rows.rows[rows.n_rows-1].klen, rows.data + rows.rows[rows.n_rows-1].off, &fs->prev_key, NULL);
}
} else { } else {
FIDX sfile = FIDX_NULL; // write the sorted rowset into a new temp file
result = extend_fileset(bl, fs, &sfile); if (fs->have_sorted_output) {
fs->have_sorted_output = FALSE;
result = brtloader_fi_close(&bl->file_infos, fs->sorted_output);
}
if (result == 0) { if (result == 0) {
if (fs->have_sorted_output) { FIDX sfile = FIDX_NULL;
fs->have_sorted_output = FALSE; result = extend_fileset(bl, fs, &sfile);
result = brtloader_fi_close(&bl->file_infos, fs->sorted_output);
}
if (result == 0) { if (result == 0) {
result = write_rowset_to_file(bl, sfile, rows); result = write_rowset_to_file(bl, sfile, rows);
if (result == 0) { if (result == 0) {
fs->have_sorted_output = TRUE; fs->sorted_output = sfile; fs->have_sorted_output = TRUE; fs->sorted_output = sfile;
// set the max key in the temp file to the max key in the sorted rowset
result = toku_dbt_set(rows.rows[rows.n_rows-1].klen, rows.data + rows.rows[rows.n_rows-1].off, &fs->prev_key, NULL); result = toku_dbt_set(rows.rows[rows.n_rows-1].klen, rows.data + rows.rows[rows.n_rows-1].off, &fs->prev_key, NULL);
} }
} }
......
...@@ -98,9 +98,9 @@ check_brtloader-test$(BINSUF): EXTRA_ARGS=dir.$@ ...@@ -98,9 +98,9 @@ check_brtloader-test$(BINSUF): EXTRA_ARGS=dir.$@
check_brtloader-test-bad-generate$(BINSUF): EXTRA_ARGS=dir.$@ check_brtloader-test-bad-generate$(BINSUF): EXTRA_ARGS=dir.$@
check_brtloader-test-extractor$(BINSUF): $(patsubst %,check_brtloader-test-extractor-%, 1 2 3 1a 2a 3a) check_brtloader-test-extractor$(BINSUF): $(patsubst %,check_brtloader-test-extractor-%, 1 2 3 1a 2a 3a 4a 5a)
true $(SUMMARIZE_CMD) true $(SUMMARIZE_CMD)
# the 1,2,3 tests don't use valgrind, the 1a,2a,3a tests use valgrind. # the 1,2,3 tests don't use valgrind, the 1a,2a,3a,4a,5a tests use valgrind.
check_brtloader-test-extractor-1: brtloader-test-extractor$(BINSUF) check_brtloader-test-extractor-1: brtloader-test-extractor$(BINSUF)
./$< $(VERBVERBOSE) -s -r 1000 --rowsets 1000 --asc dir.$@ $(SUMMARIZE_CMD) ./$< $(VERBVERBOSE) -s -r 1000 --rowsets 1000 --asc dir.$@ $(SUMMARIZE_CMD)
check_brtloader-test-extractor-2: brtloader-test-extractor$(BINSUF) check_brtloader-test-extractor-2: brtloader-test-extractor$(BINSUF)
...@@ -113,6 +113,10 @@ check_brtloader-test-extractor-2a: brtloader-test-extractor$(BINSUF) ...@@ -113,6 +113,10 @@ check_brtloader-test-extractor-2a: brtloader-test-extractor$(BINSUF)
$(VGRIND) ./$< $(VERBVERBOSE) -s -r 1000 --rowsets 1000 --dsc dir.$@ $(SUMMARIZE_CMD) $(VGRIND) ./$< $(VERBVERBOSE) -s -r 1000 --rowsets 1000 --dsc dir.$@ $(SUMMARIZE_CMD)
check_brtloader-test-extractor-3a: brtloader-test-extractor$(BINSUF) check_brtloader-test-extractor-3a: brtloader-test-extractor$(BINSUF)
$(VGRIND) ./$< $(VERBVERBOSE) -s -r 1000 --rowsets 1000 --random dir.$@ $(SUMMARIZE_CMD) $(VGRIND) ./$< $(VERBVERBOSE) -s -r 1000 --rowsets 1000 --random dir.$@ $(SUMMARIZE_CMD)
check_brtloader-test-extractor-4a: brtloader-test-extractor$(BINSUF)
$(VGRIND) ./$< $(VERBVERBOSE) -s -r 1000 --rowsets 3 --asc dir.$@ $(SUMMARIZE_CMD)
check_brtloader-test-extractor-5a: brtloader-test-extractor$(BINSUF)
$(VGRIND) ./$< $(VERBVERBOSE) -s -r 1000 --rowsets 3 --asc --asc-poison dir.$@ $(SUMMARIZE_CMD)
check_brtloader-test-extractor-errors$(BINSUF): $(patsubst %,check_brtloader-test-extractor-errors-%, 1 2) check_brtloader-test-extractor-errors$(BINSUF): $(patsubst %,check_brtloader-test-extractor-errors-%, 1 2)
true $(SUMMARIZE_CMD) true $(SUMMARIZE_CMD)
......
...@@ -277,6 +277,7 @@ static void shuffle(int a[], int n) { ...@@ -277,6 +277,7 @@ static void shuffle(int a[], int n) {
} }
static int ascending_keys = 0; static int ascending_keys = 0;
static int ascending_keys_poison = 0;
static int descending_keys = 0; static int descending_keys = 0;
static int random_keys = 0; static int random_keys = 0;
...@@ -288,7 +289,12 @@ static void test_extractor(int nrows, int nrowsets, const char *testdir) { ...@@ -288,7 +289,12 @@ static void test_extractor(int nrows, int nrowsets, const char *testdir) {
int nkeys = nrows * nrowsets; int nkeys = nrows * nrowsets;
int *keys = toku_calloc(nkeys, sizeof (int)); assert(keys); int *keys = toku_calloc(nkeys, sizeof (int)); assert(keys);
for (int i = 0; i < nkeys; i++) for (int i = 0; i < nkeys; i++)
keys[i] = ascending_keys ? i : nkeys - i; keys[i] = ascending_keys ? 2*i : nkeys - i;
if (ascending_keys_poison) {
if (verbose)
printf("poison %d %d %d\n", nrows*(nrowsets-1), keys[nrows*(nrowsets-1)], keys[nrows-1] -1);
keys[nrows*(nrowsets-1)] = keys[nrows-1] - 1;
}
if (random_keys) if (random_keys)
shuffle(keys, nkeys); shuffle(keys, nkeys);
...@@ -384,6 +390,9 @@ int test_main (int argc, const char *argv[]) { ...@@ -384,6 +390,9 @@ int test_main (int argc, const char *argv[]) {
descending_keys = 1; descending_keys = 1;
} else if (strcmp(argv[0],"--random") == 0) { } else if (strcmp(argv[0],"--random") == 0) {
random_keys = 1; random_keys = 1;
} else if (strcmp(argv[0], "--asc-poison") == 0) {
ascending_keys = 1;
ascending_keys_poison = 1;
} else if (argc!=1) { } else if (argc!=1) {
return usage(progname); return usage(progname);
exit(1); exit(1);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment