#3043 merge a new hotindex undo function to main refs[t:3043]

git-svn-id: file:///svn/toku/tokudb@25766 c7de825b-a66e-492c-adef-691d508d4ae1

src/indexer-internal.h

@@ -10,6 +10,14 @@
 #ifndef TOKU_INDEXER_INTERNAL_H
 #define TOKU_INDEXER_INTERNAL_H
 
+// the indexer_commit_keys is a set of keys described by a DBT in the keys array.
+// the array is a resizeable array with max size "max_keys" and current size "current_keys".
+struct indexer_commit_keys {
+    int max_keys;        // max number of keys
+    int current_keys;    // number of valid keys
+    DBT *keys;           // the variable length keys array
+};
+
 struct __toku_indexer_internal {
     DB_ENV *env;
     DB_TXN *txn;
@@ -27,10 +35,15 @@ struct __toku_indexer_internal {
     FILENUM  *fnums; /* [N] */
     FILENUMS filenums;
 
+    // undo state
+    struct indexer_commit_keys commit_keys; // set of keys to commit
+    DBT hotkey, hotval;                     // current hot key and value
+    DBT previous_insert_key;                // previous insert key
+
     // test functions
     int (*undo_do)(DB_INDEXER *indexer, DB *hotdb, ULEHANDLE ule);
     int (*test_is_xid_live)(DB_INDEXER *indexer, TXNID xid);
-    int (*test_maybe_lock_provisional_key)(DB_INDEXER *indexer, TXNID xid, DB *hotdb, DBT *key);
+    int (*test_lock_key)(DB_INDEXER *indexer, TXNID xid, DB *hotdb, DBT *key);
     int (*test_delete_provisional)(DB_INDEXER *indexer, DB *hotdb, DBT *hotkey, XIDS xids);
     int (*test_delete_committed)(DB_INDEXER *indexer, DB *hotdb, DBT *hotkey, XIDS xids);
     int (*test_insert_provisional)(DB_INDEXER *indexer, DB *hotdb, DBT *hotkey, DBT *hotval, XIDS xids);
@@ -41,6 +54,10 @@ struct __toku_indexer_internal {
     int test_only_flags;
 };
 
+void indexer_undo_do_init(DB_INDEXER *indexer);
+
+void indexer_undo_do_destroy(DB_INDEXER *indexer);
+
 int indexer_undo_do(DB_INDEXER *indexer, DB *hotdb, ULEHANDLE ule);
 
 #endif

src/indexer.c

@@ -92,6 +92,7 @@ free_indexer_resources(DB_INDEXER *indexer) {
             toku_free(indexer->i->fnums); 
             indexer->i->fnums = NULL;
         }
+        indexer_undo_do_destroy(indexer);
         indexer_release_refs(indexer);
         // indexer->i
         toku_free(indexer->i);
@@ -172,6 +173,7 @@ toku_indexer_create_indexer(DB_ENV *env,
 create_exit:
     if ( rval == 0 ) {
 
+        indexer_undo_do_init(indexer);
         indexer_add_refs(indexer);
         
         *indexerp = indexer;

src/tests/hotindexer-undo-do-test.c

@@ -195,14 +195,13 @@ test_is_xid_live(DB_INDEXER *indexer, TXNID xid) {
 }
 
 static int 
-test_maybe_lock_provisional_key(DB_INDEXER *indexer, TXNID xid, DB *hotdb, DBT *key) {
+test_lock_key(DB_INDEXER *indexer, TXNID xid, DB *hotdb, DBT *key) {
     invariant(indexer == test_indexer);
     invariant(hotdb == test_hotdb);
-    if (test_is_xid_live(indexer, xid)) {
+    invariant(test_is_xid_live(indexer, xid));
     printf("lock [%lu] ", xid);
     print_dbt(key);
     printf("\n");
-    }
     return 0;
 }
 
@@ -360,19 +359,19 @@ read_test(char *testname, ULE ule) {
                 ule_add_provisional(ule, &uxr_s);
                 continue;
             }
-            printf("%s???\n", line);
+            fprintf(stderr, "%s???\n", line);
             r = EINVAL;
         }
         toku_free(line);
         fclose(f);
     } else {
         r = errno;
-        printf("fopen %s errno=%d\n", testname, errno);
+        fprintf(stderr, "fopen %s errno=%d\n", testname, errno);
     }
     return r;
  }
 
-static void
+static int
 run_test(char *envdir, char *testname) {
     if (verbose)
         printf("%s\n", testname);
@@ -403,7 +402,7 @@ run_test(char *envdir, char *testname) {
 
     // set test callbacks
     indexer->i->test_is_xid_live = test_is_xid_live;
-    indexer->i->test_maybe_lock_provisional_key = test_maybe_lock_provisional_key;
+    indexer->i->test_lock_key = test_lock_key;
     indexer->i->test_delete_provisional = test_delete_provisional;
     indexer->i->test_delete_committed = test_delete_committed;
     indexer->i->test_insert_provisional = test_insert_provisional;
@@ -419,7 +418,9 @@ run_test(char *envdir, char *testname) {
     ule_init(ule);
 
     // read the test
-    r = read_test(testname, ule); assert_zero(r);
+    r = read_test(testname, ule);
+    if (r != 0)
+        return r;
 
     r = indexer->i->undo_do(indexer, dest_db, ule); assert_zero(r);
 
@@ -435,6 +436,8 @@ run_test(char *envdir, char *testname) {
     r = env->close(env, 0); assert_zero(r);
 
     live_destroy(&live_xids);
+    
+    return r;
 }
 
 int
@@ -457,7 +460,7 @@ test_main(int argc, char * const argv[]) {
         break;
     }
 
-    for ( ; i < argc; i++) {
+    for (r = 0 ; r == 0 && i < argc; i++) {
         char *testname = argv[i];
         char envdir[strlen(ENVDIR) + 1 + 32 + 1];
         sprintf(envdir, "%s.%d", ENVDIR, toku_os_getpid());
@@ -467,9 +470,9 @@ test_main(int argc, char * const argv[]) {
         r = system(syscmd); assert_zero(r);
         r = toku_os_mkdir(envdir, S_IRWXU+S_IRWXG+S_IRWXO); assert_zero(r);
 
-        run_test(envdir, testname);
+        r = run_test(envdir, testname);
     }
 
-    return 0;
+    return r;
 }
 

src/tests/hotindexer-undo-do-tests/commit.d0.d100.d200.test

+key k1
+delete committed 0
+delete committed 100
+delete committed 200

src/tests/hotindexer-undo-do-tests/commit.d0.d100.i200.result

+insert_committed [200] v100 k1 
+commit_any [200] v100 

src/tests/hotindexer-undo-do-tests/commit.d0.d100.i200.test

+key k1
+delete committed 0
+delete committed 100
+insert committed 200 v100

src/tests/hotindexer-undo-do-tests/commit.d0.d100.test

+key k1
+delete committed 0
+delete committed 100

src/tests/hotindexer-undo-do-tests/commit.d0.i100.d200.result

+insert_committed [100] v100 k1 
+commit_any [100] v100 
+delete_committed [200] v100 
+commit_any [200] v100 

src/tests/hotindexer-undo-do-tests/commit.d0.i100.d200.test

+key k1
+delete committed 0
+insert committed 100 v100
+delete committed 200

src/tests/hotindexer-undo-do-tests/commit.i0.d100.i200.result

+insert_committed [0] v0 k1 
+delete_committed [100] v0 
+commit_any [100] v0 
+insert_committed [200] v100 k1 
+commit_any [200] v100 

src/tests/hotindexer-undo-do-tests/commit.i0.d100.i200.test

+key k1
+insert committed 0   v0
+delete committed 100
+insert committed 200 v100

src/tests/hotindexer-undo-do-tests/commit.i0.d100.result

+insert_committed [0] v10 k1 
+delete_committed [100] v10 
+commit_any [100] v10 

src/tests/hotindexer-undo-do-tests/commit.i0.d100.test

+key k1
+insert committed 0 v10
+delete committed 100

src/tests/hotindexer-undo-do-tests/commit.i0.i100.d200.result

+insert_committed [0] v0 k1 
+delete_committed [100] v0 
+insert_committed [100] v100 k1 
+commit_any [100] v0 
+commit_any [100] v100 
+delete_committed [200] v100 
+commit_any [200] v100 

src/tests/hotindexer-undo-do-tests/commit.i0.i100.d200.test

+key k1
+insert committed 0   v0
+insert committed 100 v100
+delete committed 200

src/tests/hotindexer-undo-do-tests/commit.i0.i100.i200.result

+insert_committed [0] v0 k1 
+delete_committed [100] v0 
+insert_committed [100] v100 k1 
+commit_any [100] v0 
+commit_any [100] v100 
+delete_committed [200] v100 
+insert_committed [200] v200 k1 
+commit_any [200] v100 
+commit_any [200] v200 

src/tests/hotindexer-undo-do-tests/commit.i0.i100.i200.test

+key k1
+insert committed 0   v0
+insert committed 100 v100
+insert committed 200 v200

src/tests/hotindexer-undo-do-tests/commit.i0.i100.result

+insert_committed [0] v10 k1 
+delete_committed [100] v10 
+insert_committed [100] v20 k1 
+commit_any [100] v10 
+commit_any [100] v20 

src/tests/hotindexer-undo-do-tests/commit.i0.i100.test

+key k1
+insert committed 0   v10
+insert committed 100 v20

src/tests/hotindexer-undo-do-tests/insert.300.result

+insert_committed [0] v100 k1 
+delete_committed [200] v100 
+insert_committed [200] v200 k1 
+commit_any [200] v100 
+commit_any [200] v200 
+delete_committed [300] v200 
+insert_committed [300] v300 k1 
+commit_any [300] v200 
+commit_any [300] v300 

src/tests/hotindexer-undo-do-tests/insert.300.test

+key k1
+insert committed   0 v100
+insert committed 200 v200
+insert committed 300 v300

src/tests/hotindexer-undo-do-tests/placeholder.3.test

 key k1
 delete committed 0
-insert committed 100 v10
 insert provisional 300 v18
 placeholder 301
 insert provisional 302 v20

src/tests/hotindexer-undo-do-tests/prov.1.result

@@ -7,9 +7,8 @@ delete_committed [200] v20
 insert_committed [200] v10 k1 
 commit_any [200] v20 
 commit_any [200] v10 
-delete_committed [300,301] v10 
-insert_committed [300,301] v30 k1 
-commit_any [300,301] v10 
-commit_any [300,301] v30 
-delete_committed [300,301,302] v30 
-commit_any [300,301,302] v30 
+delete_committed [300] v10 
+insert_committed [300] v30 k1 
+delete_committed [300] v30 
+commit_any [300] v10 
+commit_any [300] v30 

src/tests/hotindexer-undo-do-tests/prov.d0.d100.test

+key k1
+delete committed 0
+delete provisional 100

src/tests/hotindexer-undo-do-tests/prov.insert.100.result → src/tests/hotindexer-undo-do-tests/prov.d0.i100.d200.result

 insert_committed [100] v100 k1 
+delete_committed [100] v100 
 commit_any [100] v100 

src/tests/hotindexer-undo-do-tests/prov.d0.i100.d200.test

+key k1
+delete committed 0
+insert provisional 100 v100
+delete provisional 200
+

src/tests/hotindexer-undo-do-tests/prov.d0.i100.result

+insert_committed [100] v100 k1 

src/tests/hotindexer-undo-do-tests/prov.i0.d100.result

+insert_committed [0] v10 k1 
+delete_committed [100] v10 
+commit_any [100] v10 

src/tests/hotindexer-undo-do-tests/prov.i0.d100.test

+key k1
+insert committed 0 v10
+delete provisional 100

src/tests/hotindexer-undo-do-tests/prov.i0.i100.result

+insert_committed [0] v10 k1 
+delete_committed [100] v10 
+insert_committed [100] v100 k1 
+commit_any [100] v10 

src/tests/hotindexer-undo-do-tests/prov.i0.i100.test

+key k1
+insert committed 0 v10
+insert provisional 100 v100

src/tests/hotindexer-undo-do-tests/prov.live.d0.d100.test

+live 100
+key k1
+delete committed 0
+delete provisional 100

src/tests/hotindexer-undo-do-tests/prov.live.d0.i100.d200.result

+insert_provisional [100] v100 k1 
+lock [100] v100 
+delete_provisional [100,200] v100 
+lock [100] v100 
+lock [100] v100 

src/tests/hotindexer-undo-do-tests/prov.live.d0.i100.d200.test

+live 100 200
+key k1
+delete committed 0
+insert provisional 100 v100
+delete provisional 200
+

src/tests/hotindexer-undo-do-tests/prov.live.i0.d100.result

+insert_committed [0] v10 k1 
+delete_provisional [100] v10 
+lock [100] v10 
+lock [100] v10 

src/tests/hotindexer-undo-do-tests/prov.live.i0.d100.test

+live 100
+key k1
+insert committed 0 v10
+delete provisional 100

src/tests/hotindexer-undo-do-tests/prov.live.i0.i100.result

+insert_committed [0] v10 k1 
+delete_provisional [100] v10 
+lock [100] v10 
+insert_provisional [100] v100 k1 
+lock [100] v100 

src/tests/hotindexer-undo-do-tests/prov.live.i0.i100.test

+live 100
+key k1
+insert committed 0 v10
+insert provisional 100 v100

src/tests/run-hotindexer-undo-do-tests.bash

@@ -4,10 +4,11 @@
 
 tests=""
 verbose=0
+valgrind=""
 
 for arg in $* ; do 
-    if [ $arg = "--verbose" ] ; then
-	verbose=1
+    if [[ $arg =~ "--(.*)=(.*)" ]] ; then
+	eval ${BASH_REMATCH[1]}=${BASH_REMATCH[2]}
     else
 	tests="$tests $arg"
     fi
@@ -26,7 +27,7 @@ for t in $tests ; do
     fi
     if [ $verbose != 0 ] ; then echo $testdir $testname $testfile $resultfile; fi
 
-    ./hotindexer-undo-do-test.tdb $testdir/$testfile >$testdir/$testname.run
+    $valgrind ./hotindexer-undo-do-test.tdb $testdir/$testfile >$testdir/$testname.run
 
     if [ -f $testdir/$resultfile ] ; then
 	diff -q $testdir/$testname.run $testdir/$resultfile >/dev/null 2>&1