#3043 add more hotindex undo prov tests, and fix the key locking refs[t:3043]

git-svn-id: file:///svn/toku/tokudb@25850 c7de825b-a66e-492c-adef-691d508d4ae1

src/indexer-internal.h

@@ -38,7 +38,6 @@ struct __toku_indexer_internal {
     // undo state
     struct indexer_commit_keys commit_keys; // set of keys to commit
     DBT hotkey, hotval;                     // current hot key and value
-    DBT previous_insert_key;                // previous insert key
 
     // test functions
     int (*undo_do)(DB_INDEXER *indexer, DB *hotdb, ULEHANDLE ule);

src/indexer-undo-do.c

@@ -81,7 +81,6 @@ static int indexer_brt_insert_committed(DB_INDEXER *indexer, DB *hotdb, DBT *hot
 static int indexer_brt_commit(DB_INDEXER *indexer, DB *hotdb, DBT *hotkey, XIDS xids);
 static int indexer_lock_key(DB_INDEXER *indexer, DB *hotdb, DBT *key, TXNID outermost_live_xid);
 static int indexer_is_xid_live(DB_INDEXER *indexer, TXNID xid);
-static int indexer_is_previous_insert_key_valid(DB_INDEXER *indexer);
 
 void
 indexer_undo_do_init(DB_INDEXER *indexer) {
@@ -90,16 +89,12 @@ indexer_undo_do_init(DB_INDEXER *indexer) {
     // DBTs for the key and val
     toku_init_dbt(&indexer->i->hotkey); indexer->i->hotkey.flags = DB_DBT_REALLOC;
     toku_init_dbt(&indexer->i->hotval); indexer->i->hotval.flags = DB_DBT_REALLOC;
-
-    // DBT for the previous inserted key
-    toku_init_dbt(&indexer->i->previous_insert_key); indexer->i->previous_insert_key.flags = DB_DBT_REALLOC;
 }
 
 void
 indexer_undo_do_destroy(DB_INDEXER *indexer) {
     toku_destroy_dbt(&indexer->i->hotkey);
     toku_destroy_dbt(&indexer->i->hotval);
-    toku_destroy_dbt(&indexer->i->previous_insert_key);
     indexer_commit_keys_destroy(&indexer->i->commit_keys);
 }
 
@@ -164,7 +159,6 @@ indexer_undo_do_committed(DB_INDEXER *indexer, DB *hotdb, ULEHANDLE ule) {
                 result = indexer_brt_insert_committed(indexer, hotdb, &indexer->i->hotkey, &indexer->i->hotval, xids);
                 if (result == 0) {
                     indexer_commit_keys_add(&indexer->i->commit_keys, indexer->i->hotkey.size, indexer->i->hotkey.data);
-                    result = toku_dbt_set(indexer->i->hotkey.size, indexer->i->hotkey.data, &indexer->i->previous_insert_key, NULL);
                 }
             }
         } else
@@ -249,8 +243,7 @@ indexer_undo_do_provisional(DB_INDEXER *indexer, DB *hotdb, ULEHANDLE ule) {
 
         // do
         if (uxr_is_delete(uxr)) {
-            if (outermost_is_live && indexer_is_previous_insert_key_valid(indexer))
-                result = indexer_lock_key(indexer, hotdb, &indexer->i->previous_insert_key, outermost_xid);
+            ; // do nothing
         } else if (uxr_is_insert(uxr)) {
             // generate the hot key and val
             result = indexer_generate_hot_key_val(indexer, hotdb, ule, uxr, &indexer->i->hotkey, &indexer->i->hotval);
@@ -258,13 +251,16 @@ indexer_undo_do_provisional(DB_INDEXER *indexer, DB *hotdb, ULEHANDLE ule) {
                 // send the insert message
                 if (!outermost_is_live) {
                     result = indexer_brt_insert_committed(indexer, hotdb, &indexer->i->hotkey, &indexer->i->hotval, xids);
+#if 0
+                    // no need to do this
+                    if (result == 0)
+                        indexer_commit_keys_add(&indexer->i->commit_keys, indexer->i->hotkey.size, indexer->i->hotkey.data);
+#endif
                 } else {
                     result = indexer_brt_insert_provisional(indexer, hotdb, &indexer->i->hotkey, &indexer->i->hotval, xids);
                     if (result == 0) 
                         result = indexer_lock_key(indexer, hotdb, &indexer->i->hotkey, outermost_xid);
                 }
-                if (result == 0)
-                    result = toku_dbt_set(indexer->i->hotkey.size, indexer->i->hotkey.data, &indexer->i->previous_insert_key, NULL);
             }
         } else
             invariant(0);
@@ -282,13 +278,41 @@ indexer_undo_do_provisional(DB_INDEXER *indexer, DB *hotdb, ULEHANDLE ule) {
     return result;
 }
 
+
+static int UU()
+indexer_fast_undo_do(DB_INDEXER *indexer, DB *hotdb, ULEHANDLE ule) {
+    int result = 0;
+    UXRHANDLE uxr = ule_get_uxr(ule, 0); invariant(uxr);
+    if (uxr_is_insert(uxr)) {
+        // generate the hot key and val
+        result = indexer_generate_hot_key_val(indexer, hotdb, ule, uxr, &indexer->i->hotkey, &indexer->i->hotval);
+        if (result == 0) {
+            // send the insert message
+            // TXNID this_xid = uxr_get_txnid(uxr);
+            XIDS xids = NULL; // xids init one this_xid
+            result = indexer_brt_insert_committed(indexer, hotdb, &indexer->i->hotkey, &indexer->i->hotval, xids);
+        }
+    }
+    return result;
+ }
+
 int
 indexer_undo_do(DB_INDEXER *indexer, DB *hotdb, ULEHANDLE ule) {
-    int result;
+    int result = 0;
 
+#if 0
+    int num_committed = ule_get_num_committed(ule);
+    int num_provisional = ule_get_num_provisional(ule);
+    // fast path for a simple ule with a single committed transaction record
+    if (num_committed == 1 && num_provisonal == 0) {
+        result = indexer_fast_undo_do(indexer, hotdb, ule);
+    } else 
+#endif
+    {
         result = indexer_undo_do_committed(indexer, hotdb, ule);
         if (result == 0)
             result = indexer_undo_do_provisional(indexer, hotdb, ule);
+    }
         
     if ( indexer->i->test_only_flags == INDEXER_TEST_ONLY_ERROR_CALLBACK ) 
         result = EINVAL;
@@ -526,8 +550,3 @@ indexer_brt_commit(DB_INDEXER *indexer, DB *hotdb, DBT *hotkey, XIDS xids) {
     }
     return result;
 }
-
-static int
-indexer_is_previous_insert_key_valid(DB_INDEXER *indexer) {
-    return indexer->i->previous_insert_key.size > 0;
-}

src/tests/hotindexer-undo-do-test.c

@@ -359,6 +359,13 @@ read_test(char *testname, ULE ule) {
                 ule_add_provisional(ule, &uxr_s);
                 continue;
             }
+            // placeholder provisional XID
+            if (strcmp(fields[0], "placeholder") == 0 && nfields == 3 && fields[1][0] == 'p') {
+                UXR_S uxr_s; 
+                uxr_init(&uxr_s, XR_PLACEHOLDER, NULL, 0, atoll(fields[2]));
+                ule_add_provisional(ule, &uxr_s);
+                continue;
+            }
             fprintf(stderr, "%s???\n", line);
             r = EINVAL;
         }

src/tests/hotindexer-undo-do-tests/placeholder.0.result

-insert_committed [300,301,302] v10 k1 
-commit_any [300,301,302] v10 
+insert_committed [300] v10 k1 

src/tests/hotindexer-undo-do-tests/placeholder.1.live.result

-insert_provisional [300,301,302] v10 k1 
+insert_provisional [300,301] v10 k1 
 lock [300] v10 

src/tests/hotindexer-undo-do-tests/placeholder.2.result

 insert_committed [100] v10 k1 
 commit_any [100] v10 
-delete_committed [300,301,302] v10 
-insert_committed [300,301,302] v20 k1 
-commit_any [300,301,302] v10 
-commit_any [300,301,302] v20 
+delete_committed [300] v10 
+insert_committed [300] v20 k1 
+commit_any [300] v10 

src/tests/hotindexer-undo-do-tests/placeholder.3.result

+insert_committed [300] v18 k1 
+delete_committed [300] v18 
+insert_committed [300] v20 k1 
+commit_any [300] v18 

src/tests/hotindexer-undo-do-tests/prov-2.py

+#!/usr/bin/env python
+# generate hotindexing undo provisional tests
+
+import sys
+
+def print_tr(fp, tr, trstack):
+    trtype = tr[0]
+    xid = tr[1:]
+    if trtype == 'i':
+        print >>fp, "insert", trstack, xid, "v"+xid
+    if trtype == 'd':
+        print >>fp, "delete", trstack, xid
+    if trtype == 'p':
+        print >>fp, "placeholder", trstack, xid
+
+def print_test(fp, live, commit, prov0, prov1):
+    if live != "":
+        for xid in live.split(","):
+            print >>fp, "live", xid
+    print >>fp, "key k1"
+    print_tr(fp, commit, "committed")
+    print_tr(fp, prov0, "provisional")
+    print_tr(fp, prov1, "provisional")
+
+def main():
+    for live in ["", "200", "200,201"]:
+        for commit in ["i0", "d0"]:
+            for prov0 in ["i200", "d200", "p200"]:
+                for prov1 in ["i201", "d201"]:
+                    if live == "":
+                        fname = "prov.%s.%s.%s.test" % (commit, prov0, prov1)
+                    else:
+                        fname = "prov.live%s.%s.%s.%s.test" % (live, commit, prov0, prov1)
+                    print fname
+                    fp = open(fname, "w")
+                    if fp:
+                        print_test(fp, live, commit, prov0, prov1)
+                        fp.close()
+    return 0
+
+sys.exit(main())

src/tests/hotindexer-undo-do-tests/prov.1.live.result

@@ -13,4 +13,3 @@ insert_provisional [300,301] v30 k1
 lock [300] v30 
 delete_provisional [300,301,302] v30 
 lock [300] v30 
-lock [300] v30 

src/tests/hotindexer-undo-do-tests/prov.d0.i100.d200.test → src/tests/hotindexer-undo-do-tests/prov.d0.d200.d201.test

 key k1
 delete committed 0
-insert provisional 100 v100
 delete provisional 200
-
+delete provisional 201

src/tests/hotindexer-undo-do-tests/prov.d0.d200.i201.result

+insert_committed [200] v201 k1 

src/tests/hotindexer-undo-do-tests/prov.live.d0.i100.d200.test → src/tests/hotindexer-undo-do-tests/prov.d0.d200.i201.test

-live 100 200
 key k1
 delete committed 0
-insert provisional 100 v100
 delete provisional 200
-
+insert provisional 201 v201

src/tests/hotindexer-undo-do-tests/prov.d0.i100.d200.result

-insert_committed [100] v100 k1 
-delete_committed [100] v100 
-commit_any [100] v100 

src/tests/hotindexer-undo-do-tests/prov.d0.i200.d201.result

+insert_committed [200] v200 k1 
+delete_committed [200] v200 
+commit_any [200] v200 

src/tests/hotindexer-undo-do-tests/prov.d0.i200.d201.test

+key k1
+delete committed 0
+insert provisional 200 v200
+delete provisional 201

src/tests/hotindexer-undo-do-tests/prov.d0.i200.i201.result

+insert_committed [200] v200 k1 
+delete_committed [200] v200 
+insert_committed [200] v201 k1 
+commit_any [200] v200 

src/tests/hotindexer-undo-do-tests/prov.d0.i200.i201.test

+key k1
+delete committed 0
+insert provisional 200 v200
+insert provisional 201 v201

src/tests/hotindexer-undo-do-tests/prov.d0.p200.d201.test

+key k1
+delete committed 0
+placeholder provisional 200
+delete provisional 201

src/tests/hotindexer-undo-do-tests/prov.d0.p200.i201.result

+insert_committed [200] v201 k1 

src/tests/hotindexer-undo-do-tests/prov.d0.p200.i201.test

+key k1
+delete committed 0
+placeholder provisional 200
+insert provisional 201 v201

src/tests/hotindexer-undo-do-tests/prov.i0.d200.d201.result

+insert_committed [0] v0 k1 
+delete_committed [200] v0 
+commit_any [200] v0 

src/tests/hotindexer-undo-do-tests/prov.i0.d200.d201.test

+key k1
+insert committed 0 v0
+delete provisional 200
+delete provisional 201

src/tests/hotindexer-undo-do-tests/prov.i0.d200.i201.result

+insert_committed [0] v0 k1 
+delete_committed [200] v0 
+insert_committed [200] v201 k1 
+commit_any [200] v0 

src/tests/hotindexer-undo-do-tests/prov.i0.d200.i201.test

+key k1
+insert committed 0 v0
+delete provisional 200
+insert provisional 201 v201

src/tests/hotindexer-undo-do-tests/prov.i0.i200.d201.result

+insert_committed [0] v0 k1 
+delete_committed [200] v0 
+insert_committed [200] v200 k1 
+delete_committed [200] v200 
+commit_any [200] v0 
+commit_any [200] v200 

src/tests/hotindexer-undo-do-tests/prov.i0.i200.d201.test

+key k1
+insert committed 0 v0
+insert provisional 200 v200
+delete provisional 201

src/tests/hotindexer-undo-do-tests/prov.i0.i200.i201.result

+insert_committed [0] v0 k1 
+delete_committed [200] v0 
+insert_committed [200] v200 k1 
+delete_committed [200] v200 
+insert_committed [200] v201 k1 
+commit_any [200] v0 
+commit_any [200] v200 

src/tests/hotindexer-undo-do-tests/prov.i0.i200.i201.test

+key k1
+insert committed 0 v0
+insert provisional 200 v200
+insert provisional 201 v201

src/tests/hotindexer-undo-do-tests/prov.i0.p200.d201.result

+insert_committed [0] v0 k1 
+delete_committed [200] v0 
+commit_any [200] v0 

src/tests/hotindexer-undo-do-tests/prov.i0.p200.d201.test

+key k1
+insert committed 0 v0
+placeholder provisional 200
+delete provisional 201

src/tests/hotindexer-undo-do-tests/prov.i0.p200.i201.result

+insert_committed [0] v0 k1 
+delete_committed [200] v0 
+insert_committed [200] v201 k1 
+commit_any [200] v0 

src/tests/hotindexer-undo-do-tests/prov.i0.p200.i201.test

+key k1
+insert committed 0 v0
+placeholder provisional 200
+insert provisional 201 v201

src/tests/hotindexer-undo-do-tests/prov.live.d0.i100.d200.result

-insert_provisional [100] v100 k1 
-lock [100] v100 
-delete_provisional [100,200] v100 
-lock [100] v100 
-lock [100] v100 

src/tests/hotindexer-undo-do-tests/prov.live.i0.d100.result

 insert_committed [0] v10 k1 
 delete_provisional [100] v10 
 lock [100] v10 
-lock [100] v10 

src/tests/hotindexer-undo-do-tests/prov.live200,201.d0.d200.d201.test

+live 200
+live 201
+key k1
+delete committed 0
+delete provisional 200
+delete provisional 201

src/tests/hotindexer-undo-do-tests/prov.live200,201.d0.d200.i201.result

+insert_provisional [200,201] v201 k1 
+lock [200] v201 

src/tests/hotindexer-undo-do-tests/prov.live200,201.d0.d200.i201.test

+live 200
+live 201
+key k1
+delete committed 0
+delete provisional 200
+insert provisional 201 v201

src/tests/hotindexer-undo-do-tests/prov.live200,201.d0.i200.d201.result

+insert_provisional [200] v200 k1 
+lock [200] v200 
+delete_provisional [200,201] v200 
+lock [200] v200 

src/tests/hotindexer-undo-do-tests/prov.live200,201.d0.i200.d201.test

+live 200
+live 201
+key k1
+delete committed 0
+insert provisional 200 v200
+delete provisional 201

src/tests/hotindexer-undo-do-tests/prov.live200,201.d0.i200.i201.result

+insert_provisional [200] v200 k1 
+lock [200] v200 
+delete_provisional [200,201] v200 
+lock [200] v200 
+insert_provisional [200,201] v201 k1 
+lock [200] v201 

src/tests/hotindexer-undo-do-tests/prov.live200,201.d0.i200.i201.test

+live 200
+live 201
+key k1
+delete committed 0
+insert provisional 200 v200
+insert provisional 201 v201

src/tests/hotindexer-undo-do-tests/prov.live200,201.d0.p200.d201.test

+live 200
+live 201
+key k1
+delete committed 0
+placeholder provisional 200
+delete provisional 201

src/tests/hotindexer-undo-do-tests/prov.live200,201.d0.p200.i201.result

+insert_provisional [200,201] v201 k1 
+lock [200] v201 

src/tests/hotindexer-undo-do-tests/prov.live200,201.d0.p200.i201.test

+live 200
+live 201
+key k1
+delete committed 0
+placeholder provisional 200
+insert provisional 201 v201

src/tests/hotindexer-undo-do-tests/prov.live200,201.i0.d200.d201.result

+insert_committed [0] v0 k1 
+delete_provisional [200] v0 
+lock [200] v0 

src/tests/hotindexer-undo-do-tests/prov.live200,201.i0.d200.d201.test

+live 200
+live 201
+key k1
+insert committed 0 v0
+delete provisional 200
+delete provisional 201

src/tests/hotindexer-undo-do-tests/prov.live200,201.i0.d200.i201.result

+insert_committed [0] v0 k1 
+delete_provisional [200] v0 
+lock [200] v0 
+insert_provisional [200,201] v201 k1 
+lock [200] v201 

src/tests/hotindexer-undo-do-tests/prov.live200,201.i0.d200.i201.test

+live 200
+live 201
+key k1
+insert committed 0 v0
+delete provisional 200
+insert provisional 201 v201

src/tests/hotindexer-undo-do-tests/prov.live200,201.i0.i200.d201.result

+insert_committed [0] v0 k1 
+delete_provisional [200] v0 
+lock [200] v0 
+insert_provisional [200] v200 k1 
+lock [200] v200 
+delete_provisional [200,201] v200 
+lock [200] v200 

src/tests/hotindexer-undo-do-tests/prov.live200,201.i0.i200.d201.test

+live 200
+live 201
+key k1
+insert committed 0 v0
+insert provisional 200 v200
+delete provisional 201

src/tests/hotindexer-undo-do-tests/prov.live200,201.i0.i200.i201.result

+insert_committed [0] v0 k1 
+delete_provisional [200] v0 
+lock [200] v0 
+insert_provisional [200] v200 k1 
+lock [200] v200 
+delete_provisional [200,201] v200 
+lock [200] v200 
+insert_provisional [200,201] v201 k1 
+lock [200] v201 

src/tests/hotindexer-undo-do-tests/prov.live200,201.i0.i200.i201.test

+live 200
+live 201
+key k1
+insert committed 0 v0
+insert provisional 200 v200
+insert provisional 201 v201

src/tests/hotindexer-undo-do-tests/prov.live200,201.i0.p200.d201.result

+insert_committed [0] v0 k1 
+delete_provisional [200,201] v0 
+lock [200] v0 

src/tests/hotindexer-undo-do-tests/prov.live200,201.i0.p200.d201.test

+live 200
+live 201
+key k1
+insert committed 0 v0
+placeholder provisional 200
+delete provisional 201

src/tests/hotindexer-undo-do-tests/prov.live200,201.i0.p200.i201.result

+insert_committed [0] v0 k1 
+delete_provisional [200,201] v0 
+lock [200] v0 
+insert_provisional [200,201] v201 k1 
+lock [200] v201 

src/tests/hotindexer-undo-do-tests/prov.live200,201.i0.p200.i201.test

+live 200
+live 201
+key k1
+insert committed 0 v0
+placeholder provisional 200
+insert provisional 201 v201

src/tests/hotindexer-undo-do-tests/prov.live200.d0.d200.d201.test

+live 200
+key k1
+delete committed 0
+delete provisional 200
+delete provisional 201

src/tests/hotindexer-undo-do-tests/prov.live200.d0.d200.i201.test

+live 200
+key k1
+delete committed 0
+delete provisional 200
+insert provisional 201 v201

src/tests/hotindexer-undo-do-tests/prov.live200.d0.i200.d201.test

+live 200
+key k1
+delete committed 0
+insert provisional 200 v200
+delete provisional 201

src/tests/hotindexer-undo-do-tests/prov.live200.d0.i200.i201.test

+live 200
+key k1
+delete committed 0
+insert provisional 200 v200
+insert provisional 201 v201

src/tests/hotindexer-undo-do-tests/prov.live200.d0.p200.d201.test

+live 200
+key k1
+delete committed 0
+placeholder provisional 200
+delete provisional 201

src/tests/hotindexer-undo-do-tests/prov.live200.d0.p200.i201.test

+live 200
+key k1
+delete committed 0
+placeholder provisional 200
+insert provisional 201 v201

src/tests/hotindexer-undo-do-tests/prov.live200.i0.d200.d201.test

+live 200
+key k1
+insert committed 0 v0
+delete provisional 200
+delete provisional 201

src/tests/hotindexer-undo-do-tests/prov.live200.i0.d200.i201.test

+live 200
+key k1
+insert committed 0 v0
+delete provisional 200
+insert provisional 201 v201

src/tests/hotindexer-undo-do-tests/prov.live200.i0.i200.d201.test

+live 200
+key k1
+insert committed 0 v0
+insert provisional 200 v200
+delete provisional 201

src/tests/hotindexer-undo-do-tests/prov.live200.i0.i200.i201.test

+live 200
+key k1
+insert committed 0 v0
+insert provisional 200 v200
+insert provisional 201 v201

src/tests/hotindexer-undo-do-tests/prov.live200.i0.p200.d201.test

+live 200
+key k1
+insert committed 0 v0
+placeholder provisional 200
+delete provisional 201

src/tests/hotindexer-undo-do-tests/prov.live200.i0.p200.i201.test

+live 200
+key k1
+insert committed 0 v0
+placeholder provisional 200
+insert provisional 201 v201