[t:4686] [t:5123] [t:5062] merge #4686 to main, includes fix for #5123. closes #4686.

git-svn-id: file:///svn/toku/tokudb@44814 c7de825b-a66e-492c-adef-691d508d4ae1

buildheader/make_tdb.c

@@ -465,6 +465,7 @@ static void print_db_txn_struct (void) {
 	"int (*commit_with_progress)(DB_TXN*, uint32_t, TXN_PROGRESS_POLL_FUNCTION, void*)",
 	"int (*abort_with_progress)(DB_TXN*, TXN_PROGRESS_POLL_FUNCTION, void*)",
 	"int (*xa_prepare) (DB_TXN*, TOKU_XA_XID *)",
+    "u_int64_t (*id64) (DB_TXN*)",
 	NULL};
     sort_and_dump_fields("db_txn", false, extra);
 }

ft/tests/recovery-test5123.c

+/* -*- mode: C; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+// vim: expandtab:ts=8:sw=4:softtabstop=4:
+#ident "$Id$"
+#ident "Copyright (c) 2010 Tokutek Inc.  All rights reserved."
+#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
+
+#include "test.h"
+#include "includes.h"
+#include "toku_os.h"
+#include "checkpoint.h"
+
+#define TESTDIR __SRCFILE__ ".dir"
+#define FILENAME "test0.ft"
+
+#include "test-ft-txns.h"
+
+static void test_5123(void) {
+    TOKULOGGER logger;
+    CACHETABLE ct;
+    test_setup(&logger, &ct);
+
+    int r;
+
+    TOKUTXN txn[3];
+    r = toku_txn_begin_txn((DB_TXN*)NULL, (TOKUTXN)0, &txn[0], logger, TXN_SNAPSHOT_ROOT);
+    CKERR(r);
+    r = toku_txn_begin_txn((DB_TXN*)NULL, (TOKUTXN)0, &txn[1], logger, TXN_SNAPSHOT_ROOT);
+    CKERR(r);
+    r = toku_txn_begin_txn((DB_TXN*)NULL, (TOKUTXN)0, &txn[2], logger, TXN_SNAPSHOT_ROOT);
+    CKERR(r);
+
+    toku_maybe_log_begin_txn_for_write_operation(txn[0]);
+    toku_maybe_log_begin_txn_for_write_operation(txn[2]);
+    toku_maybe_log_begin_txn_for_write_operation(txn[1]);
+
+    r = toku_txn_commit_txn(txn[1], FALSE, NULL, NULL);
+    CKERR(r);
+
+    r = toku_logger_close_rollback(logger, false);
+    CKERR(r);
+
+    r = toku_cachetable_close(&ct);
+    CKERR(r);
+    // "Crash"
+    r = toku_logger_close(&logger);
+    CKERR(r);
+    ct = NULL;
+    logger = NULL;
+
+    // "Recover"
+    test_setup_and_recover(&logger, &ct);
+
+    shutdown_after_recovery(&logger, &ct);
+}
+
+int test_main (int argc, const char *argv[]) {
+    default_parse_args(argc, argv);
+    test_5123();
+    return 0;
+}

ft/tests/test-ft-txns.h

+/* -*- mode: C; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+// vim: expandtab:ts=8:sw=4:softtabstop=4:
+#ifndef TEST_FT_TXNS_H
+#define TEST_FT_TXNS_H
+
+#ident "$Id: xid_lsn_independent.c 44752 2012-06-20 21:59:39Z yfogel $"
+#ident "Copyright (c) 2010 Tokutek Inc.  All rights reserved."
+#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
+
+#if !defined(TESTDIR) || !defined(FILENAME)
+# error "must define TESTDIR and FILENAME"
+#endif
+
+static inline void
+test_setup(TOKULOGGER *loggerp, CACHETABLE *ctp) {
+    *loggerp = NULL;
+    *ctp = NULL;
+    int r;
+    r = system("rm -rf " TESTDIR);
+    CKERR(r);
+    r = toku_os_mkdir(TESTDIR, S_IRWXU);
+    CKERR(r);
+
+    r = toku_logger_create(loggerp);
+    CKERR(r);
+    TOKULOGGER logger = *loggerp;
+
+    r = toku_logger_open(TESTDIR, logger);
+    CKERR(r);
+
+    r = toku_create_cachetable(ctp, 0, ZERO_LSN, logger);
+    CKERR(r);
+    CACHETABLE ct = *ctp;
+    toku_cachetable_set_env_dir(ct, TESTDIR);
+
+    toku_logger_set_cachetable(logger, ct);
+
+    r = toku_logger_open_rollback(logger, ct, TRUE);
+    CKERR(r);
+
+    r = toku_checkpoint(ct, logger, NULL, NULL, NULL, NULL, STARTUP_CHECKPOINT);
+    CKERR(r);
+}
+
+static inline void
+xid_lsn_keep_cachetable_callback (DB_ENV *env, CACHETABLE cachetable) {
+    CACHETABLE *ctp = (void*)env;
+    *ctp = cachetable;
+}
+
+static inline void test_setup_and_recover(TOKULOGGER *loggerp, CACHETABLE *ctp) {
+    int r;
+    TOKULOGGER logger = NULL;
+    CACHETABLE ct = NULL;
+    r = toku_logger_create(&logger);
+    CKERR(r);
+
+    void *ctv = &ct;  // Use intermediate void* to avoid compiler warning.
+    r = tokudb_recover(ctv,
+                       NULL_prepared_txn_callback,
+                       xid_lsn_keep_cachetable_callback,
+                       logger,
+                       TESTDIR, TESTDIR, 0, 0, 0, NULL, 0);
+    CKERR(r);
+    if (!toku_logger_is_open(logger)) {
+        //Did not need recovery.
+        invariant(ct==NULL);
+        r = toku_logger_open(TESTDIR, logger);
+        CKERR(r);
+        r = toku_create_cachetable(&ct, 0, ZERO_LSN, logger);
+        CKERR(r);
+        toku_logger_set_cachetable(logger, ct);
+    }
+    *ctp = ct;
+    *loggerp = logger;
+}
+
+static inline void clean_shutdown(TOKULOGGER *loggerp, CACHETABLE *ctp) {
+    int r;
+    r = toku_checkpoint(*ctp, *loggerp, NULL, NULL, NULL, NULL, SHUTDOWN_CHECKPOINT);
+    CKERR(r);
+
+    r = toku_logger_close_rollback(*loggerp, false);
+    CKERR(r);
+
+    r = toku_checkpoint(*ctp, *loggerp, NULL, NULL, NULL, NULL, SHUTDOWN_CHECKPOINT);
+    CKERR(r);
+
+    r = toku_logger_shutdown(*loggerp);
+    CKERR(r);
+
+    r = toku_cachetable_close(ctp);
+    CKERR(r);
+
+    r = toku_logger_close(loggerp);
+    CKERR(r);
+}
+
+static inline void shutdown_after_recovery(TOKULOGGER *loggerp, CACHETABLE *ctp) {
+    int r;
+    r = toku_logger_close_rollback(*loggerp, false);
+    CKERR(r);
+    r = toku_cachetable_close(ctp);
+    CKERR(r);
+    r = toku_logger_close(loggerp);
+    CKERR(r);
+}
+
+#endif /* TEST_FT_TXNS_H */

ft/tests/xid_lsn_independent.c

@@ -13,99 +13,7 @@
 #define TESTDIR __SRCFILE__ ".dir"
 #define FILENAME "test0.ft"
 
-static void test_setup(TOKULOGGER *loggerp, CACHETABLE *ctp) {
-    *loggerp = NULL;
-    *ctp = NULL;
-    int r;
-    r = system("rm -rf " TESTDIR);
-    CKERR(r);
-    r = toku_os_mkdir(TESTDIR, S_IRWXU);
-    CKERR(r);
-
-    r = toku_logger_create(loggerp);
-    CKERR(r);
-    TOKULOGGER logger = *loggerp;
-
-    r = toku_logger_open(TESTDIR, logger);
-    CKERR(r);
-
-    r = toku_create_cachetable(ctp, 0, ZERO_LSN, logger);
-    CKERR(r);
-    CACHETABLE ct = *ctp;
-    toku_cachetable_set_env_dir(ct, TESTDIR);
-
-    toku_logger_set_cachetable(logger, ct);
-
-    r = toku_logger_open_rollback(logger, ct, TRUE);
-    CKERR(r);
-
-    r = toku_checkpoint(ct, logger, NULL, NULL, NULL, NULL, STARTUP_CHECKPOINT);
-    CKERR(r);
-}
-
-static void
-xid_lsn_keep_cachetable_callback (DB_ENV *env, CACHETABLE cachetable) {
-    CACHETABLE *ctp = (void*)env;
-    *ctp = cachetable;
-}
-
-static void test_setup_and_recover(TOKULOGGER *loggerp, CACHETABLE *ctp) {
-    int r;
-    TOKULOGGER logger = NULL;
-    CACHETABLE ct = NULL;
-    r = toku_logger_create(&logger);
-    CKERR(r);
-
-    void *ctv = &ct;  // Use intermediate void* to avoid compiler warning.
-    r = tokudb_recover(ctv,
-                       NULL_prepared_txn_callback,
-                       xid_lsn_keep_cachetable_callback,
-                       logger,
-                       TESTDIR, TESTDIR, 0, 0, 0, NULL, 0);
-    CKERR(r);
-    if (!toku_logger_is_open(logger)) {
-        //Did not need recovery.
-        invariant(ct==NULL);
-        r = toku_logger_open(TESTDIR, logger);
-        CKERR(r);
-        r = toku_create_cachetable(&ct, 0, ZERO_LSN, logger);
-        CKERR(r);
-        toku_logger_set_cachetable(logger, ct);
-    }
-    *ctp = ct;
-    *loggerp = logger;
-}
-
-static void clean_shutdown(TOKULOGGER *loggerp, CACHETABLE *ctp) {
-    int r;
-    r = toku_checkpoint(*ctp, *loggerp, NULL, NULL, NULL, NULL, SHUTDOWN_CHECKPOINT);
-    CKERR(r);
-
-    r = toku_logger_close_rollback(*loggerp, false);
-    CKERR(r);
-
-    r = toku_checkpoint(*ctp, *loggerp, NULL, NULL, NULL, NULL, SHUTDOWN_CHECKPOINT);
-    CKERR(r);
-
-    r = toku_logger_shutdown(*loggerp);
-    CKERR(r);
-
-    r = toku_cachetable_close(ctp);
-    CKERR(r);
-
-    r = toku_logger_close(loggerp);
-    CKERR(r);
-}
-
-static void shutdown_after_recovery(TOKULOGGER *loggerp, CACHETABLE *ctp) {
-    int r;
-    r = toku_logger_close_rollback(*loggerp, false);
-    CKERR(r);
-    r = toku_cachetable_close(ctp);
-    CKERR(r);
-    r = toku_logger_close(loggerp);
-    CKERR(r);
-}
+#include "test-ft-txns.h"
 
 static void do_txn(TOKULOGGER logger, bool readonly) {
     int r;

ft/txn.c

@@ -63,6 +63,12 @@ toku_txn_unlock(TOKUTXN txn)
     toku_mutex_unlock(&txn->txn_lock);
 }
 
+u_int64_t
+toku_txn_get_id(TOKUTXN txn)
+{
+    return txn->txnid64;
+}
+
 int 
 toku_txn_begin_txn (
     DB_TXN  *container_db_txn,

ft/txn.h

@@ -14,6 +14,8 @@ extern "C" {
 void toku_txn_lock(TOKUTXN txn);
 void toku_txn_unlock(TOKUTXN txn);
 
+u_int64_t toku_txn_get_id(TOKUTXN txn);
+
 int toku_txn_begin_txn (
     DB_TXN  *container_db_txn,
     TOKUTXN parent_tokutxn, 

ft/txn_manager.c

@@ -314,16 +314,33 @@ setup_live_root_txn_list(TXN_MANAGER txn_manager, TOKUTXN txn) {
     return r;
 }
 
-// Add this txn to the global list of txns that have their own snapshots.
-// (Note, if a txn is a child that creates its own snapshot, then that child xid
-// is the xid stored in the global list.) 
+//Heaviside function to search through an OMT by a TXNID
 static int
-snapshot_txnids_note_txn(TXN_MANAGER txn_manager, TOKUTXN txn) {
+find_by_xid (OMTVALUE v, void *txnidv) {
+    TOKUTXN txn = v;
+    TXNID   txnidfind = (TXNID)txnidv;
+    if (txn->txnid64<txnidfind) return -1;
+    if (txn->txnid64>txnidfind) return +1;
+    return 0;
+}
+
+static void
+omt_insert_at_end_unless_recovery(OMT omt, int (*h)(OMTVALUE, void*extra), TOKUTXN txn, OMTVALUE v, bool for_recovery)
+// Effect: insert v into omt that is sorted by xid gotten from txn.
+// Rationale:
+//   During recovery, we get txns in the order that they did their first
+//   write operation, which is not necessarily monotonically increasing.
+//   During normal operation, txns are created with strictly increasing
+//   txnids, so we can always insert at the end.
+{
     int r;
-    OMT txnids = txn_manager->snapshot_txnids;
-    r = toku_omt_insert_at(txnids, (OMTVALUE) txn->txnid64, toku_omt_size(txnids));
-    assert_zero(r);
-    return r;
+    uint32_t idx = toku_omt_size(omt);
+    if (for_recovery) {
+        r = toku_omt_find_zero(omt, h, (void *) txn->txnid64, NULL, &idx);
+        invariant(r==DB_NOTFOUND);
+    }
+    r = toku_omt_insert_at(omt, v, idx);
+    lazy_assert_zero(r);
 }
 
 static TXNID
@@ -382,19 +399,25 @@ int toku_txn_manager_start_txn(
 
     toku_txn_update_xids_in_txn(txn, xid, xids);
 
+    if (!for_recovery) {
+        // TODO(leif): this would be WRONG during recovery.  We cannot
+        // assume that the first xbegin we see is the oldest, because it's
+        // just the txn that *did a write* first.  Right now, this cached
+        // value is only used for engine status, so we're not too worried.
+        // Maybe we just kill this (txn_manager->oldest_living_*).  Also
+        // see comment on txn_manager_get_oldest_living_xid_unlocked.
         if (toku_omt_size(txn_manager->live_txns) == 0) {
             assert(txn_manager->oldest_living_xid == TXNID_NONE_LIVING);
             txn_manager->oldest_living_xid = txn->txnid64;
             txn_manager->oldest_living_starttime = txn->starttime;
         }
         assert(txn_manager->oldest_living_xid <= txn->txnid64);
+    }
 
-    {
     //Add txn to list (omt) of live transactions
-        //We know it is the newest one.
-        r = toku_omt_insert_at(txn_manager->live_txns, txn, toku_omt_size(txn_manager->live_txns));
-        assert_zero(r);
+    omt_insert_at_end_unless_recovery(txn_manager->live_txns, find_by_xid, txn, (OMTVALUE) txn, for_recovery);
 
+    {
         //
         // maintain the data structures necessary for MVCC:
         //  1. add txn to list of live_root_txns if this is a root transaction
@@ -411,12 +434,7 @@ int toku_txn_manager_start_txn(
         // add ancestor information, and maintain global live root txn list
         if (parent == NULL) {
             //Add txn to list (omt) of live root txns
-            r = toku_omt_insert_at(
-                txn_manager->live_root_txns,
-                (OMTVALUE) txn->txnid64,
-                toku_omt_size(txn_manager->live_root_txns)
-                ); //We know it is the newest one.
-            assert_zero(r);
+            omt_insert_at_end_unless_recovery(txn_manager->live_root_txns, toku_find_xid_by_xid, txn, (OMTVALUE) txn->txnid64, for_recovery);
         }
 
         // setup information for snapshot reads
@@ -426,8 +444,11 @@ int toku_txn_manager_start_txn(
             if (parent == NULL || txn->snapshot_type == TXN_SNAPSHOT_CHILD) {
                 r = setup_live_root_txn_list(txn_manager, txn);  
                 assert_zero(r);
-                r = snapshot_txnids_note_txn(txn_manager, txn);
-                assert_zero(r);
+
+                // Add this txn to the global list of txns that have their own snapshots.
+                // (Note, if a txn is a child that creates its own snapshot, then that child xid
+                // is the xid stored in the global list.) 
+                omt_insert_at_end_unless_recovery(txn_manager->snapshot_txnids, toku_find_xid_by_xid, txn, (OMTVALUE) txn->txnid64, for_recovery);
             }
             // in this case, it is a child transaction that specified its snapshot to be that 
             // of the root transaction
@@ -615,6 +636,7 @@ void toku_txn_manager_finish_txn(TXN_MANAGER txn_manager, TOKUTXN txn) {
 
     assert(txn_manager->oldest_living_xid <= txn->txnid64);
     if (txn->txnid64 == txn_manager->oldest_living_xid) {
+        // oldest_living_xid is always zero during recovery
         OMTVALUE oldest_txnv;
         r = toku_omt_fetch(txn_manager->live_txns, 0, &oldest_txnv);
         if (r==0) {
@@ -659,16 +681,6 @@ void toku_txn_manager_clone_state_for_gc(
     toku_mutex_unlock(&txn_manager->txn_manager_lock);
 }
 
-//Heaviside function to search through an OMT by a TXNID
-static int
-find_by_xid (OMTVALUE v, void *txnidv) {
-    TOKUTXN txn = v;
-    TXNID   txnidfind = (TXNID)txnidv;
-    if (txn->txnid64<txnidfind) return -1;
-    if (txn->txnid64>txnidfind) return +1;
-    return 0;
-}
-
 void toku_txn_manager_id2txn_unlocked(TXN_MANAGER txn_manager, TXNID txnid, TOKUTXN *result) {
     OMTVALUE txnfound;
     int r = toku_omt_find_zero(txn_manager->live_txns, find_by_xid, (OMTVALUE)txnid, &txnfound, NULL);

src/tests/recover-test_stress1.c

@@ -89,11 +89,13 @@ stress_table(DB_ENV *env, DB **dbp, struct cli_args *cli_args) {
     for (int i = 4; i < 4 + cli_args->num_update_threads; ++i) {
         myargs[i].operation_extra = &uoe;
         myargs[i].operation = update_op;
+        myargs[i].do_prepare = true;
     }
 
     // make the guy that does point queries
     for (int i = 4 + cli_args->num_update_threads; i < num_threads; i++) {
         myargs[i].operation = ptquery_op;
+        myargs[i].do_prepare = true;
     }
 
     int num_seconds = random() % cli_args->time_of_test;

src/tests/test_stress1.c

@@ -89,11 +89,13 @@ stress_table(DB_ENV *env, DB **dbp, struct cli_args *cli_args) {
     for (int i = 4; i < 4 + cli_args->num_update_threads; ++i) {
         myargs[i].operation_extra = &uoe;
         myargs[i].operation = update_op;
+        myargs[i].do_prepare = true;
     }
 
     // make the guy that does point queries
     for (int i = 4 + cli_args->num_update_threads; i < num_threads; i++) {
         myargs[i].operation = ptquery_op;
+        myargs[i].do_prepare = true;
     }
 
     run_workers(myargs, num_threads, cli_args->time_of_test, false, cli_args);

src/tests/threaded_stress_test_helpers.h

@@ -167,6 +167,7 @@ struct arg {
     int thread_idx;
     int num_threads;
     struct cli_args *cli;
+    bool do_prepare;
 };
 
 DB_TXN * const null_txn = 0;
@@ -180,6 +181,7 @@ static void arg_init(struct arg *arg, DB **dbp, DB_ENV *env, struct cli_args *cl
     arg->lock_type = STRESS_LOCK_NONE;
     arg->txn_type = DB_TXN_SNAPSHOT;
     arg->operation_extra = NULL;
+    arg->do_prepare = false;
 }
 
 enum operation_type {
@@ -401,6 +403,15 @@ static void *worker(void *arg_v) {
             r = env->txn_begin(env, 0, &txn, arg->txn_type); CKERR(r);
         }
         r = arg->operation(txn, arg, arg->operation_extra, we->counters);
+        if (!arg->cli->single_txn && arg->do_prepare) {
+            u_int8_t gid[DB_GID_SIZE];
+            memset(gid, 0, DB_GID_SIZE);
+            u_int64_t gid_val = txn->id64(txn);
+            u_int64_t *gid_count_p = (void *)gid;  // make gcc --happy about -Wstrict-aliasing
+            *gid_count_p = gid_val;
+            int rr = txn->prepare(txn, gid);
+            assert_zero(rr);
+        }
         if (r == 0) {
             if (!arg->cli->single_txn) {
                 { int chk_r = txn->commit(txn,0); CKERR(chk_r); }
@@ -1302,6 +1313,28 @@ static int fill_tables_with_zeroes(DB **dbs, int num_DBs, int num_elements, u_in
     return 0;
 }
 
+static void do_xa_recovery(DB_ENV* env) {    
+    DB_PREPLIST preplist[1];
+    long num_recovered= 0;
+    int r = 0;
+    r = env->txn_recover(env, preplist, 1, &num_recovered, DB_NEXT);
+    while(r==0 && num_recovered > 0) {
+        DB_TXN* recovered_txn = preplist[0].txn;
+        if (verbose) {
+            printf("recovering transaction with id %"PRIu64" \n", recovered_txn->id64(recovered_txn));
+        }
+        if (random() % 2 == 0) {
+            int rr = recovered_txn->commit(recovered_txn, 0);
+            CKERR(rr);
+        }
+        else {
+            int rr = recovered_txn->abort(recovered_txn);
+            CKERR(rr);
+        }
+        r = env->txn_recover(env, preplist, 1, &num_recovered, DB_NEXT);
+    }
+}
+
 static int open_tables(DB_ENV **env_res, DB **db_res, int num_DBs,
                       int (*bt_compare)(DB *, const DBT *, const DBT *),
                       struct env_args env_args) {
@@ -1328,6 +1361,7 @@ static int open_tables(DB_ENV **env_res, DB **db_res, int num_DBs,
         CKERR(r);
     }
     r = env->open(env, env_args.envdir, DB_RECOVER|DB_INIT_LOCK|DB_INIT_LOG|DB_INIT_MPOOL|DB_INIT_TXN|DB_CREATE|DB_PRIVATE, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
+    do_xa_recovery(env);
     r = env->checkpointing_set_period(env, env_args.checkpointing_period); CKERR(r);
     r = env->cleaner_set_period(env, env_args.cleaner_period); CKERR(r);
     r = env->cleaner_set_iterations(env, env_args.cleaner_iterations); CKERR(r);

src/ydb_txn.c

@@ -141,6 +141,12 @@ toku_txn_id(DB_TXN * txn) {
     return -1;
 }
 
+static u_int64_t 
+toku_txn_id64(DB_TXN * txn) {
+    HANDLE_PANICKED_ENV(txn->mgrp);
+    return toku_txn_get_id(db_txn_struct_i(txn)->tokutxn);
+}
+
 static int 
 toku_txn_abort_only(DB_TXN * txn,
                     TXN_PROGRESS_POLL_FUNCTION poll, void *poll_extra) {
@@ -402,6 +408,7 @@ toku_txn_begin(DB_ENV *env, DB_TXN * stxn, DB_TXN ** txn, u_int32_t flags) {
     STXN(xa_prepare);
     STXN(txn_stat);
 #undef STXN
+    result->id64 = toku_txn_id64;
 
     result->parent = stxn;
 #if !TOKUDB_NATIVE_H
@@ -479,6 +486,7 @@ void toku_keep_prepared_txn_callback (DB_ENV *env, TOKUTXN tokutxn) {
     STXN(prepare);
     STXN(txn_stat);
 #undef STXN
+    result->id64 = toku_txn_id64;
     
     result->parent = NULL;