DB-823 fix lock tables + transaction accounting for subsequent operations

storage/tokudb/ha_tokudb.cc

@@ -6167,6 +6167,12 @@ int ha_tokudb::external_lock(THD * thd, int lock_type) {
         if (error) { goto cleanup; }
         thd_set_ha_data(thd, tokudb_hton, trx);
     }
+
+    if (tokudb_debug & TOKUDB_DEBUG_TXN) {
+        TOKUDB_HANDLER_TRACE("trx %p %p %p %p %u %u", trx->all, trx->stmt, trx->sp_level, trx->sub_sp_level, 
+                             trx->tokudb_lock_count, trx->create_lock_count);
+    }
+
     if (trx->all == NULL) {
         trx->sp_level = NULL;
     }
@@ -6175,22 +6181,16 @@ int ha_tokudb::external_lock(THD * thd, int lock_type) {
         if (lock_type == F_WRLCK) {
             use_write_locks = true;
         }
-        if (!trx->tokudb_lock_count++) {
-            if (trx->stmt) {
-                if (tokudb_debug & TOKUDB_DEBUG_TXN) {
-                    TOKUDB_HANDLER_TRACE("stmt already set %p %p %p %p", trx->all, trx->stmt, trx->sp_level, trx->sub_sp_level);
-                }
-            } else {
-                assert(trx->stmt == 0);
+        if (!trx->stmt) {
             transaction = NULL;    // Safety
             error = create_txn(thd, trx);
             if (error) {
-                    trx->tokudb_lock_count--;  // We didn't get the lock
                 goto cleanup;
             }
-            }
+            trx->create_lock_count = trx->tokudb_lock_count;
         }
         transaction = trx->sub_sp_level;
+        trx->tokudb_lock_count++;
     }
     else {
         tokudb_pthread_mutex_lock(&share->mutex);
@@ -6205,7 +6205,9 @@ int ha_tokudb::external_lock(THD * thd, int lock_type) {
         added_rows = 0;
         deleted_rows = 0;
         share->rows_from_locked_table = 0;
-        if (trx->tokudb_lock_count > 0 && !--trx->tokudb_lock_count) {
+        if (trx->tokudb_lock_count > 0) {
+            if (--trx->tokudb_lock_count <= trx->create_lock_count) {
+                trx->create_lock_count = 0;
                 if (trx->stmt) {
                     /*
                       F_UNLCK is done without a transaction commit / rollback.
@@ -6221,6 +6223,7 @@ int ha_tokudb::external_lock(THD * thd, int lock_type) {
             }
             transaction = NULL;
         }
+    }
 cleanup:
     if (tokudb_debug & TOKUDB_DEBUG_LOCK)
         TOKUDB_HANDLER_TRACE("error=%d", error);
@@ -6234,8 +6237,9 @@ cleanup:
 */
 int ha_tokudb::start_stmt(THD * thd, thr_lock_type lock_type) {
     TOKUDB_HANDLER_DBUG_ENTER("cmd %d lock %d %s", thd_sql_command(thd), lock_type, share->table_name);
-    if (0)
+    if (tokudb_debug & TOKUDB_DEBUG_LOCK) {
         TOKUDB_HANDLER_TRACE("q %s", thd->query());
+    }
 
     int error = 0;
     tokudb_trx_data *trx = (tokudb_trx_data *) thd_get_ha_data(thd, tokudb_hton);
@@ -6245,6 +6249,11 @@ int ha_tokudb::start_stmt(THD * thd, thr_lock_type lock_type) {
         thd_set_ha_data(thd, tokudb_hton, trx);
     }
 
+    if (tokudb_debug & TOKUDB_DEBUG_TXN) {
+        TOKUDB_HANDLER_TRACE("trx %p %p %p %p %u %u", trx->all, trx->stmt, trx->sp_level, trx->sub_sp_level, 
+                             trx->tokudb_lock_count, trx->create_lock_count);
+    }
+
     /*
        note that trx->stmt may have been already initialized as start_stmt()
        is called for *each table* not for each storage engine,
@@ -6255,9 +6264,7 @@ int ha_tokudb::start_stmt(THD * thd, thr_lock_type lock_type) {
         if (error) {
             goto cleanup;
         }
-        if (tokudb_debug & TOKUDB_DEBUG_TXN) {
-            TOKUDB_HANDLER_TRACE("%p %p %p %p %u", trx->all, trx->stmt, trx->sp_level, trx->sub_sp_level, trx->tokudb_lock_count);
-        }
+        trx->create_lock_count = trx->tokudb_lock_count;
     }
     else {
         if (tokudb_debug & TOKUDB_DEBUG_TXN) {

storage/tokudb/ha_tokudb_alter_56.cc

@@ -784,13 +784,16 @@ bool ha_tokudb::commit_inplace_alter_table(TABLE *altered_table, Alter_inplace_i
         assert(trx->tokudb_lock_count > 0);
         // for partitioned tables, we use a single transaction to do all of the partition changes.  the tokudb_lock_count
         // is a reference count for each of the handlers to the same transaction.  obviously, we want to only abort once.
-        if (!--trx->tokudb_lock_count) {
+        if (trx->tokudb_lock_count > 0) {
+            if (--trx->tokudb_lock_count <= trx->create_lock_count) {
+                trx->create_lock_count = 0;
                 abort_txn(ctx->alter_txn);
                 ctx->alter_txn = NULL;
                 trx->stmt = NULL;
                 trx->sub_sp_level = NULL;
             }
             transaction = NULL;
+        }
 
         if (ctx->add_index_changed) {
             restore_add_index(table, ha_alter_info->index_add_count, ctx->incremented_num_DBs, ctx->modified_DBs);

storage/tokudb/hatoku_defines.h

@@ -354,6 +354,7 @@ typedef struct st_tokudb_trx_data {
     DB_TXN *sp_level;
     DB_TXN *sub_sp_level;
     uint tokudb_lock_count;
+    uint create_lock_count;
     tokudb_stmt_progress stmt_progress;
     bool checkpoint_lock_taken;
     LIST *handlers;