MDEV-26903: Assertion ctx->trx->state == TRX_STATE_ACTIVE on DROP INDEX

rollback_inplace_alter_table(): Tolerate a case where the transaction
is not in an active state. If ha_innobase::commit_inplace_alter_table()
failed with a deadlock, the transaction would already have been
rolled back. This omission of error handling was introduced in
commit 1bd681c8 (MDEV-25506 part 3).

After commit c3c53926 (MDEV-26554)
it became easier to trigger DB_DEADLOCK during exclusive table lock
acquisition in ha_innobase::commit_inplace_alter_table().

lock_table_low(): Add DBUG injection "innodb_table_deadlock".

mysql-test/suite/innodb/r/innodb-alter-debug.result

@@ -132,4 +132,14 @@ ALTER TABLE t1 RENAME KEY idx TO idx1, ALGORITHM=COPY;
 disconnect con1;
 connection default;
 DROP TABLE t1;
+#
+# MDEV-26903 Assertion ctx->trx->state == TRX_STATE_ACTIVE on DROP INDEX
+#
+CREATE TABLE t1(a INT PRIMARY KEY, b INT, INDEX(b)) ENGINE=InnoDB;
+SET @save_dbug=@@debug_dbug;
+SET debug_dbug='+d,innodb_table_deadlock';
+ALTER TABLE t1 DROP INDEX b, ALGORITHM=INPLACE;
+ERROR 40001: Deadlock found when trying to get lock; try restarting transaction
+SET debug_dbug=@save_dbug;
+DROP TABLE t1;
 # End of 10.6 tests

mysql-test/suite/innodb/t/innodb-alter-debug.test

@@ -177,6 +177,18 @@ disconnect con1;
 connection default;
 DROP TABLE t1;
 
+--echo #
+--echo # MDEV-26903 Assertion ctx->trx->state == TRX_STATE_ACTIVE on DROP INDEX
+--echo #
+
+CREATE TABLE t1(a INT PRIMARY KEY, b INT, INDEX(b)) ENGINE=InnoDB;
+SET @save_dbug=@@debug_dbug;
+SET debug_dbug='+d,innodb_table_deadlock';
+--error ER_LOCK_DEADLOCK
+ALTER TABLE t1 DROP INDEX b, ALGORITHM=INPLACE;
+SET debug_dbug=@save_dbug;
+DROP TABLE t1;
+
 --echo # End of 10.6 tests
 
 # Wait till all disconnects are completed

storage/innobase/handler/handler0alter.cc

@@ -8678,6 +8678,8 @@ inline bool rollback_inplace_alter_table(Alter_inplace_info *ha_alter_info,
     /* If we have not started a transaction yet,
     (almost) nothing has been or needs to be done. */
     dict_sys.lock(SRW_LOCK_CALL);
+  else if (ctx->trx->state == TRX_STATE_NOT_STARTED)
+    goto free_and_exit;
   else if (ctx->new_table)
   {
     ut_ad(ctx->trx->state == TRX_STATE_ACTIVE);

storage/innobase/lock/lock0lock.cc

@@ -3433,6 +3433,7 @@ lock_table_other_has_incompatible(
 static dberr_t lock_table_low(dict_table_t *table, lock_mode mode,
                               que_thr_t *thr, trx_t *trx)
 {
+  DBUG_EXECUTE_IF("innodb_table_deadlock", return DB_DEADLOCK;);
   lock_t *wait_for=
     lock_table_other_has_incompatible(trx, LOCK_WAIT, table, mode);
   dberr_t err= DB_SUCCESS;