MDEV-13797 InnoDB may hang if shutdown is initiated soon after startup while...

MDEV-13797 InnoDB may hang if shutdown is initiated soon after startup while rolling back recovered incomplete transactions

trx_rollback_resurrected(): If shutdown was initiated, fake all
remaining active transactions to XA PREPARE state, so that shutdown
can proceed. Also, make the parameter "all" an output that will be
assigned to FALSE in this case.

trx_rollback_or_clean_recovered(): Remove the shutdown check
(it was moved to trx_rollback_resurrected()).

trx_undo_free_prepared(): Relax assertions.

mysql-test/suite/innodb/r/recovery_shutdown.result

+#
+# MDEV-13797 InnoDB may hang if shutdown is initiated soon after startup
+# while rolling back recovered incomplete transactions
+#
+CREATE TABLE t (a INT) ENGINE=InnoDB;
+BEGIN;
+COMMIT;
+CREATE TABLE t8 (a SERIAL, b INT UNIQUE, c INT UNIQUE) ENGINE=InnoDB;
+BEGIN;
+INSERT INTO t8 (a) SELECT NULL FROM t;
+UPDATE t8 SET a=a+100, b=a;
+DELETE FROM t8;
+CREATE TABLE t7 (a SERIAL, b INT UNIQUE, c INT UNIQUE) ENGINE=InnoDB;
+BEGIN;
+INSERT INTO t7 (a) SELECT NULL FROM t;
+UPDATE t7 SET a=a+100, b=a;
+DELETE FROM t7;
+CREATE TABLE t6 (a SERIAL, b INT UNIQUE, c INT UNIQUE) ENGINE=InnoDB;
+BEGIN;
+INSERT INTO t6 (a) SELECT NULL FROM t;
+UPDATE t6 SET a=a+100, b=a;
+DELETE FROM t6;
+CREATE TABLE t5 (a SERIAL, b INT UNIQUE, c INT UNIQUE) ENGINE=InnoDB;
+BEGIN;
+INSERT INTO t5 (a) SELECT NULL FROM t;
+UPDATE t5 SET a=a+100, b=a;
+DELETE FROM t5;
+CREATE TABLE t4 (a SERIAL, b INT UNIQUE, c INT UNIQUE) ENGINE=InnoDB;
+BEGIN;
+INSERT INTO t4 (a) SELECT NULL FROM t;
+UPDATE t4 SET a=a+100, b=a;
+DELETE FROM t4;
+CREATE TABLE t3 (a SERIAL, b INT UNIQUE, c INT UNIQUE) ENGINE=InnoDB;
+BEGIN;
+INSERT INTO t3 (a) SELECT NULL FROM t;
+UPDATE t3 SET a=a+100, b=a;
+DELETE FROM t3;
+CREATE TABLE t2 (a SERIAL, b INT UNIQUE, c INT UNIQUE) ENGINE=InnoDB;
+BEGIN;
+INSERT INTO t2 (a) SELECT NULL FROM t;
+UPDATE t2 SET a=a+100, b=a;
+DELETE FROM t2;
+CREATE TABLE t1 (a SERIAL, b INT UNIQUE, c INT UNIQUE) ENGINE=InnoDB;
+BEGIN;
+INSERT INTO t1 (a) SELECT NULL FROM t;
+UPDATE t1 SET a=a+100, b=a;
+DELETE FROM t1;
+SET GLOBAL innodb_flush_log_at_trx_commit=1;
+CREATE TABLE u(a SERIAL) ENGINE=INNODB;
+# Kill and restart
+DROP TABLE t,u;

mysql-test/suite/innodb/t/recovery_shutdown.test

+--source include/have_innodb.inc
+--source include/not_embedded.inc
+
+--echo #
+--echo # MDEV-13797 InnoDB may hang if shutdown is initiated soon after startup
+--echo # while rolling back recovered incomplete transactions
+--echo #
+
+CREATE TABLE t (a INT) ENGINE=InnoDB;
+let $size = 100;
+let $trx = 8;
+let $c = $size;
+BEGIN;
+--disable_query_log
+while ($c) {
+INSERT INTO t VALUES();
+dec $c;
+}
+--enable_query_log
+COMMIT;
+
+let $c = $trx;
+while ($c)
+{
+connect (con$c,localhost,root,,);
+eval CREATE TABLE t$c (a SERIAL, b INT UNIQUE, c INT UNIQUE) ENGINE=InnoDB;
+BEGIN;
+eval INSERT INTO t$c (a) SELECT NULL FROM t;
+eval UPDATE t$c SET a=a+$size, b=a;
+eval DELETE FROM t$c;
+dec $c;
+}
+
+--connection default
+SET GLOBAL innodb_flush_log_at_trx_commit=1;
+CREATE TABLE u(a SERIAL) ENGINE=INNODB;
+
+--source include/kill_and_restart_mysqld.inc
+--source include/restart_mysqld.inc
+
+--disable_query_log
+let $c = $trx;
+while ($c)
+{
+disconnect con$c;
+eval DROP TABLE t$c;
+dec $c;
+}
+--enable_query_log
+
+DROP TABLE t,u;

storage/innobase/trx/trx0roll.cc

@@ -672,7 +672,7 @@ ibool
 trx_rollback_resurrected(
 /*=====================*/
 	trx_t*	trx,	/*!< in: transaction to rollback or clean */
-	ibool	all)	/*!< in: FALSE=roll back dictionary transactions;
+	ibool*	all)	/*!< in/out: FALSE=roll back dictionary transactions;
 			TRUE=roll back all non-PREPARED transactions */
 {
 	ut_ad(mutex_own(&trx_sys->mutex));
@@ -683,16 +683,15 @@ trx_rollback_resurrected(
 	to accidentally clean up a non-recovered transaction here. */
 
 	trx_mutex_enter(trx);
-	bool		is_recovered	= trx->is_recovered;
-	trx_state_t	state		= trx->state;
+	if (!trx->is_recovered) {
+func_exit:
 		trx_mutex_exit(trx);
-
-	if (!is_recovered) {
 		return(FALSE);
 	}
 
-	switch (state) {
+	switch (trx->state) {
 	case TRX_STATE_COMMITTED_IN_MEMORY:
+		trx_mutex_exit(trx);
 		mutex_exit(&trx_sys->mutex);
 		fprintf(stderr,
 			"InnoDB: Cleaning up trx with id " TRX_ID_FMT "\n",
@@ -701,7 +700,17 @@ trx_rollback_resurrected(
 		trx_free_for_background(trx);
 		return(TRUE);
 	case TRX_STATE_ACTIVE:
-		if (all || trx_get_dict_operation(trx) != TRX_DICT_OP_NONE) {
+		if (srv_shutdown_state != SRV_SHUTDOWN_NONE
+		    && srv_fast_shutdown) {
+			trx->state = TRX_STATE_PREPARED;
+			trx_sys->n_prepared_trx++;
+			trx_sys->n_prepared_recovered_trx++;
+			*all = FALSE;
+			goto func_exit;
+		}
+		trx_mutex_exit(trx);
+
+		if (*all || trx_get_dict_operation(trx) != TRX_DICT_OP_NONE) {
 			mutex_exit(&trx_sys->mutex);
 			trx_rollback_active(trx);
 			trx_free_for_background(trx);
@@ -709,13 +718,13 @@ trx_rollback_resurrected(
 		}
 		return(FALSE);
 	case TRX_STATE_PREPARED:
-		return(FALSE);
+		goto func_exit;
 	case TRX_STATE_NOT_STARTED:
 		break;
 	}
 
 	ut_error;
-	return(FALSE);
+	goto func_exit;
 }
 
 /*******************************************************************//**
@@ -762,17 +771,11 @@ trx_rollback_or_clean_recovered(
 
 			assert_trx_in_rw_list(trx);
 
-			if (srv_shutdown_state != SRV_SHUTDOWN_NONE
-			    && srv_fast_shutdown != 0) {
-				all = FALSE;
-				break;
-			}
-
 			/* If this function does a cleanup or rollback
 			then it will release the trx_sys->mutex, therefore
 			we need to reacquire it before retrying the loop. */
 
-			if (trx_rollback_resurrected(trx, all)) {
+			if (trx_rollback_resurrected(trx, &all)) {
 
 				mutex_enter(&trx_sys->mutex);
 

storage/innobase/trx/trx0undo.cc

@@ -2023,9 +2023,13 @@ trx_undo_free_prepared(
 			/* fall through */
 		case TRX_UNDO_ACTIVE:
 			/* lock_trx_release_locks() assigns
-			trx->is_recovered=false */
+			trx->is_recovered=false and
+			trx->state = TRX_STATE_COMMITTED_IN_MEMORY,
+			also for transactions that we faked
+			to TRX_STATE_PREPARED in trx_rollback_resurrected(). */
 			ut_a(srv_read_only_mode
-			     || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO);
+			     || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO
+			     || srv_fast_shutdown);
 			break;
 		default:
 			ut_error;
@@ -2047,9 +2051,13 @@ trx_undo_free_prepared(
 			/* fall through */
 		case TRX_UNDO_ACTIVE:
 			/* lock_trx_release_locks() assigns
-			trx->is_recovered=false */
+			trx->is_recovered=false and
+			trx->state = TRX_STATE_COMMITTED_IN_MEMORY,
+			also for transactions that we faked
+			to TRX_STATE_PREPARED in trx_rollback_resurrected(). */
 			ut_a(srv_read_only_mode
-			     || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO);
+			     || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO
+			     || srv_fast_shutdown);
 			break;
 		default:
 			ut_error;

storage/xtradb/trx/trx0roll.cc

@@ -672,7 +672,7 @@ ibool
 trx_rollback_resurrected(
 /*=====================*/
 	trx_t*	trx,	/*!< in: transaction to rollback or clean */
-	ibool	all)	/*!< in: FALSE=roll back dictionary transactions;
+	ibool*	all)	/*!< in/out: FALSE=roll back dictionary transactions;
 			TRUE=roll back all non-PREPARED transactions */
 {
 	ut_ad(mutex_own(&trx_sys->mutex));
@@ -683,16 +683,15 @@ trx_rollback_resurrected(
 	to accidentally clean up a non-recovered transaction here. */
 
 	trx_mutex_enter(trx);
-	bool		is_recovered	= trx->is_recovered;
-	trx_state_t	state		= trx->state;
+	if (!trx->is_recovered) {
+func_exit:
 		trx_mutex_exit(trx);
-
-	if (!is_recovered) {
 		return(FALSE);
 	}
 
-	switch (state) {
+	switch (trx->state) {
 	case TRX_STATE_COMMITTED_IN_MEMORY:
+		trx_mutex_exit(trx);
 		mutex_exit(&trx_sys->mutex);
 		fprintf(stderr,
 			"InnoDB: Cleaning up trx with id " TRX_ID_FMT "\n",
@@ -701,7 +700,17 @@ trx_rollback_resurrected(
 		trx_free_for_background(trx);
 		return(TRUE);
 	case TRX_STATE_ACTIVE:
-		if (all || trx_get_dict_operation(trx) != TRX_DICT_OP_NONE) {
+		if (srv_shutdown_state != SRV_SHUTDOWN_NONE
+		    && srv_fast_shutdown) {
+			trx->state = TRX_STATE_PREPARED;
+			trx_sys->n_prepared_trx++;
+			trx_sys->n_prepared_recovered_trx++;
+			*all = FALSE;
+			goto func_exit;
+		}
+		trx_mutex_exit(trx);
+
+		if (*all || trx_get_dict_operation(trx) != TRX_DICT_OP_NONE) {
 			mutex_exit(&trx_sys->mutex);
 			trx_rollback_active(trx);
 			trx_free_for_background(trx);
@@ -709,13 +718,13 @@ trx_rollback_resurrected(
 		}
 		return(FALSE);
 	case TRX_STATE_PREPARED:
-		return(FALSE);
+		goto func_exit;
 	case TRX_STATE_NOT_STARTED:
 		break;
 	}
 
 	ut_error;
-	return(FALSE);
+	goto func_exit;
 }
 
 /*******************************************************************//**
@@ -762,17 +771,11 @@ trx_rollback_or_clean_recovered(
 
 			assert_trx_in_rw_list(trx);
 
-			if (srv_shutdown_state != SRV_SHUTDOWN_NONE
-			    && srv_fast_shutdown != 0) {
-				all = FALSE;
-				break;
-			}
-
 			/* If this function does a cleanup or rollback
 			then it will release the trx_sys->mutex, therefore
 			we need to reacquire it before retrying the loop. */
 
-			if (trx_rollback_resurrected(trx, all)) {
+			if (trx_rollback_resurrected(trx, &all)) {
 
 				mutex_enter(&trx_sys->mutex);
 

storage/xtradb/trx/trx0undo.cc

@@ -2023,9 +2023,13 @@ trx_undo_free_prepared(
 			/* fall through */
 		case TRX_UNDO_ACTIVE:
 			/* lock_trx_release_locks() assigns
-			trx->is_recovered=false */
+			trx->is_recovered=false and
+			trx->state = TRX_STATE_COMMITTED_IN_MEMORY,
+			also for transactions that we faked
+			to TRX_STATE_PREPARED in trx_rollback_resurrected(). */
 			ut_a(srv_read_only_mode
-			     || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO);
+			     || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO
+			     || srv_fast_shutdown);
 			break;
 		default:
 			ut_error;
@@ -2047,9 +2051,13 @@ trx_undo_free_prepared(
 			/* fall through */
 		case TRX_UNDO_ACTIVE:
 			/* lock_trx_release_locks() assigns
-			trx->is_recovered=false */
+			trx->is_recovered=false and
+			trx->state = TRX_STATE_COMMITTED_IN_MEMORY,
+			also for transactions that we faked
+			to TRX_STATE_PREPARED in trx_rollback_resurrected(). */
 			ut_a(srv_read_only_mode
-			     || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO);
+			     || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO
+			     || srv_fast_shutdown);
 			break;
 		default:
 			ut_error;