MDEV-34542 Assertion `lock_trx_has_sys_table_locks(trx) == __null' failed in...

MDEV-34542  Assertion `lock_trx_has_sys_table_locks(trx) == __null' failed in void row_mysql_unfreeze_data_dictionary(trx_t*)

- During XA PREPARE, InnoDB releases the non-exclusive locks.
But it fails to remove the non-exclusive table lock from the
transaction table locks. In the mean time, main thread evicts
the table from the LRU cache. While rollbacking the XA transaction,
InnoDB iterates through the table locks to check whether it
holds lock on any system tables and wrongly assumes the
evicted table as system table since the table id is 0

Fix:
===
During XA PREPARE, remove the table locks of the transaction while
releasing the non-exclusive locks.

mysql-test/suite/innodb/r/lock_release.result

+#
+# MDEV-34542  Assertion `lock_trx_has_sys_table_locks(trx) == __null'
+#   failed in void row_mysql_unfreeze_data_dictionary(trx_t*)
+#
+#
+CREATE TABLE t1 (c1 CHAR(1) ,c2 INT) ENGINE=INNODB
+PARTITION BY LINEAR HASH ((c2)) PARTITIONS 512;
+CREATE TABLE t2 (a INT) ENGINE=INNODB;
+set @old_table_open_cache= @@table_open_cache;
+XA START 'a';
+INSERT INTO mysql.innodb_index_stats SELECT * FROM mysql.innodb_index_stats WHERE table_name='';
+SET GLOBAL table_open_cache=10;
+INSERT into t2 (a) VALUES (1);
+SELECT * FROM t1;
+c1	c2
+XA END 'a';
+XA PREPARE 'a';
+SELECT sleep(3);
+sleep(3)
+0
+XA ROLLBACK 'a';
+DROP TABLE t1, t2;
+SET GLOBAL table_open_cache=@old_table_open_cache;

mysql-test/suite/innodb/t/lock_release.test

+--source include/have_innodb.inc
+--source include/have_partition.inc
+
+--echo #
+--echo # MDEV-34542  Assertion `lock_trx_has_sys_table_locks(trx) == __null'
+--echo #   failed in void row_mysql_unfreeze_data_dictionary(trx_t*)
+--echo #
+--echo #
+CREATE TABLE t1 (c1 CHAR(1) ,c2 INT) ENGINE=INNODB
+                PARTITION BY LINEAR HASH ((c2)) PARTITIONS 512;
+CREATE TABLE t2 (a INT) ENGINE=INNODB;
+
+set @old_table_open_cache= @@table_open_cache;
+XA START 'a';
+INSERT INTO mysql.innodb_index_stats SELECT * FROM mysql.innodb_index_stats WHERE table_name='';
+SET GLOBAL table_open_cache=10;
+INSERT into t2 (a) VALUES (1);
+SELECT * FROM t1;
+XA END 'a';
+XA PREPARE 'a';
+# Added sleep to make sure that InnoDB main thread is to remove
+# the innodb_index_stats from table cache
+SELECT sleep(3);
+XA ROLLBACK 'a';
+DROP TABLE t1, t2;
+SET GLOBAL table_open_cache=@old_table_open_cache;

storage/innobase/lock/lock0lock.cc

@@ -4382,6 +4382,52 @@ static void lock_rec_unlock_supremum(lock_t *lock)
   trx_mutex_exit(lock->trx);
 }
 
+/*********************************************************************//**
+Removes table locks of the transaction on a table to be dropped. */
+static
+void
+lock_trx_table_locks_remove(
+/*========================*/
+	const lock_t*	lock_to_remove)		/*!< in: lock to remove */
+{
+	trx_t*		trx = lock_to_remove->trx;
+
+	ut_ad(lock_mutex_own());
+
+	/* It is safe to read this because we are holding the lock mutex */
+	if (!trx->lock.cancel) {
+		trx_mutex_enter(trx);
+	} else {
+		ut_ad(trx_mutex_own(trx));
+	}
+
+	for (lock_list::iterator it = trx->lock.table_locks.begin(),
+             end = trx->lock.table_locks.end(); it != end; ++it) {
+		const lock_t*	lock = *it;
+
+		ut_ad(!lock || trx == lock->trx);
+		ut_ad(!lock || lock_get_type_low(lock) & LOCK_TABLE);
+		ut_ad(!lock || lock->un_member.tab_lock.table);
+
+		if (lock == lock_to_remove) {
+			*it = NULL;
+
+			if (!trx->lock.cancel) {
+				trx_mutex_exit(trx);
+			}
+
+			return;
+		}
+	}
+
+	if (!trx->lock.cancel) {
+		trx_mutex_exit(trx);
+	}
+
+	/* Lock must exist in the vector. */
+	ut_error;
+}
+
 /** Release non-exclusive locks on XA PREPARE,
 and release possible other transactions waiting because of these locks. */
 void lock_release_on_prepare(trx_t *trx)
@@ -4427,6 +4473,7 @@ void lock_release_on_prepare(trx_t *trx)
       case LOCK_IS:
       case LOCK_S:
         lock_table_dequeue(lock);
+        lock_trx_table_locks_remove(lock);
         break;
       case LOCK_IX:
       case LOCK_X:
@@ -4456,52 +4503,6 @@ void lock_release_on_prepare(trx_t *trx)
 	(lock_get_mode(lock) == LOCK_S \
 	 || lock_get_mode(lock) == LOCK_X)
 
-/*********************************************************************//**
-Removes table locks of the transaction on a table to be dropped. */
-static
-void
-lock_trx_table_locks_remove(
-/*========================*/
-	const lock_t*	lock_to_remove)		/*!< in: lock to remove */
-{
-	trx_t*		trx = lock_to_remove->trx;
-
-	ut_ad(lock_mutex_own());
-
-	/* It is safe to read this because we are holding the lock mutex */
-	if (!trx->lock.cancel) {
-		trx_mutex_enter(trx);
-	} else {
-		ut_ad(trx_mutex_own(trx));
-	}
-
-	for (lock_list::iterator it = trx->lock.table_locks.begin(),
-             end = trx->lock.table_locks.end(); it != end; ++it) {
-		const lock_t*	lock = *it;
-
-		ut_ad(!lock || trx == lock->trx);
-		ut_ad(!lock || lock_get_type_low(lock) & LOCK_TABLE);
-		ut_ad(!lock || lock->un_member.tab_lock.table);
-
-		if (lock == lock_to_remove) {
-			*it = NULL;
-
-			if (!trx->lock.cancel) {
-				trx_mutex_exit(trx);
-			}
-
-			return;
-		}
-	}
-
-	if (!trx->lock.cancel) {
-		trx_mutex_exit(trx);
-	}
-
-	/* Lock must exist in the vector. */
-	ut_error;
-}
-
 /*===================== VALIDATION AND DEBUGGING ====================*/
 
 /** Print info of a table lock.