MDEV-30423 Deadlock on Replica during BACKUP STAGE BLOCK_COMMIT on XA transactions

The user XA commit execution branch was caught not have been covered
with MDEV-21953 fixes.

The XA involved deadlock is resolved now to apply the former fixes
pattern.
Along the fixes the following changes have been implemented.
- MDL lock attribute correction
- dissociation of the externally completed XA from the current
  thread's xid_state in the error branches
- cleanup_context() preseves the prepared XA
- wait_for_prior_commit() is relocated to satisfy both
  the binlog ON (log-slave-updates and skip-log-bin)
  and OFF slave execution branches.

wolfssl @ f1e2165c

-Subproject commit 4fbd4fd36a21efd9d1a7e17aba390e91c78693b1
+Subproject commit f1e2165c591f074feb47872a8ff712713ec411e1

libmariadb @ 7fdb3eab

-Subproject commit d204e83104222844251b221e9be7eb3dd9f8d63d
+Subproject commit 7fdb3eab66384a355475704332d11cc1ab82499a

mysql-test/include/master-slave.inc

@@ -11,6 +11,7 @@
 #   [--let $rpl_skip_start_slave= 1]
 #   [--let $rpl_debug= 1]
 #   [--let $slave_timeout= NUMBER]
+#   [--let $rpl_server_skip_log_bin= 1]
 #   --source include/master-slave.inc
 #
 # Parameters:

mysql-test/include/rpl_init.inc

@@ -73,6 +73,7 @@
 #      before CHANGE MASTER and START SLAVE.  RESET MASTER and RESET
 #      SLAVE are suppressed if $rpl_skip_reset_master_and_slave is
 #      set.
+#      Also see $rpl_server_skip_log_bin.
 #
 #    $rpl_skip_change_master
 #      By default, this script issues CHANGE MASTER so that all slaves
@@ -94,6 +95,10 @@
 #      Timeout used when waiting for the slave threads to start.
 #      See include/wait_for_slave_param.inc
 #
+#    $rpl_server_skip_log_bin
+#      When $rpl_skip_reset_master_and_slave is not set
+#      RESET MASTER does not report ER_FLUSH_MASTER_BINLOG_CLOSED
+#      on any server.
 #
 # ==== Side effects ====
 #
@@ -161,7 +166,16 @@ while ($_rpl_server)
   USE test;
   if (!$rpl_skip_reset_master_and_slave)
   {
-    RESET MASTER;
+    if (!$rpl_server_skip_log_bin)
+    {
+     --error 0
+       RESET MASTER;
+    }
+    if ($rpl_server_skip_log_bin)
+    {
+     --error 0,ER_FLUSH_MASTER_BINLOG_CLOSED
+       RESET MASTER;
+    }
     SET GLOBAL gtid_slave_pos= "";
     RESET SLAVE;
   }

mysql-test/suite/rpl/r/parallel_backup.result

@@ -7,6 +7,8 @@ include/master-slave.inc
 connection master;
 CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE = innodb;
 connection slave;
+call mtr.add_suppression("Deadlock found when trying to get lock");
+call mtr.add_suppression("Commit failed due to failure of an earlier commit");
 include/stop_slave.inc
 SET @old_parallel_threads= @@GLOBAL.slave_parallel_threads;
 SET @old_parallel_mode   = @@GLOBAL.slave_parallel_mode;
@@ -30,6 +32,167 @@ connection backup_slave;
 BACKUP STAGE END;
 connection slave;
 include/diff_tables.inc [master:t1,slave:t1]
+# MDEV-30423: dealock XA COMMIT vs BACKUP
+#
+# Normal XA COMMIT
+connection slave;
+include/stop_slave.inc
+connection master;
+connection aux_slave;
+BEGIN;
+INSERT INTO t1 VALUES (4);
+connection master;
+XA START '1';
+INSERT INTO  t1 VALUES (3);
+XA END '1';
+XA PREPARE '1';
+connection master1;
+INSERT INTO t1 VALUES (4);
+connection master;
+XA COMMIT '1';
+include/save_master_gtid.inc
+connection slave;
+include/start_slave.inc
+connection aux_slave;
+# Xid '1' must be in the output:
+XA RECOVER;
+formatID	gtrid_length	bqual_length	data
+1	1	0	1
+connection backup_slave;
+BACKUP STAGE START;
+BACKUP STAGE BLOCK_COMMIT;
+connection aux_slave;
+ROLLBACK;
+connection backup_slave;
+BACKUP STAGE END;
+connection slave;
+include/sync_with_master_gtid.inc
+include/stop_slave.inc
+#
+# Normal XA ROLLBACK
+connection slave;
+include/stop_slave.inc
+Warnings:
+Note	1255	Slave already has been stopped
+connection master;
+connection aux_slave;
+BEGIN;
+INSERT INTO t1 VALUES (6);
+connection master;
+XA START '1';
+INSERT INTO  t1 VALUES (5);
+XA END '1';
+XA PREPARE '1';
+connection master1;
+INSERT INTO t1 VALUES (6);
+connection master;
+XA ROLLBACK '1';
+include/save_master_gtid.inc
+connection slave;
+include/start_slave.inc
+connection aux_slave;
+# Xid '1' must be in the output:
+XA RECOVER;
+formatID	gtrid_length	bqual_length	data
+1	1	0	1
+connection backup_slave;
+BACKUP STAGE START;
+BACKUP STAGE BLOCK_COMMIT;
+connection aux_slave;
+ROLLBACK;
+connection backup_slave;
+BACKUP STAGE END;
+connection slave;
+include/sync_with_master_gtid.inc
+include/stop_slave.inc
+#
+# Errored out XA COMMIT
+connection slave;
+include/stop_slave.inc
+Warnings:
+Note	1255	Slave already has been stopped
+connection master;
+connection aux_slave;
+BEGIN;
+INSERT INTO t1 VALUES (8);
+connection master;
+XA START '1';
+INSERT INTO  t1 VALUES (7);
+XA END '1';
+XA PREPARE '1';
+connection master1;
+INSERT INTO t1 VALUES (8);
+connection master;
+XA COMMIT '1';
+include/save_master_gtid.inc
+connection slave;
+SET @sav_innodb_lock_wait_timeout  = @@global.innodb_lock_wait_timeout;
+SET @sav_slave_transaction_retries = @@global.slave_transaction_retries;
+SET @@global.innodb_lock_wait_timeout =1;
+SET @@global.slave_transaction_retries=0;
+include/start_slave.inc
+connection aux_slave;
+# Xid '1' must be in the output:
+XA RECOVER;
+formatID	gtrid_length	bqual_length	data
+1	1	0	1
+connection backup_slave;
+BACKUP STAGE START;
+BACKUP STAGE BLOCK_COMMIT;
+connection aux_slave;
+ROLLBACK;
+connection backup_slave;
+BACKUP STAGE END;
+connection slave;
+include/stop_slave.inc
+SET @@global.innodb_lock_wait_timeout = @sav_innodb_lock_wait_timeout;
+SET @@global.slave_transaction_retries= @sav_slave_transaction_retries;
+connection slave;
+include/start_slave.inc
+include/sync_with_master_gtid.inc
+#
+# Errored out XA ROLLBACK
+connection slave;
+include/stop_slave.inc
+connection master;
+connection aux_slave;
+BEGIN;
+INSERT INTO t1 VALUES (10);
+connection master;
+XA START '1';
+INSERT INTO  t1 VALUES (9);
+XA END '1';
+XA PREPARE '1';
+connection master1;
+INSERT INTO t1 VALUES (10);
+connection master;
+XA ROLLBACK '1';
+include/save_master_gtid.inc
+connection slave;
+SET @sav_innodb_lock_wait_timeout  = @@global.innodb_lock_wait_timeout;
+SET @sav_slave_transaction_retries = @@global.slave_transaction_retries;
+SET @@global.innodb_lock_wait_timeout =1;
+SET @@global.slave_transaction_retries=0;
+include/start_slave.inc
+connection aux_slave;
+# Xid '1' must be in the output:
+XA RECOVER;
+formatID	gtrid_length	bqual_length	data
+1	1	0	1
+connection backup_slave;
+BACKUP STAGE START;
+BACKUP STAGE BLOCK_COMMIT;
+connection aux_slave;
+ROLLBACK;
+connection backup_slave;
+BACKUP STAGE END;
+connection slave;
+include/stop_slave.inc
+SET @@global.innodb_lock_wait_timeout = @sav_innodb_lock_wait_timeout;
+SET @@global.slave_transaction_retries= @sav_slave_transaction_retries;
+connection slave;
+include/start_slave.inc
+include/sync_with_master_gtid.inc
 connection slave;
 include/stop_slave.inc
 SET @@global.slave_parallel_threads= @old_parallel_threads;

mysql-test/suite/rpl/r/parallel_backup_lsu_off.result

+# Specialized --log-slave-updates = 0 version of parallel_backup test.
+# MDEV-21953: deadlock between BACKUP STAGE BLOCK_COMMIT and parallel
+# MDEV-30423: dealock XA COMMIT vs BACKUP
+include/master-slave.inc
+[connection master]
+#
+# MDEV-21953: deadlock between BACKUP STAGE BLOCK_COMMIT and parallel
+# replication
+#
+connection master;
+CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE = innodb;
+connection slave;
+call mtr.add_suppression("Deadlock found when trying to get lock");
+call mtr.add_suppression("Commit failed due to failure of an earlier commit");
+include/stop_slave.inc
+SET @old_parallel_threads= @@GLOBAL.slave_parallel_threads;
+SET @old_parallel_mode   = @@GLOBAL.slave_parallel_mode;
+SET @@global.slave_parallel_threads= 2;
+SET @@global.slave_parallel_mode   = 'optimistic';
+connection master;
+INSERT INTO  t1 VALUES (1);
+INSERT INTO  t1 VALUES (2);
+connect  aux_slave,127.0.0.1,root,,test,$SLAVE_MYPORT,;
+BEGIN;
+INSERT INTO t1 VALUES (1);
+connection slave;
+include/start_slave.inc
+connection aux_slave;
+connect  backup_slave,127.0.0.1,root,,test,$SLAVE_MYPORT,;
+BACKUP STAGE START;
+BACKUP STAGE BLOCK_COMMIT;
+connection aux_slave;
+ROLLBACK;
+connection backup_slave;
+BACKUP STAGE END;
+connection slave;
+include/diff_tables.inc [master:t1,slave:t1]
+# MDEV-30423: dealock XA COMMIT vs BACKUP
+#
+# Normal XA COMMIT
+connection slave;
+include/stop_slave.inc
+connection master;
+connection aux_slave;
+BEGIN;
+INSERT INTO t1 VALUES (4);
+connection master;
+XA START '1';
+INSERT INTO  t1 VALUES (3);
+XA END '1';
+XA PREPARE '1';
+connection master1;
+INSERT INTO t1 VALUES (4);
+connection master;
+XA COMMIT '1';
+include/save_master_gtid.inc
+connection slave;
+include/start_slave.inc
+connection aux_slave;
+# Xid '1' must be in the output:
+XA RECOVER;
+formatID	gtrid_length	bqual_length	data
+1	1	0	1
+connection backup_slave;
+BACKUP STAGE START;
+BACKUP STAGE BLOCK_COMMIT;
+connection aux_slave;
+ROLLBACK;
+connection backup_slave;
+BACKUP STAGE END;
+connection slave;
+include/sync_with_master_gtid.inc
+include/stop_slave.inc
+#
+# Normal XA ROLLBACK
+connection slave;
+include/stop_slave.inc
+Warnings:
+Note	1255	Slave already has been stopped
+connection master;
+connection aux_slave;
+BEGIN;
+INSERT INTO t1 VALUES (6);
+connection master;
+XA START '1';
+INSERT INTO  t1 VALUES (5);
+XA END '1';
+XA PREPARE '1';
+connection master1;
+INSERT INTO t1 VALUES (6);
+connection master;
+XA ROLLBACK '1';
+include/save_master_gtid.inc
+connection slave;
+include/start_slave.inc
+connection aux_slave;
+# Xid '1' must be in the output:
+XA RECOVER;
+formatID	gtrid_length	bqual_length	data
+1	1	0	1
+connection backup_slave;
+BACKUP STAGE START;
+BACKUP STAGE BLOCK_COMMIT;
+connection aux_slave;
+ROLLBACK;
+connection backup_slave;
+BACKUP STAGE END;
+connection slave;
+include/sync_with_master_gtid.inc
+include/stop_slave.inc
+#
+# Errored out XA COMMIT
+connection slave;
+include/stop_slave.inc
+Warnings:
+Note	1255	Slave already has been stopped
+connection master;
+connection aux_slave;
+BEGIN;
+INSERT INTO t1 VALUES (8);
+connection master;
+XA START '1';
+INSERT INTO  t1 VALUES (7);
+XA END '1';
+XA PREPARE '1';
+connection master1;
+INSERT INTO t1 VALUES (8);
+connection master;
+XA COMMIT '1';
+include/save_master_gtid.inc
+connection slave;
+SET @sav_innodb_lock_wait_timeout  = @@global.innodb_lock_wait_timeout;
+SET @sav_slave_transaction_retries = @@global.slave_transaction_retries;
+SET @@global.innodb_lock_wait_timeout =1;
+SET @@global.slave_transaction_retries=0;
+include/start_slave.inc
+connection aux_slave;
+# Xid '1' must be in the output:
+XA RECOVER;
+formatID	gtrid_length	bqual_length	data
+1	1	0	1
+connection backup_slave;
+BACKUP STAGE START;
+BACKUP STAGE BLOCK_COMMIT;
+connection aux_slave;
+ROLLBACK;
+connection backup_slave;
+BACKUP STAGE END;
+connection slave;
+include/stop_slave.inc
+SET @@global.innodb_lock_wait_timeout = @sav_innodb_lock_wait_timeout;
+SET @@global.slave_transaction_retries= @sav_slave_transaction_retries;
+connection slave;
+include/start_slave.inc
+include/sync_with_master_gtid.inc
+#
+# Errored out XA ROLLBACK
+connection slave;
+include/stop_slave.inc
+connection master;
+connection aux_slave;
+BEGIN;
+INSERT INTO t1 VALUES (10);
+connection master;
+XA START '1';
+INSERT INTO  t1 VALUES (9);
+XA END '1';
+XA PREPARE '1';
+connection master1;
+INSERT INTO t1 VALUES (10);
+connection master;
+XA ROLLBACK '1';
+include/save_master_gtid.inc
+connection slave;
+SET @sav_innodb_lock_wait_timeout  = @@global.innodb_lock_wait_timeout;
+SET @sav_slave_transaction_retries = @@global.slave_transaction_retries;
+SET @@global.innodb_lock_wait_timeout =1;
+SET @@global.slave_transaction_retries=0;
+include/start_slave.inc
+connection aux_slave;
+# Xid '1' must be in the output:
+XA RECOVER;
+formatID	gtrid_length	bqual_length	data
+1	1	0	1
+connection backup_slave;
+BACKUP STAGE START;
+BACKUP STAGE BLOCK_COMMIT;
+connection aux_slave;
+ROLLBACK;
+connection backup_slave;
+BACKUP STAGE END;
+connection slave;
+include/stop_slave.inc
+SET @@global.innodb_lock_wait_timeout = @sav_innodb_lock_wait_timeout;
+SET @@global.slave_transaction_retries= @sav_slave_transaction_retries;
+connection slave;
+include/start_slave.inc
+include/sync_with_master_gtid.inc
+connection slave;
+include/stop_slave.inc
+SET @@global.slave_parallel_threads= @old_parallel_threads;
+SET @@global.slave_parallel_mode   = @old_parallel_mode;
+include/start_slave.inc
+connection server_1;
+DROP TABLE t1;
+include/rpl_end.inc

mysql-test/suite/rpl/r/parallel_backup_slave_binlog_off.result

+# Specialized --skip-log-bin slave version of parallel_backup test.
+# MDEV-21953: deadlock between BACKUP STAGE BLOCK_COMMIT and parallel
+# MDEV-30423: dealock XA COMMIT vs BACKUP
+include/master-slave.inc
+[connection master]
+#
+# MDEV-21953: deadlock between BACKUP STAGE BLOCK_COMMIT and parallel
+# replication
+#
+connection master;
+CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE = innodb;
+connection slave;
+call mtr.add_suppression("Deadlock found when trying to get lock");
+call mtr.add_suppression("Commit failed due to failure of an earlier commit");
+include/stop_slave.inc
+SET @old_parallel_threads= @@GLOBAL.slave_parallel_threads;
+SET @old_parallel_mode   = @@GLOBAL.slave_parallel_mode;
+SET @@global.slave_parallel_threads= 2;
+SET @@global.slave_parallel_mode   = 'optimistic';
+connection master;
+INSERT INTO  t1 VALUES (1);
+INSERT INTO  t1 VALUES (2);
+connect  aux_slave,127.0.0.1,root,,test,$SLAVE_MYPORT,;
+BEGIN;
+INSERT INTO t1 VALUES (1);
+connection slave;
+include/start_slave.inc
+connection aux_slave;
+connect  backup_slave,127.0.0.1,root,,test,$SLAVE_MYPORT,;
+BACKUP STAGE START;
+BACKUP STAGE BLOCK_COMMIT;
+connection aux_slave;
+ROLLBACK;
+connection backup_slave;
+BACKUP STAGE END;
+connection slave;
+include/diff_tables.inc [master:t1,slave:t1]
+# MDEV-30423: dealock XA COMMIT vs BACKUP
+#
+# Normal XA COMMIT
+connection slave;
+include/stop_slave.inc
+connection master;
+connection aux_slave;
+BEGIN;
+INSERT INTO t1 VALUES (4);
+connection master;
+XA START '1';
+INSERT INTO  t1 VALUES (3);
+XA END '1';
+XA PREPARE '1';
+connection master1;
+INSERT INTO t1 VALUES (4);
+connection master;
+XA COMMIT '1';
+include/save_master_gtid.inc
+connection slave;
+include/start_slave.inc
+connection aux_slave;
+# Xid '1' must be in the output:
+XA RECOVER;
+formatID	gtrid_length	bqual_length	data
+1	1	0	1
+connection backup_slave;
+BACKUP STAGE START;
+BACKUP STAGE BLOCK_COMMIT;
+connection aux_slave;
+ROLLBACK;
+connection backup_slave;
+BACKUP STAGE END;
+connection slave;
+include/sync_with_master_gtid.inc
+include/stop_slave.inc
+#
+# Normal XA ROLLBACK
+connection slave;
+include/stop_slave.inc
+Warnings:
+Note	1255	Slave already has been stopped
+connection master;
+connection aux_slave;
+BEGIN;
+INSERT INTO t1 VALUES (6);
+connection master;
+XA START '1';
+INSERT INTO  t1 VALUES (5);
+XA END '1';
+XA PREPARE '1';
+connection master1;
+INSERT INTO t1 VALUES (6);
+connection master;
+XA ROLLBACK '1';
+include/save_master_gtid.inc
+connection slave;
+include/start_slave.inc
+connection aux_slave;
+# Xid '1' must be in the output:
+XA RECOVER;
+formatID	gtrid_length	bqual_length	data
+1	1	0	1
+connection backup_slave;
+BACKUP STAGE START;
+BACKUP STAGE BLOCK_COMMIT;
+connection aux_slave;
+ROLLBACK;
+connection backup_slave;
+BACKUP STAGE END;
+connection slave;
+include/sync_with_master_gtid.inc
+include/stop_slave.inc
+#
+# Errored out XA COMMIT
+connection slave;
+include/stop_slave.inc
+Warnings:
+Note	1255	Slave already has been stopped
+connection master;
+connection aux_slave;
+BEGIN;
+INSERT INTO t1 VALUES (8);
+connection master;
+XA START '1';
+INSERT INTO  t1 VALUES (7);
+XA END '1';
+XA PREPARE '1';
+connection master1;
+INSERT INTO t1 VALUES (8);
+connection master;
+XA COMMIT '1';
+include/save_master_gtid.inc
+connection slave;
+SET @sav_innodb_lock_wait_timeout  = @@global.innodb_lock_wait_timeout;
+SET @sav_slave_transaction_retries = @@global.slave_transaction_retries;
+SET @@global.innodb_lock_wait_timeout =1;
+SET @@global.slave_transaction_retries=0;
+include/start_slave.inc
+connection aux_slave;
+# Xid '1' must be in the output:
+XA RECOVER;
+formatID	gtrid_length	bqual_length	data
+1	1	0	1
+connection backup_slave;
+BACKUP STAGE START;
+BACKUP STAGE BLOCK_COMMIT;
+connection aux_slave;
+ROLLBACK;
+connection backup_slave;
+BACKUP STAGE END;
+connection slave;
+include/stop_slave.inc
+SET @@global.innodb_lock_wait_timeout = @sav_innodb_lock_wait_timeout;
+SET @@global.slave_transaction_retries= @sav_slave_transaction_retries;
+connection slave;
+include/start_slave.inc
+include/sync_with_master_gtid.inc
+#
+# Errored out XA ROLLBACK
+connection slave;
+include/stop_slave.inc
+connection master;
+connection aux_slave;
+BEGIN;
+INSERT INTO t1 VALUES (10);
+connection master;
+XA START '1';
+INSERT INTO  t1 VALUES (9);
+XA END '1';
+XA PREPARE '1';
+connection master1;
+INSERT INTO t1 VALUES (10);
+connection master;
+XA ROLLBACK '1';
+include/save_master_gtid.inc
+connection slave;
+SET @sav_innodb_lock_wait_timeout  = @@global.innodb_lock_wait_timeout;
+SET @sav_slave_transaction_retries = @@global.slave_transaction_retries;
+SET @@global.innodb_lock_wait_timeout =1;
+SET @@global.slave_transaction_retries=0;
+include/start_slave.inc
+connection aux_slave;
+# Xid '1' must be in the output:
+XA RECOVER;
+formatID	gtrid_length	bqual_length	data
+1	1	0	1
+connection backup_slave;
+BACKUP STAGE START;
+BACKUP STAGE BLOCK_COMMIT;
+connection aux_slave;
+ROLLBACK;
+connection backup_slave;
+BACKUP STAGE END;
+connection slave;
+include/stop_slave.inc
+SET @@global.innodb_lock_wait_timeout = @sav_innodb_lock_wait_timeout;
+SET @@global.slave_transaction_retries= @sav_slave_transaction_retries;
+connection slave;
+include/start_slave.inc
+include/sync_with_master_gtid.inc
+connection slave;
+include/stop_slave.inc
+SET @@global.slave_parallel_threads= @old_parallel_threads;
+SET @@global.slave_parallel_mode   = @old_parallel_mode;
+include/start_slave.inc
+connection server_1;
+DROP TABLE t1;
+include/rpl_end.inc

mysql-test/suite/rpl/t/parallel_backup.test

@@ -12,6 +12,9 @@
 CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE = innodb;
 
 --sync_slave_with_master
+call mtr.add_suppression("Deadlock found when trying to get lock");
+call mtr.add_suppression("Commit failed due to failure of an earlier commit");
+
 --source include/stop_slave.inc
 SET @old_parallel_threads= @@GLOBAL.slave_parallel_threads;
 SET @old_parallel_mode   = @@GLOBAL.slave_parallel_mode;
@@ -61,6 +64,32 @@ BACKUP STAGE END;
 --let $diff_tables= master:t1,slave:t1
 --source include/diff_tables.inc
 
+#
+--echo # MDEV-30423: dealock XA COMMIT vs BACKUP
+# Prove XA "COMPLETE" 'xid' does not dealock similary to the normal trx case.
+# The slave binlog group commit leader is blocked by a local trx like in
+# the above normal trx case.
+# [Notice a reuse of t1,aux_conn from above.]
+#
+--let $complete = COMMIT
+--source parallel_backup_xa.inc
+--let $complete = ROLLBACK
+--source parallel_backup_xa.inc
+
+--let $slave_ooo_error = 1
+--let $complete = COMMIT
+--source parallel_backup_xa.inc
+--connection slave
+--source include/start_slave.inc
+--source include/sync_with_master_gtid.inc
+
+--let $slave_ooo_error = 1
+--let $complete = ROLLBACK
+--source parallel_backup_xa.inc
+--connection slave
+--source include/start_slave.inc
+--source include/sync_with_master_gtid.inc
+
 
 # Clean up.
 --connection slave

mysql-test/suite/rpl/t/parallel_backup_lsu_off-slave.opt

+--log-slave-updates=0
+

mysql-test/suite/rpl/t/parallel_backup_lsu_off.test

+#
+--echo # Specialized --log-slave-updates = 0 version of parallel_backup test.
+#
+--echo # MDEV-21953: deadlock between BACKUP STAGE BLOCK_COMMIT and parallel
+--echo # MDEV-30423: dealock XA COMMIT vs BACKUP
+--let $rpl_skip_reset_master_and_slave = 1
+--source parallel_backup.test

mysql-test/suite/rpl/t/parallel_backup_slave_binlog_off-slave.opt

+--skip-log-bin
\ No newline at end of file

mysql-test/suite/rpl/t/parallel_backup_slave_binlog_off.test

+#
+--echo # Specialized --skip-log-bin slave version of parallel_backup test.
+#
+--echo # MDEV-21953: deadlock between BACKUP STAGE BLOCK_COMMIT and parallel
+--echo # MDEV-30423: dealock XA COMMIT vs BACKUP
+--let $rpl_server_skip_log_bin= 1
+--source parallel_backup.test

mysql-test/suite/rpl/t/parallel_backup_xa.inc

+# Invoked from parallel_backup.test
+# Parameters:
+# $complete        = COMMIT or ROLLBACK
+# $slave_ooo_error = 1 means slave group commit did not succeed
+#
+--let $kind = Normal
+if ($slave_ooo_error)
+{
+  --let $kind = Errored out
+}
+--echo #
+--echo # $kind XA $complete
+
+--connection slave
+--source include/stop_slave.inc
+
+--connection master
+# val_0 is the first value to insert on master in prepared xa
+# val_1 is the next one to insert which is the value to block on slave
+--let $val_0 = `SELECT max(a)+1 FROM t1`
+--let $val_1 = $val_0
+--inc $val_1
+
+--connection aux_slave
+BEGIN;
+--eval INSERT INTO t1 VALUES ($val_1)
+
+--connection master
+XA START '1';
+--eval INSERT INTO  t1 VALUES ($val_0)
+XA END '1';
+XA PREPARE '1';
+--connection master1
+--eval INSERT INTO t1 VALUES ($val_1)
+--connection master
+--eval XA $complete '1'
+--source include/save_master_gtid.inc
+
+--connection slave
+if ($slave_ooo_error)
+{
+  SET @sav_innodb_lock_wait_timeout  = @@global.innodb_lock_wait_timeout;
+  SET @sav_slave_transaction_retries = @@global.slave_transaction_retries;
+  SET @@global.innodb_lock_wait_timeout =1;
+  SET @@global.slave_transaction_retries=0;
+}
+--source include/start_slave.inc
+--connection aux_slave
+--let $wait_condition= SELECT COUNT(*) = 1 FROM information_schema.processlist WHERE state = "Waiting for prior transaction to commit"
+--source include/wait_condition.inc
+--echo # Xid '1' must be in the output:
+XA RECOVER;
+--connection backup_slave
+  BACKUP STAGE START;
+--send BACKUP STAGE BLOCK_COMMIT
+--connection aux_slave
+  --sleep 1
+  if ($slave_ooo_error)
+  {
+    --let $wait_condition= SELECT COUNT(*) = 0 FROM information_schema.processlist WHERE state = "Waiting for prior transaction to commit"
+    --source include/wait_condition.inc
+  }
+  ROLLBACK;
+--let $wait_condition= SELECT COUNT(*) = 1 FROM information_schema.processlist WHERE state = "Waiting for backup lock"
+--source include/wait_condition.inc
+--connection backup_slave
+  --reap
+  BACKUP STAGE END;
+--connection slave
+if (!$slave_ooo_error)
+{
+  --source include/sync_with_master_gtid.inc
+}  
+--source include/stop_slave.inc
+if ($slave_ooo_error)
+{
+  SET @@global.innodb_lock_wait_timeout = @sav_innodb_lock_wait_timeout;
+  SET @@global.slave_transaction_retries= @sav_slave_transaction_retries;
+}

sql/handler.cc

@@ -2165,7 +2165,7 @@ static my_bool xacommit_handlerton(THD *unused1, plugin_ref plugin,
                                    void *arg)
 {
   handlerton *hton= plugin_hton(plugin);
-  if (hton->recover)
+  if (hton->recover || (hton == binlog_hton && current_thd->rgi_slave))
   {
     hton->commit_by_xid(hton, ((struct xahton_st *)arg)->xid);
     ((struct xahton_st *)arg)->result= 0;
@@ -2177,7 +2177,7 @@ static my_bool xarollback_handlerton(THD *unused1, plugin_ref plugin,
                                      void *arg)
 {
   handlerton *hton= plugin_hton(plugin);
-  if (hton->recover)
+  if (hton->recover || (hton == binlog_hton && current_thd->rgi_slave))
   {
     hton->rollback_by_xid(hton, ((struct xahton_st *)arg)->xid);
     ((struct xahton_st *)arg)->result= 0;

sql/log.cc

@@ -1677,11 +1677,12 @@ int binlog_init(void *p)
   {
     binlog_hton->prepare= binlog_prepare;
     binlog_hton->start_consistent_snapshot= binlog_start_consistent_snapshot;
-    binlog_hton->commit_by_xid= binlog_commit_by_xid;
-    binlog_hton->rollback_by_xid= binlog_rollback_by_xid;
     // recover needs to be set to make xa{commit,rollback}_handlerton effective
     binlog_hton->recover= binlog_xa_recover_dummy;
   }
+  binlog_hton->commit_by_xid= binlog_commit_by_xid;
+  binlog_hton->rollback_by_xid= binlog_rollback_by_xid;
+
   binlog_hton->flags= HTON_NOT_USER_SELECTABLE | HTON_HIDDEN | HTON_NO_ROLLBACK;
   return 0;
 }
@@ -2006,6 +2007,11 @@ static int binlog_commit_by_xid(handlerton *hton, XID *xid)
   int rc= 0;
   THD *thd= current_thd;
 
+  if (thd->is_current_stmt_binlog_disabled())
+  {
+    return thd->wait_for_prior_commit();
+  }
+
   /* the asserted state can't be reachable with xa commit */
   DBUG_ASSERT(!thd->get_stmt_da()->is_error() ||
               thd->get_stmt_da()->sql_errno() != ER_XA_RBROLLBACK);
@@ -2035,6 +2041,11 @@ static int binlog_rollback_by_xid(handlerton *hton, XID *xid)
   int rc= 0;
   THD *thd= current_thd;
 
+  if (thd->is_current_stmt_binlog_disabled())
+  {
+    return thd->wait_for_prior_commit();
+  }
+
   if (thd->get_stmt_da()->is_error() &&
       thd->get_stmt_da()->sql_errno() == ER_XA_RBROLLBACK)
     return rc;

sql/rpl_rli.cc

@@ -2285,11 +2285,9 @@ void rpl_group_info::cleanup_context(THD *thd, bool error)
 
   if (unlikely(error))
   {
-    /*
-      trans_rollback above does not rollback XA transactions
-      (todo/fixme consider to do so.
-    */
-    if (thd->transaction->xid_state.is_explicit_XA())
+    // leave alone any XA prepared transactions
+    if (thd->transaction->xid_state.is_explicit_XA() &&
+        thd->transaction->xid_state.get_state_code() != XA_PREPARED)
       xa_trans_force_rollback(thd);
 
     thd->release_transactional_locks();

sql/xa.cc

@@ -600,6 +600,7 @@ bool trans_xa_commit(THD *thd)
 
     if (auto xs= xid_cache_search(thd, thd->lex->xid))
     {
+      bool xid_deleted= false;
       res= xa_trans_rolled_back(xs);
       /*
         Acquire metadata lock which will ensure that COMMIT is blocked
@@ -610,7 +611,7 @@ bool trans_xa_commit(THD *thd)
       */
       MDL_request mdl_request;
       MDL_REQUEST_INIT(&mdl_request, MDL_key::BACKUP, "", "", MDL_BACKUP_COMMIT,
-                       MDL_STATEMENT);
+                       MDL_EXPLICIT);
       if (thd->mdl_context.acquire_lock(&mdl_request,
                                         thd->variables.lock_wait_timeout))
       {
@@ -621,25 +622,37 @@ bool trans_xa_commit(THD *thd)
         */
         DBUG_ASSERT(thd->is_error());
 
-        xs->acquired_to_recovered();
-        DBUG_RETURN(true);
+        res= true;
+        goto _end_external_xid;
       }
-      DBUG_ASSERT(!xid_state.xid_cache_element);
-
-      if (thd->wait_for_prior_commit())
+      else
       {
-        DBUG_ASSERT(thd->is_error());
-
-        xs->acquired_to_recovered();
-        DBUG_RETURN(true);
+        thd->backup_commit_lock= &mdl_request;
       }
+      DBUG_ASSERT(!xid_state.xid_cache_element);
 
       xid_state.xid_cache_element= xs;
       ha_commit_or_rollback_by_xid(thd->lex->xid, !res);
-      xid_state.xid_cache_element= 0;
+      if (!res && thd->is_error())
+      {
+        // hton completion error retains xs/xid in the cache,
+        // unless there had been already one as reflected by `res`.
+        res= true;
+        goto _end_external_xid;
+      }
+      xid_cache_delete(thd, xs);
+      xid_deleted= true;
 
+  _end_external_xid:
+      xid_state.xid_cache_element= 0;
       res= res || thd->is_error();
-      xid_cache_delete(thd, xs);
+      if (!xid_deleted)
+        xs->acquired_to_recovered();
+      if (mdl_request.ticket)
+      {
+        thd->mdl_context.release_lock(mdl_request.ticket);
+        thd->backup_commit_lock= 0;
+      }
     }
     else
       my_error(ER_XAER_NOTA, MYF(0));
@@ -761,9 +774,11 @@ bool trans_xa_rollback(THD *thd)
 
     if (auto xs= xid_cache_search(thd, thd->lex->xid))
     {
+      bool res;
+      bool xid_deleted= false;
       MDL_request mdl_request;
       MDL_REQUEST_INIT(&mdl_request, MDL_key::BACKUP, "", "", MDL_BACKUP_COMMIT,
-                       MDL_STATEMENT);
+                       MDL_EXPLICIT);
       if (thd->mdl_context.acquire_lock(&mdl_request,
                                         thd->variables.lock_wait_timeout))
       {
@@ -774,23 +789,33 @@ bool trans_xa_rollback(THD *thd)
         */
         DBUG_ASSERT(thd->is_error());
 
-        xs->acquired_to_recovered();
-        DBUG_RETURN(true);
+        goto _end_external_xid;
       }
-      xa_trans_rolled_back(xs);
-      DBUG_ASSERT(!xid_state.xid_cache_element);
-
-      if (thd->wait_for_prior_commit())
+      else
       {
-        DBUG_ASSERT(thd->is_error());
-        xs->acquired_to_recovered();
-        DBUG_RETURN(true);
+        thd->backup_commit_lock= &mdl_request;
       }
+      res= xa_trans_rolled_back(xs);
+      DBUG_ASSERT(!xid_state.xid_cache_element);
 
       xid_state.xid_cache_element= xs;
       ha_commit_or_rollback_by_xid(thd->lex->xid, 0);
-      xid_state.xid_cache_element= 0;
+      if (!res && thd->is_error())
+      {
+        goto _end_external_xid;
+      }
       xid_cache_delete(thd, xs);
+      xid_deleted= true;
+
+  _end_external_xid:
+      xid_state.xid_cache_element= 0;
+      if (!xid_deleted)
+        xs->acquired_to_recovered();
+      if (mdl_request.ticket)
+      {
+        thd->mdl_context.release_lock(mdl_request.ticket);
+        thd->backup_commit_lock= 0;
+      }
     }
     else
       my_error(ER_XAER_NOTA, MYF(0));