MDEV-31949 IV. Recovery

MDEV-33168 XA crash-recovery base on engines prepare first rule

This commit address XA transaction's prepare, commit and rollback
commands' crash-recovery both in the normal and semisync slave mode.

Key changes include:
- xid_recovery_member is extended to keep track of XID transaction
  "state" change. The state associated with its XID that when being
  reused could exists in binlog in multiple instance
- xarecover_handlerton is extended to register the user prepare xid
  similarly to the normal transactions
- xarecover_do_commit_or_rollback calls now a new
  xarecover_decide_xa() to decide on a user xid at the
  end of recovery
- TC_LOG_BINLOG::recover and few members of Recovery_context class are
  extended to scan binlog for the user XIDs and keep track of them, and
  their marking as truncatable or contrary - that is durable

Todo:
1. integrate with Xid_list_log_event
2. tests with multiple-engines

mysql-test/suite/binlog/r/binlog_truncate_active_log.result

 call mtr.add_suppression("Can.t init tc log");
 call mtr.add_suppression("Aborting");
+call mtr.add_suppression("Found.*prepared [XA ]*transaction[s]*");
 RESET MASTER;
 SET @@global.sync_binlog=1;
 CREATE TABLE t (f INT) ENGINE=INNODB;
@@ -249,7 +250,7 @@ disconnect master2;
 disconnect master3;
 disconnect master4;
 # restart: --rpl-semi-sync-slave-enabled=1 --sync-binlog=1 --log-warnings=3
-FOUND 1 /Successfully truncated.*to remove transactions starting from GTID 0-1-21/ in mysqld.1.err
+FOUND 1 /Successfully truncated.*to remove transactions starting from GTID 0-1-20/ in mysqld.1.err
 Pre-crash binlog file content:
 include/show_binlog_events.inc
 Log_name	Pos	Event_type	Server_id	End_log_pos	Info
@@ -272,13 +273,9 @@ master-bin.000004	#	Xid	#	#	COMMIT /* XID */
 master-bin.000004	#	Gtid	#	#	BEGIN GTID #-#-#
 master-bin.000004	#	Query	#	#	use `test`; INSERT INTO tm VALUES (10)
 master-bin.000004	#	Query	#	#	COMMIT
-master-bin.000004	#	Gtid	#	#	XA START X'786964',X'',1 GTID #-#-#
-master-bin.000004	#	Query	#	#	use `test`; DELETE FROM t WHERE f = 10
-master-bin.000004	#	Query	#	#	XA END X'786964',X'',1
-master-bin.000004	#	XA_prepare	#	#	XA PREPARE X'786964',X'',1
 SELECT @@global.gtid_binlog_pos as 'After the crash';
 After the crash
-0-1-20
+0-1-19
 "One row should be present in table 't'"
 SELECT * FROM t;
 f
@@ -287,6 +284,202 @@ f
 SELECT * FROM t4;
 f
 DELETE FROM t;
+# no 'xid' in
+XA RECOVER;
+formatID	gtrid_length	bqual_length	data
+SELECT count(*) > 0 as "because of rolled back" FROM t WHERE f = 10;
+because of rolled back
+0
+# Case E.
+connect master1,localhost,root,,;
+connect master2,localhost,root,,;
+connect master3,localhost,root,,;
+connect master4,localhost,root,,;
+connection default;
+INSERT INTO t  VALUES (10);
+INSERT INTO tm VALUES (10);
+connection master1;
+CALL sp_xa;
+connection master2;
+SET DEBUG_SYNC= "commit_before_get_LOCK_after_binlog_sync SIGNAL master2_ready";
+INSERT INTO t2 VALUES (20);
+connection master3;
+SET DEBUG_SYNC= "now WAIT_FOR master2_ready";
+SELECT @@global.gtid_binlog_pos as 'Before the crash';
+Before the crash
+0-1-24
+connection master4;
+SET DEBUG_SYNC= "ha_commit_trans_before_log_and_order SIGNAL master4_ready WAIT_FOR signal_never_arrives";
+INSERT INTO t4  VALUES (13);
+connection master3;
+SET DEBUG_SYNC= "now WAIT_FOR master4_ready";
+SELECT @@global.gtid_binlog_pos as 'Before the crash and never logged trx';
+Before the crash and never logged trx
+0-1-24
+connection default;
+# Kill the server
+disconnect master1;
+disconnect master2;
+disconnect master3;
+disconnect master4;
+# restart: --rpl-semi-sync-slave-enabled=1 --sync-binlog=1 --log-warnings=3
+# *** Expect NOT FOUND /Successfully truncated ... which is correct. ***
+NOT FOUND /Successfully truncated.*to remove transactions starting from GTID 0-1-2[1-4]/ in mysqld.1.err
+Pre-crash binlog file content:
+include/show_binlog_events.inc
+Log_name	Pos	Event_type	Server_id	End_log_pos	Info
+master-bin.000005	#	Gtid	#	#	BEGIN GTID #-#-#
+master-bin.000005	#	Query	#	#	use `test`; DELETE FROM t
+master-bin.000005	#	Xid	#	#	COMMIT /* XID */
+master-bin.000005	#	Gtid	#	#	BEGIN GTID #-#-#
+master-bin.000005	#	Query	#	#	use `test`; INSERT INTO t  VALUES (10)
+master-bin.000005	#	Xid	#	#	COMMIT /* XID */
+master-bin.000005	#	Gtid	#	#	BEGIN GTID #-#-#
+master-bin.000005	#	Query	#	#	use `test`; INSERT INTO tm VALUES (10)
+master-bin.000005	#	Query	#	#	COMMIT
+master-bin.000005	#	Gtid	#	#	XA START X'786964',X'',1 GTID #-#-#
+master-bin.000005	#	Query	#	#	use `test`; DELETE FROM t WHERE f = 10
+master-bin.000005	#	Query	#	#	XA END X'786964',X'',1
+master-bin.000005	#	XA_prepare	#	#	XA PREPARE X'786964',X'',1
+master-bin.000005	#	Gtid	#	#	BEGIN GTID #-#-#
+master-bin.000005	#	Query	#	#	use `test`; INSERT INTO t2 VALUES (20)
+master-bin.000005	#	Xid	#	#	COMMIT /* XID */
+SELECT @@global.gtid_binlog_pos as 'After the crash';
+After the crash
+0-1-24
+"One row should be present in table 't'"
+SELECT * FROM t;
+f
+10
+"No row should be present in table 't4'"
+SELECT * FROM t4;
+f
+# 'xid' exists in
+XA RECOVER;
+formatID	gtrid_length	bqual_length	data
+1	3	0	xid
+# Case F.
+connect master1,localhost,root,,;
+connect master2,localhost,root,,;
+connect master3,localhost,root,,;
+connect master4,localhost,root,,;
+connection default;
+INSERT INTO t  VALUES (10);
+INSERT INTO tm VALUES (10);
+connection master1;
+SET DEBUG_SYNC= "commit_before_get_LOCK_commit_ordered SIGNAL master1_ready WAIT_FOR signal_never_arrives";
+XA ROLLBACK 'xid';;
+connection master2;
+SET DEBUG_SYNC= "now WAIT_FOR master1_ready";
+SET DEBUG_SYNC= "commit_before_get_LOCK_after_binlog_sync SIGNAL master2_ready";
+INSERT INTO t2 VALUES (30);
+connection master3;
+SET DEBUG_SYNC= "now WAIT_FOR master2_ready";
+SELECT @@global.gtid_binlog_pos as 'Before the crash';
+Before the crash
+0-1-28
+connection master4;
+SET DEBUG_SYNC= "ha_commit_trans_before_log_and_order SIGNAL master4_ready WAIT_FOR signal_never_arrives";
+INSERT INTO t4  VALUES (13);
+connection master3;
+SET DEBUG_SYNC= "now WAIT_FOR master4_ready";
+SELECT @@global.gtid_binlog_pos as 'Before the crash and never logged trx';
+Before the crash and never logged trx
+0-1-28
+connection default;
+# Kill the server
+disconnect master1;
+disconnect master2;
+disconnect master3;
+disconnect master4;
+# restart: --rpl-semi-sync-slave-enabled=1 --sync-binlog=1 --log-warnings=3
+FOUND 1 /Successfully truncated.*to remove transactions starting from GTID 0-1-27/ in mysqld.1.err
+Pre-crash binlog file content:
+include/show_binlog_events.inc
+Log_name	Pos	Event_type	Server_id	End_log_pos	Info
+master-bin.000006	#	Gtid	#	#	BEGIN GTID #-#-#
+master-bin.000006	#	Query	#	#	use `test`; INSERT INTO t  VALUES (10)
+master-bin.000006	#	Xid	#	#	COMMIT /* XID */
+master-bin.000006	#	Gtid	#	#	BEGIN GTID #-#-#
+master-bin.000006	#	Query	#	#	use `test`; INSERT INTO tm VALUES (10)
+master-bin.000006	#	Query	#	#	COMMIT
+SELECT @@global.gtid_binlog_pos as 'After the crash';
+After the crash
+0-1-26
+"One row should be present in table 't'"
+SELECT * FROM t;
+f
+10
+10
+"No row should be present in table 't4'"
+SELECT * FROM t4;
+f
+# 'xid' exists in
+XA RECOVER;
+formatID	gtrid_length	bqual_length	data
+1	3	0	xid
+# Case G.
+connect master1,localhost,root,,;
+connect master2,localhost,root,,;
+connect master3,localhost,root,,;
+connect master4,localhost,root,,;
+connection default;
+INSERT INTO t  VALUES (10);
+INSERT INTO tm VALUES (10);
+connection master1;
+SET DEBUG_SYNC= "commit_before_get_LOCK_commit_ordered SIGNAL master1_ready WAIT_FOR signal_never_arrives";
+XA COMMIT 'xid';;
+connection master2;
+SET DEBUG_SYNC= "now WAIT_FOR master1_ready";
+SET DEBUG_SYNC= "commit_before_get_LOCK_after_binlog_sync SIGNAL master2_ready";
+INSERT INTO t2 VALUES (30);
+connection master3;
+SET DEBUG_SYNC= "now WAIT_FOR master2_ready";
+SELECT @@global.gtid_binlog_pos as 'Before the crash';
+Before the crash
+0-1-30
+connection master4;
+SET DEBUG_SYNC= "ha_commit_trans_before_log_and_order SIGNAL master4_ready WAIT_FOR signal_never_arrives";
+INSERT INTO t4  VALUES (13);
+connection master3;
+SET DEBUG_SYNC= "now WAIT_FOR master4_ready";
+SELECT @@global.gtid_binlog_pos as 'Before the crash and never logged trx';
+Before the crash and never logged trx
+0-1-30
+connection default;
+# Kill the server
+disconnect master1;
+disconnect master2;
+disconnect master3;
+disconnect master4;
+# restart: --rpl-semi-sync-slave-enabled=1 --sync-binlog=1 --log-warnings=3
+FOUND 1 /Successfully truncated.*to remove transactions starting from GTID 0-1-29/ in mysqld.1.err
+Pre-crash binlog file content:
+include/show_binlog_events.inc
+Log_name	Pos	Event_type	Server_id	End_log_pos	Info
+master-bin.000007	#	Gtid	#	#	BEGIN GTID #-#-#
+master-bin.000007	#	Query	#	#	use `test`; INSERT INTO t  VALUES (10)
+master-bin.000007	#	Xid	#	#	COMMIT /* XID */
+master-bin.000007	#	Gtid	#	#	BEGIN GTID #-#-#
+master-bin.000007	#	Query	#	#	use `test`; INSERT INTO tm VALUES (10)
+master-bin.000007	#	Query	#	#	COMMIT
+SELECT @@global.gtid_binlog_pos as 'After the crash';
+After the crash
+0-1-28
+"One row should be present in table 't'"
+SELECT * FROM t;
+f
+10
+10
+10
+"No row should be present in table 't4'"
+SELECT * FROM t4;
+f
+# 'xid' exists in
+XA RECOVER;
+formatID	gtrid_length	bqual_length	data
+1	3	0	xid
+XA COMMIT 'xid';
 DROP PROCEDURE sp_xa;
 # Cleanup
 DROP TABLE t,t2,tm;

mysql-test/suite/binlog/r/binlog_truncate_multi_log_xa.result

+call mtr.add_suppression("Can.t init tc log");
+call mtr.add_suppression("Aborting");
+call mtr.add_suppression("Found.*prepared [XA ]*transaction[s]*");
+SET @@global.max_binlog_size= 4096;
+SET @@global.sync_binlog= 1;
+RESET MASTER;
+FLUSH LOGS;
+CREATE TABLE ti (a INT PRIMARY KEY, b MEDIUMTEXT) ENGINE=Innodb;
+CREATE PROCEDURE sp_xa()
+BEGIN
+XA START   'xid';
+INSERT INTO ti VALUES (2, REPEAT("x", 4100));
+XA END     'xid';
+XA PREPARE 'xid';
+END|
+# case A. binlog input p1,g2,g3: p1 gets truncated along with g2,g3
+connect master1,localhost,root,,;
+"List of binary logs before rotation"
+show binary logs;
+Log_name	File_size
+master-bin.000001	#
+master-bin.000002	#
+INSERT INTO ti VALUES(1,"I am gonna survive");
+SET DEBUG_SYNC= "commit_after_release_LOCK_after_binlog_sync SIGNAL master1_ready WAIT_FOR master1_go_never_arrives";
+call sp_xa();
+connect master2,localhost,root,,;
+SET DEBUG_SYNC= "now WAIT_FOR master1_ready";
+SET DEBUG_SYNC= "commit_before_get_LOCK_commit_ordered SIGNAL master2_ready WAIT_FOR master2_go_never_arrives";
+INSERT INTO ti VALUES (3, "not gonna survive");
+connection default;
+SET DEBUG_SYNC= "now WAIT_FOR master2_ready";
+connect master3,localhost,root,,;
+SET DEBUG_SYNC= "ha_commit_trans_before_log_and_order SIGNAL master3_ready WAIT_FOR master3_go_never_arrives";
+INSERT INTO ti VALUES (4, "not gonna be logged therefore survive"),(5, "ditto");
+connection default;
+SET DEBUG_SYNC= "now WAIT_FOR master3_ready";
+"List of binary logs before crash"
+show binary logs;
+Log_name	File_size
+master-bin.000001	#
+master-bin.000002	#
+master-bin.000003	#
+# The gtid binlog state prior the crash will be truncated at the end of the test
+SELECT @@global.gtid_binlog_state;
+@@global.gtid_binlog_state
+0-1-5
+connection default;
+# Kill the server
+disconnect master1;
+disconnect master2;
+disconnect master3;
+# restart: --rpl-semi-sync-slave-enabled=1 --sync-binlog=1 --log-warnings=3
+FOUND 1 /Successfully truncated.*to remove transactions starting from GTID 0-1-4/ in mysqld.1.err
+FOUND 1 /truncated binlog file:.*master.*000002/ in mysqld.1.err
+# One record should be present in table
+SELECT * FROM ti;
+a	b
+1	I am gonna survive
+# The truncated gtid binlog state
+SELECT @@global.gtid_binlog_state;
+@@global.gtid_binlog_state
+0-1-3
+SELECT @@global.gtid_binlog_pos;
+@@global.gtid_binlog_pos
+0-1-3
+DELETE FROM ti;
+# case B. binlog input g1,p2,g3: p2 gets truncated along with g1,g3
+connect master1,localhost,root,,;
+"List of binary logs before rotation"
+show binary logs;
+Log_name	File_size
+master-bin.000001	#
+master-bin.000002	#
+master-bin.000003	#
+INSERT INTO ti VALUES(1,"I am gonna survive");
+SET DEBUG_SYNC= "commit_after_release_LOCK_after_binlog_sync SIGNAL master1_ready WAIT_FOR master1_go_never_arrives";
+INSERT INTO ti VALUES (3, "not gonna survive");
+connect master2,localhost,root,,;
+SET DEBUG_SYNC= "now WAIT_FOR master1_ready";
+SET DEBUG_SYNC= "commit_before_get_LOCK_commit_ordered SIGNAL master2_ready WAIT_FOR master2_go_never_arrives";
+call sp_xa();
+connection default;
+SET DEBUG_SYNC= "now WAIT_FOR master2_ready";
+connect master3,localhost,root,,;
+SET DEBUG_SYNC= "ha_commit_trans_before_log_and_order SIGNAL master3_ready WAIT_FOR master3_go_never_arrives";
+INSERT INTO ti VALUES (4, "not gonna be logged therefore survive"),(5, "ditto");
+connection default;
+SET DEBUG_SYNC= "now WAIT_FOR master3_ready";
+"List of binary logs before crash"
+show binary logs;
+Log_name	File_size
+master-bin.000001	#
+master-bin.000002	#
+master-bin.000003	#
+# The gtid binlog state prior the crash will be truncated at the end of the test
+SELECT @@global.gtid_binlog_state;
+@@global.gtid_binlog_state
+0-1-7
+connection default;
+# Kill the server
+disconnect master1;
+disconnect master2;
+disconnect master3;
+# restart: --rpl-semi-sync-slave-enabled=1 --sync-binlog=1 --log-warnings=3
+FOUND 1 /Successfully truncated.*to remove transactions starting from GTID 0-1-6/ in mysqld.1.err
+FOUND 1 /truncated binlog file:.*master.*000003/ in mysqld.1.err
+# One record should be present in table
+SELECT * FROM ti;
+a	b
+1	I am gonna survive
+# The truncated gtid binlog state
+SELECT @@global.gtid_binlog_state;
+@@global.gtid_binlog_state
+0-1-5
+SELECT @@global.gtid_binlog_pos;
+@@global.gtid_binlog_pos
+0-1-5
+DELETE FROM ti;
+# case C. binlog input G1,P2,G3: P2 remains prepared.
+connect master1,localhost,root,,;
+INSERT INTO ti VALUES(1,"I am gonna survive");
+connect master2,localhost,root,,;
+call sp_xa();
+connect master3,localhost,root,,;
+INSERT INTO ti VALUES(3,"Me too");
+connection default;
+# restart: --rpl-semi-sync-slave-enabled=1 --sync-binlog=1 --log-warnings=3
+disconnect master1;
+disconnect master2;
+disconnect master3;
+# restart: --rpl-semi-sync-slave-enabled=1 --sync-binlog=1 --log-warnings=3
+# xid must be in
+XA RECOVER;
+formatID	gtrid_length	bqual_length	data
+1	3	0	xid
+# case D. prove Binlog-checkpoint based recovery for XA-"complete"
+XA COMMIT 'xid';
+FLUSH LOGS;
+# Kill the server
+# restart: --rpl-semi-sync-slave-enabled=1 --sync-binlog=1 --log-warnings=3
+# no xid must be in
+XA RECOVER;
+formatID	gtrid_length	bqual_length	data
+SELECT count(*) = 3 FROM ti;
+count(*) = 3
+1
+SELECT @@global.gtid_binlog_state;
+@@global.gtid_binlog_state
+0-1-10
+SELECT @@global.gtid_binlog_pos;
+@@global.gtid_binlog_pos
+0-1-10
+# Cleanup
+DROP PROCEDURE sp_xa;
+DROP TABLE ti;
+SET @@global.sync_binlog= default;
+# End of the tests

mysql-test/suite/binlog/r/binlog_xa_mdev-31949.result

@@ -2,61 +2,110 @@ call mtr.add_suppression("Found 1 prepared XA transactions");
 call mtr.add_suppression("unknown option");
 CREATE TABLE t31949 (a INT PRIMARY KEY) ENGINE=Innodb;
 CREATE VIEW v_processlist  as SELECT * FROM performance_schema.threads where type = 'FOREGROUND';
-XA START '1';
+# case 1. only in engine prepared XA is rolled back
+connect master1,localhost,root,,;
+XA START 'A';
 INSERT INTO t31949 SET a=1;
-XA END '1';
-XA PREPARE '1';
+XA END 'A';
+SET DEBUG_SYNC= "commit_before_get_LOCK_log  SIGNAL master1_ready WAIT_FOR signal_never_arrives";
+XA PREPARE 'A';
+connection default;
+SET DEBUG_SYNC= "now WAIT_FOR master1_ready";
+# Kill and restart
+disconnect master1;
+# Proof 1:
+XA RECOVER;
+formatID	gtrid_length	bqual_length	data
+SELECT count(*) = 0 from t31949;
+count(*) = 0
+1
+# case 2. both engine prepared and binlog written XA remains
+connect master1,localhost,root,,;
+XA START 'A';
+INSERT INTO t31949 SET a=1;
+XA END 'A';
+XA PREPARE 'A';
+connection default;
 # Kill and restart
-# xid '1' most be recovered
+disconnect master1;
+# Proof 2:
+# xid 'A' must be recovered
 XA RECOVER;
 formatID	gtrid_length	bqual_length	data
-1	1	0	1
-# COMMIT from an external connection
-XA COMMIT '1';
+1	1	0	A
+SELECT count(*) = 0 from t31949;
+count(*) = 0
+1
+# case 3. XA-COMMIT from an external connection after binlogging recovers to COMPLETE
+connect master1,localhost,root,,;
+SET DEBUG_SYNC= "commit_before_get_LOCK_commit_ordered  SIGNAL master1_ready WAIT_FOR signal_never_arrives";
+XA COMMIT 'A';
+connection default;
+SET DEBUG_SYNC= "now WAIT_FOR master1_ready";
 # Kill and restart
-# xid '1' must be committed
+disconnect master1;
+# Proof 3:
+# xid 'A' must be recovered to commit
 XA RECOVER;
 formatID	gtrid_length	bqual_length	data
 select count(*) = 1 from t31949;
 count(*) = 1
 1
-XA START '2';
+# case 4. XA-ROLLBACK from the native connection after binlogging recovers to COMPLETE
+connect master2,localhost,root,,;
+XA START 'B';
 INSERT INTO t31949 SET a=2;
-XA END '2';
-XA PREPARE '2';
-# ROLLBACK from the native connection
-XA ROLLBACK '2';
+XA END 'B';
+XA PREPARE 'B';
+SET DEBUG_SYNC= "commit_before_get_LOCK_commit_ordered  SIGNAL master1_ready WAIT_FOR signal_never_arrives";
+XA ROLLBACK 'B';
+connection default;
+SET DEBUG_SYNC= "now WAIT_FOR master1_ready";
 # Kill and restart
-# xid '1' must be rolled back
+disconnect master2;
+# Proof 4:
+# xid 'B' must be recovered to roll back
 XA RECOVER;
 formatID	gtrid_length	bqual_length	data
 select count(*) = 1 from t31949;
 count(*) = 1
 1
-XA START '3';
+# case 5. ditto to XA-COMMIT
+connect master3,localhost,root,,;
+XA START 'C';
 INSERT INTO t31949 SET a=3;
-XA END '3';
-XA PREPARE '3';
-# COMMIT from the native connection
-XA COMMIT '3';
+XA END 'C';
+XA PREPARE 'C';
+SET DEBUG_SYNC= "commit_before_get_LOCK_commit_ordered  SIGNAL master1_ready WAIT_FOR signal_never_arrives";
+XA COMMIT 'C';
+connection default;
+SET DEBUG_SYNC= "now WAIT_FOR master1_ready";
 # Kill and restart
-# xid '3' must be committed
+disconnect master3;
+# Proof 5:
+# xid 'C' must be recovered to commit
 XA RECOVER;
 formatID	gtrid_length	bqual_length	data
 select count(*) = 2 from t31949;
 count(*) = 2
 1
-connect con4,127.0.0.1,root,,test,$MASTER_MYPORT,;
-XA START '4';
+# case 6. XA-ROLLBACK from an external connection  after binlogging recovers to COMPLETE
+connect conn$case,localhost,root,,;
+XA START 'D';
 INSERT INTO t31949 SET a=4;
-XA END '4';
-XA PREPARE '4';
-disconnect con4;
+XA END 'D';
+XA PREPARE 'D';
+disconnect conn6;
+connection default;
+connect master4,localhost,root,,;
+SET DEBUG_SYNC= "commit_before_get_LOCK_commit_ordered  SIGNAL master1_ready WAIT_FOR signal_never_arrives";
+XA ROLLBACK 'D';
 connection default;
-# ROLLBACK from an external connection
-XA ROLLBACK '4';
+SET DEBUG_SYNC= "now WAIT_FOR master1_ready";
 # Kill and restart
-# xid '4' must be rolled back
+disconnect master4;
+# Proof 6:
+# xid 'D' must be recovered to roll back
 XA RECOVER;
 formatID	gtrid_length	bqual_length	data
 select count(*) = 2 from t31949;

mysql-test/suite/binlog/t/binlog_truncate_active_log.inc

+#
+# Meanings of parameters:
+#
+# $query1       a query that creates a group of events that goes into binlog
+# $query2       a query that follows $query1 in binlog without having done
+#                 engine commit
+# $query20      an optinal $query2 that durably commits in engine
+# $delete       when 1 deletes from t
+# $no_truncate  when 1 NOT FOUND off search_pattern_in_file.inc is "good".
+#
 connect(master1,localhost,root,,);
 connect(master2,localhost,root,,);
 connect(master3,localhost,root,,);
@@ -10,14 +20,29 @@ INSERT INTO t  VALUES (10);
 INSERT INTO tm VALUES (10);
 
 --connection master1
-# Hold insert after write to binlog and before "run_commit_ordered" in engine
-SET DEBUG_SYNC= "commit_before_get_LOCK_commit_ordered SIGNAL master1_ready WAIT_FOR signal_never_arrives";
---send_eval $query1
+if (!$query20)
+{
+  # Hold insert after write to binlog and before "run_commit_ordered" in engine
+  SET DEBUG_SYNC= "commit_before_get_LOCK_commit_ordered SIGNAL master1_ready WAIT_FOR signal_never_arrives";
+  --send_eval $query1
+}
+if ($query20)
+{
+  # do not hold the $query2 insert to let it binlog past $query1 "us"
+  --eval $query1
+}
 
 --connection master2
-SET DEBUG_SYNC= "now WAIT_FOR master1_ready";
-SET DEBUG_SYNC= "commit_before_get_LOCK_after_binlog_sync SIGNAL master2_ready";
---send_eval $query2
+if (!$query20)
+{
+  SET DEBUG_SYNC= "now WAIT_FOR master1_ready";
+  SET DEBUG_SYNC= "commit_before_get_LOCK_after_binlog_sync SIGNAL master2_ready";  --send_eval $query2
+}
+if ($query20)
+{
+  SET DEBUG_SYNC= "commit_before_get_LOCK_after_binlog_sync SIGNAL master2_ready";
+  --eval $query20
+}
 
 --connection master3
 SET DEBUG_SYNC= "now WAIT_FOR master2_ready";
@@ -46,6 +71,10 @@ SELECT @@global.gtid_binlog_pos as 'Before the crash and never logged trx';
 --source include/start_mysqld.inc
 
 # Check error log for a successful truncate message.
+if ($no_truncate)
+{
+  --echo # *** Expect NOT FOUND /Successfully truncated ... which is correct. ***
+}
 --let $log_error_ = $MYSQLTEST_VARDIR/log/mysqld.1.err
 
 --let SEARCH_FILE=$log_error_
@@ -67,4 +96,7 @@ SELECT * FROM t4;
 --inc $binlog_file_index
 
 # Local cleanup
-DELETE FROM t;
+if ($delete)
+{
+  DELETE FROM t;
+}

mysql-test/suite/binlog/t/binlog_truncate_active_log.test

@@ -5,6 +5,7 @@
 # ==== References ====
 #
 # MDEV-21117: recovery for --rpl-semi-sync-slave-enabled server
+# MDEV-33168 XA crash-recovery base on engines prepare first rule
 
 --source include/have_innodb.inc
 --source include/have_aria.inc
@@ -15,11 +16,14 @@
 
 call mtr.add_suppression("Can.t init tc log");
 call mtr.add_suppression("Aborting");
+call mtr.add_suppression("Found.*prepared [XA ]*transaction[s]*");
 
 # The following cases are tested:
 # A. 2pc transaction is followed by a blank "zero-engines" one
 # B. 2pc transaction follows the blank one
 # C. Similarly to A, with the XA blank transaction
+#
+# D-G verify MDEV-33168.
 
 RESET MASTER;
 SET @@global.sync_binlog=1;
@@ -31,6 +35,8 @@ CREATE TABLE tm (f INT) ENGINE=Aria;
 # Old (pre-crash) binlog file index initial value.
 # It keeps incremented at the end of each case.
 --let $binlog_file_index=1
+# parameter to binlog_truncate_active_log.inc
+--let $delete=1
 
 --echo # Case A.
 # Using 'debug_sync' hold 'query1' execution after 'query1' is flushed and
@@ -88,15 +94,71 @@ BEGIN
 END|
 delimiter ;|
 
-# The same as in B with $query1 being the prepared XA transaction.
-# Truncation must occurs at $query2.
---let $truncate_gtid_pos = 0-1-21
+# $query1 being the prepared XA transaction.
+# Truncation must occurs at $query1 because $query2 is not going to commit
+# therefore at recovery it will be in-doubt and thus be rolled back by
+#  --rpl-semi-sync-slave-enabled=1. As $query2 could not be proved
+# as acknowlegded by slave, $query1 has to be pessimistically regarded the same.
+--let $truncate_gtid_pos = 0-1-20
 --let $query1 = CALL sp_xa
 --let $query2 = INSERT INTO t2 VALUES (20)
 --source binlog_truncate_active_log.inc
+# Proof:
+--echo # no 'xid' in
+XA RECOVER;
+SELECT count(*) > 0 as "because of rolled back" FROM t WHERE f = 10;
 
-DROP PROCEDURE sp_xa;
+--echo # Case E.
+# The same as D, but XAP of $query1 survives the crash-recovery thanks to
+# $query20 is going to durably commit prior to the crash.
+
+--let $truncate_gtid_pos = 0-1-2[1-4]
+--let $query1 = CALL sp_xa
+let $query2 =;
+--let $query20 = INSERT INTO t2 VALUES (20)
+--let $delete=0
+--let $no_truncate=1
+--source binlog_truncate_active_log.inc
+# Proof:
+--echo # 'xid' exists in
+XA RECOVER;
+
+# restore to the default
+let $query20 =;
+--let $no_truncate=0
+
+--echo # Case F.
+# Recovery of XA-ROLLBACK. The conditions are like in D. The outcome
+# is truncation of both $query1,2.
+# Therefore `xid` must remain in prepared state.
+--let $truncate_gtid_pos = 0-1-27
+
+--let $query1 = XA ROLLBACK 'xid';
+--let $query2 = INSERT INTO t2 VALUES (30)
+--let $delete=0
+--source binlog_truncate_active_log.inc
 
+# Proof:
+--echo # 'xid' exists in
+XA RECOVER;
+
+--echo # Case G.
+# Recovery of XA-COMMIT. The conditions are like in F. The outcome
+# is truncation of both $query1,2.
+# Therefore `xid` must remain in prepared state.
+--let $truncate_gtid_pos = 0-1-29
+
+--let $query1 = XA COMMIT 'xid';
+--let $query2 = INSERT INTO t2 VALUES (30)
+--let $delete=0
+--source binlog_truncate_active_log.inc
+
+# Proof:
+--echo # 'xid' exists in
+XA RECOVER;
+
+XA COMMIT 'xid';
+DROP PROCEDURE sp_xa;
 
 --echo # Cleanup
 DROP TABLE t,t2,tm;

mysql-test/suite/binlog/t/binlog_truncate_multi_log_xa.inc

+# $query1          a query that creates a group of events that goes into binlog
+# $query2          a query that follows $query1 in binlog without having done
+#                    engine commit
+# $truncate_index  binlog file index that undergoes truncation
+# $truncate_gtid   the first gtid in the cut off tail part of
+#                    the binlog index' file
+
+connect(master1,localhost,root,,);
+--echo "List of binary logs before rotation"
+--source include/show_binary_logs.inc
+
+# Some load to either non- and transactional egines
+# that should not affect the following recovery:
+INSERT INTO ti VALUES(1,"I am gonna survive");
+
+# XAP_1 will be holding a mutex ..
+SET DEBUG_SYNC= "commit_after_release_LOCK_after_binlog_sync SIGNAL master1_ready WAIT_FOR master1_go_never_arrives";
+--send_eval $query1
+
+connect(master2,localhost,root,,);
+# .. to not let the 2nd trx to commit.
+SET DEBUG_SYNC= "now WAIT_FOR master1_ready";
+SET DEBUG_SYNC= "commit_before_get_LOCK_commit_ordered SIGNAL master2_ready WAIT_FOR master2_go_never_arrives";
+--send_eval $query2
+
+--connection default
+SET DEBUG_SYNC= "now WAIT_FOR master2_ready";
+
+connect(master3,localhost,root,,);
+# The 3nd trx will be considered at recovery, but as it won't get into binlog
+# it won't recover.
+SET DEBUG_SYNC= "ha_commit_trans_before_log_and_order SIGNAL master3_ready WAIT_FOR master3_go_never_arrives";
+--send INSERT INTO ti VALUES (4, "not gonna be logged therefore survive"),(5, "ditto")
+
+--connection default
+SET DEBUG_SYNC= "now WAIT_FOR master3_ready";
+
+--echo "List of binary logs before crash"
+--source include/show_binary_logs.inc
+--echo # The gtid binlog state prior the crash will be truncated at the end of the test
+SELECT @@global.gtid_binlog_state;
+
+--connection default
+--source include/kill_mysqld.inc
+--disconnect master1
+--disconnect master2
+--disconnect master3
+
+#
+# Server restart
+#
+--let $restart_parameters= --rpl-semi-sync-slave-enabled=1 --sync-binlog=1 --log-warnings=3
+--source include/start_mysqld.inc
+
+# Check error log for a successful truncate message.
+let $log_error_ = $MYSQLTEST_VARDIR/log/mysqld.1.err;
+
+--let SEARCH_FILE=$log_error_
+--let SEARCH_PATTERN=Successfully truncated.*to remove transactions starting from GTID $truncate_gtid
+--source include/search_pattern_in_file.inc
+--let SEARCH_PATTERN=truncated binlog file:.*master.*00000$truncate_index
+--source include/search_pattern_in_file.inc
+
+--echo # One record should be present in table
+SELECT * FROM ti;
+--echo # The truncated gtid binlog state
+SELECT @@global.gtid_binlog_state;
+SELECT @@global.gtid_binlog_pos;
+DELETE FROM ti;
+

mysql-test/suite/binlog/t/binlog_truncate_multi_log_xa.test

+# ==== Purpose ====
+#
+# Test verifies truncation of multiple binary logs.
+#
+# ==== References ====
+# MDEV-21117: recovery for --rpl-semi-sync-slave-enabled server
+# MDEV-33168 XA crash-recovery base on engines prepare first rule
+
+--source include/have_innodb.inc
+--source include/have_debug_sync.inc
+--source include/have_binlog_format_row.inc
+
+call mtr.add_suppression("Can.t init tc log");
+call mtr.add_suppression("Aborting");
+call mtr.add_suppression("Found.*prepared [XA ]*transaction[s]*");
+
+SET @@global.max_binlog_size= 4096;
+SET @@global.sync_binlog= 1;
+RESET MASTER;
+FLUSH LOGS;
+CREATE TABLE ti (a INT PRIMARY KEY, b MEDIUMTEXT) ENGINE=Innodb;
+
+delimiter |;
+CREATE PROCEDURE sp_xa()
+BEGIN
+ XA START   'xid';
+ INSERT INTO ti VALUES (2, REPEAT("x", 4100));
+ XA END     'xid';
+ XA PREPARE 'xid';
+END|
+delimiter ;|
+
+# Notations from MDEV-21117:
+# the small lettercase `g` stands for non-committed in doubt transaction
+# Similarly `p` stands for in-doubt user prepared XA.
+# The uppercased G:s,P that stand for the fact of resolved status (to complete
+# the transaction (both kinds) or keep xa as prepared).
+
+--echo # case A. binlog input p1,g2,g3: p1 gets truncated along with g2,g3
+--let $truncate_index=2
+--let $truncate_gtid=0-1-4
+--let $query1=call sp_xa()
+--let $query2=INSERT INTO ti VALUES (3, "not gonna survive")
+--source binlog_truncate_multi_log_xa.inc
+
+--echo # case B. binlog input g1,p2,g3: p2 gets truncated along with g1,g3
+--let $truncate_index=3
+--let $truncate_gtid=0-1-6
+--let $query1=INSERT INTO ti VALUES (3, "not gonna survive")
+--let $query2=call sp_xa()
+--source binlog_truncate_multi_log_xa.inc
+
+
+--echo # case C. binlog input G1,P2,G3: P2 remains prepared.
+connect(master1,localhost,root,,);
+INSERT INTO ti VALUES(1,"I am gonna survive");
+
+connect(master2,localhost,root,,);
+call sp_xa();
+
+connect(master3,localhost,root,,);
+INSERT INTO ti VALUES(3,"Me too");
+
+--connection default
+--source include/restart_mysqld.inc
+--disconnect master1
+--disconnect master2
+--disconnect master3
+
+#
+# Server restart
+#
+--let $restart_parameters= --rpl-semi-sync-slave-enabled=1 --sync-binlog=1 --log-warnings=3
+--source include/start_mysqld.inc
+--echo # xid must be in
+XA RECOVER;
+
+--echo # case D. prove Binlog-checkpoint based recovery for XA-"complete"
+# crash after XA-COMMIT followed by Rotate and Binlog-Checkpoint
+# must be sufficient for the XA transaction be durably committed.
+# It can't be found in prepared state at recovery.
+XA COMMIT 'xid';
+FLUSH LOGS;
+--source include/wait_for_binlog_checkpoint.inc
+
+--source include/kill_mysqld.inc
+--let $restart_parameters= --rpl-semi-sync-slave-enabled=1 --sync-binlog=1 --log-warnings=3
+--source include/start_mysqld.inc
+
+--echo # no xid must be in
+XA RECOVER;
+
+SELECT count(*) = 3 FROM ti;
+SELECT @@global.gtid_binlog_state;
+SELECT @@global.gtid_binlog_pos;
+
+
+--echo # Cleanup
+DROP PROCEDURE sp_xa;
+DROP TABLE ti;
+SET @@global.sync_binlog= default;
+--echo # End of the tests

mysql-test/suite/binlog/t/binlog_xa_mdev-31949.test

 --source include/have_innodb.inc
+--source include/have_debug.inc
 --source include/have_binlog_format_row.inc
 
 # MDEV-31949 slow xa on parallel slave
@@ -11,68 +12,130 @@ call mtr.add_suppression("unknown option");
 CREATE TABLE t31949 (a INT PRIMARY KEY) ENGINE=Innodb;
 CREATE VIEW v_processlist  as SELECT * FROM performance_schema.threads where type = 'FOREGROUND';
 
-XA START '1';
+--let $case=1
+--echo # case $case. only in engine prepared XA is rolled back
+
+--connect(master1,localhost,root,,)
+XA START 'A';
 INSERT INTO t31949 SET a=1;
-XA END '1';
-XA PREPARE '1';
+XA END 'A';
+SET DEBUG_SYNC= "commit_before_get_LOCK_log  SIGNAL master1_ready WAIT_FOR signal_never_arrives";
+--send XA PREPARE 'A'
+
+--connection default
+SET DEBUG_SYNC= "now WAIT_FOR master1_ready";
 --source include/kill_and_restart_mysqld.inc
+--disconnect master1
 
---echo # xid '1' most be recovered
+--echo # Proof $case:
 XA RECOVER;
---echo # COMMIT from an external connection
-XA COMMIT '1';
+SELECT count(*) = 0 from t31949;
+
+
+--inc $case
+--echo # case $case. both engine prepared and binlog written XA remains
+--connect(master1,localhost,root,,)
+XA START 'A';
+INSERT INTO t31949 SET a=1;
+XA END 'A';
+XA PREPARE 'A';
+
+--connection default
 --source include/kill_and_restart_mysqld.inc
+--disconnect master1
 
---echo # xid '1' must be committed
+--echo # Proof $case:
+--echo # xid 'A' must be recovered
 XA RECOVER;
-select count(*) = 1 from t31949;
+SELECT count(*) = 0 from t31949;
+
 
+--inc $case
+--echo # case $case. XA-COMMIT from an external connection after binlogging recovers to COMPLETE
+--connect(master1,localhost,root,,)
+SET DEBUG_SYNC= "commit_before_get_LOCK_commit_ordered  SIGNAL master1_ready WAIT_FOR signal_never_arrives";
+--send XA COMMIT 'A'
+
+--connection default
+SET DEBUG_SYNC= "now WAIT_FOR master1_ready";
+--source include/kill_and_restart_mysqld.inc
+--disconnect master1
 
-XA START '2';
+--echo # Proof $case:
+--echo # xid 'A' must be recovered to commit
+XA RECOVER;
+select count(*) = 1 from t31949;
+
+--inc $case
+--echo # case $case. XA-ROLLBACK from the native connection after binlogging recovers to COMPLETE
+--connect(master2,localhost,root,,)
+XA START 'B';
 INSERT INTO t31949 SET a=2;
-XA END '2';
-XA PREPARE '2';
---echo # ROLLBACK from the native connection
-XA ROLLBACK '2';
+XA END 'B';
+XA PREPARE 'B';
+SET DEBUG_SYNC= "commit_before_get_LOCK_commit_ordered  SIGNAL master1_ready WAIT_FOR signal_never_arrives";
+--send XA ROLLBACK 'B'
 
+--connection default
+SET DEBUG_SYNC= "now WAIT_FOR master1_ready";
 --source include/kill_and_restart_mysqld.inc
---echo # xid '1' must be rolled back
+--disconnect master2
+
+--echo # Proof $case:
+--echo # xid 'B' must be recovered to roll back
 XA RECOVER;
 select count(*) = 1 from t31949;
 
-XA START '3';
+--inc $case
+--echo # case $case. ditto to XA-COMMIT
+--connect(master3,localhost,root,,)
+XA START 'C';
 INSERT INTO t31949 SET a=3;
-XA END '3';
-XA PREPARE '3';
---echo # COMMIT from the native connection
-XA COMMIT '3';
+XA END 'C';
+XA PREPARE 'C';
+SET DEBUG_SYNC= "commit_before_get_LOCK_commit_ordered  SIGNAL master1_ready WAIT_FOR signal_never_arrives";
+--send XA COMMIT 'C'
+
+--connection default
+SET DEBUG_SYNC= "now WAIT_FOR master1_ready";
 --source include/kill_and_restart_mysqld.inc
+--disconnect master3
 
---echo # xid '3' must be committed
+--echo # Proof $case:
+--echo # xid 'C' must be recovered to commit
 XA RECOVER;
 select count(*) = 2 from t31949;
 
---connect(con4,127.0.0.1,root,,test,$MASTER_MYPORT,)
---let $conn1_id=`SELECT connection_id()`
-XA START '4';
+
+--inc $case
+--echo # case $case. XA-ROLLBACK from an external connection  after binlogging recovers to COMPLETE
+
+--connect(conn$case,localhost,root,,)
+--let $conn_id=`SELECT connection_id()`
+XA START 'D';
 INSERT INTO t31949 SET a=4;
-XA END '4';
-XA PREPARE '4';
+XA END 'D';
+XA PREPARE 'D';
+--disconnect conn$case
 
---disconnect con4
 --connection default
---let $wait_condition= SELECT count(*) = 0 FROM v_processlist WHERE PROCESSLIST_ID = $conn1_id
+--let $wait_condition= SELECT count(*) = 0 FROM v_processlist WHERE PROCESSLIST_ID = $conn_id
 --source include/wait_condition.inc
 
---echo # ROLLBACK from an external connection
-XA ROLLBACK '4';
+--connect(master4,localhost,root,,)
+SET DEBUG_SYNC= "commit_before_get_LOCK_commit_ordered  SIGNAL master1_ready WAIT_FOR signal_never_arrives";
+--send XA ROLLBACK 'D'
+
+--connection default
+SET DEBUG_SYNC= "now WAIT_FOR master1_ready";
 --source include/kill_and_restart_mysqld.inc
+--disconnect master4
 
---echo # xid '4' must be rolled back
+--echo # Proof $case:
+--echo # xid 'D' must be recovered to roll back
 XA RECOVER;
 select count(*) = 2 from t31949;
 
-
 # cleanup
 DROP TABLE t31949;
 DROP VIEW v_processlist;

mysql-test/suite/rpl/r/rpl_semi_sync_fail_over.result

@@ -35,11 +35,16 @@ connection server_1;
 # 0-1-4 (Not committed)  |  0-1-4 (Received through semi-sync    |
 #                        |         replication and applied)      |
 #=================================================================
+connect  conn_client_2,127.0.0.1,root,,test,$SERVER_MYPORT_1,;
+connect  conn_client_3,127.0.0.1,root,,test,$SERVER_MYPORT_1,;
 connect  conn_client,127.0.0.1,root,,test,$SERVER_MYPORT_1,;
 SET DEBUG_SYNC= "commit_after_release_LOCK_after_binlog_sync SIGNAL con1_ready WAIT_FOR con1_go";
 INSERT INTO t1 VALUES (2, REPEAT("x", 4100));
 connection server_1;
 SET DEBUG_SYNC= "now WAIT_FOR con1_ready";
+select @@global.gtid_binlog_pos as "server_1 state";
+server_1 state
+0-1-4
 # Kill the server
 connection server_2;
 include/wait_for_slave_param.inc [Slave_SQL_Running_State]
@@ -48,11 +53,13 @@ include/assert.inc [Table t1 should have 2 rows.]
 SELECT @@GLOBAL.gtid_current_pos;
 @@GLOBAL.gtid_current_pos
 0-1-4
-# restart: --skip-slave-start=1 --rpl-semi-sync-slave-enabled=1
+# restart: --skip-slave-start=1 --rpl-semi-sync-slave-enabled=1  --max-binlog-size=4096
 connection server_1;
 include/assert.inc [Table t1 should have 1 rows.]
 FOUND 1 /truncated binlog file:.*master.*000001/ in mysqld.1.err
 disconnect conn_client;
+disconnect conn_client_2;
+disconnect conn_client_3;
 connection server_2;
 set global rpl_semi_sync_master_enabled = 1;
 set global rpl_semi_sync_master_wait_point=AFTER_SYNC;
@@ -60,6 +67,9 @@ connection server_1;
 CHANGE MASTER TO master_host='127.0.0.1', master_port=$new_master_port, master_user='root', master_use_gtid=SLAVE_POS;
 set global rpl_semi_sync_slave_enabled = 1;
 set @@global.gtid_slave_pos=@@global.gtid_binlog_pos;
+select @@global.gtid_slave_pos;
+@@global.gtid_slave_pos
+0-1-3
 include/start_slave.inc
 #
 # Server_2 promoted as master will send 0-1-4 to new slave Server_1
@@ -109,10 +119,12 @@ connection server_2;
 # 0-2-7 (Not commited)  |  0-2-7 (Received through semi-sync     |
 #                       |         replication and applied)       |
 #=================================================================
+connect  conn_client_2,127.0.0.1,root,,test,$SERVER_MYPORT_2,;
+connect  conn_client_3,127.0.0.1,root,,test,$SERVER_MYPORT_2,;
 connect  conn_client,127.0.0.1,root,,test,$SERVER_MYPORT_2,;
 SET DEBUG_SYNC= "commit_before_get_LOCK_commit_ordered SIGNAL con1_ready WAIT_FOR con1_go";
 SET STATEMENT server_id=1 FOR INSERT INTO t1 VALUES (4, REPEAT("x", 4100));
-connect  conn_client_2,127.0.0.1,root,,test,$SERVER_MYPORT_2,;
+connection conn_client_2;
 SET DEBUG_SYNC= "now WAIT_FOR con1_ready";
 SET GLOBAL debug_dbug="d,Notify_binlog_EOF";
 INSERT INTO t1 VALUES (5, REPEAT("x", 4100));
@@ -126,11 +138,13 @@ include/assert.inc [Table t1 should have 5 rows.]
 SELECT @@GLOBAL.gtid_current_pos;
 @@GLOBAL.gtid_current_pos
 0-2-7
-# restart: --skip-slave-start=1 --rpl-semi-sync-slave-enabled=1
+# restart: --skip-slave-start=1 --rpl-semi-sync-slave-enabled=1  --max-binlog-size=4096
 connection server_2;
 include/assert.inc [Table t1 should have 3 rows.]
 FOUND 1 /truncated binlog file:.*slave.*000002.* to remove transactions starting from GTID 0-1-6/ in mysqld.2.err
 disconnect conn_client;
+disconnect conn_client_2;
+disconnect conn_client_3;
 connection server_1;
 set global rpl_semi_sync_master_enabled = 1;
 set global rpl_semi_sync_master_wait_point=AFTER_SYNC;
@@ -138,6 +152,9 @@ connection server_2;
 CHANGE MASTER TO master_host='127.0.0.1', master_port=$new_master_port, master_user='root', master_use_gtid=SLAVE_POS;
 set global rpl_semi_sync_slave_enabled = 1;
 set @@global.gtid_slave_pos=@@global.gtid_binlog_pos;
+select @@global.gtid_slave_pos;
+@@global.gtid_slave_pos
+0-2-5
 include/start_slave.inc
 #
 # Server_1 promoted as master will send 0-1-6  and 0-2-7 to slave Server_2
@@ -188,15 +205,22 @@ connection server_1;
 # 0-1-10 (Not commited -  |                                      |
 #  never sent to slave)   |                                      |
 #=================================================================
+connect  conn_client_2,127.0.0.1,root,,test,$SERVER_MYPORT_1,;
+connect  conn_client_3,127.0.0.1,root,,test,$SERVER_MYPORT_1,;
 connect  conn_client,127.0.0.1,root,,test,$SERVER_MYPORT_1,;
 SET DEBUG_SYNC= "commit_before_get_LOCK_commit_ordered SIGNAL con1_ready WAIT_FOR con1_go";
 INSERT INTO t1 VALUES (7, REPEAT("x", 4100));
-connect  conn_client_3,127.0.0.1,root,,test,$SERVER_MYPORT_1,;
+connection conn_client_3;
 SET DEBUG_SYNC= "now WAIT_FOR con1_ready";
+include/save_master_gtid.inc
+connection server_2;
+include/sync_with_master_gtid.inc
+connection conn_client_3;
 SET DEBUG_SYNC= "commit_before_update_binlog_end_pos SIGNAL con3_ready WAIT_FOR con1_go";
 INSERT INTO t1 VALUES (8, REPEAT("x", 4100));
 connection server_1;
 SET DEBUG_SYNC= "now WAIT_FOR con3_ready";
+connection server_1;
 # Kill the server
 connection server_2;
 include/wait_for_slave_param.inc [Slave_SQL_Running_State]
@@ -205,11 +229,13 @@ include/assert.inc [Table t1 should have 7 rows.]
 SELECT @@GLOBAL.gtid_current_pos;
 @@GLOBAL.gtid_current_pos
 0-1-9
-# restart: --skip-slave-start=1 --rpl-semi-sync-slave-enabled=1
+# restart: --skip-slave-start=1 --rpl-semi-sync-slave-enabled=1  --max-binlog-size=4096
 connection server_1;
 include/assert.inc [Table t1 should have 6 rows.]
-FOUND 1 /truncated binlog file:.*master.*000002.* to remove transactions starting from GTID 0-1-9/ in mysqld.1.err
+FOUND 1 /truncated binlog file:.*master.* to remove transactions starting from GTID 0-1-9/ in mysqld.1.err
 disconnect conn_client;
+disconnect conn_client_2;
+disconnect conn_client_3;
 connection server_2;
 set global rpl_semi_sync_master_enabled = 1;
 set global rpl_semi_sync_master_wait_point=AFTER_SYNC;
@@ -217,6 +243,9 @@ connection server_1;
 CHANGE MASTER TO master_host='127.0.0.1', master_port=$new_master_port, master_user='root', master_use_gtid=SLAVE_POS;
 set global rpl_semi_sync_slave_enabled = 1;
 set @@global.gtid_slave_pos=@@global.gtid_binlog_pos;
+select @@global.gtid_slave_pos;
+@@global.gtid_slave_pos
+0-1-8
 include/start_slave.inc
 #
 # Server_2 promoted as master will send 0-1-9 to slave Server_1
@@ -249,10 +278,11 @@ gtid_binlog_pos	0-2-10
 SHOW VARIABLES LIKE 'gtid_binlog_state';
 Variable_name	Value
 gtid_binlog_state	0-1-9,0-2-10
+include/stop_slave.inc
 #
 # Cleanup
 #
-include/stop_slave.inc
+connection server_1;
 set global rpl_semi_sync_slave_enabled = 0;
 set global rpl_semi_sync_master_enabled = 0;
 set global rpl_semi_sync_master_wait_point=default;

mysql-test/suite/rpl/t/rpl_semi_sync_crash.inc

+if ($primary)
+{
+  --eval select @@global.gtid_binlog_pos as "initial $primary state"
+}
 if ($failover_to_slave)
 {
  --let $server_to_crash=1
  --let $server_to_promote=2
  --let $new_master_port=$SERVER_MYPORT_2
  --let $client_port=$SERVER_MYPORT_1
-
+ --connect (conn_client_2,127.0.0.1,root,,test,$SERVER_MYPORT_1,)
+ --connect (conn_client_3,127.0.0.1,root,,test,$SERVER_MYPORT_1,)
  --connect (conn_client,127.0.0.1,root,,test,$SERVER_MYPORT_1,)
 }
 if (!$failover_to_slave)
@@ -14,6 +19,8 @@ if (!$failover_to_slave)
  --let $new_master_port=$SERVER_MYPORT_1
  --let $client_port=$SERVER_MYPORT_2
 
+ --connect (conn_client_2,127.0.0.1,root,,test,$SERVER_MYPORT_2,)
+ --connect (conn_client_3,127.0.0.1,root,,test,$SERVER_MYPORT_2,)
  --connect (conn_client,127.0.0.1,root,,test,$SERVER_MYPORT_2,)
 }
 
@@ -23,10 +30,15 @@ if (!$failover_to_slave)
 
 if ($case == 1)
 {
+  if ($pre_query_to_crash)
+  {
+    --eval $pre_query_to_crash
+  }
   SET DEBUG_SYNC= "commit_after_release_LOCK_after_binlog_sync SIGNAL con1_ready WAIT_FOR con1_go";
   --send_eval $query_to_crash
   --connection server_$server_to_crash
   SET DEBUG_SYNC= "now WAIT_FOR con1_ready";
+  --eval select @@global.gtid_binlog_pos as "server_$server_to_crash state"
   --source include/kill_mysqld.inc
 }
 
@@ -35,7 +47,7 @@ if ($case == 2)
 {
   SET DEBUG_SYNC= "commit_before_get_LOCK_commit_ordered SIGNAL con1_ready WAIT_FOR con1_go";
   --send_eval $query_to_crash
-  --connect (conn_client_2,127.0.0.1,root,,test,$SERVER_MYPORT_2,)
+  --connection conn_client_2
   # use the same signal with $query_to_crash
   SET DEBUG_SYNC= "now WAIT_FOR con1_ready";
   SET GLOBAL debug_dbug="d,Notify_binlog_EOF";
@@ -48,15 +60,40 @@ if ($case == 2)
 # complicate recovery with an extra binlog file
 if ($case == 3)
 {
+  if ($pre_query_to_crash)
+  {
+    --connection conn_client_3
+    --eval $pre_query_to_crash
+  if ($pre_query_to_crash_2)
+  {
+    --eval $pre_query_to_crash_2
+  }
+    --connection conn_client
+  }
   SET DEBUG_SYNC= "commit_before_get_LOCK_commit_ordered SIGNAL con1_ready WAIT_FOR con1_go";
   --send_eval $query_to_crash
-  --connect (conn_client_3,127.0.0.1,root,,test,$SERVER_MYPORT_1,)
+  --connection conn_client_3
   SET DEBUG_SYNC= "now WAIT_FOR con1_ready";
+  --source include/save_master_gtid.inc
+
+# This guarantees the $expected_rows_on_slave assert
+  --connection server_$server_to_promote
+  --source include/sync_with_master_gtid.inc
+
+  --connection conn_client_3
   # use the same signal with $query_to_crash
   SET DEBUG_SYNC= "commit_before_update_binlog_end_pos SIGNAL con3_ready WAIT_FOR con1_go";
   --send_eval $query2_to_crash
   --connection server_$server_to_crash
   SET DEBUG_SYNC= "now WAIT_FOR con3_ready";
+--disable_query_log
+--disable_result_log
+  if ($debug_pre_crash)
+  {
+      --eval $debug_pre_crash
+  }
+--enable_result_log
+--enable_query_log
    --source include/kill_mysqld.inc
 }
 
@@ -67,14 +104,13 @@ source include/wait_for_slave_param.inc;
 
 --error 2003
 --source include/stop_slave.inc
-
 --let $assert_cond= COUNT(*) = $expected_rows_on_slave FROM t1
 --let $assert_text= Table t1 should have $expected_rows_on_slave rows.
 --source include/assert.inc
 
 SELECT @@GLOBAL.gtid_current_pos;
 
---let $restart_parameters=--skip-slave-start=1 --rpl-semi-sync-slave-enabled=1
+--let $restart_parameters=--skip-slave-start=1 --rpl-semi-sync-slave-enabled=1 $dbug_restart --max-binlog-size=4096
 --let $allow_rpl_inited=1
 --source include/start_mysqld.inc
 --connection server_$server_to_crash
@@ -92,6 +128,8 @@ let $log_error_ = $MYSQLTEST_VARDIR/log/mysqld.$server_to_crash.err;
 --source include/search_pattern_in_file.inc
 
 --disconnect conn_client
+--disconnect conn_client_2
+--disconnect conn_client_3
 
 #
 # FAIL OVER now to new master
@@ -109,4 +147,12 @@ if (`select $server_to_crash = 2`)
 evalp CHANGE MASTER TO master_host='127.0.0.1', master_port=$new_master_port, master_user='root', master_use_gtid=SLAVE_POS;
 set global rpl_semi_sync_slave_enabled = 1;
 set @@global.gtid_slave_pos=@@global.gtid_binlog_pos;
+select @@global.gtid_slave_pos;
 --source include/start_slave.inc
+
+if ($primary)
+{
+  --let $tmp=$primary
+  --let $primary=$replica
+  --let $replica=$tmp
+}

mysql-test/suite/rpl/t/rpl_semi_sync_fail_over.inc

+--connection server_2
+--sync_with_master
+
+--connection server_1
+--let $case = 1
+--echo #
+--echo #  Case:$case
+--echo #
+--echo # CRASH the original master, and FAILOVER to the new
+# value 1 for server id 1 -> 2 failover
+--let $failover_to_slave=1
+--let $query_to_crash= INSERT INTO t1 VALUES (2, REPEAT("x", 4100))
+--echo # $query_to_crash
+--echo # Row - 2 will be in master's binlog but not committed, gets replicated
+--echo # to slave and applied. On crash master should have 1 row and slave
+--echo # should have 2 rows.
+--echo #
+--echo # Expected State post crash:
+--echo #=================================================================
+--echo # Master                 |  Slave                                |
+--echo # 0-1-4 (Not committed)  |  0-1-4 (Received through semi-sync    |
+--echo #                        |         replication and applied)      |
+--echo #=================================================================
+--let $log_search_pattern=truncated binlog file:.*master.*000001
+--let $expected_rows_on_master= 1
+--let $expected_rows_on_slave= 2
+--source rpl_semi_sync_crash.inc
+
+--echo #
+--echo # Server_2 promoted as master will send 0-1-4 to new slave Server_1
+--echo #
+--connection server_2
+--let $rows_so_far=3
+--eval INSERT INTO t1 VALUES ($rows_so_far, 'dummy3')
+--save_master_pos
+--echo # The gtid state on current master must be equal to ...
+SHOW VARIABLES LIKE 'gtid_binlog_pos';
+SHOW VARIABLES LIKE 'gtid_binlog_state';
+SHOW VARIABLES LIKE 'gtid_slave_pos';
+
+--connection server_1
+--sync_with_master
+--eval SELECT COUNT(*) = $rows_so_far as 'true' FROM t1
+--echo # ... the gtid states on the slave:
+SHOW VARIABLES LIKE 'gtid_slave_pos';
+SHOW VARIABLES LIKE 'gtid_binlog_pos';
+SHOW VARIABLES LIKE 'gtid_binlog_state';
+
+--connection server_2
+--let $case = 2
+--echo #
+--echo #  Case:$case
+--echo #
+--echo # CRASH the new master, and FAILOVER back to the original
+# value 0 for the reverse server id 2 -> 1 failover
+--let $failover_to_slave=0
+# Additionally through "foreign" server_id verify MDEV-27760's acceptance
+# policy on the recient (to be promoted into master) server.
+--let $query_to_crash = SET STATEMENT server_id=1 FOR INSERT INTO t1 VALUES (4, REPEAT("x", 4100))
+--let $query2_to_crash= INSERT INTO t1 VALUES (5, REPEAT("x", 4100))
+--echo # $query_to_crash
+--echo # $query2_to_crash
+--echo # Rows 4 and 5 will be in master's binlog but not committed, they get
+--echo # replicated to slave and applied. On crash master should have 3 rows
+--echo # and slave should have 5 rows.
+--echo #
+--echo # Expected State post crash:
+--echo #=================================================================
+--echo # Master                |  Slave                                 |
+--echo # 0-1-6 (Not commited)  |  0-1-6 (Received through semi-sync     |
+--echo #                       |         replication and applied)       |
+--echo # 0-2-7 (Not commited)  |  0-2-7 (Received through semi-sync     |
+--echo #                       |         replication and applied)       |
+--echo #=================================================================
+--let $log_search_pattern=truncated binlog file:.*slave.*000002.* to remove transactions starting from GTID 0-1-6
+--let $expected_rows_on_master= 3
+--let $expected_rows_on_slave= 5
+--source rpl_semi_sync_crash.inc
+
+--echo #
+--echo # Server_1 promoted as master will send 0-1-6  and 0-2-7 to slave Server_2
+--echo #
+--connection server_1
+--let $rows_so_far=6
+--eval INSERT INTO t1 VALUES ($rows_so_far, 'dummy6')
+--save_master_pos
+--echo # The gtid state on current master must be equal to ...
+SHOW VARIABLES LIKE 'gtid_binlog_pos';
+SHOW VARIABLES LIKE 'gtid_binlog_state';
+SHOW VARIABLES LIKE 'gtid_slave_pos';
+
+--connection server_2
+--sync_with_master
+--eval SELECT COUNT(*) = $rows_so_far as 'true' FROM t1
+--echo # ... the gtid states on the slave:
+SHOW VARIABLES LIKE 'gtid_slave_pos';
+SHOW VARIABLES LIKE 'gtid_binlog_pos';
+SHOW VARIABLES LIKE 'gtid_binlog_state';
+
+--let $diff_tables=server_1:t1, server_2:t1
+--source include/diff_tables.inc
+
+--connection server_1
+--let $case = 3
+--echo #
+--echo #  Case:$case
+--echo #
+--echo # CRASH the master and FAILOVER to slave
+--let $failover_to_slave=1
+--let $query_to_crash = INSERT INTO t1 VALUES (7, REPEAT("x", 4100))
+--let $query2_to_crash= INSERT INTO t1 VALUES (8, REPEAT("x", 4100))
+--echo # $query_to_crash
+--echo # $query2_to_crash
+--echo # Rows 7 and 8 will be in master's binlog but not committed, only 7
+--echo # gets replicated to slave and applied. On crash master should have 6
+--echo # rows and slave should have 7 rows.
+--echo #
+--echo # Expected State post crash:
+--echo #=================================================================
+--echo # Master                  | Slave                                |
+--echo # 0-1-9  (Not commited)   | 0-1-9 (Received through semi-sync    |
+--echo #                         |  replication and applied)            |
+--echo # 0-1-10 (Not commited -  |                                      |
+--echo #  never sent to slave)   |                                      |
+--echo #=================================================================
+--let $log_search_pattern=truncated binlog file:.*master.* to remove transactions starting from GTID 0-1-9
+--let $expected_rows_on_master= 6
+--let $expected_rows_on_slave= 7
+--source rpl_semi_sync_crash.inc
+
+--echo #
+--echo # Server_2 promoted as master will send 0-1-9 to slave Server_1
+--echo #
+--connection server_2
+--let $rows_so_far=8
+--eval INSERT INTO t1 VALUES ($rows_so_far, 'Done')
+--source include/save_master_gtid.inc
+--echo # The gtid state on current master must be equal to ...
+SHOW VARIABLES LIKE 'gtid_binlog_pos';
+SHOW VARIABLES LIKE 'gtid_binlog_state';
+SHOW VARIABLES LIKE 'gtid_slave_pos';
+
+--connection server_1
+--source include/sync_with_master_gtid.inc
+--eval SELECT COUNT(*) = $rows_so_far as 'true' FROM t1
+--echo # ... the gtid states on the slave:
+SHOW VARIABLES LIKE 'gtid_slave_pos';
+SHOW VARIABLES LIKE 'gtid_binlog_pos';
+SHOW VARIABLES LIKE 'gtid_binlog_state';
+--source include/stop_slave.inc

mysql-test/suite/rpl/t/rpl_semi_sync_fail_over.test

@@ -10,212 +10,7 @@
 --source include/have_binlog_format_row.inc
 --source include/master-slave.inc
 
-# Initial slave
---connection server_2
---source include/stop_slave.inc
-
-# Initial master
---connection server_1
-RESET MASTER;
-SET @@global.max_binlog_size= 4096;
-
---connection server_2
-RESET MASTER;
-SET @@global.max_binlog_size= 4096;
-set @@global.rpl_semi_sync_slave_enabled = 1;
-set @@global.gtid_slave_pos = "";
-CHANGE MASTER TO master_use_gtid= slave_pos;
---source include/start_slave.inc
-
---connection server_1
-ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB;
-set @@global.rpl_semi_sync_master_enabled = 1;
-set @@global.rpl_semi_sync_master_wait_point=AFTER_SYNC;
-
-CREATE TABLE t1 (a INT PRIMARY KEY, b MEDIUMTEXT) ENGINE=Innodb;
-INSERT INTO t1 VALUES (1, 'dummy1');
---save_master_pos
-
---connection server_2
---sync_with_master
-
---connection server_1
---let $case = 1
---echo #
---echo #  Case:$case
---echo #
---echo # CRASH the original master, and FAILOVER to the new
-# value 1 for server id 1 -> 2 failover
---let $failover_to_slave=1
---let $query_to_crash= INSERT INTO t1 VALUES (2, REPEAT("x", 4100))
---echo # $query_to_crash
---echo # Row - 2 will be in master's binlog but not committed, gets replicated
---echo # to slave and applied. On crash master should have 1 row and slave
---echo # should have 2 rows.
---echo #
---echo # Expected State post crash:
---echo #=================================================================
---echo # Master                 |  Slave                                |
---echo # 0-1-4 (Not committed)  |  0-1-4 (Received through semi-sync    |
---echo #                        |         replication and applied)      |
---echo #=================================================================
---let $log_search_pattern=truncated binlog file:.*master.*000001
---let $expected_rows_on_master= 1
---let $expected_rows_on_slave= 2
---source rpl_semi_sync_crash.inc
-
---echo #
---echo # Server_2 promoted as master will send 0-1-4 to new slave Server_1
---echo #
---connection server_2
---let $rows_so_far=3
---eval INSERT INTO t1 VALUES ($rows_so_far, 'dummy3')
---save_master_pos
---echo # The gtid state on current master must be equal to ...
-SHOW VARIABLES LIKE 'gtid_binlog_pos';
-SHOW VARIABLES LIKE 'gtid_binlog_state';
-SHOW VARIABLES LIKE 'gtid_slave_pos';
-
---connection server_1
---sync_with_master
---eval SELECT COUNT(*) = $rows_so_far as 'true' FROM t1
---echo # ... the gtid states on the slave:
-SHOW VARIABLES LIKE 'gtid_slave_pos';
-SHOW VARIABLES LIKE 'gtid_binlog_pos';
-SHOW VARIABLES LIKE 'gtid_binlog_state';
-
---connection server_2
---let $case = 2
---echo #
---echo #  Case:$case
---echo #
---echo # CRASH the new master, and FAILOVER back to the original
-# value 0 for the reverse server id 2 -> 1 failover
---let $failover_to_slave=0
-# Additionally through "foreign" server_id verify MDEV-27760's acceptance
-# policy on the recient (to be promoted into master) server.
---let $query_to_crash = SET STATEMENT server_id=1 FOR INSERT INTO t1 VALUES (4, REPEAT("x", 4100))
---let $query2_to_crash= INSERT INTO t1 VALUES (5, REPEAT("x", 4100))
---echo # $query_to_crash
---echo # $query2_to_crash
---echo # Rows 4 and 5 will be in master's binlog but not committed, they get
---echo # replicated to slave and applied. On crash master should have 3 rows
---echo # and slave should have 5 rows.
---echo #
---echo # Expected State post crash:
---echo #=================================================================
---echo # Master                |  Slave                                 |
---echo # 0-1-6 (Not commited)  |  0-1-6 (Received through semi-sync     |
---echo #                       |         replication and applied)       |
---echo # 0-2-7 (Not commited)  |  0-2-7 (Received through semi-sync     |
---echo #                       |         replication and applied)       |
---echo #=================================================================
---let $log_search_pattern=truncated binlog file:.*slave.*000002.* to remove transactions starting from GTID 0-1-6
---let $expected_rows_on_master= 3
---let $expected_rows_on_slave= 5
---source rpl_semi_sync_crash.inc
-
---echo #
---echo # Server_1 promoted as master will send 0-1-6  and 0-2-7 to slave Server_2
---echo #
---connection server_1
---let $rows_so_far=6
---eval INSERT INTO t1 VALUES ($rows_so_far, 'dummy6')
---save_master_pos
---echo # The gtid state on current master must be equal to ...
-SHOW VARIABLES LIKE 'gtid_binlog_pos';
-SHOW VARIABLES LIKE 'gtid_binlog_state';
-SHOW VARIABLES LIKE 'gtid_slave_pos';
-
---connection server_2
---sync_with_master
---eval SELECT COUNT(*) = $rows_so_far as 'true' FROM t1
---echo # ... the gtid states on the slave:
-SHOW VARIABLES LIKE 'gtid_slave_pos';
-SHOW VARIABLES LIKE 'gtid_binlog_pos';
-SHOW VARIABLES LIKE 'gtid_binlog_state';
-
---let $diff_tables=server_1:t1, server_2:t1
---source include/diff_tables.inc
-
---connection server_1
---let $case = 3
---echo #
---echo #  Case:$case
---echo #
---echo # CRASH the master and FAILOVER to slave
---let $failover_to_slave=1
---let $query_to_crash = INSERT INTO t1 VALUES (7, REPEAT("x", 4100))
---let $query2_to_crash= INSERT INTO t1 VALUES (8, REPEAT("x", 4100))
---echo # $query_to_crash
---echo # $query2_to_crash
---echo # Rows 7 and 8 will be in master's binlog but not committed, only 7
---echo # gets replicated to slave and applied. On crash master should have 6
---echo # rows and slave should have 7 rows.
---echo #
---echo # Expected State post crash:
---echo #=================================================================
---echo # Master                  | Slave                                |
---echo # 0-1-9  (Not commited)   | 0-1-9 (Received through semi-sync    |
---echo #                         |  replication and applied)            |
---echo # 0-1-10 (Not commited -  |                                      |
---echo #  never sent to slave)   |                                      |
---echo #=================================================================
---let $log_search_pattern=truncated binlog file:.*master.*000002.* to remove transactions starting from GTID 0-1-9
---let $expected_rows_on_master= 6
---let $expected_rows_on_slave= 7
---source rpl_semi_sync_crash.inc
-
---echo #
---echo # Server_2 promoted as master will send 0-1-9 to slave Server_1
---echo #
---connection server_2
---let $rows_so_far=8
---eval INSERT INTO t1 VALUES ($rows_so_far, 'Done')
---source include/save_master_gtid.inc
---echo # The gtid state on current master must be equal to ...
-SHOW VARIABLES LIKE 'gtid_binlog_pos';
-SHOW VARIABLES LIKE 'gtid_binlog_state';
-SHOW VARIABLES LIKE 'gtid_slave_pos';
-
---connection server_1
---source include/sync_with_master_gtid.inc
---eval SELECT COUNT(*) = $rows_so_far as 'true' FROM t1
---echo # ... the gtid states on the slave:
-SHOW VARIABLES LIKE 'gtid_slave_pos';
-SHOW VARIABLES LIKE 'gtid_binlog_pos';
-SHOW VARIABLES LIKE 'gtid_binlog_state';
-
---echo #
---echo # Cleanup
---echo #
-
---source include/stop_slave.inc
-set global rpl_semi_sync_slave_enabled = 0;
-set global rpl_semi_sync_master_enabled = 0;
-set global rpl_semi_sync_master_wait_point=default;
-RESET MASTER;
-RESET SLAVE;
-
---connection server_2
-RESET MASTER;
-RESET SLAVE;
-set @@global.rpl_semi_sync_master_enabled = 0;
-set @@global.rpl_semi_sync_slave_enabled = 0;
-set @@global.rpl_semi_sync_master_wait_point=default;
-evalp CHANGE MASTER TO master_host='127.0.0.1', master_port=$SERVER_MYPORT_1, master_user='root', master_use_gtid=SLAVE_POS;
-set @@global.gtid_slave_pos=@@global.gtid_binlog_pos;
---source include/start_slave.inc
-
---connection server_1
-DROP TABLE t1;
---save_master_pos
-
---connection server_2
---sync_with_master
-
-connection default;
---enable_reconnect
---source include/wait_until_connected_again.inc
+--let $scenario=rpl_semi_sync_fail_over.inc
+--source rpl_semi_sync_fail_over_gen.inc
 
 --source include/rpl_end.inc

mysql-test/suite/rpl/t/rpl_semi_sync_fail_over_gen.inc

+# Initial slave
+--connection server_2
+--source include/stop_slave.inc
+
+# Initial master
+--connection server_1
+RESET MASTER;
+SET @@global.max_binlog_size= 4096;
+
+--connection server_2
+RESET MASTER;
+SET @@global.max_binlog_size= 4096;
+set @@global.rpl_semi_sync_slave_enabled = 1;
+set @@global.gtid_slave_pos = "";
+CHANGE MASTER TO master_use_gtid= slave_pos;
+--source include/start_slave.inc
+
+--connection server_1
+ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB;
+set @@global.rpl_semi_sync_master_enabled = 1;
+set @@global.rpl_semi_sync_master_wait_point=AFTER_SYNC;
+
+CREATE TABLE t1 (a INT PRIMARY KEY, b MEDIUMTEXT) ENGINE=Innodb;
+INSERT INTO t1 VALUES (1, 'dummy1');
+--save_master_pos
+
+# the actual slave server will have stopped the slave service at exit
+--source $scenario
+
+--echo #
+--echo # Cleanup
+--echo #
+# current slave
+--connection server_1
+set global rpl_semi_sync_slave_enabled = 0;
+set global rpl_semi_sync_master_enabled = 0;
+set global rpl_semi_sync_master_wait_point=default;
+RESET MASTER;
+RESET SLAVE;
+
+--connection server_2
+RESET MASTER;
+RESET SLAVE;
+set @@global.rpl_semi_sync_master_enabled = 0;
+set @@global.rpl_semi_sync_slave_enabled = 0;
+set @@global.rpl_semi_sync_master_wait_point=default;
+evalp CHANGE MASTER TO master_host='127.0.0.1', master_port=$SERVER_MYPORT_1, master_user='root', master_use_gtid=SLAVE_POS;
+set @@global.gtid_slave_pos=@@global.gtid_binlog_pos;
+--source include/start_slave.inc
+
+--connection server_1
+DROP TABLE t1;
+--save_master_pos
+
+--connection server_2
+--sync_with_master
+
+connection default;
+--enable_reconnect
+--source include/wait_until_connected_again.inc
+

mysql-test/suite/rpl/t/rpl_semi_sync_fail_over_xa.cnf

+!include suite/rpl/rpl_1slave_base.cnf
+!include include/default_client.cnf
+
+
+[mysqld.1]
+log-slave-updates
+gtid-strict-mode=1
+sync-binlog=1
+
+[mysqld.2]
+log-slave-updates
+gtid-strict-mode=1
+sync-binlog=1

mysql-test/suite/rpl/t/rpl_semi_sync_fail_over_xa.test

+# ==== References ====
+#
+# MDEV-21117 recovery for --rpl-semi-sync-slave-enabled server
+# MDEV-27760 event may non stop replicate in circular semisync setup
+#
+
+--source include/have_innodb.inc
+--source include/have_debug.inc
+--source include/have_debug_sync.inc
+--source include/have_binlog_format_row.inc
+--source include/master-slave.inc
+
+--let $scenario=rpl_semi_sync_fail_over_xa.inc
+--source rpl_semi_sync_fail_over_gen.inc
+
+--source include/rpl_end.inc

sql/handler.cc

@@ -2545,6 +2545,7 @@ struct xarecover_st
   int len, found_foreign_xids, found_my_xids;
   XID *list;
   HASH *commit_list;
+  HASH *xa_prepared_list; // prepared user xa list
   bool dry_run;
   MEM_ROOT *mem_root;
   bool error;
@@ -2553,13 +2554,14 @@ struct xarecover_st
 /**
   Inserts a new hash member.
 
-  returns a successfully created and inserted @c xid_recovery_member
+  @return a successfully created and inserted @c xid_recovery_member
             into hash @c hash_arg,
           or NULL.
 */
 static xid_recovery_member*
 xid_member_insert(HASH *hash_arg, my_xid xid_arg, MEM_ROOT *ptr_mem_root,
-                  XID *full_xid_arg, decltype(::server_id) server_id_arg)
+                  XID *full_xid_arg, decltype(::server_id) server_id_arg,
+                  xid_recovery_member::enum_xa_binlog_state xa_binlog_state_arg)
 {
   xid_recovery_member *member= (xid_recovery_member *)
     alloc_root(ptr_mem_root, sizeof(xid_recovery_member));
@@ -2573,32 +2575,61 @@ xid_member_insert(HASH *hash_arg, my_xid xid_arg, MEM_ROOT *ptr_mem_root,
 
   if (full_xid_arg)
     *xid_full= *full_xid_arg;
-  *member= xid_recovery_member(xid_arg, 1, false, xid_full, server_id_arg);
-
+  *member=
+    xid_recovery_member(xid_arg,
+                        xa_binlog_state_arg == xid_recovery_member::XA_NONE ?
+                        1 : 0, false, xid_full, server_id_arg,
+                        xa_binlog_state_arg);
   return
     my_hash_insert(hash_arg, (uchar*) member) ? NULL : member;
 }
 
-/*
+/**
   Inserts a new or updates an existing hash member to increment
-  the member's prepare counter.
+  the member's engine prepare counter or binlog transaction state.
+  For normal transactions @c xid_arg must be non-zero, and it's zero
+  for the user XA.
+  @c xa_binlog_state_arg is meaningful only for the latter.
 
-  returns false  on success,
-           true   otherwise.
+  @return xid_recovery_member pointer to when success,
+          NULL                           otherwise.
 */
-static bool xid_member_replace(HASH *hash_arg, my_xid xid_arg,
+xid_recovery_member*
+xid_member_replace(HASH *hash_arg, my_xid xid_arg,
                    MEM_ROOT *ptr_mem_root,
                    XID *full_xid_arg,
-                               decltype(::server_id) server_id_arg)
+                   decltype(::server_id) server_id_arg,
+                   xid_recovery_member::enum_xa_binlog_state
+                   xa_binlog_state_arg)
 {
   xid_recovery_member* member;
   if ((member= (xid_recovery_member *)
-       my_hash_search(hash_arg, (uchar *)& xid_arg, sizeof(xid_arg))))
+       my_hash_search(hash_arg,
+                      (xid_arg == 0) ? full_xid_arg->key() : (uchar *)& xid_arg,
+                      (xid_arg == 0) ?
+                      full_xid_arg->key_length() : sizeof(xid_arg))))
+  {
+    if (xa_binlog_state_arg == xid_recovery_member::XA_NONE)
+    {
+      DBUG_ASSERT(member->in_engine_prepare > 0);
+
       member->in_engine_prepare++;
+    }
     else
-    member= xid_member_insert(hash_arg, xid_arg, ptr_mem_root, full_xid_arg,  server_id_arg);
+    {
+      DBUG_ASSERT(member->xid == 0);
+      DBUG_ASSERT(xa_binlog_state_arg == xid_recovery_member::XA_COMPLETE ||
+                  xa_binlog_state_arg == xid_recovery_member::XA_PREPARE);
 
-  return member == NULL;
+      member->is_state_valid= member->xa_binlog_state != xa_binlog_state_arg;
+      member->xa_binlog_state= xa_binlog_state_arg;
+    }
+  }
+  else
+    member= xid_member_insert(hash_arg, xid_arg, ptr_mem_root, full_xid_arg,
+                              server_id_arg, xa_binlog_state_arg);
+
+  return member;
 }
 
 /*
@@ -2634,6 +2665,75 @@ static bool xarecover_decide_to_commit(xid_recovery_member* member,
      true : false);
 }
 
+
+/*
+  Conduct decisions on the user XA:s according to the xa state and
+  recovery configuration.
+  ptr_commit_max is NULL implies the normal recovery in which case
+  the decision is computed along the normal transaction recovery rules.
+
+  Otherwise in the semisync recovery when XA is prepared in Engine
+  and there's no intent in binlog to complete it, then its fate depends on
+  whether its xid is present in Xid_list_log.
+  When it's there the prepared XA survives, else it's rolled back.
+  Completion operations over a prepared XA recorded in binlog are
+  decided similarly to the normal transaction case, basing on
+  the operation's binlog offset that is compared against a computed
+  truncate position.
+
+  Returns  0     as do nothing
+           -1,1  as perform according to xa_binlog_state
+*/
+static int xarecover_decide_xa(xid_recovery_member* member,
+                                Binlog_offset *ptr_commit_max)
+{
+  int rc= -1; // todo: account Xlle
+
+  if (member->xa_binlog_state > xid_recovery_member::XA_NONE)
+  {
+    if (member->xa_binlog_state == xid_recovery_member::XA_PREPARE)
+    {//if (member->in_engine_prepare == 0) {member->is_state_valid= false; return 0;}
+      if (member->decided_to_commit)
+      {
+        rc= 0;
+        xid_cache_insert(member->full_xid);
+      }
+      else if (!ptr_commit_max)
+      {
+        member->xa_binlog_state= xid_recovery_member::XA_ROLLBACK;
+        rc= -1;
+      }
+      else if (member->binlog_coord >= *ptr_commit_max)
+      {
+        member->xa_binlog_state= xid_recovery_member::XA_ROLLBACK;
+        rc= -1;
+      }
+      else
+      {
+        /*
+          In the semisync slave recovery XA-PREPARE was followed in binlog
+          with a committed transaction.
+        */
+        DBUG_ASSERT(member->in_engine_prepare > 0);
+
+        rc= 0;
+        xid_cache_insert(member->full_xid);
+      }
+    }
+    else
+    {
+      DBUG_ASSERT(member->xa_binlog_state > xid_recovery_member::XA_COMPLETE);
+
+      rc= xarecover_decide_to_commit(member, ptr_commit_max) ?
+        (member->xa_binlog_state == xid_recovery_member::XA_COMMIT ? 1 : -1) : 0;
+      if (rc == 0)
+        xid_cache_insert(member->full_xid);
+    }
+  }
+
+  return rc;
+}
+
 /*
   Helper function for xarecover_do_commit_or_rollback_handlerton.
   For a given hton decides what to do with a xid passed in the 2nd arg
@@ -2653,9 +2753,10 @@ static void xarecover_do_commit_or_rollback(handlerton *hton,
   else
     x= *member->full_xid;
 
+  if (member->xid > 0)
+  {
     rc= xarecover_decide_to_commit(member, ptr_commit_max) ?
       hton->commit_by_xid(hton, &x) : hton->rollback_by_xid(hton, &x);
-
     /*
       It's fine to have non-zero rc which would be from transaction
       non-participant hton:s.
@@ -2674,6 +2775,15 @@ static void xarecover_do_commit_or_rollback(handlerton *hton,
                               member->decided_to_commit ? "Committed" :
                               "Rolled back", (ulonglong) member->xid);
     }
+  }
+  else
+  {
+    int do_it= xarecover_decide_xa(member, ptr_commit_max);
+
+    rc = do_it == 0 ? FALSE :
+      (do_it == 1 ?
+       hton->commit_by_xid(hton, &x) : hton->rollback_by_xid(hton, &x));
+  }
 }
 
 /*
@@ -2713,7 +2823,7 @@ static my_bool xarecover_complete_and_count(void *member_arg,
   if (member->in_engine_prepare)
   {
     complete_params->count++;
-    if (global_system_variables.log_warnings > 2)
+    if (global_system_variables.log_warnings > 2) // todo: relax to info for xa
       sql_print_warning("Found prepared transaction with xid %llu",
                         (ulonglong) member->xid);
   }
@@ -2786,8 +2896,17 @@ static my_bool xarecover_handlerton(THD *unused, plugin_ref plugin,
             char buf[XIDDATASIZE*4+6];
             _db_doprnt_("ignore xid %s", xid_to_str(buf, info->list[i]));
             });
+          if (!info->dry_run)
+          {
+            /* The user xid:s are decided for insertion with binlog */
+            xid_member_replace(info->xa_prepared_list, x, info->mem_root,
+                               &info->list[i], server_id);
+          }
+          else
+          {
             xid_cache_insert(info->list + i);
             info->found_foreign_xids++;
+          }
           continue;
         }
         if (IF_WSREP(!(wsrep_emulate_bin_log &&
@@ -2808,9 +2927,10 @@ static my_bool xarecover_handlerton(THD *unused, plugin_ref plugin,
           // remember "full" xid too when it's not in mysql format.
           // Also record the transaction's original server_id. It will be used for
           // populating the input XID to be searched in hash.
-          if (xid_member_replace(info->commit_list, x, info->mem_root,
+          if (!xid_member_replace(info->commit_list, x, info->mem_root,
                                   is_server_xid? NULL : &info->list[i],
-                                 is_server_xid? info->list[i].get_trx_server_id() : server_id))
+                                  is_server_xid?
+                                  info->list[i].get_trx_server_id() : server_id))
           {
             info->error= true;
             sql_print_error("Error in memory allocation at xarecover_handlerton");
@@ -2850,12 +2970,13 @@ static my_bool xarecover_handlerton(THD *unused, plugin_ref plugin,
   return FALSE;
 }
 
-int ha_recover(HASH *commit_list, MEM_ROOT *arg_mem_root)
+int ha_recover(HASH *commit_list, HASH *xa_prepared_list, MEM_ROOT *arg_mem_root)
 {
   struct xarecover_st info;
   DBUG_ENTER("ha_recover");
   info.found_foreign_xids= info.found_my_xids= 0;
   info.commit_list= commit_list;
+  info.xa_prepared_list= xa_prepared_list;
   info.dry_run= (info.commit_list==0 && tc_heuristic_recover==0);
   info.list= NULL;
   info.mem_root= arg_mem_root;
@@ -2907,7 +3028,7 @@ int ha_recover(HASH *commit_list, MEM_ROOT *arg_mem_root)
   if (info.error)
     DBUG_RETURN(1);
 
-  if (info.commit_list)
+  if (info.commit_list && !info.found_foreign_xids)
     sql_print_information("Crash table recovery finished.");
   DBUG_RETURN(0);
 }

sql/handler.h

@@ -965,7 +965,7 @@ typedef struct xid_t XID;
 */
 typedef uint Binlog_file_id;
 const Binlog_file_id MAX_binlog_id= UINT_MAX;
-const my_off_t       MAX_off_t    = (~(my_off_t) 0);
+const my_off_t       MAX_binlog_offset    = (~(my_off_t) 0);
 /*
   Compound binlog-id and byte offset of transaction's first event
   in a sequence (e.g the recovery sequence) of binlog files.
@@ -989,15 +989,30 @@ struct xid_recovery_member
   Binlog_offset binlog_coord;
   XID *full_xid;           // needed by wsrep or past it recovery
   decltype(::server_id) server_id;         // server id of orginal server
+  enum enum_xa_binlog_state
+  { XA_NONE= 0, XA_PREPARE, XA_COMPLETE, XA_COMMIT, XA_ROLLBACK }
+    xa_binlog_state;
+  bool is_state_valid;
 
   xid_recovery_member(my_xid xid_arg, uint prepare_arg, bool decided_arg,
-                      XID *full_xid_arg, decltype(::server_id) server_id_arg)
+                      XID *full_xid_arg, decltype(::server_id) server_id_arg,
+                      enum_xa_binlog_state xa_binlog_state_arg)
     : xid(xid_arg), in_engine_prepare(prepare_arg),
       decided_to_commit(decided_arg),
-      binlog_coord(Binlog_offset(MAX_binlog_id, MAX_off_t)),
-      full_xid(full_xid_arg), server_id(server_id_arg) {};
+      binlog_coord(Binlog_offset(MAX_binlog_id, MAX_binlog_offset)),
+      full_xid(full_xid_arg), server_id(server_id_arg),
+      xa_binlog_state(xa_binlog_state_arg), is_state_valid(true) {};
+  bool is_binlog_set() { return binlog_coord.second != MAX_binlog_offset; }
 };
 
+xid_recovery_member*
+xid_member_replace(HASH *hash_arg, my_xid xid_arg,
+                   MEM_ROOT *ptr_mem_root,
+                   XID *full_xid_arg,
+                   decltype(::server_id) server_id_arg,
+                   xid_recovery_member::enum_xa_binlog_state
+                   xa_binlog_state= xid_recovery_member::XA_NONE);
+
 /* for recover() handlerton call */
 #define MIN_XID_LIST_SIZE  128
 #define MAX_XID_LIST_SIZE  (1024*128)
@@ -5383,7 +5398,8 @@ int ha_commit_one_phase(THD *thd, bool all);
 int ha_commit_trans(THD *thd, bool all);
 int ha_rollback_trans(THD *thd, bool all);
 int ha_prepare(THD *thd);
-int ha_recover(HASH *commit_list, MEM_ROOT *mem_root= NULL);
+int ha_recover(HASH *commit_list, HASH *xa_recover_list= NULL,
+               MEM_ROOT *mem_root= NULL);
 uint ha_recover_complete(HASH *commit_list, Binlog_offset *coord= NULL);
 
 /* transactions: these functions never call handlerton functions directly */

sql/log_event.h

@@ -2207,6 +2207,10 @@ class Query_log_event: public Log_event
   virtual bool is_begin()    { return !strcmp(query, "BEGIN"); }
   virtual bool is_commit()   { return !strcmp(query, "COMMIT"); }
   virtual bool is_rollback() { return !strcmp(query, "ROLLBACK"); }
+  virtual bool is_xa_commit()
+  {
+    return !strncasecmp(query, C_STRING_WITH_LEN("XA COMMIT"));
+  }
 };
 
 class Query_compressed_log_event:public Query_log_event{

sql/log_event_server.cc

@@ -3324,6 +3324,11 @@ Gtid_log_event::Gtid_log_event(THD *thd_arg, uint64 seq_no_arg,
       flags2|= thd->lex->sql_command == SQLCOM_XA_PREPARE ?
         FL_PREPARED_XA : FL_COMPLETED_XA;
       xid.set(xid_state.get_xid());
+
+      // multi-engine external completion is unaware of the prepared engine #.
+      extra_engines= thd->transaction->all.ha_list ?
+        ha_count_rw_2pc(thd_arg,
+                        thd_arg->in_multi_stmt_transaction_mode()) - 1 : 0;
     }
     /* count non-zero extra recoverable engines; total = extra + 1 */
     if (has_xid)
@@ -3338,13 +3343,6 @@ Gtid_log_event::Gtid_log_event(THD *thd_arg, uint64 seq_no_arg,
     {
       extra_engines= UCHAR_MAX;
     }
-    else if (thd->lex->sql_command == SQLCOM_XA_PREPARE)
-    {
-      DBUG_ASSERT(thd_arg->in_multi_stmt_transaction_mode());
-
-      uint8 count= ha_count_rw_2pc(thd_arg, true);
-      extra_engines= count > 1 ? 0 : UCHAR_MAX;
-    }
     if (extra_engines > 0)
       flags_extra|= FL_EXTRA_MULTI_ENGINE;
   }