BUG#43171 - Assertion failed: thd->transaction.xid_state.xid.is_null()

XA START may cause assertion failure/server crash when it is called
after unilateral roll back issued by the Resource Manager (both
in regular transaction and after XA transaction).

The problem was that rm_error variable wasn't set/reset properly.

mysql-test/r/xa.result:
  A test case for BUG#43171.
mysql-test/t/xa.test:
  A test case for BUG#43171.
sql/handler.cc:
  Setting rm_error when we're out of XA transaction has no
  special meaning. But it blocks reset of thd->transaction.xid
  structure later.
sql/sql_parse.cc:
  Reset rm_error before we enter ha_rollback(), so
  thd->transaction.xid strucure is reinitialized.

mysql-test/r/xa.result

@@ -89,3 +89,28 @@ xa start 'a';
 xa end 'a';
 xa prepare 'a';
 xa commit 'a';
+CREATE TABLE t1(a INT, KEY(a)) ENGINE=InnoDB;
+INSERT INTO t1 VALUES(1),(2);
+BEGIN;
+UPDATE t1 SET a=3 WHERE a=1;
+BEGIN;
+UPDATE t1 SET a=4 WHERE a=2;
+UPDATE t1 SET a=5 WHERE a=2;
+UPDATE t1 SET a=5 WHERE a=1;
+ERROR 40001: Deadlock found when trying to get lock; try restarting transaction
+ROLLBACK;
+ROLLBACK;
+BEGIN;
+UPDATE t1 SET a=3 WHERE a=1;
+XA START 'xid1';
+UPDATE t1 SET a=4 WHERE a=2;
+UPDATE t1 SET a=5 WHERE a=2;
+UPDATE t1 SET a=5 WHERE a=1;
+ERROR 40001: Deadlock found when trying to get lock; try restarting transaction
+XA END 'xid1';
+ERROR XA102: XA_RBDEADLOCK: Transaction branch was rolled back: deadlock was detected
+XA ROLLBACK 'xid1';
+XA START 'xid1';
+XA END 'xid1';
+XA ROLLBACK 'xid1';
+DROP TABLE t1;

mysql-test/t/xa.test

@@ -149,6 +149,68 @@ xa end 'a';
 xa prepare 'a';
 xa commit 'a';
 
+#
+# BUG#43171 - Assertion failed: thd->transaction.xid_state.xid.is_null()
+#
+CREATE TABLE t1(a INT, KEY(a)) ENGINE=InnoDB;
+INSERT INTO t1 VALUES(1),(2);
+connect(con1,localhost,root,,);
+
+# Part 1: Prepare to test XA START after regular transaction deadlock
+BEGIN;
+UPDATE t1 SET a=3 WHERE a=1;
+
+connection default;
+BEGIN;
+UPDATE t1 SET a=4 WHERE a=2;
+
+connection con1;
+let $conn_id= `SELECT CONNECTION_ID()`;
+SEND UPDATE t1 SET a=5 WHERE a=2;
+
+connection default;
+let $wait_timeout= 2;
+let $wait_condition= SELECT 1 FROM INFORMATION_SCHEMA.PROCESSLIST
+WHERE ID=$conn_id AND STATE='Searching rows for update';
+--source include/wait_condition.inc
+
+--error ER_LOCK_DEADLOCK
+UPDATE t1 SET a=5 WHERE a=1;
+ROLLBACK;
+
+# Part 2: Prepare to test XA START after XA transaction deadlock
+connection con1;
+REAP;
+ROLLBACK;
+BEGIN;
+UPDATE t1 SET a=3 WHERE a=1;
+
+connection default;
+XA START 'xid1';
+UPDATE t1 SET a=4 WHERE a=2;
+
+connection con1;
+SEND UPDATE t1 SET a=5 WHERE a=2;
+
+connection default;
+let $wait_timeout= 2;
+let $wait_condition= SELECT 1 FROM INFORMATION_SCHEMA.PROCESSLIST
+WHERE ID=$conn_id AND STATE='Searching rows for update';
+--source include/wait_condition.inc
+
+--error ER_LOCK_DEADLOCK
+UPDATE t1 SET a=5 WHERE a=1;
+--error ER_XA_RBDEADLOCK
+XA END 'xid1';
+XA ROLLBACK 'xid1';
+
+XA START 'xid1';
+XA END 'xid1';
+XA ROLLBACK 'xid1';
+
+disconnect con1;
+DROP TABLE t1;
+
 # Wait till all disconnects are completed
 --source include/wait_until_count_sessions.inc
 

sql/handler.cc

@@ -1311,7 +1311,8 @@ int ha_rollback_trans(THD *thd, bool all)
     }
     trans->ha_list= 0;
     trans->no_2pc=0;
-    if (is_real_trans && thd->transaction_rollback_request)
+    if (is_real_trans && thd->transaction_rollback_request &&
+        thd->transaction.xid_state.xa_state != XA_NOTR)
       thd->transaction.xid_state.rm_error= thd->main_da.sql_errno();
     if (all)
       thd->variables.tx_isolation=thd->session_tx_isolation;

sql/sql_parse.cc

@@ -122,6 +122,14 @@ static bool xa_trans_rolled_back(XID_STATE *xid_state)
 */
 static bool xa_trans_rollback(THD *thd)
 {
+  /*
+    Resource Manager error is meaningless at this point, as we perform
+    explicit rollback request by user. We must reset rm_error before
+    calling ha_rollback(), so thd->transaction.xid structure gets reset
+    by ha_rollback()/THD::transaction::cleanup().
+  */
+  thd->transaction.xid_state.rm_error= 0;
+
   bool status= test(ha_rollback(thd));
 
   thd->options&= ~(ulong) OPTION_BEGIN;
@@ -129,7 +137,6 @@ static bool xa_trans_rollback(THD *thd)
   thd->server_status&= ~SERVER_STATUS_IN_TRANS;
   xid_cache_delete(&thd->transaction.xid_state);
   thd->transaction.xid_state.xa_state= XA_NOTR;
-  thd->transaction.xid_state.rm_error= 0;
 
   return status;
 }