Bug #56448 Assertion failed: ! is_set() with second xa end

The problem was that issuing XA END when the XA transaction was
already ended, caused an assertion. This assertion tests that 
the server does not try to send OK to the client if there has
already been an error reported. The bug was only noticeable on
debug versions of the server.

The reason for the problem was that the trans_xa_end() function
reported success if the transaction was at XA_IDLE state at the
end regardless of any errors occured during processing of
trans_xa_end(). So if the transaction state was XA_IDLE already,
reported errors would be ignored.

This patch fixes the problem by having trans_xa_end() take into
consideration any reported errors. The patch also fixes a similar
bug with XA PREPARE.

Test case added to xa.test.

mysql-test/r/xa.result

@@ -131,3 +131,14 @@ XA START 'xid1';
 XA END 'xid1';
 XA ROLLBACK 'xid1';
 DROP TABLE t1;
+#
+# Bug#56448 Assertion failed: ! is_set() with second xa end
+#
+XA START 'x';
+XA END 'x';
+XA END 'x';
+ERROR XAE07: XAER_RMFAIL: The command cannot be executed when global transaction is in the  IDLE state
+XA PREPARE 'x';
+XA PREPARE 'x';
+ERROR XAE07: XAER_RMFAIL: The command cannot be executed when global transaction is in the  PREPARED state
+XA ROLLBACK 'x';

mysql-test/t/xa.test

@@ -228,6 +228,23 @@ XA ROLLBACK 'xid1';
 disconnect con1;
 DROP TABLE t1;
 
+
+--echo #
+--echo # Bug#56448 Assertion failed: ! is_set() with second xa end
+--echo #
+
+XA START 'x';
+XA END 'x';
+# Second XA END caused an assertion.
+--error ER_XAER_RMFAIL
+XA END 'x';
+XA PREPARE 'x';
+# Second XA PREPARE also caused an assertion.
+--error ER_XAER_RMFAIL
+XA PREPARE 'x';
+XA ROLLBACK 'x';
+
+
 # Wait till all disconnects are completed
 --source include/wait_until_count_sessions.inc
 

sql/transaction.cc

@@ -565,7 +565,8 @@ bool trans_xa_end(THD *thd)
   else if (!xa_trans_rolled_back(&thd->transaction.xid_state))
     thd->transaction.xid_state.xa_state= XA_IDLE;
 
-  DBUG_RETURN(thd->transaction.xid_state.xa_state != XA_IDLE);
+  DBUG_RETURN(thd->is_error() ||
+              thd->transaction.xid_state.xa_state != XA_IDLE);
 }
 
 
@@ -596,7 +597,8 @@ bool trans_xa_prepare(THD *thd)
   else
     thd->transaction.xid_state.xa_state= XA_PREPARED;
 
-  DBUG_RETURN(thd->transaction.xid_state.xa_state != XA_PREPARED);
+  DBUG_RETURN(thd->is_error() ||
+              thd->transaction.xid_state.xa_state != XA_PREPARED);
 }