Bug #11766788 (former bug 59986)

Assert in Diagnostics_area::set_ok_status() for XA COMMIT

This assert was triggered if XA COMMIT was issued when an XA transaction
already had encountered an error (e.g. a deadlock) which required
the XA transaction to be rolled back.

In general, the assert is triggered if a statement tries to send OK to
the client when an error has already been reported. It was triggered
in this case because the trans_xa_commit() function first reported an
error, then rolled back the transaction and finally returned FALSE,
indicating success. Since trans_xa_commit() reported success,
mysql_execute_command() tried to report OK, triggering the assert.

This patch fixes the problem by fixing trans_xa_commit() to return TRUE
if it encounters an error that requires rollback, even if the rollback
itself is successful.

Test case added to xa.test.

mysql-test/r/xa.result

@@ -142,3 +142,27 @@ XA PREPARE 'x';
 XA PREPARE 'x';
 ERROR XAE07: XAER_RMFAIL: The command cannot be executed when global transaction is in the  PREPARED state
 XA ROLLBACK 'x';
+#
+# Bug#59986 Assert in Diagnostics_area::set_ok_status() for XA COMMIT
+#
+DROP TABLE IF EXISTS t1;
+CREATE TABLE t1(a INT, b INT, PRIMARY KEY(a)) engine=InnoDB;
+INSERT INTO t1 VALUES (1, 1), (2, 2);
+# Connection con1
+XA START 'a';
+UPDATE t1 SET b= 3 WHERE a=1;
+# Connection default
+XA START 'b';
+UPDATE t1 SET b=4 WHERE a=2;
+# Sending:
+UPDATE t1 SET b=5 WHERE a=1;
+# Connection con1
+UPDATE t1 SET b=6 WHERE a=2;
+ERROR 40001: Deadlock found when trying to get lock; try restarting transaction
+XA COMMIT 'a';
+ERROR XA102: XA_RBDEADLOCK: Transaction branch was rolled back: deadlock was detected
+# Connection default
+# Reaping: UPDATE t1 SET b=5 WHERE a=1
+XA END 'b';
+XA ROLLBACK 'b';
+DROP TABLE t1;

mysql-test/t/xa.test

@@ -245,6 +245,48 @@ XA PREPARE 'x';
 XA ROLLBACK 'x';
 
 
+--echo #
+--echo # Bug#59986 Assert in Diagnostics_area::set_ok_status() for XA COMMIT
+--echo #
+
+--disable_warnings
+DROP TABLE IF EXISTS t1;
+--enable_warnings
+
+CREATE TABLE t1(a INT, b INT, PRIMARY KEY(a)) engine=InnoDB;
+INSERT INTO t1 VALUES (1, 1), (2, 2);
+
+--echo # Connection con1
+connect (con1, localhost, root);
+XA START 'a';
+UPDATE t1 SET b= 3 WHERE a=1;
+
+--echo # Connection default
+connection default;
+XA START 'b';
+UPDATE t1 SET b=4 WHERE a=2;
+--echo # Sending:
+--send UPDATE t1 SET b=5 WHERE a=1
+
+--echo # Connection con1
+connection con1;
+--sleep 1
+--error ER_LOCK_DEADLOCK
+UPDATE t1 SET b=6 WHERE a=2;
+# This used to trigger the assert
+--error ER_XA_RBDEADLOCK
+XA COMMIT 'a';
+
+--echo # Connection default
+connection default;
+--echo # Reaping: UPDATE t1 SET b=5 WHERE a=1
+--reap
+XA END 'b';
+XA ROLLBACK 'b';
+DROP TABLE t1;
+disconnect con1;
+
+
 # Wait till all disconnects are completed
 --source include/wait_until_count_sessions.inc
 

sql/transaction.cc

-/* Copyright (C) 2008 Sun/MySQL
+/* Copyright (c) 2000, 2011, Oracle and/or its affiliates. All rights reserved.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -635,8 +635,9 @@ bool trans_xa_commit(THD *thd)
 
   if (xa_trans_rolled_back(&thd->transaction.xid_state))
   {
-    if ((res= test(ha_rollback_trans(thd, TRUE))))
+    if (ha_rollback_trans(thd, TRUE))
       my_error(ER_XAER_RMERR, MYF(0));
+    res= thd->is_error();
   }
   else if (xa_state == XA_IDLE && thd->lex->xa_opt == XA_ONE_PHASE)
   {