Bug#28323: Server crashed in xid cache operations

The problem was that the server did not robustly handle a
unilateral roll back issued by the Resource Manager (RM)
due to a resource deadlock within the transaction branch.
By not acknowledging the roll back, the server (TM) would
eventually corrupt the XA transaction state and crash.

The solution is to mark the transaction as rollback-only
if the RM indicates that it rolled back its branch of the
transaction.

mysql-test/r/xa.result:
  Add test case result for Bug#28323
mysql-test/t/xa.test:
  Add test case for Bug#28323
sql/handler.cc:
  Reset XID only at the end of the global transaction.
sql/share/errmsg.txt:
  Add new error codes.
sql/sql_class.h:
  Remember the error reported by the Resource Manager.
sql/sql_parse.cc:
  Rollback the transaction if the Resource Manager reported
  a error and rolled back its branch of the transaction.

mysql-test/r/xa.result

@@ -55,3 +55,22 @@ select * from t1;
 a
 20
 drop table t1;
+drop table if exists t1;
+create table t1(a int, b int, c varchar(20), primary key(a)) engine = innodb;
+insert into t1 values(1, 1, 'a');
+insert into t1 values(2, 2, 'b');
+xa start 'a','b';
+update t1 set c = 'aa' where a = 1;
+xa start 'a','c';
+update t1 set c = 'bb' where a = 2;
+update t1 set c = 'bb' where a = 2;
+update t1 set c = 'aa' where a = 1;
+ERROR 40001: Deadlock found when trying to get lock; try restarting transaction
+select count(*) from t1;
+count(*)
+2
+xa end 'a','c';
+ERROR XA102: XA_RBDEADLOCK: Transaction branch was rolled back: deadlock was detected
+xa rollback 'a','c';
+xa start 'a','c';
+End of 5.0 tests

mysql-test/t/xa.test

@@ -74,3 +74,47 @@ xa start 'zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz';
 select * from t1;
 drop table t1;
 
+disconnect con1;
+
+#
+# Bug#28323: Server crashed in xid cache operations
+#
+
+--disable_warnings
+drop table if exists t1;
+--enable_warnings
+
+create table t1(a int, b int, c varchar(20), primary key(a)) engine = innodb;
+insert into t1 values(1, 1, 'a');
+insert into t1 values(2, 2, 'b');
+
+connect (con1,localhost,root,,);
+connect (con2,localhost,root,,);
+
+--connection con1
+xa start 'a','b';
+update t1 set c = 'aa' where a = 1;
+--connection con2
+xa start 'a','c';
+update t1 set c = 'bb' where a = 2;
+--connection con1
+--send update t1 set c = 'bb' where a = 2
+--connection con2
+--sleep 1
+--error ER_LOCK_DEADLOCK
+update t1 set c = 'aa' where a = 1;
+select count(*) from t1;
+--error ER_XA_RBDEADLOCK
+xa end 'a','c';
+xa rollback 'a','c';
+--disconnect con2
+
+connect (con3,localhost,root,,);
+--connection con3
+xa start 'a','c';
+
+--disconnect con1
+--disconnect con3
+--connection default
+
+--echo End of 5.0 tests

sql/handler.cc

@@ -817,7 +817,12 @@ int ha_rollback_trans(THD *thd, bool all)
     trans->nht=0;
     trans->no_2pc=0;
     if (is_real_trans)
+    {
+      if (thd->transaction_rollback_request)
+        thd->transaction.xid_state.rm_error= thd->net.last_errno;
+      else
         thd->transaction.xid_state.xid.null();
+    }
     if (all)
     {
       thd->variables.tx_isolation=thd->session_tx_isolation;

sql/share/errmsg.txt

@@ -5645,3 +5645,7 @@ ER_LOAD_DATA_INVALID_COLUMN
   eng "Invalid column reference (%-.64s) in LOAD DATA"
 ER_LOG_PURGE_NO_FILE  
 	eng "Being purged log %s was not found"
+ER_XA_RBTIMEOUT XA106
+	eng "XA_RBTIMEOUT: Transaction branch was rolled back: took too long"
+ER_XA_RBDEADLOCK XA102
+	eng "XA_RBDEADLOCK: Transaction branch was rolled back: deadlock was detected"

sql/sql_class.h

@@ -920,7 +920,7 @@ struct st_savepoint {
   uint                 length, nht;
 };
 
-enum xa_states {XA_NOTR=0, XA_ACTIVE, XA_IDLE, XA_PREPARED};
+enum xa_states {XA_NOTR=0, XA_ACTIVE, XA_IDLE, XA_PREPARED, XA_ROLLBACK_ONLY};
 extern const char *xa_state_names[];
 
 typedef struct st_xid_state {
@@ -928,6 +928,8 @@ typedef struct st_xid_state {
   XID  xid;                           // transaction identifier
   enum xa_states xa_state;            // used by external XA only
   bool in_thd;
+  /* Error reported by the Resource Manager (RM) to the Transaction Manager. */
+  uint rm_error;
 } XID_STATE;
 
 extern pthread_mutex_t LOCK_xid_cache;

sql/sql_parse.cc

@@ -90,9 +90,57 @@ const char *command_name[]={
 };
 
 const char *xa_state_names[]={
-  "NON-EXISTING", "ACTIVE", "IDLE", "PREPARED"
+  "NON-EXISTING", "ACTIVE", "IDLE", "PREPARED", "ROLLBACK ONLY"
 };
 
+/**
+  Mark a XA transaction as rollback-only if the RM unilaterally
+  rolled back the transaction branch.
+
+  @note If a rollback was requested by the RM, this function sets
+        the appropriate rollback error code and transits the state
+        to XA_ROLLBACK_ONLY.
+
+  @return TRUE if transaction was rolled back or if the transaction
+          state is XA_ROLLBACK_ONLY. FALSE otherwise.
+*/
+static bool xa_trans_rolled_back(XID_STATE *xid_state)
+{
+  if (xid_state->rm_error)
+  {
+    switch (xid_state->rm_error) {
+    case ER_LOCK_WAIT_TIMEOUT:
+      my_error(ER_XA_RBTIMEOUT, MYF(0));
+      break;
+    case ER_LOCK_DEADLOCK:
+      my_error(ER_XA_RBDEADLOCK, MYF(0));
+      break;
+    default:
+      my_error(ER_XA_RBROLLBACK, MYF(0));
+    }
+    xid_state->xa_state= XA_ROLLBACK_ONLY;
+  }
+
+  return (xid_state->xa_state == XA_ROLLBACK_ONLY);
+}
+
+/**
+  Rollback work done on behalf of at ransaction branch.
+*/
+static bool xa_trans_rollback(THD *thd)
+{
+  bool status= test(ha_rollback(thd));
+
+  thd->options&= ~(ulong) OPTION_BEGIN;
+  thd->transaction.all.modified_non_trans_table= FALSE;
+  thd->server_status&= ~SERVER_STATUS_IN_TRANS;
+  xid_cache_delete(&thd->transaction.xid_state);
+  thd->transaction.xid_state.xa_state= XA_NOTR;
+  thd->transaction.xid_state.rm_error= 0;
+
+  return status;
+}
+
 #ifndef EMBEDDED_LIBRARY
 static bool do_command(THD *thd);
 #endif // EMBEDDED_LIBRARY
@@ -5049,6 +5097,7 @@ create_sp_error:
     }
     DBUG_ASSERT(thd->transaction.xid_state.xid.is_null());
     thd->transaction.xid_state.xa_state=XA_ACTIVE;
+    thd->transaction.xid_state.rm_error= 0;
     thd->transaction.xid_state.xid.set(thd->lex->xid);
     xid_cache_insert(&thd->transaction.xid_state);
     thd->transaction.all.modified_non_trans_table= FALSE;
@@ -5074,6 +5123,8 @@ create_sp_error:
       my_error(ER_XAER_NOTA, MYF(0));
       break;
     }
+    if (xa_trans_rolled_back(&thd->transaction.xid_state))
+      break;
     thd->transaction.xid_state.xa_state=XA_IDLE;
     send_ok(thd);
     break;
@@ -5105,6 +5156,12 @@ create_sp_error:
       XID_STATE *xs=xid_cache_search(thd->lex->xid);
       if (!xs || xs->in_thd)
         my_error(ER_XAER_NOTA, MYF(0));
+      else if (xa_trans_rolled_back(xs))
+      {
+        ha_commit_or_rollback_by_xid(thd->lex->xid, 0);
+        xid_cache_delete(xs);
+        break;
+      }
       else
       {
         ha_commit_or_rollback_by_xid(thd->lex->xid, 1);
@@ -5113,6 +5170,11 @@ create_sp_error:
       }
       break;
     }
+    if (xa_trans_rolled_back(&thd->transaction.xid_state))
+    {
+      xa_trans_rollback(thd);
+      break;
+    }
     if (thd->transaction.xid_state.xa_state == XA_IDLE &&
         thd->lex->xa_opt == XA_ONE_PHASE)
     {
@@ -5159,28 +5221,26 @@ create_sp_error:
         my_error(ER_XAER_NOTA, MYF(0));
       else
       {
+        bool ok= !xa_trans_rolled_back(xs);
         ha_commit_or_rollback_by_xid(thd->lex->xid, 0);
         xid_cache_delete(xs);
+        if (ok)
           send_ok(thd);
       }
       break;
     }
     if (thd->transaction.xid_state.xa_state != XA_IDLE &&
-        thd->transaction.xid_state.xa_state != XA_PREPARED)
+        thd->transaction.xid_state.xa_state != XA_PREPARED &&
+        thd->transaction.xid_state.xa_state != XA_ROLLBACK_ONLY)
     {
       my_error(ER_XAER_RMFAIL, MYF(0),
                xa_state_names[thd->transaction.xid_state.xa_state]);
       break;
     }
-    if (ha_rollback(thd))
+    if (xa_trans_rollback(thd))
       my_error(ER_XAER_RMERR, MYF(0));
     else
       send_ok(thd);
-    thd->options&= ~(ulong) OPTION_BEGIN;
-    thd->transaction.all.modified_non_trans_table= FALSE;
-    thd->server_status&= ~SERVER_STATUS_IN_TRANS;
-    xid_cache_delete(&thd->transaction.xid_state);
-    thd->transaction.xid_state.xa_state=XA_NOTR;
     break;
   case SQLCOM_XA_RECOVER:
     res= mysql_xa_recover(thd);