MDEV-29684 Fixes for cluster wide write conflict resolving

Cluster conflict victim's THD is marked with wsrep_aborter.
THD::wsrep_aorter holds the thread ID of the hight priority tread,
which is currently carrying out BF aborting for this victim.

However, the BF abort operation is not always successful,
and in such case the wsrep_aborter mark should be removed.
In the old code, this wsrep_aborter resetting did not happen,
and this could lead to a situation where the sticky wsrep_aborter
mark prevents any further attempt to BF abort this transaction.

This commit fixes this issue, and resets wsrep_aborter after
unsuccesful BF abort attempt.
Reviewed-by: Jan Lindström <jan.lindstrom@mariadb.com>

sql/service_wsrep.cc

@@ -349,13 +349,20 @@ extern "C" void wsrep_commit_ordered(THD *thd)
 
 extern "C" bool wsrep_thd_set_wsrep_aborter(THD *bf_thd, THD *victim_thd)
 {
-  WSREP_DEBUG("wsrep_thd_set_wsrep_aborter called");
   mysql_mutex_assert_owner(&victim_thd->LOCK_thd_data);
+  if (!bf_thd)
+  {
+    victim_thd->wsrep_aborter= 0;
+    WSREP_DEBUG("wsrep_thd_set_wsrep_aborter resetting wsrep_aborter");
+    return false;
+  }
   if (victim_thd->wsrep_aborter && victim_thd->wsrep_aborter != bf_thd->thread_id)
   {
     return true;
   }
-  victim_thd->wsrep_aborter = bf_thd->thread_id;
+  victim_thd->wsrep_aborter= bf_thd->thread_id;
+  WSREP_DEBUG("wsrep_thd_set_wsrep_aborter setting wsrep_aborter %u",
+              victim_thd->wsrep_aborter);
   return false;
 }
 

sql/wsrep_thd.cc

-/* Copyright (C) 2013-2022 Codership Oy <info@codership.com>
+/* Copyright (C) 2013-2023 Codership Oy <info@codership.com>
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -308,11 +308,11 @@ void wsrep_fire_rollbacker(THD *thd)
 }
 
 
-int wsrep_abort_thd(THD *bf_thd_ptr, THD *victim_thd_ptr, my_bool signal)
+int wsrep_abort_thd(THD *bf_thd,
+                    THD *victim_thd,
+                    my_bool signal)
 {
   DBUG_ENTER("wsrep_abort_thd");
-  THD *victim_thd= (THD *) victim_thd_ptr;
-  THD *bf_thd= (THD *) bf_thd_ptr;
 
   mysql_mutex_assert_owner(&victim_thd->LOCK_thd_data);
   mysql_mutex_assert_owner(&victim_thd->LOCK_thd_kill);
@@ -323,16 +323,21 @@ int wsrep_abort_thd(THD *bf_thd_ptr, THD *victim_thd_ptr, my_bool signal)
   if ((WSREP(bf_thd) ||
        ((WSREP_ON || bf_thd->variables.wsrep_OSU_method == WSREP_OSU_RSU) &&
 	 wsrep_thd_is_toi(bf_thd))) &&
-       victim_thd &&
        !wsrep_thd_is_aborting(victim_thd))
   {
-      WSREP_DEBUG("wsrep_abort_thd, by: %llu, victim: %llu", (bf_thd) ?
-                  (long long)bf_thd->real_id : 0, (long long)victim_thd->real_id);
+      WSREP_DEBUG("wsrep_abort_thd, by: %llu, victim: %llu",
+                  (long long)bf_thd->real_id, (long long)victim_thd->real_id);
       ha_abort_transaction(bf_thd, victim_thd, signal);
   }
   else
   {
-    WSREP_DEBUG("wsrep_abort_thd not effective: %p %p", bf_thd, victim_thd);
+    WSREP_DEBUG("wsrep_abort_thd not effective: bf %llu victim %llu "
+                "wsrep %d wsrep_on %d RSU %d TOI %d aborting %d",
+                (long long)bf_thd->real_id, (long long)victim_thd->real_id,
+                WSREP_NNULL(bf_thd), WSREP_ON,
+                bf_thd->variables.wsrep_OSU_method == WSREP_OSU_RSU,
+                wsrep_thd_is_toi(bf_thd),
+                wsrep_thd_is_aborting(victim_thd));
     wsrep_thd_UNLOCK(victim_thd);
   }
 

sql/wsrep_thd.h

-/* Copyright (C) 2013-2022 Codership Oy <info@codership.com>
+/* Copyright (C) 2013-2023 Codership Oy <info@codership.com>
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -88,7 +88,9 @@ bool wsrep_create_appliers(long threads, bool mutex_protected=false);
 void wsrep_create_rollbacker();
 
 bool wsrep_bf_abort(THD* bf_thd, THD* victim_thd);
-int  wsrep_abort_thd(THD *bf_thd_ptr, THD *victim_thd_ptr, my_bool signal);
+int  wsrep_abort_thd(THD *bf_thd,
+                     THD *victim_thd,
+                     my_bool signal) __attribute__((nonnull(1,2)));
 
 /*
   Helper methods to deal with thread local storage.

storage/innobase/handler/ha_innodb.cc

@@ -18748,6 +18748,16 @@ wsrep_kill_victim(
       lock_cancel_waiting_and_release(wait_lock);
     }
   }
+  else
+  {
+    wsrep_thd_LOCK(thd);
+    victim_trx->lock.was_chosen_as_wsrep_victim= false;
+    wsrep_thd_set_wsrep_aborter(NULL, thd);
+    wsrep_thd_UNLOCK(thd);
+
+    WSREP_DEBUG("wsrep_thd_bf_abort has failed, victim %lu will survive",
+                thd_get_thread_id(thd));
+  }
 
   DBUG_VOID_RETURN;
 }