10.4-MDEV-29684 Fixes for cluster wide write conflict resolving

If two high priority threads have lock conflict, we look at the
order of these transactions and honor the earlier transaction.
for_locking parameter in lock_rec_has_to_wait() has become
obsolete and it is now removed from the code .
Reviewed-by: Jan Lindström <jan.lindstrom@mariadb.com>

sql/service_wsrep.cc

-/* Copyright 2018-2021 Codership Oy <info@codership.com>
+/* Copyright 2018-2023 Codership Oy <info@codership.com>
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -274,7 +274,9 @@ extern "C" my_bool wsrep_thd_skip_locking(const THD *thd)
 
 extern "C" my_bool wsrep_thd_order_before(const THD *left, const THD *right)
 {
-  if (wsrep_thd_trx_seqno(left) < wsrep_thd_trx_seqno(right)) {
+  if (wsrep_thd_is_BF(left, false) &&
+      wsrep_thd_is_BF(right, false) &&
+      wsrep_thd_trx_seqno(left) < wsrep_thd_trx_seqno(right)) {
     WSREP_DEBUG("BF conflict, order: %lld %lld\n",
                 (long long)wsrep_thd_trx_seqno(left),
                 (long long)wsrep_thd_trx_seqno(right));

sql/sql_class.cc

@@ -5230,8 +5230,6 @@ thd_need_ordering_with(const MYSQL_THD thd, const MYSQL_THD other_thd)
      (e.g. InnoDB does it by keeping lock_sys.mutex locked)
   */
   if (WSREP_ON &&
-      wsrep_thd_is_BF(thd, false) &&
-      wsrep_thd_is_BF(other_thd, false) &&
       wsrep_thd_order_before(thd, other_thd))
     return 0;
 #endif /* WITH_WSREP */

storage/innobase/lock/lock0lock.cc

@@ -819,25 +819,34 @@ lock_rec_has_to_wait(
 	}
 
 #ifdef WITH_WSREP
-		/* New lock request from a transaction is using unique key
-		scan and this transaction is a wsrep high priority transaction
-		(brute force). If conflicting transaction is also wsrep high
-		priority transaction we should avoid lock conflict because
-		ordering of these transactions is already decided and
-		conflicting transaction will be later replayed. Note
-		that thread holding conflicting lock can't be
-		committed or rolled back while we hold
-		lock_sys->mutex. */
-		if (trx->is_wsrep_UK_scan()
-		    && wsrep_thd_is_BF(lock2->trx->mysql_thd, false)) {
-			return false;
-		}
+	/* New lock request from a transaction is using unique key
+	scan and this transaction is a wsrep high priority transaction
+	(brute force). If conflicting transaction is also wsrep high
+	priority transaction we should avoid lock conflict because
+	ordering of these transactions is already decided and
+	conflicting transaction will be later replayed. Note
+	that thread holding conflicting lock can't be
+	committed or rolled back while we hold
+	lock_sys->mutex. */
+	if (trx->is_wsrep_UK_scan()
+	    && wsrep_thd_is_BF(lock2->trx->mysql_thd, false)) {
+		return false;
+	}
 
-		/* We very well can let bf to wait normally as other
-		BF will be replayed in case of conflict. For debug
-		builds we will do additional sanity checks to catch
-		unsupported bf wait if any. */
-		ut_d(wsrep_assert_no_bf_bf_wait(lock2, trx));
+	/* If BF-BF conflict, we have to look at write set order */
+	if (trx->is_wsrep()
+	    && (type_mode & LOCK_MODE_MASK) == LOCK_X
+	    && (lock2->type_mode & LOCK_MODE_MASK) == LOCK_X
+	    && wsrep_thd_order_before(trx->mysql_thd,
+				      lock2->trx->mysql_thd)) {
+		return false;
+	}
+
+	/* We very well can let bf to wait normally as other
+	BF will be replayed in case of conflict. For debug
+	builds we will do additional sanity checks to catch
+	unsupported bf wait if any. */
+	ut_d(wsrep_assert_no_bf_bf_wait(lock2, trx));
 #endif /* WITH_WSREP */
 
 	return true;
@@ -2043,6 +2052,14 @@ lock_rec_has_to_wait_in_queue(
 		if (heap_no < lock_rec_get_n_bits(lock)
 		    && (p[bit_offset] & bit_mask)
 		    && lock_has_to_wait(wait_lock, lock)) {
+#ifdef WITH_WSREP
+			if (lock->trx->is_wsrep()
+			    && wsrep_thd_order_before(wait_lock->trx->mysql_thd,
+						      lock->trx->mysql_thd)) {
+				/* don't wait for another BF lock */
+				continue;
+			}
+#endif
 			return(lock);
 		}
 	}