MDEV-30413 : run sequence nextval got [Note] WSREP: MDL BF-BF conflict and [ERROR] Aborting

Sequence objects are implemented using special tables.
These tables do not have primary key and only one row.
NEXTVAL is basically update from existing value to new
value. In Galera this could mean that two write-sets
from different nodes do not conflict and this could
lead situation where write-sets are executed
concurrently and possibly in wrong seqno order.

This is fixed by using table-level exclusive key for
SEQUENCE updates. Note that this naturally works
correctly only if InnoDB storage engine is used for
sequence.

This fix does not contain a test case because while
it is possible to syncronize appliers using dbug_sync
it was too hard to syncronize MDL-lock requests to
exact objects. Testing done for this fix is documented
on MDEV.
Signed-off-by: Julius Goryavsky <julius.goryavsky@mariadb.com>

storage/innobase/handler/ha_innodb.cc

@@ -8543,6 +8543,37 @@ wsrep_calc_row_hash(
 
 	return(0);
 }
+
+/** Append table-level exclusive key.
+@param thd   MySQL thread handle
+@param table table
+@retval false on success
+@retval true on failure */
+ATTRIBUTE_COLD bool wsrep_append_table_key(MYSQL_THD thd, const dict_table_t &table)
+{
+  char db_buf[NAME_LEN + 1];
+  char tbl_buf[NAME_LEN + 1];
+  ulint db_buf_len, tbl_buf_len;
+
+  if (!table.parse_name(db_buf, tbl_buf, &db_buf_len, &tbl_buf_len))
+  {
+    WSREP_ERROR("Parse_name for table key append failed: %s",
+                wsrep_thd_query(thd));
+    return true;
+  }
+
+  /* Append table-level exclusive key */
+  const int rcode = wsrep_thd_append_table_key(thd, db_buf,
+                                               tbl_buf, WSREP_SERVICE_KEY_EXCLUSIVE);
+  if (rcode)
+  {
+    WSREP_ERROR("Appending table key failed: %s, %d",
+                wsrep_thd_query(thd), rcode);
+    return true;
+  }
+
+  return false;
+}
 #endif /* WITH_WSREP */
 
 /**
@@ -8713,11 +8744,16 @@ ha_innobase::update_row(
 	    && !wsrep_thd_ignore_table(m_user_thd)) {
 		DBUG_PRINT("wsrep", ("update row key"));
 
-		if (wsrep_append_keys(m_user_thd,
-				      wsrep_protocol_version >= 4
-				      ? WSREP_SERVICE_KEY_UPDATE
-				      : WSREP_SERVICE_KEY_EXCLUSIVE,
-				      old_row, new_row)){
+		/* We use table-level exclusive key for SEQUENCES
+		   and normal key append for others. */
+		if (table->s->table_type == TABLE_TYPE_SEQUENCE) {
+			if (wsrep_append_table_key(m_user_thd, *m_prebuilt->table))
+				DBUG_RETURN(HA_ERR_INTERNAL_ERROR);
+		} else if (wsrep_append_keys(m_user_thd,
+					     wsrep_protocol_version >= 4
+					     ? WSREP_SERVICE_KEY_UPDATE
+					     : WSREP_SERVICE_KEY_EXCLUSIVE,
+					     old_row, new_row)) {
 			WSREP_DEBUG("WSREP: UPDATE_ROW_KEY FAILED");
 			DBUG_PRINT("wsrep", ("row key failed"));
 			DBUG_RETURN(HA_ERR_INTERNAL_ERROR);

storage/innobase/include/ha_prototypes.h

@@ -462,5 +462,15 @@ void destroy_background_thd(MYSQL_THD thd);
 void
 innobase_reset_background_thd(MYSQL_THD);
 
+#ifdef WITH_WSREP
+/** Append table-level exclusive key.
+@param thd   MySQL thread handle
+@param table table
+@retval false on success
+@retval true on failure */
+struct dict_table_t;
+bool wsrep_append_table_key(MYSQL_THD thd, const dict_table_t &table);
+#endif /* WITH_WSREP */
+
 #endif /* !UNIV_INNOCHECKSUM */
 #endif /* HA_INNODB_PROTOTYPES_H */

storage/innobase/row/row0ins.cc

@@ -49,6 +49,7 @@ Created 4/20/1996 Heikki Tuuri
 #ifdef WITH_WSREP
 #include <wsrep.h>
 #include <mysql/service_wsrep.h>
+#include "ha_prototypes.h"
 #endif /* WITH_WSREP */
 
 /*************************************************************************
@@ -2574,42 +2575,6 @@ but GCC 4.8.5 does not support pop_options. */
 # pragma GCC optimize ("O0")
 #endif
 
-#ifdef WITH_WSREP
-/** Start bulk insert operation for Galera by appending
-table-level exclusive key for bulk insert.
-@param trx transaction
-@param index index
-@retval false on success
-@retval true on failure */
-ATTRIBUTE_COLD static bool row_ins_wsrep_start_bulk(trx_t *trx, const dict_index_t &index)
-{
-  char db_buf[NAME_LEN + 1];
-  char tbl_buf[NAME_LEN + 1];
-  ulint	db_buf_len, tbl_buf_len;
-
-  if (!index.table->parse_name(db_buf, tbl_buf, &db_buf_len, &tbl_buf_len))
-  {
-    WSREP_ERROR("Parse_name for bulk insert failed: %s",
-                wsrep_thd_query(trx->mysql_thd));
-    trx->error_state = DB_ROLLBACK;
-    return true;
-  }
-
-  /* Append table-level exclusive key for bulk insert. */
-  const int rcode = wsrep_thd_append_table_key(trx->mysql_thd, db_buf,
-                                               tbl_buf, WSREP_SERVICE_KEY_EXCLUSIVE);
-  if (rcode)
-  {
-    WSREP_ERROR("Appending table key for bulk insert failed: %s, %d",
-                wsrep_thd_query(trx->mysql_thd), rcode);
-    trx->error_state = DB_ROLLBACK;
-    return true;
-  }
-
-  return false;
-}
-#endif
-
 /***************************************************************//**
 Tries to insert an entry into a clustered index, ignoring foreign key
 constraints. If a record with the same unique key is found, the other
@@ -2770,10 +2735,13 @@ row_ins_clust_index_entry_low(
 #ifdef WITH_WSREP
 			if (trx->is_wsrep())
 			{
-			    if (!wsrep_thd_is_local_transaction(trx->mysql_thd))
-				    goto skip_bulk_insert;
-			    if (row_ins_wsrep_start_bulk(trx, *index))
-				    goto err_exit;
+				if (!wsrep_thd_is_local_transaction(trx->mysql_thd))
+					goto skip_bulk_insert;
+				if (wsrep_append_table_key(trx->mysql_thd, *index->table))
+				{
+					trx->error_state = DB_ROLLBACK;
+					goto err_exit;
+				}
 			}
 #endif /* WITH_WSREP */