MDEV-31737 : Node never returns from Donor/Desynced to Synced when wsrep_mode...

MDEV-31737 : Node never returns from Donor/Desynced to Synced when wsrep_mode = BF_ABORT_MARIABACKUP

Problem was incorrect condition when node should have
resumed and resync at backup_end. Simplified condition
to fix the problem and added missing test case for
this wsrep_mode = BF_ABORT_MARIABACKUP.
Signed-off-by: Julius Goryavsky <julius.goryavsky@mariadb.com>

mysql-test/suite/galera/r/galera_bf_abort_mariabackup.result

+connection node_2;
+connection node_1;
+connection node_1;
+connection node_2;
+# Case 1 : MariaBackup SST
+connection node_1;
+CREATE TABLE t(i INT NOT NULL PRIMARY KEY) ENGINE INNODB;
+INSERT INTO t VALUES(1);
+# Restart node_2, force SST.
+connection node_2;
+connection node_1;
+connection node_2;
+Starting server ...
+connection node_1;
+# Both should return FOUND 2 as we have bootstrap and SST
+FOUND 2 /Desyncing and pausing the provider/ in mysqld.1.err
+FOUND 2 /Resuming and resyncing the provider/ in mysqld.1.err
+connection node_1;
+SET GLOBAL wsrep_mode = "BF_ABORT_MARIABACKUP";
+# Restart node_2, force SST.
+connection node_2;
+connection node_1;
+INSERT INTO t VALUES(2);
+connection node_2;
+Starting server ...
+connection node_2;
+connection node_1;
+# Both should return FOUND 3 as we have 1 new SST
+FOUND 3 /Desyncing and pausing the provider/ in mysqld.1.err
+FOUND 3 /Resuming and resyncing the provider/ in mysqld.1.err
+SET GLOBAL wsrep_mode = "";
+DROP TABLE t;
+# Case 2: MariaBackup backup from node_2
+connection node_1;
+CREATE TABLE t(i INT NOT NULL PRIMARY KEY) ENGINE INNODB;
+INSERT INTO t VALUES(1),(2),(3),(4),(5);
+connection node_2;
+SET GLOBAL wsrep_mode = "";
+SELECT @@wsrep_mode;
+@@wsrep_mode
+
+# Both should return FOUND 1 as we have backup
+FOUND 1 /Desyncing and pausing the provider/ in mysqld.2.err
+FOUND 1 /Resuming and resyncing the provider/ in mysqld.2.err
+SET GLOBAL wsrep_mode = "BF_ABORT_MARIABACKUP";
+SELECT @@wsrep_mode;
+@@wsrep_mode
+BF_ABORT_MARIABACKUP
+# Both should return FOUND 1 as node should not desync
+FOUND 1 /Desyncing and pausing the provider/ in mysqld.2.err
+FOUND 1 /Resuming and resyncing the provider/ in mysqld.2.err
+# Should return FOUND 1 because only last backup does not desync
+FOUND 1 /Server not desynched from group because WSREP_MODE_BF_MARIABACKUP used./ in mysqld.2.err
+SET GLOBAL wsrep_mode = "";
+connection node_1;
+DROP TABLE t;
+disconnect node_2;
+disconnect node_1;

mysql-test/suite/galera/t/galera_bf_abort_mariabackup.cnf

+!include ../galera_2nodes.cnf
+
+[mysqld]
+wsrep_sst_method=mariabackup
+wsrep_sst_auth="root:"
+wsrep_debug=1
+
+[mysqld.1]
+wsrep_provider_options='base_port=@mysqld.1.#galera_port;gcache.size=1;pc.ignore_sb=true'
+
+[mysqld.2]
+wsrep_provider_options='base_port=@mysqld.2.#galera_port;gcache.size=1;pc.ignore_sb=true'
+
+[sst]
+transferfmt=@ENV.MTR_GALERA_TFMT
+streamfmt=mbstream

mysql-test/suite/galera/t/galera_bf_abort_mariabackup.test

+--source include/galera_cluster.inc
+--source include/have_mariabackup.inc
+
+# Save original auto_increment_offset values.
+--let $node_1=node_1
+--let $node_2=node_2
+--source include/auto_increment_offset_save.inc
+
+#
+--echo # Case 1 : MariaBackup SST
+#
+--connection node_1
+CREATE TABLE t(i INT NOT NULL PRIMARY KEY) ENGINE INNODB;
+INSERT INTO t VALUES(1);
+#
+# In default settings donor should desync
+#
+--echo # Restart node_2, force SST.
+--connection node_2
+--source include/shutdown_mysqld.inc
+--remove_file $MYSQLTEST_VARDIR/mysqld.2/data/grastate.dat
+
+--connection node_1
+--let $wait_condition = SELECT VARIABLE_VALUE = 1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'
+--source include/wait_condition.inc
+
+--connection node_2
+--echo Starting server ...
+let $restart_noprint=2;
+--source include/start_mysqld.inc
+
+--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size';
+--source include/wait_condition.inc
+
+--let $wait_condition = SELECT VARIABLE_VALUE = 'ON' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_ready';
+--source include/wait_condition.inc
+
+--connection node_1
+let SEARCH_FILE = $MYSQLTEST_VARDIR/log/mysqld.1.err;
+--echo # Both should return FOUND 2 as we have bootstrap and SST
+let SEARCH_PATTERN = Desyncing and pausing the provider;
+--source include/search_pattern_in_file.inc
+let SEARCH_PATTERN = Resuming and resyncing the provider;
+--source include/search_pattern_in_file.inc
+
+#
+# Now we set wsrep_mode = BF_ABORT_MARIABACKUP
+#
+--connection node_1
+SET GLOBAL wsrep_mode = "BF_ABORT_MARIABACKUP";
+
+--echo # Restart node_2, force SST.
+--connection node_2
+--source include/shutdown_mysqld.inc
+--remove_file $MYSQLTEST_VARDIR/mysqld.2/data/grastate.dat
+
+--connection node_1
+--let $wait_condition = SELECT VARIABLE_VALUE = 1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'
+--source include/wait_condition.inc
+INSERT INTO t VALUES(2);
+
+--connection node_2
+--echo Starting server ...
+let $restart_noprint=2;
+--source include/start_mysqld.inc
+
+--connection node_2
+--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size';
+--source include/wait_condition.inc
+
+--let $wait_condition = SELECT VARIABLE_VALUE = 'ON' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_ready';
+--source include/wait_condition.inc
+
+# Restore original auto_increment_offset values.
+--source include/auto_increment_offset_restore.inc
+
+--connection node_1
+let SEARCH_FILE = $MYSQLTEST_VARDIR/log/mysqld.1.err;
+--echo # Both should return FOUND 3 as we have 1 new SST
+let SEARCH_PATTERN = Desyncing and pausing the provider;
+--source include/search_pattern_in_file.inc
+let SEARCH_PATTERN = Resuming and resyncing the provider;
+--source include/search_pattern_in_file.inc
+SET GLOBAL wsrep_mode = "";
+DROP TABLE t;
+#
+--echo # Case 2: MariaBackup backup from node_2
+#
+--connection node_1
+CREATE TABLE t(i INT NOT NULL PRIMARY KEY) ENGINE INNODB;
+INSERT INTO t VALUES(1),(2),(3),(4),(5);
+
+--connection node_2
+SET GLOBAL wsrep_mode = "";
+SELECT @@wsrep_mode;
+let $targetdir=$MYSQLTEST_VARDIR/tmp/backup;
+--let $backup_log=$MYSQLTEST_VARDIR/tmp/backup.log
+--disable_result_log
+--exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --defaults-group-suffix=.2 --backup --target-dir=$targetdir > $backup_log 2>&1;
+--enable_result_log
+
+let SEARCH_FILE = $MYSQLTEST_VARDIR/log/mysqld.2.err;
+--echo # Both should return FOUND 1 as we have backup
+let SEARCH_PATTERN = Desyncing and pausing the provider;
+--source include/search_pattern_in_file.inc
+let SEARCH_PATTERN = Resuming and resyncing the provider;
+--source include/search_pattern_in_file.inc
+#
+# Now we set wsrep_mode = BF_ABORT_MARIABACKUP
+#
+SET GLOBAL wsrep_mode = "BF_ABORT_MARIABACKUP";
+SELECT @@wsrep_mode;
+let $targetdir=$MYSQLTEST_VARDIR/tmp/backup2;
+--let $backup_log=$MYSQLTEST_VARDIR/tmp/backup2.log
+--disable_result_log
+--exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --defaults-group-suffix=.2 --backup --target-dir=$targetdir --mysqld-args --wsrep-mode="BF_ABORT_MARIABACKUP" > $backup_log 2>&1;
+--enable_result_log
+
+let SEARCH_FILE = $MYSQLTEST_VARDIR/log/mysqld.2.err;
+--echo # Both should return FOUND 1 as node should not desync
+let SEARCH_PATTERN = Desyncing and pausing the provider;
+--source include/search_pattern_in_file.inc
+let SEARCH_PATTERN = Resuming and resyncing the provider;
+--source include/search_pattern_in_file.inc
+--echo # Should return FOUND 1 because only last backup does not desync
+let SEARCH_PATTERN = Server not desynched from group because WSREP_MODE_BF_MARIABACKUP used.;
+--source include/search_pattern_in_file.inc
+
+SET GLOBAL wsrep_mode = "";
+
+--connection node_1
+DROP TABLE t;
+
+--source include/auto_increment_offset_restore.inc
+
+--source include/galera_end.inc

sql/backup.cc

@@ -292,6 +292,7 @@ static bool backup_block_ddl(THD *thd)
   thd->clear_error();
 
 #ifdef WITH_WSREP
+  DBUG_ASSERT(thd->wsrep_desynced_backup_stage == false);
   /*
     if user is specifically choosing to allow BF aborting for BACKUP STAGE BLOCK_DDL lock
     holder, then do not desync and pause the node from cluster replication.
@@ -303,6 +304,7 @@ static bool backup_block_ddl(THD *thd)
   if (WSREP_NNULL(thd))
   {
     Wsrep_server_state &server_state= Wsrep_server_state::instance();
+
     if (!wsrep_check_mode(WSREP_MODE_BF_MARIABACKUP) ||
         server_state.state() == Wsrep_server_state::s_donor)
     {
@@ -352,17 +354,17 @@ static bool backup_block_ddl(THD *thd)
   /* There can't be anything more that needs to be logged to ddl log */
   THD_STAGE_INFO(thd, org_stage);
   stop_ddl_logging();
-#ifdef WITH_WSREP
-  // Allow tests to block the applier thread using the DBUG facilities
-  DBUG_EXECUTE_IF("sync.wsrep_after_mdl_block_ddl",
+
+  // Allow tests to block the backup thread
+  DBUG_EXECUTE_IF("sync.after_mdl_block_ddl",
                   {
                    const char act[]=
                       "now "
-                     "signal signal.wsrep_apply_toi";
+                     "SIGNAL sync.after_mdl_block_ddl_reached "
+                     "WAIT_FOR signal.after_mdl_block_ddl_continue";
                    DBUG_ASSERT(!debug_sync_set_action(thd,
                                                       STRING_WITH_LEN(act)));
                   };);
-#endif /* WITH_WSREP */
 
   DBUG_RETURN(0);
 err:
@@ -423,8 +425,8 @@ bool backup_end(THD *thd)
     thd->current_backup_stage= BACKUP_FINISHED;
     thd->mdl_context.release_lock(old_ticket);
 #ifdef WITH_WSREP
-    if (WSREP_NNULL(thd) && thd->wsrep_desynced_backup_stage &&
-	!wsrep_check_mode(WSREP_MODE_BF_MARIABACKUP))
+    // If node was desynced, resume and resync
+    if (thd->wsrep_desynced_backup_stage)
     {
       Wsrep_server_state &server_state= Wsrep_server_state::instance();
       server_state.resume_and_resync();