MDEV-28205: SST via mariabackup stops on failure while archiving logs

Currenly SST script for mariabackup stops on any failure while archiving
logs, e.g. when unable to create directory, insufficient permissions, gzip
failure, etc. However, in case of such problems, the script should issue
a warning and continue without archiving, but not exit with a fatal error.

This commit adds this fix to the SST script for mariabackup.

mysql-test/suite/galera/r/galera_sst_mariabackup_logarchive,debug.rdiff

+--- r/galera_sst_mariabackup_logarachive.result
++++ r/galera_sst_mariabackup_logarachive.reject
+@@ -286,5 +286,113 @@
+ DROP TABLE t1;
+ COMMIT;
+ SET AUTOCOMMIT=ON;
++Performing State Transfer on a server that has been killed and restarted
++while a DDL was in progress on it
++connection node_1;
++CREATE TABLE t1 (f1 CHAR(255)) ENGINE=InnoDB;
++SET AUTOCOMMIT=OFF;
++START TRANSACTION;
++INSERT INTO t1 VALUES ('node1_committed_before');
++INSERT INTO t1 VALUES ('node1_committed_before');
++INSERT INTO t1 VALUES ('node1_committed_before');
++INSERT INTO t1 VALUES ('node1_committed_before');
++INSERT INTO t1 VALUES ('node1_committed_before');
++connection node_2;
++START TRANSACTION;
++INSERT INTO t1 VALUES ('node2_committed_before');
++INSERT INTO t1 VALUES ('node2_committed_before');
++INSERT INTO t1 VALUES ('node2_committed_before');
++INSERT INTO t1 VALUES ('node2_committed_before');
++INSERT INTO t1 VALUES ('node2_committed_before');
++COMMIT;
++SET GLOBAL debug_dbug = 'd,sync.alter_opened_table';
++connection node_1;
++ALTER TABLE t1 ADD COLUMN f2 INTEGER;
++connection node_2;
++SET wsrep_sync_wait = 0;
++Killing server ...
++connection node_1;
++SET AUTOCOMMIT=OFF;
++START TRANSACTION;
++INSERT INTO t1 (f1) VALUES ('node1_committed_during');
++INSERT INTO t1 (f1) VALUES ('node1_committed_during');
++INSERT INTO t1 (f1) VALUES ('node1_committed_during');
++INSERT INTO t1 (f1) VALUES ('node1_committed_during');
++INSERT INTO t1 (f1) VALUES ('node1_committed_during');
++COMMIT;
++START TRANSACTION;
++INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after');
++INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after');
++INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after');
++INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after');
++INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after');
++connect node_1a_galera_st_kill_slave_ddl, 127.0.0.1, root, , test, $NODE_MYPORT_1;
++SET AUTOCOMMIT=OFF;
++START TRANSACTION;
++INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after');
++INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after');
++INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after');
++INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after');
++INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after');
++connection node_2;
++Performing --wsrep-recover ...
++connection node_2;
++Starting server ...
++Using --wsrep-start-position when starting mysqld ...
++SET AUTOCOMMIT=OFF;
++START TRANSACTION;
++INSERT INTO t1 (f1) VALUES ('node2_committed_after');
++INSERT INTO t1 (f1) VALUES ('node2_committed_after');
++INSERT INTO t1 (f1) VALUES ('node2_committed_after');
++INSERT INTO t1 (f1) VALUES ('node2_committed_after');
++INSERT INTO t1 (f1) VALUES ('node2_committed_after');
++COMMIT;
++connection node_1;
++INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after');
++INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after');
++INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after');
++INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after');
++INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after');
++COMMIT;
++SET AUTOCOMMIT=OFF;
++START TRANSACTION;
++INSERT INTO t1 (f1) VALUES ('node1_committed_after');
++INSERT INTO t1 (f1) VALUES ('node1_committed_after');
++INSERT INTO t1 (f1) VALUES ('node1_committed_after');
++INSERT INTO t1 (f1) VALUES ('node1_committed_after');
++INSERT INTO t1 (f1) VALUES ('node1_committed_after');
++COMMIT;
++connection node_1a_galera_st_kill_slave_ddl;
++INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after');
++INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after');
++INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after');
++INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after');
++INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after');
++ROLLBACK;
++SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1';
++COUNT(*) = 2
++1
++SELECT COUNT(*) = 35 FROM t1;
++COUNT(*) = 35
++1
++SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1;
++COUNT(*) = 0
++1
++COMMIT;
++SET AUTOCOMMIT=ON;
++connection node_1;
++SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1';
++COUNT(*) = 2
++1
++SELECT COUNT(*) = 35 FROM t1;
++COUNT(*) = 35
++1
++SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1;
++COUNT(*) = 0
++1
++DROP TABLE t1;
++COMMIT;
++SET AUTOCOMMIT=ON;
++SET GLOBAL debug_dbug = $debug_orig;
+ disconnect node_2;
+ disconnect node_1;

mysql-test/suite/galera/t/galera_sst_mariabackup_logarchive.cnf

+!include ../galera_2nodes.cnf
+
+[mysqld]
+wsrep_sst_method=mariabackup
+wsrep_sst_auth="root:"
+wsrep_debug=ON
+
+[mysqld.1]
+wsrep_provider_options='base_port=@mysqld.1.#galera_port;gcache.size=1;pc.ignore_sb=true'
+
+[mysqld.2]
+wsrep_provider_options='base_port=@mysqld.2.#galera_port;gcache.size=1;pc.ignore_sb=true'
+
+[sst]
+transferfmt=@ENV.MTR_GALERA_TFMT
+sst-log-archive=1
+sst-log-archive-dir=@ENV.MYSQLTEST_VARDIR/tmp/logarchive

mysql-test/suite/galera/t/galera_sst_mariabackup_logarchive.test

+--source include/big_test.inc
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+--source include/have_mariabackup.inc
+
+# Save original auto_increment_offset values.
+--let $node_1=node_1
+--let $node_2=node_2
+--source include/auto_increment_offset_save.inc
+
+--connection node_2
+
+--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size';
+--source include/wait_condition.inc
+
+--let $wait_condition = SELECT VARIABLE_VALUE = 'Synced' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_state_comment';
+--source include/wait_condition.inc
+
+--connection node_1
+
+--let $wait_condition = SELECT VARIABLE_VALUE = 'Synced' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_state_comment';
+--source include/wait_condition.inc
+
+--remove_files_wildcard $MYSQLTEST_VARDIR/tmp/logarchive *
+
+--source suite/galera/include/galera_st_shutdown_slave.inc
+--source suite/galera/include/galera_st_clean_slave.inc
+
+--source suite/galera/include/galera_st_kill_slave.inc
+--source suite/galera/include/galera_st_kill_slave_ddl.inc
+
+# Restore original auto_increment_offset values.
+--source include/auto_increment_offset_restore.inc
+
+--let sst_test_true_count=7
+if (`select version() like '%debug%'`)
+{
+  --let sst_test_true_count=10
+}
+
+--source include/galera_end.inc
+
+--list_files_write_file $MYSQLTEST_VARDIR/tmp/logarchive_list $MYSQLTEST_VARDIR/tmp/logarchive mariabackup.*.log.*.gz
+
+--perl
+  use strict;
+  use warnings;
+  my $file= $ENV{'MYSQLTEST_VARDIR'}.'/tmp/logarchive_list';
+  my $count= 0;
+  open(FILE, "$file") or die("Error $? opening $file: $!\n");
+  while (<FILE>) {
+    my $line = $_;
+    $count++;
+  }
+  close(FILE) or die("Error $? closing $file: $!");
+  if ($count != $ENV{'sst_test_true_count'}) {
+    die("Wrong log archives counter: $count");
+  }
+EOF
+
+--remove_file $MYSQLTEST_VARDIR/tmp/logarchive_list
+--remove_files_wildcard $MYSQLTEST_VARDIR/tmp/logarchive *
+--rmdir $MYSQLTEST_VARDIR/tmp/logarchive

scripts/wsrep_sst_mariabackup.sh

@@ -535,7 +535,14 @@ read_cnf()
     ssystag=$(parse_cnf mysqld_safe syslog-tag "${SST_SYSLOG_TAG:-}")
     ssystag="$ssystag-"
     sstlogarchive=$(parse_cnf sst sst-log-archive 1)
-    sstlogarchivedir=$(parse_cnf sst sst-log-archive-dir '/tmp/sst_log_archive')
+    sstlogarchivedir=""
+    if [ $sstlogarchive -ne 0 ]; then
+        sstlogarchivedir=$(parse_cnf sst sst-log-archive-dir \
+                           '/tmp/sst_log_archive')
+        if [ -n "$sstlogarchivedir" ]; then
+            sstlogarchivedir=$(trim_dir "$sstlogarchivedir")
+        fi
+    fi
 
     if [ $speciald -eq 0 ]; then
         wsrep_log_error \
@@ -879,7 +886,15 @@ else
 
         if [ -n "$sstlogarchivedir" ]; then
             if [ ! -d "$sstlogarchivedir" ]; then
-                mkdir -p "$sstlogarchivedir"
+                if ! mkdir -p "$sstlogarchivedir"; then
+                    sstlogarchivedir=""
+                    wsrep_log_warning \
+                       "Unable to create '$sstlogarchivedir' directory"
+                fi
+            elif [ ! -w "$sstlogarchivedir" ]; then
+                sstlogarchivedir=""
+                wsrep_log_warning \
+                  "The '$sstlogarchivedir' directory is not writtable"
             fi
         fi
 
@@ -891,8 +906,8 @@ else
                 newfile="$INNOAPPLYLOG.$ARCHIVETIMESTAMP"
             fi
             wsrep_log_info "Moving '$INNOAPPLYLOG' to '$newfile'"
-            mv "$INNOAPPLYLOG" "$newfile"
-            gzip "$newfile"
+            mv "$INNOAPPLYLOG" "$newfile" && gzip "$newfile" || \
+                wsrep_log_warning "Failed to archive log file ('$newfile')"
         fi
 
         if [ -e "$INNOMOVELOG" ]; then
@@ -903,8 +918,8 @@ else
                 newfile="$INNOMOVELOG.$ARCHIVETIMESTAMP"
             fi
             wsrep_log_info "Moving '$INNOMOVELOG' to '$newfile'"
-            mv "$INNOMOVELOG" "$newfile"
-            gzip "$newfile"
+            mv "$INNOMOVELOG" "$newfile" && gzip "$newfile" ||
+                wsrep_log_warning "Failed to archive log file ('$newfile')"
         fi
 
         if [ -e "$INNOBACKUPLOG" ]; then
@@ -915,8 +930,8 @@ else
                 newfile="$INNOBACKUPLOG.$ARCHIVETIMESTAMP"
             fi
             wsrep_log_info "Moving '$INNOBACKUPLOG' to '$newfile'"
-            mv "$INNOBACKUPLOG" "$newfile"
-            gzip "$newfile"
+            mv "$INNOBACKUPLOG" "$newfile" && gzip "$newfile" ||
+                wsrep_log_warning "Failed to archive log file ('$newfile')"
         fi
     fi
     INNOAPPLY="> '$INNOAPPLYLOG' 2>&1"