MDEV-26064: mariabackup SST fails when starting with --innodb-force-recovery

If the server is started with the --innodb-force-recovery argument
on the command line, then during SST this argument can be passed to
mariabackup only at the --prepare stage, and accordingly it must be
removed from the --mysqld-args list (and it is not should be passed
to mariabackup otherwise).

This commit fixes a flaw in the SST scripts and add a test that
checks the ability to run the joiner node in a configuration that
uses --innodb-force-recovery=1.

mysql-test/suite/galera/r/galera_sst_mariabackup_force_recovery,debug.rdiff

+--- r/galera_sst_mariabackup.result	2018-11-21 16:50:35.766982279 +0200
++++ r/galera_sst_mariabackup.reject	2018-11-22 09:20:10.344408266 +0200
+@@ -286,5 +286,113 @@
+ DROP TABLE t1;
+ COMMIT;
+ SET AUTOCOMMIT=ON;
++Performing State Transfer on a server that has been killed and restarted
++while a DDL was in progress on it
++connection node_1;
++CREATE TABLE t1 (f1 CHAR(255)) ENGINE=InnoDB;
++SET AUTOCOMMIT=OFF;
++START TRANSACTION;
++INSERT INTO t1 VALUES ('node1_committed_before');
++INSERT INTO t1 VALUES ('node1_committed_before');
++INSERT INTO t1 VALUES ('node1_committed_before');
++INSERT INTO t1 VALUES ('node1_committed_before');
++INSERT INTO t1 VALUES ('node1_committed_before');
++connection node_2;
++START TRANSACTION;
++INSERT INTO t1 VALUES ('node2_committed_before');
++INSERT INTO t1 VALUES ('node2_committed_before');
++INSERT INTO t1 VALUES ('node2_committed_before');
++INSERT INTO t1 VALUES ('node2_committed_before');
++INSERT INTO t1 VALUES ('node2_committed_before');
++COMMIT;
++SET GLOBAL debug_dbug = 'd,sync.alter_opened_table';
++connection node_1;
++ALTER TABLE t1 ADD COLUMN f2 INTEGER;
++connection node_2;
++SET wsrep_sync_wait = 0;
++Killing server ...
++connection node_1;
++SET AUTOCOMMIT=OFF;
++START TRANSACTION;
++INSERT INTO t1 (f1) VALUES ('node1_committed_during');
++INSERT INTO t1 (f1) VALUES ('node1_committed_during');
++INSERT INTO t1 (f1) VALUES ('node1_committed_during');
++INSERT INTO t1 (f1) VALUES ('node1_committed_during');
++INSERT INTO t1 (f1) VALUES ('node1_committed_during');
++COMMIT;
++START TRANSACTION;
++INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after');
++INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after');
++INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after');
++INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after');
++INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after');
++connect node_1a_galera_st_kill_slave_ddl, 127.0.0.1, root, , test, $NODE_MYPORT_1;
++SET AUTOCOMMIT=OFF;
++START TRANSACTION;
++INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after');
++INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after');
++INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after');
++INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after');
++INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after');
++connection node_2;
++Performing --wsrep-recover ...
++connection node_2;
++Starting server ...
++Using --wsrep-start-position when starting mysqld ...
++SET AUTOCOMMIT=OFF;
++START TRANSACTION;
++INSERT INTO t1 (f1) VALUES ('node2_committed_after');
++INSERT INTO t1 (f1) VALUES ('node2_committed_after');
++INSERT INTO t1 (f1) VALUES ('node2_committed_after');
++INSERT INTO t1 (f1) VALUES ('node2_committed_after');
++INSERT INTO t1 (f1) VALUES ('node2_committed_after');
++COMMIT;
++connection node_1;
++INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after');
++INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after');
++INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after');
++INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after');
++INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after');
++COMMIT;
++SET AUTOCOMMIT=OFF;
++START TRANSACTION;
++INSERT INTO t1 (f1) VALUES ('node1_committed_after');
++INSERT INTO t1 (f1) VALUES ('node1_committed_after');
++INSERT INTO t1 (f1) VALUES ('node1_committed_after');
++INSERT INTO t1 (f1) VALUES ('node1_committed_after');
++INSERT INTO t1 (f1) VALUES ('node1_committed_after');
++COMMIT;
++connection node_1a_galera_st_kill_slave_ddl;
++INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after');
++INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after');
++INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after');
++INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after');
++INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after');
++ROLLBACK;
++SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1';
++COUNT(*) = 2
++1
++SELECT COUNT(*) = 35 FROM t1;
++COUNT(*) = 35
++1
++SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1;
++COUNT(*) = 0
++1
++COMMIT;
++SET AUTOCOMMIT=ON;
++connection node_1;
++SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1';
++COUNT(*) = 2
++1
++SELECT COUNT(*) = 35 FROM t1;
++COUNT(*) = 35
++1
++SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1;
++COUNT(*) = 0
++1
++DROP TABLE t1;
++COMMIT;
++SET AUTOCOMMIT=ON;
++SET GLOBAL debug_dbug = $debug_orig;
+ disconnect node_2;
+ disconnect node_1;

mysql-test/suite/galera/t/galera_sst_mariabackup_force_recovery-master.opt

+--innodb-force-recovery=1

mysql-test/suite/galera/t/galera_sst_mariabackup_force_recovery.cnf

+!include ../galera_2nodes.cnf
+
+[mysqld]
+wsrep_sst_method=mariabackup
+wsrep_sst_auth="root:"
+wsrep_debug=ON
+
+[mysqld.1]
+wsrep_provider_options='base_port=@mysqld.1.#galera_port;gcache.size=1;pc.ignore_sb=true'
+
+[mysqld.2]
+wsrep_provider_options='base_port=@mysqld.2.#galera_port;gcache.size=1;pc.ignore_sb=true'
+
+[sst]
+transferfmt=@ENV.MTR_GALERA_TFMT
+streamfmt=mbstream

mysql-test/suite/galera/t/galera_sst_mariabackup_force_recovery.test

+--source include/big_test.inc
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+--source include/have_mariabackup.inc
+
+# Save original auto_increment_offset values.
+--let $node_1=node_1
+--let $node_2=node_2
+--source include/auto_increment_offset_save.inc
+
+--source suite/galera/include/galera_st_shutdown_slave.inc
+--source suite/galera/include/galera_st_clean_slave.inc
+
+--source suite/galera/include/galera_st_kill_slave.inc
+--source suite/galera/include/galera_st_kill_slave_ddl.inc
+
+# Restore original auto_increment_offset values.
+--source include/auto_increment_offset_restore.inc
+
+--source include/galera_end.inc

scripts/wsrep_sst_common.sh

@@ -44,6 +44,7 @@ WSREP_SST_OPT_HOST_ESCAPED=""
 INNODB_DATA_HOME_DIR="${INNODB_DATA_HOME_DIR:-}"
 INNODB_LOG_GROUP_HOME="${INNODB_LOG_GROUP_HOME:-}"
 INNODB_UNDO_DIR="${INNODB_UNDO_DIR:-}"
+INNODB_FORCE_RECOVERY=""
 INNOEXTRA=""
 
 while [ $# -gt 0 ]; do
@@ -382,6 +383,14 @@ case "$1" in
                        fi
                        skip_mysqld_arg=1
                        ;;
+                   '--innodb-force-recovery')
+                       if [ -n "$value" ]; then
+                           if [ "$value" -ne 0 ]; then
+                               INNODB_FORCE_RECOVERY="$value"
+                           fi
+                       fi
+                       skip_mysqld_arg=1
+                       ;;
                    '--log-bin')
                        if [ -z "$WSREP_SST_OPT_BINLOG" ]; then
                            MYSQLD_OPT_LOG_BIN="$value"
@@ -499,6 +508,7 @@ if [ -n "$WSREP_SST_OPT_BINLOG" ]; then
     fi
 fi
 
+readonly INNODB_FORCE_RECOVERY
 readonly WSREP_SST_OPT_MYSQLD
 
 get_binlog()

scripts/wsrep_sst_mariabackup.sh

@@ -951,7 +951,11 @@ setup_commands()
     if [ -n "$WSREP_SST_OPT_MYSQLD" ]; then
         mysqld_args="--mysqld-args $WSREP_SST_OPT_MYSQLD"
     fi
-    INNOAPPLY="$BACKUP_BIN --prepare $disver $iapts $INNOEXTRA --target-dir='$DATA' --datadir='$DATA' $mysqld_args $INNOAPPLY"
+    if [ -z "$INNODB_FORCE_RECOVERY" ]; then
+        INNOAPPLY="$BACKUP_BIN --prepare $disver $iapts $INNOEXTRA --target-dir='$DATA' --datadir='$DATA' $mysqld_args $INNOAPPLY"
+    else
+        INNOAPPLY="$BACKUP_BIN --prepare $disver $iapts $INNOEXTRA --innodb-force-recovery=$INNODB_FORCE_RECOVERY --target-dir='$DATA' --datadir='$DATA' $mysqld_args $INNOAPPLY"
+    fi
     INNOMOVE="$BACKUP_BIN $WSREP_SST_OPT_CONF --move-back $disver $impts --force-non-empty-directories --target-dir='$DATA' --datadir='${TDATA:-$DATA}' $INNOMOVE"
     INNOBACKUP="$BACKUP_BIN $WSREP_SST_OPT_CONF --backup $disver $iopts $tmpopts $INNOEXTRA --galera-info --stream=$sfmt --target-dir='$itmpdir' --datadir='$DATA' $mysqld_args $INNOBACKUP"
 }

scripts/wsrep_sst_xtrabackup-v2.sh

@@ -973,7 +973,11 @@ fi
 
 setup_commands()
 {
-    INNOAPPLY="$BACKUP_BIN $disver $iapts $INNOEXTRA --apply-log $rebuildcmd '$DATA' $INNOAPPLY"
+    if [ -z "$INNODB_FORCE_RECOVERY" ]; then
+        INNOAPPLY="$BACKUP_BIN $disver $iapts $INNOEXTRA --apply-log $rebuildcmd '$DATA' $INNOAPPLY"
+    else
+        INNOAPPLY="$BACKUP_BIN $disver $iapts $INNOEXTRA --innodb-force-recovery=$INNODB_FORCE_RECOVERY --apply-log $rebuildcmd '$DATA' $INNOAPPLY"
+    fi
     INNOMOVE="$BACKUP_BIN $WSREP_SST_OPT_CONF --move-back $disver $impts --force-non-empty-directories '$DATA' $INNOMOVE"
     sfmt_work="$sfmt"
     if [ "$sfmt" = 'mbstream' ]; then