MDEV-26064: mariabackup SST fails when starting with --innodb-force-recovery

If the server is started with the --innodb-force-recovery argument
on the command line, then during SST this argument can be passed to
mariabackup only at the --prepare stage, and accordingly it must be
removed from the --mysqld-args list (and it is not should be passed
to mariabackup otherwise).

This commit fixes a flaw in the SST scripts and add a test that
checks the ability to run the joiner node in a configuration that
uses --innodb-force-recovery=1.

mysql-test/suite/galera/r/galera_sst_mariabackup_force_recovery,debug.rdiff

+--- r/galera_sst_mariabackup.result	2021-04-10 14:25:04.142716409 +0300
++++ r/galera_sst_mariabackup,debug.reject	2021-04-10 14:53:30.033162191 +0300
+@@ -516,5 +516,189 @@
+ 1
+ DROP TABLE t1;
+ COMMIT;
++Performing State Transfer on a server that has been killed and restarted
++while a DDL was in progress on it
++connection node_1;
++CREATE TABLE t1 (id int not null primary key,f1 CHAR(255)) ENGINE=InnoDB;
++SET AUTOCOMMIT=OFF;
++START TRANSACTION;
++INSERT INTO t1 VALUES (1,'node1_committed_before');
++INSERT INTO t1 VALUES (2,'node1_committed_before');
++INSERT INTO t1 VALUES (3,'node1_committed_before');
++INSERT INTO t1 VALUES (4,'node1_committed_before');
++INSERT INTO t1 VALUES (5,'node1_committed_before');
++connection node_2;
++START TRANSACTION;
++INSERT INTO t1 VALUES (6,'node2_committed_before');
++INSERT INTO t1 VALUES (7,'node2_committed_before');
++INSERT INTO t1 VALUES (8,'node2_committed_before');
++INSERT INTO t1 VALUES (9,'node2_committed_before');
++INSERT INTO t1 VALUES (10,'node2_committed_before');
++COMMIT;
++SET GLOBAL debug_dbug = 'd,sync.alter_opened_table';
++connection node_1;
++ALTER TABLE t1 ADD COLUMN f2 INTEGER;
++connection node_2;
++SET wsrep_sync_wait = 0;
++Killing server ...
++connection node_1;
++SET AUTOCOMMIT=OFF;
++START TRANSACTION;
++INSERT INTO t1 (id,f1) VALUES (11,'node1_committed_during');
++INSERT INTO t1 (id,f1) VALUES (12,'node1_committed_during');
++INSERT INTO t1 (id,f1) VALUES (13,'node1_committed_during');
++INSERT INTO t1 (id,f1) VALUES (14,'node1_committed_during');
++INSERT INTO t1 (id,f1) VALUES (15,'node1_committed_during');
++COMMIT;
++START TRANSACTION;
++INSERT INTO t1 (id,f1) VALUES (16,'node1_to_be_committed_after');
++INSERT INTO t1 (id,f1) VALUES (17,'node1_to_be_committed_after');
++INSERT INTO t1 (id,f1) VALUES (18,'node1_to_be_committed_after');
++INSERT INTO t1 (id,f1) VALUES (19,'node1_to_be_committed_after');
++INSERT INTO t1 (id,f1) VALUES (20,'node1_to_be_committed_after');
++connect node_1a_galera_st_kill_slave_ddl, 127.0.0.1, root, , test, $NODE_MYPORT_1;
++SET AUTOCOMMIT=OFF;
++START TRANSACTION;
++INSERT INTO t1 (id,f1) VALUES (21,'node1_to_be_rollbacked_after');
++INSERT INTO t1 (id,f1) VALUES (22,'node1_to_be_rollbacked_after');
++INSERT INTO t1 (id,f1) VALUES (23,'node1_to_be_rollbacked_after');
++INSERT INTO t1 (id,f1) VALUES (24,'node1_to_be_rollbacked_after');
++INSERT INTO t1 (id,f1) VALUES (25,'node1_to_be_rollbacked_after');
++connection node_2;
++Performing --wsrep-recover ...
++connection node_2;
++Starting server ...
++Using --wsrep-start-position when starting mysqld ...
++SET AUTOCOMMIT=OFF;
++START TRANSACTION;
++INSERT INTO t1 (id,f1) VALUES (26,'node2_committed_after');
++INSERT INTO t1 (id,f1) VALUES (27,'node2_committed_after');
++INSERT INTO t1 (id,f1) VALUES (28,'node2_committed_after');
++INSERT INTO t1 (id,f1) VALUES (29,'node2_committed_after');
++INSERT INTO t1 (id,f1) VALUES (30,'node2_committed_after');
++COMMIT;
++connection node_1;
++INSERT INTO t1 (id,f1) VALUES (31,'node1_to_be_committed_after');
++INSERT INTO t1 (id,f1) VALUES (32,'node1_to_be_committed_after');
++INSERT INTO t1 (id,f1) VALUES (33,'node1_to_be_committed_after');
++INSERT INTO t1 (id,f1) VALUES (34,'node1_to_be_committed_after');
++INSERT INTO t1 (id,f1) VALUES (35,'node1_to_be_committed_after');
++COMMIT;
++SET AUTOCOMMIT=OFF;
++START TRANSACTION;
++INSERT INTO t1 (id,f1) VALUES (36,'node1_committed_after');
++INSERT INTO t1 (id,f1) VALUES (37,'node1_committed_after');
++INSERT INTO t1 (id,f1) VALUES (38,'node1_committed_after');
++INSERT INTO t1 (id,f1) VALUES (39,'node1_committed_after');
++INSERT INTO t1 (id,f1) VALUES (40,'node1_committed_after');
++COMMIT;
++connection node_1a_galera_st_kill_slave_ddl;
++INSERT INTO t1 (id,f1) VALUES (41,'node1_to_be_rollbacked_after');
++INSERT INTO t1 (id,f1) VALUES (42,'node1_to_be_rollbacked_after');
++INSERT INTO t1 (id,f1) VALUES (43,'node1_to_be_rollbacked_after');
++INSERT INTO t1 (id,f1) VALUES (44,'node1_to_be_rollbacked_after');
++INSERT INTO t1 (id,f1) VALUES (45,'node1_to_be_rollbacked_after');
++ROLLBACK;
++SET AUTOCOMMIT=ON;
++SET SESSION wsrep_sync_wait=15;
++SELECT COUNT(*) AS EXPECT_3 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1';
++EXPECT_3
++3
++SELECT COUNT(*) AS EXPECT_35 FROM t1;
++EXPECT_35
++35
++SELECT * FROM t1;
++id	f1	f2
++1	node1_committed_before	NULL
++2	node1_committed_before	NULL
++3	node1_committed_before	NULL
++4	node1_committed_before	NULL
++5	node1_committed_before	NULL
++6	node2_committed_before	NULL
++7	node2_committed_before	NULL
++8	node2_committed_before	NULL
++9	node2_committed_before	NULL
++10	node2_committed_before	NULL
++11	node1_committed_during	NULL
++12	node1_committed_during	NULL
++13	node1_committed_during	NULL
++14	node1_committed_during	NULL
++15	node1_committed_during	NULL
++16	node1_to_be_committed_after	NULL
++17	node1_to_be_committed_after	NULL
++18	node1_to_be_committed_after	NULL
++19	node1_to_be_committed_after	NULL
++20	node1_to_be_committed_after	NULL
++26	node2_committed_after	NULL
++27	node2_committed_after	NULL
++28	node2_committed_after	NULL
++29	node2_committed_after	NULL
++30	node2_committed_after	NULL
++31	node1_to_be_committed_after	NULL
++32	node1_to_be_committed_after	NULL
++33	node1_to_be_committed_after	NULL
++34	node1_to_be_committed_after	NULL
++35	node1_to_be_committed_after	NULL
++36	node1_committed_after	NULL
++37	node1_committed_after	NULL
++38	node1_committed_after	NULL
++39	node1_committed_after	NULL
++40	node1_committed_after	NULL
++SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1;
++COUNT(*) = 0
++1
++COMMIT;
++connection node_1;
++SET AUTOCOMMIT=ON;
++SET SESSION wsrep_sync_wait=15;
++SELECT COUNT(*) AS EXPECT_3 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1';
++EXPECT_3
++3
++SELECT COUNT(*) AS EXPECT_35 FROM t1;
++EXPECT_35
++35
++SELECT * FROM t1;
++id	f1	f2
++1	node1_committed_before	NULL
++2	node1_committed_before	NULL
++3	node1_committed_before	NULL
++4	node1_committed_before	NULL
++5	node1_committed_before	NULL
++6	node2_committed_before	NULL
++7	node2_committed_before	NULL
++8	node2_committed_before	NULL
++9	node2_committed_before	NULL
++10	node2_committed_before	NULL
++11	node1_committed_during	NULL
++12	node1_committed_during	NULL
++13	node1_committed_during	NULL
++14	node1_committed_during	NULL
++15	node1_committed_during	NULL
++16	node1_to_be_committed_after	NULL
++17	node1_to_be_committed_after	NULL
++18	node1_to_be_committed_after	NULL
++19	node1_to_be_committed_after	NULL
++20	node1_to_be_committed_after	NULL
++26	node2_committed_after	NULL
++27	node2_committed_after	NULL
++28	node2_committed_after	NULL
++29	node2_committed_after	NULL
++30	node2_committed_after	NULL
++31	node1_to_be_committed_after	NULL
++32	node1_to_be_committed_after	NULL
++33	node1_to_be_committed_after	NULL
++34	node1_to_be_committed_after	NULL
++35	node1_to_be_committed_after	NULL
++36	node1_committed_after	NULL
++37	node1_committed_after	NULL
++38	node1_committed_after	NULL
++39	node1_committed_after	NULL
++40	node1_committed_after	NULL
++SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1;
++COUNT(*) = 0
++1
++DROP TABLE t1;
++COMMIT;
++SET GLOBAL debug_dbug = $debug_orig;
+ disconnect node_2;
+ disconnect node_1;

mysql-test/suite/galera/t/galera_sst_mariabackup_force_recovery-master.opt

+--innodb-force-recovery=1

mysql-test/suite/galera/t/galera_sst_mariabackup_force_recovery.cnf

+!include ../galera_2nodes.cnf
+
+[mysqld]
+wsrep_sst_method=mariabackup
+wsrep_sst_auth="root:"
+wsrep_debug=1
+
+[mysqld.1]
+wsrep_provider_options='base_port=@mysqld.1.#galera_port;gcache.size=1;pc.ignore_sb=true'
+
+[mysqld.2]
+wsrep_provider_options='base_port=@mysqld.2.#galera_port;gcache.size=1;pc.ignore_sb=true'
+
+[sst]
+transferfmt=@ENV.MTR_GALERA_TFMT
+streamfmt=mbstream

mysql-test/suite/galera/t/galera_sst_mariabackup_force_recovery.test

+--source include/big_test.inc
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+--source include/have_mariabackup.inc
+
+# Save original auto_increment_offset values.
+--let $node_1=node_1
+--let $node_2=node_2
+--source include/auto_increment_offset_save.inc
+
+--source suite/galera/include/galera_st_shutdown_slave.inc
+--source suite/galera/include/galera_st_clean_slave.inc
+
+--source suite/galera/include/galera_st_kill_slave.inc
+--source suite/galera/include/galera_st_kill_slave_ddl.inc
+
+# Restore original auto_increment_offset values.
+--source include/auto_increment_offset_restore.inc
+
+--source include/galera_end.inc

scripts/wsrep_sst_common.sh

@@ -44,6 +44,7 @@ WSREP_SST_OPT_HOST_ESCAPED=""
 INNODB_DATA_HOME_DIR="${INNODB_DATA_HOME_DIR:-}"
 INNODB_LOG_GROUP_HOME="${INNODB_LOG_GROUP_HOME:-}"
 INNODB_UNDO_DIR="${INNODB_UNDO_DIR:-}"
+INNODB_FORCE_RECOVERY=""
 INNOEXTRA=""
 
 while [ $# -gt 0 ]; do
@@ -382,6 +383,14 @@ case "$1" in
                        fi
                        skip_mysqld_arg=1
                        ;;
+                   '--innodb-force-recovery')
+                       if [ -n "$value" ]; then
+                           if [ "$value" -ne 0 ]; then
+                               INNODB_FORCE_RECOVERY="$value"
+                           fi
+                       fi
+                       skip_mysqld_arg=1
+                       ;;
                    '--log-bin')
                        if [ -z "$WSREP_SST_OPT_BINLOG" ]; then
                            MYSQLD_OPT_LOG_BIN="$value"
@@ -499,6 +508,7 @@ if [ -n "$WSREP_SST_OPT_BINLOG" ]; then
     fi
 fi
 
+readonly INNODB_FORCE_RECOVERY
 readonly WSREP_SST_OPT_MYSQLD
 
 get_binlog()

scripts/wsrep_sst_mariabackup.sh

@@ -951,7 +951,11 @@ setup_commands()
     if [ -n "$WSREP_SST_OPT_MYSQLD" ]; then
         mysqld_args="--mysqld-args $WSREP_SST_OPT_MYSQLD"
     fi
-    INNOAPPLY="$BACKUP_BIN --prepare $disver $iapts $INNOEXTRA --target-dir='$DATA' --datadir='$DATA' $mysqld_args $INNOAPPLY"
+    if [ -z "$INNODB_FORCE_RECOVERY" ]; then
+        INNOAPPLY="$BACKUP_BIN --prepare $disver $iapts $INNOEXTRA --target-dir='$DATA' --datadir='$DATA' $mysqld_args $INNOAPPLY"
+    else
+        INNOAPPLY="$BACKUP_BIN --prepare $disver $iapts $INNOEXTRA --innodb-force-recovery=$INNODB_FORCE_RECOVERY --target-dir='$DATA' --datadir='$DATA' $mysqld_args $INNOAPPLY"
+    fi
     INNOMOVE="$BACKUP_BIN $WSREP_SST_OPT_CONF --move-back $disver $impts --force-non-empty-directories --target-dir='$DATA' --datadir='${TDATA:-$DATA}' $INNOMOVE"
     INNOBACKUP="$BACKUP_BIN $WSREP_SST_OPT_CONF --backup $disver $iopts $tmpopts $INNOEXTRA --galera-info --stream=$sfmt --target-dir='$itmpdir' --datadir='$DATA' $mysqld_args $INNOBACKUP"
 }