MDEV-32344: IST failed with ssl-mode=VERIFY_CA

This commit fixes a bug where IST could be rejected in favor of SST
when ssl-mode=VERIFY_CA and when mariabackup is used. It also contains
a test and small code simplifications that will make it easier to find
bugs in the future.

mysql-test/suite/galera/r/galera_ist_mariabackup_verify_ca.result

+connection node_2;
+connection node_1;
+connection node_1;
+connection node_2;
+CREATE TABLE t1 (f1 INT PRIMARY KEY);
+connection node_2;
+# Verify that graceful shutdown succeeds...
+connection node_1;
+INSERT INTO t1 VALUES (1);
+INSERT INTO t1 VALUES (2);
+INSERT INTO t1 VALUES (3);
+connection node_2;
+# Start node_2 again...
+SELECT * FROM t1;
+f1
+1
+2
+3
+connection node_1;
+include/assert_grep.inc [mariabackup IST completed on joiner]
+DROP TABLE t1;

mysql-test/suite/galera/r/galera_ist_rsync_verify_ca.result

+connection node_2;
+connection node_1;
+connection node_1;
+connection node_2;
+CREATE TABLE t1 (f1 INT PRIMARY KEY);
+connection node_2;
+# Verify that graceful shutdown succeeds...
+connection node_1;
+INSERT INTO t1 VALUES (1);
+INSERT INTO t1 VALUES (2);
+INSERT INTO t1 VALUES (3);
+connection node_2;
+# Start node_2 again...
+SELECT * FROM t1;
+f1
+1
+2
+3
+connection node_1;
+include/assert_grep.inc [rsync IST completed on joiner]
+DROP TABLE t1;

mysql-test/suite/galera/t/galera_ist_mariabackup_verify_ca.cnf

+!include ../galera_2nodes.cnf
+
+[mysqld]
+wsrep_sst_method=mariabackup
+wsrep_sst_auth=root:
+
+ssl-cert=@ENV.MYSQL_TEST_DIR/std_data/server-cert.pem
+ssl-key=@ENV.MYSQL_TEST_DIR/std_data/server-key.pem
+ssl-ca=@ENV.MYSQL_TEST_DIR/std_data/cacert.pem
+
+[mysqld.1]
+wsrep_provider_options='base_port=@mysqld.1.#galera_port;pc.ignore_sb=true'
+
+[mysqld.2]
+wsrep_provider_options='base_port=@mysqld.2.#galera_port;pc.ignore_sb=true'
+
+[sst]
+ssl-mode=VERIFY_CA
+transferfmt=@ENV.MTR_GALERA_TFMT
+streamfmt=mbstream

mysql-test/suite/galera/t/galera_ist_mariabackup_verify_ca.test

+--source include/big_test.inc
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+--source include/have_mariabackup.inc
+
+--let $node_1=node_1
+--let $node_2=node_2
+--source include/auto_increment_offset_save.inc
+
+--let $LOG_FILE=$MYSQL_TMP_DIR/galera_node2.log
+--error 0,1
+--remove_file $LOG_FILE
+
+CREATE TABLE t1 (f1 INT PRIMARY KEY);
+
+--connection node_2
+
+--echo # Verify that graceful shutdown succeeds...
+--source include/shutdown_mysqld.inc
+
+--connection node_1
+
+--let $wait_condition = SELECT VARIABLE_VALUE = 1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'
+--source include/wait_condition.inc
+
+INSERT INTO t1 VALUES (1);
+INSERT INTO t1 VALUES (2);
+INSERT INTO t1 VALUES (3);
+
+--connection node_2
+--echo # Start node_2 again...
+--let $restart_noprint=2
+--let $start_mysqld_params=--log-error=$LOG_FILE
+--source include/start_mysqld.inc
+
+--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'
+--source include/wait_condition.inc
+
+SELECT * FROM t1;
+
+--let $start_mysqld_params=
+--source include/restart_mysqld.inc
+
+--connection node_1
+
+--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'
+--source include/wait_condition.inc
+
+# Confirm that IST took place
+--let $assert_text = mariabackup IST completed on joiner
+--let $assert_select = mariabackup IST completed on joiner
+--let $assert_count = 1
+--let $assert_file = $LOG_FILE
+--let $assert_only_after = Prepared IST receiver for
+--source include/assert_grep.inc
+
+DROP TABLE t1;
+
+--source include/auto_increment_offset_restore.inc
+
+--remove_file $LOG_FILE

mysql-test/suite/galera/t/galera_ist_rsync_verify_ca.cnf

+!include ../galera_2nodes.cnf
+
+[mysqld]
+wsrep_sst_method=rsync
+
+ssl-cert=@ENV.MYSQL_TEST_DIR/std_data/server-cert.pem
+ssl-key=@ENV.MYSQL_TEST_DIR/std_data/server-key.pem
+ssl-ca=@ENV.MYSQL_TEST_DIR/std_data/cacert.pem
+
+[mysqld.1]
+wsrep_provider_options='base_port=@mysqld.1.#galera_port;pc.ignore_sb=true'
+
+[mysqld.2]
+wsrep_provider_options='base_port=@mysqld.2.#galera_port;pc.ignore_sb=true'
+
+[sst]
+ssl-mode=VERIFY_CA

mysql-test/suite/galera/t/galera_ist_rsync_verify_ca.test

+--source include/big_test.inc
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+
+--let $node_1=node_1
+--let $node_2=node_2
+--source include/auto_increment_offset_save.inc
+
+--let $LOG_FILE=$MYSQL_TMP_DIR/galera_node2.log
+--error 0,1
+--remove_file $LOG_FILE
+
+CREATE TABLE t1 (f1 INT PRIMARY KEY);
+
+--connection node_2
+
+--echo # Verify that graceful shutdown succeeds...
+--source include/shutdown_mysqld.inc
+
+--connection node_1
+
+--let $wait_condition = SELECT VARIABLE_VALUE = 1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'
+--source include/wait_condition.inc
+
+INSERT INTO t1 VALUES (1);
+INSERT INTO t1 VALUES (2);
+INSERT INTO t1 VALUES (3);
+
+--connection node_2
+--echo # Start node_2 again...
+--let $restart_noprint=2
+--let $start_mysqld_params=--log-error=$LOG_FILE
+--source include/start_mysqld.inc
+
+--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'
+--source include/wait_condition.inc
+
+SELECT * FROM t1;
+
+--let $start_mysqld_params=
+--source include/restart_mysqld.inc
+
+--connection node_1
+
+--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'
+--source include/wait_condition.inc
+
+# Confirm that IST took place
+--let $assert_text = rsync IST completed on joiner
+--let $assert_select = rsync IST completed on joiner
+--let $assert_count = 1
+--let $assert_file = $LOG_FILE
+--let $assert_only_after = Prepared IST receiver for
+--source include/assert_grep.inc
+
+DROP TABLE t1;
+
+--source include/auto_increment_offset_restore.inc
+
+--remove_file $LOG_FILE

scripts/wsrep_sst_mariabackup.sh

@@ -102,6 +102,7 @@ if [ -z "$BACKUP_BIN" ]; then
 fi
 
 DATA="$WSREP_SST_OPT_DATA"
+
 INFO_FILE='xtrabackup_galera_info'
 IST_FILE='xtrabackup_ist'
 MAGIC_FILE="$DATA/$INFO_FILE"
@@ -1042,6 +1043,23 @@ setup_commands()
     INNOBACKUP="$BACKUP_BIN$WSREP_SST_OPT_CONF --backup$disver${iopts:+ }$iopts$tmpopts$INNOEXTRA --galera-info --stream=$sfmt --target-dir='$itmpdir' --datadir='$DATA'$mysqld_args $INNOBACKUP"
 }
 
+send_magic()
+{
+    # Store donor's wsrep GTID (state ID) and wsrep_gtid_domain_id
+    # (separated by a space).
+    echo "$WSREP_SST_OPT_GTID $WSREP_SST_OPT_GTID_DOMAIN_ID" > "$MAGIC_FILE"
+
+    if [ -n "$WSREP_SST_OPT_REMOTE_PSWD" ]; then
+        # Let joiner know that we know its secret
+        echo "$SECRET_TAG $WSREP_SST_OPT_REMOTE_PSWD" >> "$MAGIC_FILE"
+    fi
+
+    if [ $WSREP_SST_OPT_BYPASS -eq 0 -a $WSREP_SST_OPT_PROGRESS -eq 1 ]; then
+        # Tell joiner what to expect:
+        echo "$TOTAL_TAG $payload" >> "$MAGIC_FILE"
+    fi
+}
+
 get_stream
 get_transfer
 
@@ -1099,20 +1117,7 @@ if [ "$WSREP_SST_OPT_ROLE" = 'donor' ]; then
         fi
 
         wsrep_log_info "Streaming GTID file before SST"
-
-        # Store donor's wsrep GTID (state ID) and wsrep_gtid_domain_id
-        # (separated by a space).
-        echo "$WSREP_SST_OPT_GTID $WSREP_SST_OPT_GTID_DOMAIN_ID" > "$MAGIC_FILE"
-
-        if [ -n "$WSREP_SST_OPT_REMOTE_PSWD" ]; then
-            # Let joiner know that we know its secret
-            echo "$SECRET_TAG $WSREP_SST_OPT_REMOTE_PSWD" >> "$MAGIC_FILE"
-        fi
-
-        if [ $WSREP_SST_OPT_PROGRESS -eq 1 ]; then
-            # Tell joiner what to expect:
-            echo "$TOTAL_TAG $payload" >> "$MAGIC_FILE"
-        fi
+        send_magic
 
         ttcmd="$tcmd"
 
@@ -1202,9 +1207,8 @@ if [ "$WSREP_SST_OPT_ROLE" = 'donor' ]; then
         wsrep_log_info "Bypassing the SST for IST"
         echo "continue" # now server can resume updating data
 
-        # Store donor's wsrep GTID (state ID) and wsrep_gtid_domain_id
-        # (separated by a space).
-        echo "$WSREP_SST_OPT_GTID $WSREP_SST_OPT_GTID_DOMAIN_ID" > "$MAGIC_FILE"
+        send_magic
+
         echo "1" > "$DATA/$IST_FILE"
 
         if [ -n "$scomp" ]; then
@@ -1310,7 +1314,7 @@ else # joiner
         impts="--parallel=$backup_threads${impts:+ }$impts"
     fi
 
-    SST_PID="$WSREP_SST_OPT_DATA/wsrep_sst.pid"
+    SST_PID="$DATA/wsrep_sst.pid"
 
     # give some time for previous SST to complete:
     check_round=0
@@ -1451,8 +1455,8 @@ else # joiner
 
         TDATA="$DATA"
         DATA="$DATA/.sst"
-
         MAGIC_FILE="$DATA/$INFO_FILE"
+
         wsrep_log_info "Waiting for SST streaming to complete!"
         monitor_process $jpid
 

scripts/wsrep_sst_rsync.sh

@@ -149,10 +149,12 @@ check_pid_and_port()
     check_pid "$pid_file" && [ $CHECK_PID -eq $pid ]
 }
 
-STUNNEL_CONF="$WSREP_SST_OPT_DATA/stunnel.conf"
-STUNNEL_PID="$WSREP_SST_OPT_DATA/stunnel.pid"
+DATA="$WSREP_SST_OPT_DATA"
+
+STUNNEL_CONF="$DATA/stunnel.conf"
+STUNNEL_PID="$DATA/stunnel.pid"
 
-MAGIC_FILE="$WSREP_SST_OPT_DATA/rsync_sst_complete"
+MAGIC_FILE="$DATA/rsync_sst_complete"
 
 get_binlog
 
@@ -163,7 +165,6 @@ fi
 
 OLD_PWD="$(pwd)"
 
-DATA="$WSREP_SST_OPT_DATA"
 if [ -n "$DATA" -a "$DATA" != '.' ]; then
     [ ! -d "$DATA" ] && mkdir -p "$DATA"
     cd "$DATA"
@@ -347,7 +348,7 @@ fi
 readonly SECRET_TAG='secret'
 readonly BYPASS_TAG='bypass'
 
-SST_PID="$WSREP_SST_OPT_DATA/wsrep_sst.pid"
+SST_PID="$DATA/wsrep_sst.pid"
 
 # give some time for previous SST to complete:
 check_round=0
@@ -379,8 +380,8 @@ done
 
 MODULE="${WSREP_SST_OPT_MODULE:-rsync_sst}"
 
-RSYNC_PID="$WSREP_SST_OPT_DATA/$MODULE.pid"
-RSYNC_CONF="$WSREP_SST_OPT_DATA/$MODULE.conf"
+RSYNC_PID="$DATA/$MODULE.pid"
+RSYNC_CONF="$DATA/$MODULE.conf"
 
 # give some time for rsync from the previous SST to complete:
 check_round=0
@@ -422,8 +423,8 @@ EOF
 
     if [ $WSREP_SST_OPT_BYPASS -eq 0 ]; then
 
-        FLUSHED="$WSREP_SST_OPT_DATA/tables_flushed"
-        ERROR="$WSREP_SST_OPT_DATA/sst_error"
+        FLUSHED="$DATA/tables_flushed"
+        ERROR="$DATA/sst_error"
 
         [ -f "$FLUSHED" ] && rm -f "$FLUSHED"
         [ -f "$ERROR" ]   && rm -f "$ERROR"
@@ -580,7 +581,7 @@ FILTER="-f '- /lost+found'
         eval rsync ${STUNNEL:+"--rsh='$STUNNEL'"} \
               --owner --group --perms --links --specials \
               --ignore-times --inplace --dirs --delete --quiet \
-              $WHOLE_FILE_OPT $FILTER "'$WSREP_SST_OPT_DATA/'" \
+              $WHOLE_FILE_OPT $FILTER "'$DATA/'" \
               "'rsync://$WSREP_SST_OPT_ADDR'" >&2 || RC=$?
 
         if [ $RC -ne 0 ]; then
@@ -688,7 +689,7 @@ FILTER="-f '- /lost+found'
              -f '- $ib_log_dir/ib_logfile[0-9]*' \
              -f '- $ar_log_dir/aria_log_control' \
              -f '- $ar_log_dir/aria_log.*' \
-             "$WSREP_SST_OPT_DATA/{}/" \
+             "$DATA/{}/" \
              "rsync://$WSREP_SST_OPT_ADDR/{}" >&2 || RC=$?
 
         cd "$OLD_PWD"
@@ -770,7 +771,7 @@ read only = no
 timeout = 300
 $SILENT
 [$MODULE]
-    path = $WSREP_SST_OPT_DATA
+    path = $DATA
     exclude = .zfs
 [$MODULE-log_dir]
     path = $ib_log_dir