Skip to content

M
MariaDB
Commit 8c8a6ed3 authored by Julius Goryavsky's avatar Julius Goryavsky
Browse files
Options

MDEV-25719: stunnel uses "verifyChain" without subject checks 

Another batch of changes that should make the SST process
more reliable in all scenarios:

 1) Added hostname or CN verification when stunnel is used
    with certificate chain verification (verifyChain = yes);
 2) Added check for the absence of the stunnel utility for
    mtr tests;
 3) Deletion of working files before and after SST is done
    more accurately;
 4) rsync on joiner can be run even if the path to its
    configuration file contains spaces;
 5) More accurate directory creation (for data files and
    for logs);
 6) IST with mysqldump no longer turns off statement logging;
 7) Reset password for mysqldump when password is empty but
    username is specified;
 8) More reliable quoting when generating statements in
    wsrep_sst_mysqldump;
 9) Added explicit generation of 2048-bit Diffie-Hellman
    parameters for sockat < 1.7.3, by analogy with xtrabackup;
10) Compression parameters for qpress are read from all
    suitable server groups in configuration file, as well as
    from the [sst] and [xtrabackup] groups;
11) Added a test that checks compression using qpress;
12) Checking for optional utilities is modified to work even
    if they implemented as built-in shell commands (unlikely
    on real systems, but more reliable).
parent 62944917
Expand all Show whitespace changes
Inline Side-by-side
Showing with 367 additions and 233 deletions
mysql-test/suite.pm
View file @ 8c8a6ed3
......@@ -63,7 +63,7 @@ sub skip_combinations {
unless ::have_mariabackup();
$skip{'include/have_mariabackup.inc'} = 'Need socket statistics utility'
unless IS_WINDOWS || ::which("ss");
unless IS_WINDOWS || ! ::have_wsrep() || ::which("lsof") || ::which("sockstat") || ::which("ss");
$skip{'include/have_mariabackup.inc'} = 'Need socat or nc'
unless IS_WINDOWS || $ENV{MTR_GALERA_TFMT};
......@@ -95,7 +95,6 @@ sub skip_combinations {
unless $::mysqld_variables{'version-ssl-library'} =~ /OpenSSL (\S+)/
and $1 ge "1.0.1d" and $1 lt "1.1.1";
$skip{'t/ssl_7937.combinations'} = [ 'x509v3' ]
unless $::mysqld_variables{'version-ssl-library'} =~ /OpenSSL (\S+)/
and $1 ge "1.0.2";
......@@ -108,4 +107,3 @@ sub skip_combinations {
}
bless { };
mysql-test/suite/galera/disabled.def
View file @ 8c8a6ed3
......@@ -21,10 +21,10 @@ galera_parallel_simple : MDEV-20318 galera.galera_parallel_simple fails
galera_partition : MDEV-21806: galera.galera_partition MTR failed: failed to recover from DONOR state
galera_shutdown_nonprim : MDEV-21493 galera.galera_shutdown_nonprim
galera_var_node_address : MDEV-20485 Galera test failure
galera_wan : MDEV-17259 Test failure on galera.galera_wan
#galera_wan : MDEV-17259 Test failure on galera.galera_wan
partition : MDEV-19958 Galera test failure on galera.partition
query_cache: MDEV-15805 Test failure on galera.query_cache
sql_log_bin : MDEV-21491 galera.sql_log_bin
#sql_log_bin : MDEV-21491 galera.sql_log_bin
versioning_trx_id: MDEV-18590: galera.versioning_trx_id: Test failure: mysqltest: Result content mismatch
galera_wsrep_provider_unset_set: wsrep_provider is read-only for security reasons
pxc-421: wsrep_provider is read-only for security reasons
......
mysql-test/suite/galera/include/have_qpress.inc 0 → 100644
View file @ 8c8a6ed3
#
# suite.pm will make sure that all tests including this file
# will be skipped as needed
#
mysql-test/suite/galera/include/have_stunnel.inc 0 → 100644
View file @ 8c8a6ed3
#
# suite.pm will make sure that all tests including this file
# will be skipped as needed
#
mysql-test/suite/galera/r/galera_sst_mariabackup_qpress.result 0 → 100644
View file @ 8c8a6ed3
SELECT 1;
1
1
include/assert_grep.inc [Compressed qpress files found]
mysql-test/suite/galera/suite.pm
View file @ 8c8a6ed3
......@@ -69,4 +69,13 @@ push @::global_suppressions,
qr|WSREP: Trying to continue unpaused monitor|,
);
sub skip_combinations {
my %skip = ();
$skip{'include/have_stunnel.inc'} = "Need 'stunnel' utility"
unless ::which("stunnel");
$skip{'include/have_qpress.inc'} = "Need 'qpress' utility"
unless ::which("qpress");
%skip;
}
bless { };
mysql-test/suite/galera/t/galera_ist_innodb_flush_logs.cnf
View file @ 8c8a6ed3
......@@ -12,4 +12,3 @@ wsrep_provider_options='base_port=@mysqld.1.#galera_port;pc.ignore_sb=true'
[mysqld.2]
wsrep_provider_options='base_port=@mysqld.2.#galera_port;pc.ignore_sb=true'
mysql-test/suite/galera/t/galera_ist_rsync.cnf
View file @ 8c8a6ed3
......@@ -10,4 +10,3 @@ wsrep_sync_wait=1
[mysqld.2]
wsrep_provider_options='base_port=@mysqld.2.#galera_port;pc.ignore_sb=true'
wsrep_sync_wait=1
mysql-test/suite/galera/t/galera_log_bin.cnf
View file @ 8c8a6ed3
......@@ -7,4 +7,3 @@ log-slave-updates
[mysqld.2]
log-bin
log-slave-updates
mysql-test/suite/galera/t/galera_log_bin.inc
View file @ 8c8a6ed3
......@@ -39,4 +39,3 @@ DROP TABLE t2;
--echo #cleanup
--connection node_1
RESET MASTER;
mysql-test/suite/galera/t/galera_log_bin_opt.cnf
View file @ 8c8a6ed3
......@@ -12,4 +12,4 @@ wsrep_provider_options='base_port=@mysqld.2.#galera_port;gcache.size=1;pc.ignore
[sst]
transferfmt=@ENV.MTR_GALERA_TFMT
streamfmt=xbstream
streamfmt=mbstream
mysql-test/suite/galera/t/galera_sst_mariabackup_qpress.cnf 0 → 100644
View file @ 8c8a6ed3
!include ../galera_2nodes.cnf
[mysqld]
wsrep_sst_method=mariabackup
wsrep_sst_auth="root:"
wsrep_debug=ON
[sst]
transferfmt=@ENV.MTR_GALERA_TFMT
compress=quicklz
compress-threads=2
compress-chunk-size=32768
mysql-test/suite/galera/t/galera_sst_mariabackup_qpress.test 0 → 100644
View file @ 8c8a6ed3
#
# This test checks that qpress compression works with mariabackup
# Initial SST happens via mariabackup, so there is not much to do in the body
# of the test
#
--source include/big_test.inc
--source include/galera_cluster.inc
--source include/have_innodb.inc
--source include/have_mariabackup.inc
--source include/have_qpress.inc
SELECT 1;
--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size';
--source include/wait_condition.inc
# Confirm that transfer was compressed using qpress algorithm
--let $assert_text = Compressed qpress files found
--let $assert_select = Compressed qpress files found
--let $assert_count = 1
--let $assert_file = $MYSQLTEST_VARDIR/log/mysqld.2.err
--let $assert_only_after = CURRENT_TEST
--source include/assert_grep.inc
mysql-test/suite/galera/t/galera_sst_rsync_encrypt_with_key.test
View file @ 8c8a6ed3
--source include/big_test.inc
--source include/galera_cluster.inc
--source include/have_debug.inc
--source include/have_stunnel.inc
# Save original auto_increment_offset values.
--let $node_1=node_1
......
mysql-test/suite/galera/t/galera_sst_rsync_encrypt_with_server.test
View file @ 8c8a6ed3
--source include/big_test.inc
--source include/galera_cluster.inc
--source include/have_debug.inc
--source include/have_stunnel.inc
# Save original auto_increment_offset values.
--let $node_1=node_1
......
mysql-test/suite/galera/t/galera_wan_restart_sst.test
View file @ 8c8a6ed3
......@@ -120,7 +120,6 @@ INSERT INTO t1 VALUES (33);
INSERT INTO t1 VALUES (341);
#
# Check all nodes
#
......
scripts/wsrep_sst_common.sh
View file @ 8c8a6ed3
# Copyright (C) 2012-2015 Codership Oy
# Copyright (C) 2017-2021 MariaDB
# Copyright (C) 2012-2015 Codership Oy
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
......@@ -822,14 +822,15 @@ wsrep_log_info()
wsrep_cleanup_progress_file()
{
[ -n "$SST_PROGRESS_FILE" ] && rm -f "$SST_PROGRESS_FILE" 2>/dev/null || true
[ -n "$SST_PROGRESS_FILE" -a \
-f "$SST_PROGRESS_FILE" ] && rm -f "$SST_PROGRESS_FILE" 2>/dev/null || true
}
wsrep_check_program()
{
local prog="$1"
local cmd=$(command -v "$prog")
if [ ! -x "$cmd" ]; then
if [ -z "$cmd" ]; then
echo "'$prog' not found in PATH"
return 2 # no such file or directory
fi
......@@ -865,9 +866,9 @@ get_openssl()
fi
# Let's look for openssl:
OPENSSL_BINARY="$(command -v openssl)"
if [ ! -x "$OPENSSL_BINARY" ]; then
if [ -z "$OPENSSL_BINARY" ]; then
OPENSSL_BINARY='/usr/bin/openssl'
if [ ! -x "$OPENSSL_BINARY" ]; then
if [ -z "$OPENSSL_BINARY" ]; then
OPENSSL_BINARY=""
fi
fi
......@@ -899,14 +900,14 @@ is_local_ip()
[ "$1" = "$(hostname -d)" ] && return 0
local ip_util="$(command -v ip)"
if [ -x "$ip_util" ]; then
if [ -n "$ip_util" ]; then
# ip address show ouput format is " inet[6] <address>/<mask>":
"$ip_util" address show \
| grep -E "^[[:space:]]*inet.? [^[:space:]]+/" -o \
| grep -F " $1/" >/dev/null && return 0
else
local ifconfig_util="$(command -v ifconfig)"
if [ -x "$ifconfig_util" ]; then
if [ -n "$ifconfig_util" ]; then
# ifconfig output format is " inet[6] <address> ...":
"$ifconfig_util" \
| grep -E "^[[:space:]]*inet.? [^[:space:]]+ " -o \
......@@ -923,16 +924,79 @@ check_sockets_utils()
sockstat_available=0
ss_available=0
[ -x "$(command -v lsof)" ] && lsof_available=1
[ -x "$(command -v sockstat)" ] && sockstat_available=1
[ -x "$(command -v ss)" ] && ss_available=1
[ -n "$(command -v lsof)" ] && lsof_available=1
[ -n "$(command -v sockstat)" ] && sockstat_available=1
[ -n "$(command -v ss)" ] && ss_available=1
if [ $lsof_available -eq 0 -a \
$sockstat_available -eq 0 -a \
$ss_available -eq 0 ]
then
wsrep_log_error "Neither lsof tool, nor ss or sockstat was found in " \
"the PATH! Make sure you have it installed."
wsrep_log_error "Neither lsof, nor sockstat or ss tool was found in " \
"the PATH. Make sure you have it installed."
exit 2 # ENOENT
fi
}
#
# If the ssl_dhparams variable is already set, uses that as a source
# of dh parameters for OpenSSL. Otherwise, looks for dhparams.pem in
# the datadir, and creates it there if it can't find the file.
#
check_for_dhparams()
{
if [ -z "$ssl_dhparams" ]; then
ssl_dhparams="$DATA/dhparams.pem"
if [ ! -r "$ssl_dhparams" ]; then
get_openssl
if [ -n "$OPENSSL_BINARY" ]; then
wsrep_log_info "Could not find dhparams file, creating $ssl_dhparams"
if ! "$OPENSSL_BINARY" dhparam -out "$ssl_dhparams" 2048 >/dev/null 2>&1
then
wsrep_log_error "******** ERROR *****************************************"
wsrep_log_error "* Could not create the dhparams.pem file with OpenSSL. *"
wsrep_log_error "********************************************************"
ssl_dhparams=""
fi
else
# Rollback: if openssl is not installed, then use
# the default parameters:
ssl_dhparams=""
fi
fi
fi
}
#
# Compares two version strings.
# The first parameter is the version to be checked;
# The second parameter is the minimum version required;
# Returns 1 (failure) if $1 >= $2, 0 (success) otherwise.
#
check_for_version()
{
y1=${1#*.}
[ "$y1" = "$1" ] && y1=""
z1=${y1#*.}
[ "$z1" = "$y1" ] && z1=""
x1=${1%%.*}
y1=${y1%%.*}
z1=${z1%%.*}
[ -z "$y1" ] && y1=0
[ -z "$z1" ] && z1=0
y2=${2#*.}
[ "$y2" = "$2" ] && y2=""
z2=${y2#*.}
[ "$z2" = "$y2" ] && z2=""
x2=${2%%.*}
y2=${y2%%.*}
z2=${z2%%.*}
[ -z "$y2" ] && y2=0
[ -z "$z2" ] && z2=0
[ $x1 -lt $x2 ] && return 1
[ $x1 -gt $x2 ] && return 0
[ $y1 -lt $y2 ] && return 1
[ $y1 -gt $y2 ] && return 0
[ $z1 -lt $z2 ] && return 1
return 0
}
scripts/wsrep_sst_mariabackup.sh
View file @ 8c8a6ed3
This diff is collapsed.
scripts/wsrep_sst_mysqldump.sh
View file @ 8c8a6ed3
......@@ -45,7 +45,12 @@ then
exit $EINVAL
fi
[ -n "$WSREP_SST_OPT_USER" ] && AUTH="-u$WSREP_SST_OPT_USER" || AUTH=
AUTH=""
usrst=0
if [ -n "$WSREP_SST_OPT_USER" ]; then
AUTH="-u$WSREP_SST_OPT_USER"
usrst=1
fi
# Refs https://github.com/codership/mysql-wsrep/issues/141
# Passing password in MYSQL_PWD environment variable is considered
......@@ -56,9 +61,14 @@ fi
# whereas (at least on Linux) unprivileged user can't see process environment
# that he does not own. So while it may be not secure in the NSA sense of the
# word, it is arguably more secure than passing password on the command line.
[ -n "$WSREP_SST_OPT_PSWD" ] && export MYSQL_PWD="$WSREP_SST_OPT_PSWD"
if [ -n "$WSREP_SST_OPT_PSWD" ]; then
export MYSQL_PWD="$WSREP_SST_OPT_PSWD"
elif [ $usrst -eq 1 ]; then
# Empty password, used for testing, debugging etc.
unset MYSQL_PWD
fi
STOP_WSREP="SET wsrep_on=OFF;"
STOP_WSREP='SET wsrep_on=OFF;'
# mysqldump cannot restore CSV tables, fix this issue
CSV_TABLES_FIX="
......@@ -68,13 +78,13 @@ USE mysql;
SET @cond = (SELECT (SUPPORT = 'YES' or SUPPORT = 'DEFAULT') FROM INFORMATION_SCHEMA.ENGINES WHERE ENGINE = 'csv');
SET @stmt = IF (@cond = '1', 'CREATE TABLE IF NOT EXISTS general_log ( event_time timestamp(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), user_host mediumtext NOT NULL, thread_id bigint(21) unsigned NOT NULL, server_id int(10) unsigned NOT NULL, command_type varchar(64) NOT NULL, argument mediumtext NOT NULL) ENGINE=CSV DEFAULT CHARSET=utf8 COMMENT=\"General log\"', 'SET @dummy = 0');
SET @stmt = IF (@cond = '1', 'CREATE TABLE IF NOT EXISTS general_log ( event_time timestamp(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), user_host mediumtext NOT NULL, thread_id bigint(21) unsigned NOT NULL, server_id int(10) unsigned NOT NULL, command_type varchar(64) NOT NULL, argument mediumtext NOT NULL) ENGINE=CSV DEFAULT CHARSET=utf8mb3 COMMENT=\"General log\"', 'SET @dummy = 0');
PREPARE stmt FROM @stmt;
EXECUTE stmt;
DROP PREPARE stmt;
SET @stmt = IF (@cond = '1', 'CREATE TABLE IF NOT EXISTS slow_log ( start_time timestamp(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), user_host mediumtext NOT NULL, query_time time(6) NOT NULL, lock_time time(6) NOT NULL, rows_sent int(11) NOT NULL, rows_examined int(11) NOT NULL, db varchar(512) NOT NULL, last_insert_id int(11) NOT NULL, insert_id int(11) NOT NULL, server_id int(10) unsigned NOT NULL, sql_text mediumtext NOT NULL, thread_id bigint(21) unsigned NOT NULL) ENGINE=CSV DEFAULT CHARSET=utf8 COMMENT=\"Slow log\"', 'SET @dummy = 0');
SET @stmt = IF (@cond = '1', 'CREATE TABLE IF NOT EXISTS slow_log ( start_time timestamp(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), user_host mediumtext NOT NULL, query_time time(6) NOT NULL, lock_time time(6) NOT NULL, rows_sent int(11) NOT NULL, rows_examined int(11) NOT NULL, db varchar(512) NOT NULL, last_insert_id int(11) NOT NULL, insert_id int(11) NOT NULL, server_id int(10) unsigned NOT NULL, sql_text mediumtext NOT NULL, thread_id bigint(21) unsigned NOT NULL) ENGINE=CSV DEFAULT CHARSET=utf8mb3 COMMENT=\"Slow log\"', 'SET @dummy = 0');
PREPARE stmt FROM @stmt;
EXECUTE stmt;
......@@ -99,15 +109,15 @@ MYSQL="$MYSQL_CLIENT $WSREP_SST_OPT_CONF "\
# Check if binary logging is enabled on the joiner node.
# Note: SELECT cannot be used at this point.
LOG_BIN=$(echo "set statement wsrep_sync_wait=0 for SHOW VARIABLES LIKE 'log_bin'" | $MYSQL |\
LOG_BIN=$(echo "set statement wsrep_sync_wait=0 for SHOW VARIABLES LIKE 'log_bin'" | $MYSQL | \
tail -1 | awk -F ' ' '{ print $2 }')
# Check the joiner node's server version.
SERVER_VERSION=$(echo "set statement wsrep_sync_wait=0 for SHOW VARIABLES LIKE 'version'" | $MYSQL |\
SERVER_VERSION=$(echo "set statement wsrep_sync_wait=0 for SHOW VARIABLES LIKE 'version'" | $MYSQL | \
tail -1 | awk -F ' ' '{ print $2 }')
# Retrieve the donor's @@global.gtid_binlog_state.
GTID_BINLOG_STATE=$(echo "SHOW GLOBAL VARIABLES LIKE 'gtid_binlog_state'" | $MYSQL |\
GTID_BINLOG_STATE=$(echo "SHOW GLOBAL VARIABLES LIKE 'gtid_binlog_state'" | $MYSQL | \
tail -1 | awk -F ' ' '{ print $2 }')
RESET_MASTER=""
......@@ -115,7 +125,7 @@ SET_GTID_BINLOG_STATE=""
SQL_LOG_BIN_OFF=""
# Safety check
if [ "${SERVER_VERSION%%.*}" != '5' ]
if [ ${SERVER_VERSION%%.*} -gt 5 ]
then
# If binary logging is enabled on the joiner node, we need to copy donor's
# gtid_binlog_state to joiner. In order to do that, a RESET MASTER must be
......@@ -135,29 +145,29 @@ MYSQLDUMP="$MYSQLDUMP $WSREP_SST_OPT_CONF $AUTH -S$WSREP_SST_OPT_SOCKET \
--disable-keys --extended-insert --skip-lock-tables --quick --set-charset \
--skip-comments --flush-privileges --all-databases --events"
# need to disable logging when loading the dump
# reason is that dump contains ALTER TABLE for log tables, and
# this causes an error if logging is enabled
GENERAL_LOG_OPT=`$MYSQL --skip-column-names -e "$STOP_WSREP SELECT @@GENERAL_LOG"`
SLOW_LOG_OPT=`$MYSQL --skip-column-names -e "$STOP_WSREP SELECT @@SLOW_QUERY_LOG"`
$MYSQL -e "$STOP_WSREP SET GLOBAL GENERAL_LOG=OFF"
$MYSQL -e "$STOP_WSREP SET GLOBAL SLOW_QUERY_LOG=OFF"
# commands to restore log settings
RESTORE_GENERAL_LOG="SET GLOBAL GENERAL_LOG=$GENERAL_LOG_OPT;"
RESTORE_SLOW_QUERY_LOG="SET GLOBAL SLOW_QUERY_LOG=$SLOW_LOG_OPT;"
if [ $WSREP_SST_OPT_BYPASS -eq 0 ]
then
(echo $STOP_WSREP && echo $RESET_MASTER && \
echo $SET_GTID_BINLOG_STATE && echo $SQL_LOG_BIN_OFF && \
echo $STOP_WSREP && $MYSQLDUMP && echo $CSV_TABLES_FIX && \
echo $RESTORE_GENERAL_LOG && echo $RESTORE_SLOW_QUERY_LOG && \
echo $SET_START_POSITION && echo $SET_WSREP_GTID_DOMAIN_ID \
# need to disable logging when loading the dump
# reason is that dump contains ALTER TABLE for log tables, and
# this causes an error if logging is enabled
GENERAL_LOG_OPT=$($MYSQL --skip-column-names -e "$STOP_WSREP SELECT @@GENERAL_LOG")
SLOW_LOG_OPT=$($MYSQL --skip-column-names -e "$STOP_WSREP SELECT @@SLOW_QUERY_LOG")
LOG_OFF="SET GLOBAL GENERAL_LOG=OFF; SET GLOBAL SLOW_QUERY_LOG=OFF;"
# commands to restore log settings
RESTORE_GENERAL_LOG="SET GLOBAL GENERAL_LOG=$GENERAL_LOG_OPT;"
RESTORE_SLOW_QUERY_LOG="SET GLOBAL SLOW_QUERY_LOG=$SLOW_LOG_OPT;"
(echo "$STOP_WSREP" && echo "$LOG_OFF" && echo "$RESET_MASTER" && \
echo "$SET_GTID_BINLOG_STATE" && echo "$SQL_LOG_BIN_OFF" && \
echo "$STOP_WSREP" && $MYSQLDUMP && echo "$CSV_TABLES_FIX" && \
echo "$RESTORE_GENERAL_LOG" && echo "$RESTORE_SLOW_QUERY_LOG" && \
echo "$SET_START_POSITION" && echo "$SET_WSREP_GTID_DOMAIN_ID" \
|| echo "SST failed to complete;") | $MYSQL
else
wsrep_log_info "Bypassing state dump."
echo $SET_START_POSITION | $MYSQL
echo "$SET_START_POSITION" | $MYSQL
fi
#
scripts/wsrep_sst_rsync.sh
View file @ 8c8a6ed3
......@@ -40,13 +40,12 @@ cleanup_joiner()
[ "0" != "$RSYNC_REAL_PID" ] && \
kill $RSYNC_REAL_PID && \
sleep 0.5 && \
kill -9 $RSYNC_REAL_PID >/dev/null 2>&1 || \
:
rm -rf "$RSYNC_CONF"
rm -f "$STUNNEL_CONF"
rm -f "$STUNNEL_PID"
rm -rf "$MAGIC_FILE"
rm -rf "$RSYNC_PID"
kill -9 $RSYNC_REAL_PID >/dev/null 2>&1 || :
[ -f "$RSYNC_CONF" ] && rm -f "$RSYNC_CONF"
[ -f "$STUNNEL_CONF" ] && rm -f "$STUNNEL_CONF"
[ -f "$STUNNEL_PID" ] && rm -f "$STUNNEL_PID"
[ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE"
[ -f "$RSYNC_PID" ] && rm -f "$RSYNC_PID"
wsrep_log_info "Joiner cleanup done."
if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then
wsrep_cleanup_progress_file
......@@ -125,17 +124,13 @@ check_pid_and_port()
}
STUNNEL_CONF="$WSREP_SST_OPT_DATA/stunnel.conf"
rm -f "$STUNNEL_CONF"
STUNNEL_PID="$WSREP_SST_OPT_DATA/stunnel.pid"
rm -f "$STUNNEL_PID"
MAGIC_FILE="$WSREP_SST_OPT_DATA/rsync_sst_complete"
rm -rf "$MAGIC_FILE"
BINLOG_TAR_FILE="$WSREP_SST_OPT_DATA/wsrep_sst_binlog.tar"
BINLOG_N_FILES=1
rm -f "$BINLOG_TAR_FILE" || :
get_binlog
......@@ -154,13 +149,13 @@ OLD_PWD="$(pwd)"
WSREP_LOG_DIR="$INNODB_LOG_GROUP_HOME"
cd "$WSREP_SST_OPT_DATA"
if [ -n "$WSREP_LOG_DIR" ]; then
# handle both relative and absolute paths
WSREP_LOG_DIR=$(cd "$WSREP_SST_OPT_DATA"; mkdir -p "$WSREP_LOG_DIR"; cd "$WSREP_LOG_DIR"; pwd -P)
else
# default to datadir
WSREP_LOG_DIR=$(cd "$WSREP_SST_OPT_DATA"; pwd -P)
[ ! -d "$WSREP_LOG_DIR" ] && mkdir -p "$WSREP_LOG_DIR"
cd "$WSREP_LOG_DIR"
fi
WSREP_LOG_DIR=$(pwd -P)
cd "$OLD_PWD"
......@@ -170,13 +165,13 @@ if [ -z "$INNODB_DATA_HOME_DIR" ]; then
INNODB_DATA_HOME_DIR=$(parse_cnf '--mysqld' 'innodb-data-home-dir')
fi
cd "$WSREP_SST_OPT_DATA"
if [ -n "$INNODB_DATA_HOME_DIR" ]; then
# handle both relative and absolute paths
INNODB_DATA_HOME_DIR=$(cd "$WSREP_SST_OPT_DATA"; mkdir -p "$INNODB_DATA_HOME_DIR"; cd "$INNODB_DATA_HOME_DIR"; pwd -P)
else
# default to datadir
INNODB_DATA_HOME_DIR=$(cd "$WSREP_SST_OPT_DATA"; pwd -P)
[ ! -d "$INNODB_DATA_HOME_DIR" ] && mkdir -p "$INNODB_DATA_HOME_DIR"
cd "$INNODB_DATA_HOME_DIR"
fi
INNODB_DATA_HOME_DIR=$(pwd -P)
cd "$OLD_PWD"
......@@ -185,13 +180,13 @@ if [ -z "$INNODB_UNDO_DIR" ]; then
INNODB_UNDO_DIR=$(parse_cnf '--mysqld' 'innodb-undo-directory')
fi
cd "$WSREP_SST_OPT_DATA"
if [ -n "$INNODB_UNDO_DIR" ]; then
# handle both relative and absolute paths
INNODB_UNDO_DIR=$(cd "$WSREP_SST_OPT_DATA"; mkdir -p "$INNODB_UNDO_DIR"; cd "$INNODB_UNDO_DIR"; pwd -P)
else
# default to datadir
INNODB_UNDO_DIR=$(cd "$WSREP_SST_OPT_DATA"; pwd -P)
[ ! -d "$INNODB_UNDO_DIR" ] && mkdir -p "$INNODB_UNDO_DIR"
cd "$INNODB_UNDO_DIR"
fi
INNODB_UNDO_DIR=$(pwd -P)
cd "$OLD_PWD"
......@@ -239,7 +234,7 @@ if [ -z "$SSLMODE" ]; then
# Implicit verification if CA is set and the SSL mode
# is not specified by user:
if [ -n "$SSTCA" ]; then
if [ -x "$(command -v stunnel)" ]; then
if [ -n "$(command -v stunnel)" ]; then
SSLMODE='VERIFY_CA'
fi
# Require SSL by default if SSL key and cert are present:
......@@ -260,28 +255,36 @@ then
case "$SSLMODE" in
'VERIFY_IDENTITY')
VERIFY_OPT='verifyPeer = yes'
CHECK_OPT=""
;;
'VERIFY_CA')
VERIFY_OPT='verifyChain = yes'
if is_local_ip "$WSREP_SST_OPT_HOST_UNESCAPED"; then
CHECK_OPT='checkHost = localhost'
else
CHECK_OPT='checkHost = $WSREP_SST_OPT_HOST_UNESCAPED'
fi
;;
*)
wsrep_log_error "Unrecognized ssl-mode option: '$SSLMODE'"
exit 22 # EINVAL
esac
if [ -z "$CAFILE_OPT" ]
then
wsrep_log_error "Can't have ssl-mode=$SSLMODE without CA file"
if [ -z "$CAFILE_OPT" ]; then
wsrep_log_error "Can't have ssl-mode='$SSLMODE' without CA file"
exit 22 # EINVAL
fi
else
VERIFY_OPT=""
CHECK_OPT=""
fi
STUNNEL=""
if [ -n "$SSLMODE" -a "$SSLMODE" != 'DISABLED' ] && wsrep_check_programs stunnel
then
wsrep_log_info "Using stunnel for SSL encryption: CAfile: '$SSTCA', SSLMODE: '$SSLMODE'"
STUNNEL="stunnel $STUNNEL_CONF"
if [ -n "$SSLMODE" -a "$SSLMODE" != 'DISABLED' ]; then
STUNNEL_BIN="$(command -v stunnel)"
if [ -n "$STUNNEL_BIN" ]; then
wsrep_log_info "Using stunnel for SSL encryption: CAfile: '$SSTCA', ssl-mode='$SSLMODE'"
STUNNEL="$STUNNEL_BIN $STUNNEL_CONF"
fi
fi
readonly SECRET_TAG="secret"
......@@ -289,7 +292,13 @@ readonly SECRET_TAG="secret"
if [ "$WSREP_SST_OPT_ROLE" = 'donor' ]
then
cat << EOF > "$STUNNEL_CONF"
[ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE"
[ -f "$BINLOG_TAR_FILE" ] && rm -f "$BINLOG_TAR_FILE"
if [ -n "$STUNNEL" ]
then
[ -f "$STUNNEL_PID" ] && rm -f "$STUNNEL_PID"
cat << EOF > "$STUNNEL_CONF"
key = $SSTKEY
cert = $SSTCERT
${CAFILE_OPT}
......@@ -300,7 +309,9 @@ client = yes
connect = $WSREP_SST_OPT_HOST_UNESCAPED:$WSREP_SST_OPT_PORT
TIMEOUTclose = 0
${VERIFY_OPT}
${CHECK_OPT}
EOF
fi
if [ $WSREP_SST_OPT_BYPASS -eq 0 ]
then
......@@ -366,7 +377,7 @@ EOF
# first, the normal directories, so that we can detect incompatible protocol
RC=0
eval rsync ${STUNNEL:+--rsh=\"$STUNNEL\"} \
eval rsync ${STUNNEL:+"'--rsh=$STUNNEL'"} \
--owner --group --perms --links --specials \
--ignore-times --inplace --dirs --delete --quiet \
$WHOLE_FILE_OPT $FILTER "'$WSREP_SST_OPT_DATA/'" \
......@@ -449,7 +460,7 @@ EOF
fi
echo "continue" # now server can resume updating data
echo 'continue' # now server can resume updating data
echo "$STATE" > "$MAGIC_FILE"
......@@ -487,7 +498,10 @@ then
wsrep_log_error "rsync daemon already running."
exit 114 # EALREADY
fi
rm -rf "$RSYNC_PID"
[ -f "$RSYNC_PID" ] && rm -f "$RSYNC_PID"
[ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE"
[ -f "$BINLOG_TAR_FILE" ] && rm -f "$BINLOG_TAR_FILE"
ADDR="$WSREP_SST_OPT_ADDR"
RSYNC_PORT="$WSREP_SST_OPT_PORT"
......@@ -541,6 +555,9 @@ EOF
rsync --daemon --no-detach --port "$RSYNC_PORT" --config "$RSYNC_CONF" $RSYNC_EXTRA_ARGS &
RSYNC_REAL_PID=$!
else
[ -f "$STUNNEL_PID" ] && rm -f "$STUNNEL_PID"
# Let's check if the path to the config file contains a space?
if [ "${RSYNC_CONF#* }" = "$RSYNC_CONF" ]; then
cat << EOF > "$STUNNEL_CONF"
key = $SSTKEY
cert = $SSTCERT
......@@ -548,13 +565,30 @@ ${CAFILE_OPT}
foreground = yes
pid = $STUNNEL_PID
debug = warning
debug = 6
client = no
[rsync]
accept = $STUNNEL_ACCEPT
exec = $(command -v rsync)
execargs = rsync --server --daemon --config=$RSYNC_CONF .
EOF
else
# The path contains a space, so we will run it via
# shell with "eval" command:
export RSYNC_CMD="eval $(command -v rsync) --server --daemon --config='$RSYNC_CONF' ."
cat << EOF > "$STUNNEL_CONF"
key = $SSTKEY
cert = $SSTCERT
${CAFILE_OPT}
foreground = yes
pid = $STUNNEL_PID
debug = warning
client = no
[rsync]
accept = $STUNNEL_ACCEPT
exec = $SHELL
execargs = $SHELL -c \$RSYNC_CMD
EOF
fi
stunnel "$STUNNEL_CONF" &
RSYNC_REAL_PID=$!
RSYNC_PID="$STUNNEL_PID"
......@@ -655,6 +689,6 @@ else
exit 22 # EINVAL
fi
rm -f "$BINLOG_TAR_FILE" || :
[ -f "$BINLOG_TAR_FILE" ] && rm -f "$BINLOG_TAR_FILE"
exit 0
scripts/wsrep_sst_xtrabackup-v2.sh
View file @ 8c8a6ed3
......@@ -55,8 +55,8 @@ tcmd=""
rebuild=0
rebuildcmd=""
payload=0
pvformat="-F '%N => Rate:%r Avg:%a Elapsed:%t %e Bytes: %b %p' "
pvopts="-f -i 10 -N $WSREP_SST_OPT_ROLE "
pvformat="-F '%N => Rate:%r Avg:%a Elapsed:%t %e Bytes: %b %p'"
pvopts="-f -i 10 -N $WSREP_SST_OPT_ROLE"
STATDIR=""
uextra=0
disver=""
......@@ -73,7 +73,7 @@ ssl_cert=""
ssl_ca=""
ssl_key=""
if [ -x "$(command -v pv)" ] && pv --help | grep -qw -- '-F'; then
if [ -n "$(command -v pv)" ] && pv --help | grep -qw -- '-F'; then
pvopts+=$pvformat
fi
pcmd="pv $pvopts"
......@@ -90,7 +90,7 @@ export PATH="/usr/sbin:/sbin:$PATH"
OS="$(uname)"
if [ ! -x "$(command -v lsof)" ]; then
if [ -z "$(command -v lsof)" ]; then
wsrep_log_error "lsof tool not found in PATH! Make sure you have it installed."
exit 2 # ENOENT
fi
......@@ -167,31 +167,6 @@ get_keys()
stagemsg+="-XB-Encrypted"
}
#
# If the ssl_dhparams variable is already set, uses that as a source
# of dh parameters for OpenSSL. Otherwise, looks for dhparams.pem in the
# datadir, and creates it there if it can't find the file.
# No input parameters
#
check_for_dhparams()
{
if [[ -z "$ssl_dhparams" ]]; then
if ! [[ -r "$DATA/dhparams.pem" ]]; then
wsrep_check_programs openssl
wsrep_log_info "Could not find dhparams file, creating $DATA/dhparams.pem"
if ! openssl dhparam -out "$DATA/dhparams.pem" 2048 >/dev/null 2>&1
then
wsrep_log_error "******** FATAL ERROR ********************************* "
wsrep_log_error "* Could not create the dhparams.pem file with OpenSSL. "
wsrep_log_error "****************************************************** "
exit 22
fi
fi
ssl_dhparams="$DATA/dhparams.pem"
fi
}
#
# verifies that the certificate matches the private key
# doing this will save us having to wait for a timeout that would
......@@ -322,27 +297,25 @@ get_transfer()
fi
# Determine the socat version
SOCAT_VERSION=`socat -V 2>&1 | grep -oe '[0-9]\.[0-9][\.0-9]*' | head -n1`
if [[ -z "$SOCAT_VERSION" ]]; then
wsrep_log_error "******** FATAL ERROR **************** "
wsrep_log_error "* Cannot determine the socat version. "
wsrep_log_error "************************************* "
SOCAT_VERSION=$(socat -V 2>&1 | grep -m1 -oe '[0-9]\.[0-9][\.0-9]*')
if [ -z "$SOCAT_VERSION" ]; then
wsrep_log_error "******** FATAL ERROR ******************"
wsrep_log_error "* Cannot determine the socat version. *"
wsrep_log_error "***************************************"
exit 2
fi
# socat versions < 1.7.3 will have 512-bit dhparams (too small)
# so create 2048-bit dhparams and send that as a parameter
# socat version >= 1.7.3, checks to see if the peername matches the hostname
# set commonname="" to disable the peername checks
#
if ! check_for_version "$SOCAT_VERSION" "1.7.3"; then
# socat versions < 1.7.3 will have 512-bit dhparams (too small)
# so create 2048-bit dhparams and send that as a parameter:
if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]]; then
# dhparams check (will create ssl_dhparams if needed)
check_for_dhparams
joiner_extra=",dhparam='$ssl_dhparams'"
fi
fi
if check_for_version "$SOCAT_VERSION" "1.7.3"; then
else
# socat version >= 1.7.3, checks to see if the peername matches
# the hostname, then set commonname="" to disable the peername
# checks:
donor_extra=',commonname=""'
fi
fi
......@@ -420,7 +393,7 @@ get_footprint()
{
pushd "$WSREP_SST_OPT_DATA" 1>/dev/null
payload=$(find . -regex '.*\.ibd$\|.*\.MYI$\|.*\.MYD$\|.*ibdata1$' -type f -print0 | du --files0-from=- --block-size=1 -c -s | awk 'END { print $1 }')
if $MY_PRINT_DEFAULTS xtrabackup | grep -q -- "--compress";then
if [ $(in_config 'xtrabackup' 'compress') -eq 1 ]; then
# QuickLZ has around 50% compression ratio
# When compression/compaction used, the progress is only an approximate.
payload=$(( payload*1/2 ))
......@@ -432,7 +405,7 @@ get_footprint()
adjust_progress()
{
if [ ! -x "$(command -v pv)" ]; then
if [ -z "$(command -v pv)" ]; then
wsrep_log_error "pv not found in path: $PATH"
wsrep_log_error "Disabling all progress/rate-limiting"
pcmd=""
......@@ -709,7 +682,7 @@ recv_joiner()
pushd "${dir}" 1>/dev/null
set +e
if [ $tmt -gt 0 -a -x "$(command -v timeout)" ]; then
if [ $tmt -gt 0 -a -n "$(command -v timeout)" ]; then
if timeout --help | grep -qw -- '-k';then
ltcmd="timeout -k $(( tmt+10 )) $tmt $tcmd"
else
......@@ -765,42 +738,6 @@ send_donor()
done
}
# Returns the version string in a standardized format
# Input "1.2.3" => echoes "010203"
# Wrongly formatted values => echoes "000000"
normalize_version()
{
local major=0
local minor=0
local patch=0
# Only parses purely numeric version numbers, 1.2.3
# Everything after the first three values are ignored
if [[ $1 =~ ^([0-9]+)\.([0-9]+)\.?([0-9]*)([\.0-9])*$ ]]; then
major=${BASH_REMATCH[1]}
minor=${BASH_REMATCH[2]}
patch=${BASH_REMATCH[3]}
fi
printf %02d%02d%02d $major $minor $patch
}
# Compares two version strings
# The first parameter is the version to be checked
# The second parameter is the minimum version required
# Returns 1 (failure) if $1 >= $2, 0 (success) otherwise
check_for_version()
{
local local_version_str=$(normalize_version "$1")
local required_version_str=$(normalize_version "$2")
if [[ "$local_version_str" < "$required_version_str" ]]; then
return 1
else
return 0
fi
}
monitor_process()
{
local sst_stream_pid=$1
......@@ -864,7 +801,7 @@ if [ ${FORCE_FTWRL:-0} -eq 1 ]; then
fi
if [[ $ssyslog -eq 1 ]];then
if [ ! -x "$(command -v logger)" ]; then
if [ -z "$(command -v logger)" ]; then
wsrep_log_error "logger not in path: $PATH. Ignoring"
else
wsrep_log_info "Logging all stderr of SST/Innobackupex to syslog"
......@@ -902,13 +839,13 @@ fi
OLD_PWD="$(pwd)"
cd "$WSREP_SST_OPT_DATA"
if [ -n "$INNODB_DATA_HOME_DIR" ]; then
# handle both relative and absolute paths
INNODB_DATA_HOME_DIR=$(cd "$DATA"; mkdir -p "$INNODB_DATA_HOME_DIR"; cd "$INNODB_DATA_HOME_DIR"; pwd -P)
else
# default to datadir
INNODB_DATA_HOME_DIR=$(cd "$DATA"; pwd -P)
[ ! -d "$INNODB_DATA_HOME_DIR" ] && mkdir -p "$INNODB_DATA_HOME_DIR"
cd "$INNODB_DATA_HOME_DIR"
fi
INNODB_DATA_HOME_DIR=$(pwd -P)
cd "$OLD_PWD"
......@@ -924,7 +861,6 @@ then
if [ $WSREP_SST_OPT_BYPASS -eq 0 ]
then
usrst=0
if [ -z "$WSREP_SST_OPT_SST_VER" ]; then
wsrep_log_error "Upgrade joiner to 5.6.21 or higher for backup locks support"
wsrep_log_error "The joiner is not supported for this version of donor"
......@@ -941,8 +877,9 @@ then
itmpdir=$(mktemp -d)
wsrep_log_info "Using $itmpdir as innobackupex temporary directory"
usrst=0
if [ -n "$WSREP_SST_OPT_USER" ]; then
INNOEXTRA+=" --user='$WSREP_SST_OPT_USER'"
INNOEXTRA="$INNOEXTRA --user='$WSREP_SST_OPT_USER'"
usrst=1
fi
......@@ -1172,11 +1109,13 @@ then
if [ -n "$qpfiles" ]; then
wsrep_log_info "Compressed qpress files found"
if [ ! -x "$(command -v qpress)" ]; then
if [ -z "$(command -v qpress)" ]; then
wsrep_log_error "qpress not found in path: $PATH"
exit 22
fi
dcmd="xargs -n 2 qpress -dT$nproc"
if [[ -n "$progress" ]] && pv --help | grep -qw '--line-mode';then
count=$(find "${DATA}" -type f -name '*.qp' | wc -l)
count=$(( count*2 ))
......@@ -1187,9 +1126,7 @@ then
fi
pcmd="pv $pvopts"
adjust_progress
dcmd="$pcmd | xargs -n 2 qpress -T${nproc}d"
else
dcmd="xargs -n 2 qpress -T${nproc}d"
dcmd="$pcmd | $dcmd"
fi
# Decompress the qpress files
......
scripts/wsrep_sst_xtrabackup.sh
View file @ 8c8a6ed3
......@@ -52,7 +52,7 @@ pvformat="-F '%N => Rate:%r Avg:%a Elapsed:%t %e Bytes: %b %p' "
pvopts="-f -i 10 -N $WSREP_SST_OPT_ROLE "
uextra=0
if [ -x "$(command -v pv)" ] && pv --help | grep -qw -- '-F'; then
if [ -n "$(command -v pv)" ] && pv --help | grep -qw -- '-F'; then
pvopts+=$pvformat
fi
pcmd="pv $pvopts"
......@@ -218,7 +218,7 @@ get_footprint()
{
pushd "$WSREP_SST_OPT_DATA" 1>/dev/null
payload=$(find . -regex '.*\.ibd$\|.*\.MYI$\|.*\.MYD$\|.*ibdata1$' -type f -print0 | du --files0-from=- --block-size=1 -c -s | awk 'END { print $1 }')
if $MY_PRINT_DEFAULTS xtrabackup | grep -q -- "--compress";then
if [ $(in_config 'xtrabackup' 'compress') -eq 1 ]; then
# QuickLZ has around 50% compression ratio
# When compression/compaction used, the progress is only an approximate.
payload=$(( payload*1/2 ))
......@@ -637,11 +637,13 @@ then
wsrep_log_info "Compressed qpress files found"
if [ ! -x "$(command -v qpress)" ]; then
if [ -z "$(command -v qpress)" ]; then
wsrep_log_error "qpress not found in path: $PATH"
exit 22
fi
dcmd="xargs -n 2 qpress -dT$nproc"
if [[ -n $progress ]] && pv --help | grep -qw -- '--line-mode';then
count=$(find "${DATA}" -type f -name '*.qp' | wc -l)
count=$(( count*2 ))
......@@ -652,9 +654,7 @@ then
fi
pcmd="pv $pvopts"
adjust_progress
dcmd="$pcmd | xargs -n 2 qpress -T${nproc}d"
else
dcmd="xargs -n 2 qpress -T${nproc}d"
dcmd="$pcmd | $dcmd"
fi
wsrep_log_info "Removing existing ibdata1 file"
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7