Commit fd9ca569 authored by Alexey Yurchenko's avatar Alexey Yurchenko Committed by Jan Lindström

MDEV-25359 : Improve mariabackup SST script compliance with native MariaDB SSL...

MDEV-25359 : Improve mariabackup SST script compliance with native MariaDB SSL practices and configuration.

1. Pass joiner's authentication information to donor together with address
   in State Transfer Request. This allows joiner to authenticate donor on
   connection. Previously joiner would accept data from anywhere.

2. Deprecate custom SSL configuration variables tca, tcert and tkey in favor
   of more familiar ssl-ca, ssl-cert and ssl-key. For backward compatibility
   tca, tcert and tkey are still supported.

3. Allow falling back to server-wide SSL configuration in [mysqld] if no SSL
   configuration is found in [sst] section of the config file.

4. Introduce ssl-mode variable in [sst] section that takes standard values
   and has following effects:
    - old-style SSL configuration present in [sst]: no effect
      otherwise:
    - ssl-mode=DISABLED or absent: retains old, backward compatible behavior
      and ignores any other SSL configuration
    - ssl-mode=VERIFY*: verify joiner's certificate and CN on donor,
                        verify donor's secret on joiner
                        (passed to donor via State Transfer Request)
                        BACKWARD INCOMPATIBLE BEHAVIOR
    - anything else enables new SSL configuration convetions but does not
      require verification

    ssl-mode should be set to VERIFY only in a fully upgraded cluster.

    Examples:

    [mysqld]
    ssl-cert=/path/to/cert
    ssl-key=/path/to/key
    ssl-ca=/path/to/ca

    [sst]

     -- server-wide SSL configuration is ignored, SST does not use SSL

    [mysqld]
    ssl-cert=/path/to/cert
    ssl-key=/path/to/key
    ssl-ca=/path/to/ca

    [sst]
    ssl-mode=REQUIRED

     -- use server-wide SSL configuration for SST but don't attempt to
        verify the peer identity

    [sst]
    ssl-cert=/path/to/cert
    ssl-key=/path/to/key
    ssl-ca=/path/to/ca
    ssl-mode=VERIFY_CA

     -- use SST-specific SSL configuration for SST and require verification
        on both sides
Reviewed-by: default avatarJan Lindström <jan.lindstrom@mariadb.com>
parent 2656e876
......@@ -3,3 +3,4 @@ connection node_1;
SELECT 1;
1
1
include/assert_grep.inc [Using openssl based encryption with socat]
connection node_2;
connection node_1;
SELECT 1;
1
1
include/assert_grep.inc [Using openssl based encryption with socat]
#
# This test checks that key and cert encryption options can be passed to mariabackup via the my.cnf file
# Initial SST happens via mariabackup, so there is not much to do in the body of the test
# This test checks that key and cert encryption options can be passed to
# mariabackup via the my.cnf file
# Initial SST happens via mariabackup, so there is not much to do in the body
# of the test
#
--source include/big_test.inc
......@@ -12,3 +14,11 @@ SELECT 1;
--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size';
--source include/wait_condition.inc
# Confirm that transfer was SSL-encrypted
--let $assert_text = Using openssl based encryption with socat
--let $assert_select = Using openssl based encryption with socat: with key and crt
--let $assert_count = 1
--let $assert_file = $MYSQLTEST_VARDIR/log/mysqld.1.err
--let $assert_only_after = CURRENT_TEST
--source include/assert_grep.inc
......@@ -25,6 +25,7 @@ WSREP_SST_OPT_DATA=""
WSREP_SST_OPT_AUTH=${WSREP_SST_OPT_AUTH:-}
WSREP_SST_OPT_USER=${WSREP_SST_OPT_USER:-}
WSREP_SST_OPT_PSWD=${WSREP_SST_OPT_PSWD:-}
WSREP_SST_OPT_REMOTE_AUTH=${WSREP_SST_OPT_REMOTE_AUTH:-}
WSREP_SST_OPT_DEFAULT=""
WSREP_SST_OPT_EXTRA_DEFAULT=""
WSREP_SST_OPT_SUFFIX_DEFAULT=""
......@@ -34,6 +35,7 @@ INNODB_DATA_HOME_DIR_ARG=""
INNODB_LOG_GROUP_HOME_ARG=""
INNODB_UNDO_DIR_ARG=""
LOG_BIN_ARG=""
readonly WSREP_SST_OPT_REMOTE_AUTH
while [ $# -gt 0 ]; do
case "$1" in
......@@ -121,7 +123,8 @@ case "$1" in
WSREP_SST_OPT_BYPASS=1
;;
'--datadir')
readonly WSREP_SST_OPT_DATA="$2"
# strip trailing '/'
readonly WSREP_SST_OPT_DATA="${2%/}"
shift
;;
'--innodb-data-home-dir')
......@@ -323,12 +326,22 @@ readonly WSREP_SST_OPT_AUTH
# Splitting AUTH into potential user:password pair
if ! wsrep_auth_not_set
then
WSREP_SST_OPT_USER="${WSREP_SST_OPT_AUTH%%:*}"
WSREP_SST_OPT_USER="${WSREP_SST_OPT_AUTH%:*}"
WSREP_SST_OPT_PSWD="${WSREP_SST_OPT_AUTH##*:}"
fi
readonly WSREP_SST_OPT_USER
readonly WSREP_SST_OPT_PSWD
if [ -n "$WSREP_SST_OPT_REMOTE_AUTH" ]
then
# Split auth string at the last ':'
readonly WSREP_SST_OPT_REMOTE_USER="${WSREP_SST_OPT_REMOTE_AUTH%:*}"
readonly WSREP_SST_OPT_REMOTE_PSWD="${WSREP_SST_OPT_REMOTE_AUTH##*:}"
else
readonly WSREP_SST_OPT_REMOTE_USER=
readonly WSREP_SST_OPT_REMOTE_PSWD=
fi
if [ -n "${WSREP_SST_OPT_DATA:-}" ]
then
SST_PROGRESS_FILE="$WSREP_SST_OPT_DATA/sst_in_progress"
......@@ -386,6 +399,19 @@ wsrep_check_programs()
done
}
# Generate a string equivalent to 16 random bytes
wsrep_gen_secret()
{
if [ -x /usr/bin/openssl ]
then
echo `/usr/bin/openssl rand -hex 16`
else
printf "%04x%04x%04x%04x%04x%04x%04x%04x" \
$RANDOM $RANDOM $RANDOM $RANDOM \
$RANDOM $RANDOM $RANDOM $RANDOM
fi
}
#
# user can specify mariabackup specific settings that will be used during sst
# process like encryption, etc.....
......@@ -397,14 +423,22 @@ wsrep_check_programs()
parse_cnf()
{
local group=$1
local var=$2
local var=${2//_/-} # normalize variable name by replacing all '_' with '-'
local reval=""
# normalize the variable names specified in cnf file (user can use _ or - for example log-bin or log_bin)
# first normalize output variable names specified in cnf file:
# user can use _ or - (for example log-bin or log_bin) and/or prefix
# variable with --loose-
# then search for needed variable
# finally get the variable value (if variables has been specified multiple time use the last value only)
reval=$($MY_PRINT_DEFAULTS "${group}" | awk -v var="${var}" 'BEGIN { OFS=FS="=" } { gsub(/_/,"-",$1); if ( $1=="--"var) lastval=substr($0,length($1)+2) } END { print lastval}')
reval=$($MY_PRINT_DEFAULTS "${group}" | \
awk -v var="${var}" 'BEGIN { OFS=FS="=" } \
{ sub(/^--loose/,"-",$0); \
gsub(/_/,"-",$1); \
if ( $1=="--"var) \
lastval=substr($0,length($1)+2) } \
END { print lastval}')
# use default if we haven't found a value
if [ -z "$reval" ]; then
......
......@@ -37,6 +37,7 @@ REMOTEIP=""
tcert=""
tpem=""
tkey=""
tmode="DISABLED"
sockopt=""
progress=""
ttime=0
......@@ -70,6 +71,8 @@ xtmpdir=""
scomp=""
sdecomp=""
readonly SECRET_TAG="secret"
# Required for backup locks
# For backup locks it is 1 sent by joiner
# 5.6.21 PXC and later can't donate to an older joiner
......@@ -261,24 +264,30 @@ get_transfer()
exit 22
fi
stagemsg+="-OpenSSL-Encrypted-3"
if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then
if [[ -z $tcert ]];then
if [[ -z $tcert ]];then
# no verification
if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then
wsrep_log_info "Decrypting with cert=${tpem}, key=${tkey}, verify=0"
tcmd="socat -u openssl-listen:${TSST_PORT},reuseaddr,cert=${tpem},key=${tkey},verify=0${sockopt} stdio"
else
wsrep_log_info "Decrypting with cert=${tpem}, key=${tkey}, cafile=${tcert}"
tcmd="socat -u openssl-listen:${TSST_PORT},reuseaddr,cert=${tpem},key=${tkey},cafile=${tcert}${sockopt} stdio"
fi
else
if [[ -z $tcert ]];then
wsrep_log_info "Encrypting with cert=${tpem}, key=${tkey}, verify=0"
tcmd="socat -u stdio openssl-connect:${REMOTEIP}:${TSST_PORT},cert=${tpem},key=${tkey},verify=0${sockopt}"
fi
else
# CA verification
if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then
wsrep_log_info "Decrypting with cert=${tpem}, key=${tkey}, cafile=${tcert}"
tcmd="socat -u openssl-listen:${TSST_PORT},reuseaddr,cert=${tpem},key=${tkey},cafile=${tcert}${sockopt} stdio"
else
if [ -n "$WSREP_SST_OPT_REMOTE_USER" ]; then
CN_option=",commonname=$WSREP_SST_OPT_REMOTE_USER"
else
CN_option=""
fi
wsrep_log_info "Encrypting with cert=${tpem}, key=${tkey}, cafile=${tcert}"
tcmd="socat -u stdio openssl-connect:${REMOTEIP}:${TSST_PORT},cert=${tpem},key=${tkey},cafile=${tcert}${sockopt}"
tcmd="socat -u stdio openssl-connect:${REMOTEIP}:${TSST_PORT},cert=${tpem},key=${tkey},cafile=${tcert}${CN_option}${sockopt}"
fi
fi
else
if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then
tcmd="socat -u TCP-LISTEN:${TSST_PORT},reuseaddr${sockopt} stdio"
......@@ -287,7 +296,6 @@ get_transfer()
fi
fi
fi
}
parse_cnf()
......@@ -295,10 +303,17 @@ parse_cnf()
local group=$1
local var=$2
# print the default settings for given group using my_print_default.
# remove possible 'loose' variable name prefix
# normalize the variable names specified in cnf file (user can use _ or - for example log-bin or log_bin)
# then grep for needed variable
# finally get the variable value (if variables has been specified multiple time use the last value only)
reval=$($MY_PRINT_DEFAULTS $group | awk -F= '{if ($1 ~ /_/) { gsub(/_/,"-",$1); print $1"="$2 } else { print $0 }}' | grep -- "--$var=" | cut -d= -f2- | tail -1)
reval=$($MY_PRINT_DEFAULTS $group | \
awk -F= '{ sub(/^--loose/,"-",$0); \
if ($1 ~ /_/) \
{ gsub(/_/,"-",$1); print $1"="$2 } \
else \
{ print $0 }}' | \
grep -- "--$var=" | cut -d= -f2- | tail -1)
if [[ -z $reval ]];then
[[ -n $3 ]] && reval=$3
fi
......@@ -348,6 +363,24 @@ adjust_progress()
fi
}
check_server_ssl_config()
{
local section=$1
tcert=$(parse_cnf $section ssl-ca "")
tpem=$(parse_cnf $section ssl-cert "")
tkey=$(parse_cnf $section ssl-key "")
if [ 0 -eq $encrypt -a -n "$tpem" -a -n "$tkey" ]
then
encrypt=3 # enable cert/key SSL encyption
# avoid CA verification if not set explicitly:
# nodes may happen to have different CA if self-generated
# zeroing up tcert does the trick
local mode=$(parse_cnf SST ssl-mode "")
[[ ${mode} = *VERIFY* ]] || tcert=""
fi
}
read_cnf()
{
sfmt=$(parse_cnf sst streamfmt "xbstream")
......@@ -356,6 +389,26 @@ read_cnf()
tpem=$(parse_cnf sst tcert "")
tkey=$(parse_cnf sst tkey "")
encrypt=$(parse_cnf sst encrypt 0)
tmode=$(parse_cnf sst ssl-mode "DISABLED" | tr [:lower:] [:upper:])
if [ -z "$tpem" -a -z "$tkey" -a -z "$tcert" ]
then # no old-style SSL config in [sst]
if [ "$tmode" != "DISABLED" ]
then # backward-incompatible behavior
check_server_ssl_config "sst"
if [ -z "$tpem" -a -z "$tkey" -a -z "$tcert" ]
then # no new-stype SSL config in [sst], try server-wide SSL config
check_server_ssl_config "mysqld.$WSREP_SST_OPT_SUFFIX_VALUE"
if [ -z "$tpem" -a -z "$tkey" -a -z "$tcert" ]
then
check_server_ssl_config "mysqld"
fi
fi
fi
fi
wsrep_log_info "SSL configuration: CA='"$tcert"', CERT='"$tpem"'," \
"KEY='"$tkey"', MODE='"$tmode"', encrypt="$encrypt
sockopt=$(parse_cnf sst sockopt "")
progress=$(parse_cnf sst progress "")
ttime=$(parse_cnf sst time 0)
......@@ -400,7 +453,7 @@ read_cnf()
if [[ $encrypt -eq 1 ]]; then
wsrep_log_error "Xtrabackup-based encryption is currently not" \
"supported with MariaBackup"
"supported with MariaBackup"
exit 2
fi
}
......@@ -627,8 +680,8 @@ recv_joiner()
popd 1>/dev/null
if [[ ${RC[0]} -eq 124 ]];then
wsrep_log_error "Possible timeout in receiving first data from "
"donor in gtid stage: exit codes: ${RC[@]}"
wsrep_log_error "Possible timeout in receiving first data from " \
"donor in gtid stage: exit codes: ${RC[@]}"
exit 32
fi
......@@ -640,12 +693,27 @@ recv_joiner()
fi
done
if [[ $checkf -eq 1 && ! -r "${MAGIC_FILE}" ]];then
# this message should cause joiner to abort
wsrep_log_error "xtrabackup process ended without creating '${MAGIC_FILE}'"
wsrep_log_info "Contents of datadir"
wsrep_log_info "$(ls -l ${dir}/*)"
exit 32
if [[ $checkf -eq 1 ]]; then
if [[ ! -r "${MAGIC_FILE}" ]];then
# this message should cause joiner to abort
wsrep_log_error "receiving process ended without creating " \
"'${MAGIC_FILE}'"
wsrep_log_info "Contents of datadir"
wsrep_log_info "$(ls -l ${dir}/*)"
exit 32
fi
# check donor supplied secret
SECRET=$(grep "$SECRET_TAG " ${MAGIC_FILE} 2>/dev/null | cut -d ' ' -f 2)
if [[ $SECRET != $MY_SECRET ]]; then
wsrep_log_error "Donor does not know my secret!"
wsrep_log_info "Donor:'$SECRET', my:'$MY_SECRET'"
exit 32
fi
# remove secret from magic file
grep -v "$SECRET_TAG " ${MAGIC_FILE} > ${MAGIC_FILE}.new
mv ${MAGIC_FILE}.new ${MAGIC_FILE}
fi
}
......@@ -661,10 +729,9 @@ send_donor()
set -e
popd 1>/dev/null
for ecode in "${RC[@]}";do
if [[ $ecode -ne 0 ]];then
wsrep_log_error "Error while getting data from donor node: " \
wsrep_log_error "Error while sending data to joiner node: " \
"exit codes: ${RC[@]}"
exit 32
fi
......@@ -878,6 +945,11 @@ then
# (separated by a space).
echo "${WSREP_SST_OPT_GTID} ${WSREP_SST_OPT_GTID_DOMAIN_ID}" > "${MAGIC_FILE}"
if [[ -n ${WSREP_SST_OPT_REMOTE_PSWD} ]]; then
# Let joiner know that we know its secret
echo "$SECRET_TAG ${WSREP_SST_OPT_REMOTE_PSWD}" >> ${MAGIC_FILE}
fi
ttcmd="$tcmd"
if [[ $encrypt -eq 1 ]];then
......@@ -990,7 +1062,6 @@ then
stagemsg="Joiner-Recv"
sencrypted=1
nthreads=1
......@@ -1012,7 +1083,26 @@ then
fi
fi
wait_for_listen ${SST_PORT} ${ADDR} ${MODULE} &
if [[ "$tmode" = *"VERIFY"* ]]
then # backward-incompatible behavior
if [ -n "$tpem" ]
then
# find out my Common Name
wsrep_check_programs openssl
CN=$(openssl x509 -noout -subject -in $tpem | \
tr "," "\n" | grep "CN =" | cut -d= -f2 | sed s/^\ // | \
sed s/\ %//)
else
CN=""
fi
MY_SECRET=$(wsrep_gen_secret)
# Add authentication data to address
ADDR="$CN:$MY_SECRET@$ADDR"
else
MY_SECRET="" # for check down in recv_joiner()
fi # tmode == *VERIFY*
wait_for_listen ${SST_PORT} "${ADDR}" ${MODULE} &
trap sig_joiner_cleanup HUP PIPE INT TERM
trap cleanup_joiner EXIT
......
......@@ -702,49 +702,30 @@ static void* sst_joiner_thread (void* a)
return NULL;
}
#define WSREP_SST_AUTH_ENV "WSREP_SST_OPT_AUTH"
#define WSREP_SST_AUTH_ENV "WSREP_SST_OPT_AUTH"
#define WSREP_SST_REMOTE_AUTH_ENV "WSREP_SST_OPT_REMOTE_AUTH"
#define DATA_HOME_DIR_ENV "INNODB_DATA_HOME_DIR"
static int sst_append_auth_env(wsp::env& env, const char* sst_auth)
static int sst_append_env_var(wsp::env& env,
const char* const var,
const char* const val)
{
int const sst_auth_size= strlen(WSREP_SST_AUTH_ENV) + 1 /* = */
+ (sst_auth ? strlen(sst_auth) : 0) + 1 /* \0 */;
int const env_str_size= strlen(var) + 1 /* = */
+ (val ? strlen(val) : 0) + 1 /* \0 */;
wsp::string sst_auth_str(sst_auth_size); // for automatic cleanup on return
if (!sst_auth_str()) return -ENOMEM;
wsp::string env_str(env_str_size); // for automatic cleanup on return
if (!env_str()) return -ENOMEM;
int ret= snprintf(sst_auth_str(), sst_auth_size, "%s=%s",
WSREP_SST_AUTH_ENV, sst_auth ? sst_auth : "");
int ret= snprintf(env_str(), env_str_size, "%s=%s", var, val ? val : "");
if (ret < 0 || ret >= sst_auth_size)
if (ret < 0 || ret >= env_str_size)
{
WSREP_ERROR("sst_append_auth_env(): snprintf() failed: %d", ret);
WSREP_ERROR("sst_append_env_var(): snprintf(%s=%s) failed: %d",
var, val, ret);
return (ret < 0 ? ret : -EMSGSIZE);
}
env.append(sst_auth_str());
return -env.error();
}
#define DATA_HOME_DIR_ENV "INNODB_DATA_HOME_DIR"
static int sst_append_data_dir(wsp::env& env, const char* data_dir)
{
int const data_dir_size= strlen(DATA_HOME_DIR_ENV) + 1 /* = */
+ (data_dir ? strlen(data_dir) : 0) + 1 /* \0 */;
wsp::string data_dir_str(data_dir_size); // for automatic cleanup on return
if (!data_dir_str()) return -ENOMEM;
int ret= snprintf(data_dir_str(), data_dir_size, "%s=%s",
DATA_HOME_DIR_ENV, data_dir ? data_dir : "");
if (ret < 0 || ret >= data_dir_size)
{
WSREP_ERROR("sst_append_data_dir(): snprintf() failed: %d", ret);
return (ret < 0 ? ret : -EMSGSIZE);
}
env.append(data_dir_str());
env.append(env_str());
return -env.error();
}
......@@ -1155,7 +1136,7 @@ static ssize_t sst_prepare_other (const char* method,
return -env.error();
}
if ((ret= sst_append_auth_env(env, sst_auth)))
if ((ret= sst_append_env_var(env, WSREP_SST_AUTH_ENV, sst_auth)))
{
WSREP_ERROR("sst_prepare_other(): appending auth failed: %d", ret);
return ret;
......@@ -1163,7 +1144,7 @@ static ssize_t sst_prepare_other (const char* method,
if (data_home_dir)
{
if ((ret= sst_append_data_dir(env, data_home_dir)))
if ((ret= sst_append_env_var(env, DATA_HOME_DIR_ENV, data_home_dir)))
{
WSREP_ERROR("sst_prepare_other(): appending data "
"directory failed: %d", ret);
......@@ -1478,6 +1459,8 @@ static int sst_donate_mysqldump (const char* addr,
wsrep::seqno::undefined()));
Wsrep_server_state::instance().sst_sent(sst_sent_gtid, ret);
wsrep_donor_monitor_end();
return ret;
}
......@@ -1859,6 +1842,7 @@ static int sst_donate_other (const char* method,
"wsrep_sst_%s "
WSREP_SST_OPT_ROLE " 'donor' "
WSREP_SST_OPT_ADDR " '%s' "
WSREP_SST_OPT_LPORT " '%u' "
WSREP_SST_OPT_SOCKET " '%s' "
WSREP_SST_OPT_DATA " '%s' "
"%s"
......@@ -1867,7 +1851,8 @@ static int sst_donate_other (const char* method,
"%s"
"%s"
"%s",
method, addr, mysqld_unix_port, mysql_real_data_home,
method, addr, mysqld_port, mysqld_unix_port,
mysql_real_data_home,
wsrep_defaults_file,
uuid_oss.str().c_str(), gtid.seqno().get(), wsrep_gtid_server.domain_id,
binlog_opt_val, binlog_index_opt_val,
......@@ -1956,7 +1941,21 @@ int wsrep_sst_donate(const std::string& msg,
const char* data= method + method_len + 1;
if (check_request_str(data, address_char))
/* check for auth@addr separator */
const char* addr= strrchr(data, '@');
wsp::string remote_auth;
if (addr)
{
remote_auth.set(strndup(data, addr - data));
addr++;
}
else
{
// no auth part
addr= data;
}
if (check_request_str(addr, address_char))
{
WSREP_ERROR("Bad SST address string. SST canceled.");
return WSREP_CB_FAILURE;
......@@ -1970,15 +1969,25 @@ int wsrep_sst_donate(const std::string& msg,
}
int ret;
if ((ret= sst_append_auth_env(env, sst_auth_real)))
if ((ret= sst_append_env_var(env, WSREP_SST_AUTH_ENV, sst_auth_real)))
{
WSREP_ERROR("wsrep_sst_donate_cb(): appending auth env failed: %d", ret);
return WSREP_CB_FAILURE;
}
if (remote_auth())
{
if ((ret= sst_append_env_var(env, WSREP_SST_REMOTE_AUTH_ENV,remote_auth())))
{
WSREP_ERROR("wsrep_sst_donate_cb(): appending remote auth env failed: "
"%d", ret);
return WSREP_CB_FAILURE;
}
}
if (data_home_dir)
{
if ((ret= sst_append_data_dir(env, data_home_dir)))
if ((ret= sst_append_env_var(env, DATA_HOME_DIR_ENV, data_home_dir)))
{
WSREP_ERROR("wsrep_sst_donate_cb(): appending data "
"directory failed: %d", ret);
......@@ -2000,11 +2009,11 @@ int wsrep_sst_donate(const std::string& msg,
if (!strcmp (WSREP_SST_MYSQLDUMP, method))
{
ret= sst_donate_mysqldump(data, current_gtid, bypass, env());
ret= sst_donate_mysqldump(addr, current_gtid, bypass, env());
}
else
{
ret= sst_donate_other(method, data, current_gtid, bypass, env());
ret= sst_donate_other(method, addr, current_gtid, bypass, env());
}
return (ret >= 0 ? 0 : 1);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment