From 0892c7ec5d7bbca00e042047c4220e6830491c37 Mon Sep 17 00:00:00 2001 From: Seppo Jaakola <seppo.jaakola@codership.com> Date: Mon, 17 Sep 2012 11:34:57 +0300 Subject: [PATCH] References lp:1051808 - merged with lp:codership-mysql 5.5.27 based trunk patched with: bzr diff lp:codership-mysql/5.5 -r3779..3790 --- .bzrignore | 5 + Docs/README-wsrep | 18 ++- scripts/CMakeLists.txt | 2 +- scripts/mysqld_safe.sh | 34 +++- scripts/wsrep_sst_common.sh | 106 ++++++++++++ scripts/wsrep_sst_mysqldump.sh | 52 +++--- scripts/wsrep_sst_rsync.sh | 48 +++--- scripts/wsrep_sst_xtrabackup.sh | 222 ++++++++++++++++++++++++++ sql/handler.cc | 4 +- sql/mysqld.cc | 49 ++++++ sql/sql_parse.cc | 6 +- sql/wsrep_mysqld.cc | 38 ++++- sql/wsrep_mysqld.h | 3 +- sql/wsrep_sst.cc | 98 ++++++++---- storage/innobase/handler/ha_innodb.cc | 5 +- support-files/mysql.spec.sh | 11 +- support-files/wsrep.cnf.sh | 3 +- 17 files changed, 598 insertions(+), 106 deletions(-) create mode 100644 scripts/wsrep_sst_common.sh create mode 100644 scripts/wsrep_sst_xtrabackup.sh diff --git a/.bzrignore b/.bzrignore index 409b6148125..943c9bf8c50 100644 --- a/.bzrignore +++ b/.bzrignore @@ -969,6 +969,8 @@ support-files/mysql.server support-files/mysql.spec support-files/mysqld_multi.server support-files/ndb-config-2-node.ini +support-files/wsrep.cnf +support-files/wsrep_notify TAGS test/ndbapi/bank/bankCreator test/ndbapi/bank/bankMakeGL @@ -1092,6 +1094,9 @@ sql/share/slovak sql/share/spanish sql/share/swedish sql/share/ukrainian +scripts/wsrep_sst_mysqldump +scripts/wsrep_sst_rsync +scripts/wsrep_sst_xtrabackup CPackConfig.cmake CPackSourceConfig.cmake Docs/INFO_BIN diff --git a/Docs/README-wsrep b/Docs/README-wsrep index 78934811984..025379764b2 100644 --- a/Docs/README-wsrep +++ b/Docs/README-wsrep @@ -137,6 +137,7 @@ Additional packages to consider (if not yet installed): * galera (multi-master replication provider, https://launchpad.net/galera) * MySQL-client-community (for connecting to server and mysqldump-based SST) * rsync (for rsync-based SST) + * xtrabackup and nc (for xtrabackup-based SST) 2.2 Upgrade system tables. @@ -379,9 +380,20 @@ to join or start a cluster. wsrep_sst_method=mysqldump What method to use to copy database state to a newly joined node. Supported methods: - - mysqldump: slow (except for small datasets) but most tested. - - rsync: much faster on large datasets. - - rsync_wan: same as rsync but with deltaxfer to minimize network traffic. + - mysqldump: slow (except for small datasets) but most tested. + - rsync: much faster on large datasets. + - rsync_wan: same as rsync but with deltaxfer to minimize network traffic. + - xtrabackup: very fast and practically non-blocking SST method based on + Percona's xtrabackup tool. + + (for xtrabackup to work the following settings must be present in my.cnf + on all nodes: + [mysqld] + wsrep_sst_auth=root:<root password> + datadir=<path to data dir> + [client] + socket=<path to socket> + ) wsrep_sst_receive_address= Address (hostname:port) at which this node wants to receive state snapshot. diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt index 5576f32e0c4..b7559b2358d 100644 --- a/scripts/CMakeLists.txt +++ b/scripts/CMakeLists.txt @@ -311,7 +311,7 @@ IF(WIN32) ENDFOREACH() ELSE() IF(WITH_WSREP) - SET(WSREP_BINARIES wsrep_sst_mysqldump wsrep_sst_rsync) + SET(WSREP_BINARIES wsrep_sst_common wsrep_sst_mysqldump wsrep_sst_rsync wsrep_sst_xtrabackup) ENDIF() # On Unix, most of the files end up in the bin directory SET(mysql_config_COMPONENT COMPONENT Development) diff --git a/scripts/mysqld_safe.sh b/scripts/mysqld_safe.sh index 5a390a32f26..72430cd19f6 100644 --- a/scripts/mysqld_safe.sh +++ b/scripts/mysqld_safe.sh @@ -190,6 +190,31 @@ wsrep_pick_url() { echo $url } +# Run mysqld with --wsrep-recover and parse recovered position from log. +# Position will be stored in wsrep_start_position_opt global. +wsrep_recovery() { + cmd="$@" + wr_logfile=$(mktemp) + log_notice "WSREP: Running position recovery" + $cmd --log_error=$wr_logfile --wsrep-recover + rp=$(grep "WSREP: Recovered position:" $wr_logfile) + if [ -z "$rp" ]; then + skipped=$(grep WSREP $wr_logfile | grep "skipping position recovery") + if [ -z "$skipped" ]; then + log_error "WSREP: Failed to recover position: " \ + `cat $wr_logfile`; + else + log_notice "WSREP: Position recovery skipped" + fi + else + start_pos=$(echo $rp | sed 's/.*WSREP\:\ Recovered\ position://' \ + | sed 's/^[ \t]*//') + wsrep_start_position_opt="--wsrep_start_position=$start_pos" + log_notice "WSREP: Recovered position $start_pos" + fi + rm $wr_logfile +} + parse_arguments() { # We only need to pass arguments through to the server if we don't # handle them here. So, we collect unrecognized options (passed on @@ -787,7 +812,8 @@ do done cmd="$cmd $args" # Avoid 'nohup: ignoring input' warning -test -n "$NOHUP_NICENESS" && cmd="$cmd < /dev/null" +nohup_redir="" +test -n "$NOHUP_NICENESS" && nohup_redir=" < /dev/null" log_notice "Starting $MYSQLD daemon with databases from $DATADIR" @@ -808,9 +834,11 @@ do if [ -z "$url" ] then - eval_log_error "$cmd" + wsrep_recovery "$cmd" + eval_log_error "$cmd $wsrep_start_position_opt $nohup_redir" else - eval_log_error "$cmd --wsrep_cluster_address=$url" + wsrep_recovery "$cmd" + eval_log_error "$cmd $wsrep_start_position_opt --wsrep_cluster_address=$url $nohup_redir" fi end_time=`date +%M%S` diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh new file mode 100644 index 00000000000..a3ad90769e9 --- /dev/null +++ b/scripts/wsrep_sst_common.sh @@ -0,0 +1,106 @@ +# Copyright (C) 2010 Codership Oy +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; see the file COPYING. If not, write to the +# Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston +# MA 02110-1301 USA. + +# This is a common command line parser to be sourced by other SST scripts + +set -u + +WSREP_SST_OPT_BYPASS=0 + +while [ $# -gt 0 ]; do +case "$1" in + '--address') + readonly WSREP_SST_OPT_ADDR="$2" + shift + ;; + '--auth') + readonly WSREP_SST_OPT_AUTH="$2" + shift + ;; + '--bypass') + WSREP_SST_OPT_BYPASS=1 + ;; + '--datadir') + readonly WSREP_SST_OPT_DATA="$2" + shift + ;; + '--defaults-file') + readonly WSREP_SST_OPT_CONF="$2" + shift + ;; + '--host') + readonly WSREP_SST_OPT_HOST="$2" + shift + ;; + '--local-port') + readonly WSREP_SST_OPT_LPORT="$2" + shift + ;; + '--parent') + readonly WSREP_SST_OPT_PARENT="$2" + shift + ;; + '--password') + readonly WSREP_SST_OPT_PSWD="$2" + shift + ;; + '--port') + readonly WSREP_SST_OPT_PORT="$2" + shift + ;; + '--role') + readonly WSREP_SST_OPT_ROLE="$2" + shift + ;; + '--socket') + readonly WSREP_SST_OPT_SOCKET="$2" + shift + ;; + '--user') + readonly WSREP_SST_OPT_USER="$2" + shift + ;; + '--gtid') + readonly WSREP_SST_OPT_GTID="$2" + shift + ;; + *) # must be command + # usage + # exit 1 + ;; +esac +shift +done +readonly WSREP_SST_OPT_BYPASS + +wsrep_log() +{ + # echo everything to stderr so that it gets into common error log + # deliberately made to look different from the rest of the log + local readonly tst="$(date +%Y%m%d\ %H:%M:%S.%N | cut -b -21)" + echo "WSREP_SST: $* ($tst)" >/dev/stderr +} + +wsrep_log_error() +{ + wsrep_log "[ERROR] $*" +} + +wsrep_log_info() +{ + wsrep_log "[INFO] $*" +} + diff --git a/scripts/wsrep_sst_mysqldump.sh b/scripts/wsrep_sst_mysqldump.sh index 8106850e918..120533edc4e 100644 --- a/scripts/wsrep_sst_mysqldump.sh +++ b/scripts/wsrep_sst_mysqldump.sh @@ -17,23 +17,10 @@ # This is a reference script for mysqldump-based state snapshot tansfer -USER=$1 -PSWD=$2 -HOST=$3 -PORT=$4 -LOCAL_HOST="127.0.0.1" -LOCAL_PORT=$5 -UUID=$6 -SEQNO=$7 -BYPASS=$8 +. $(dirname $0)/wsrep_sst_common EINVAL=22 -err() -{ - echo "SST error: $*" >&2 -} - local_ip() { PATH=$PATH:/usr/sbin:/usr/bin:/sbin:/bin @@ -51,16 +38,18 @@ local_ip() return 1 } -if test -z "$USER"; then err "USER cannot be nil"; exit $EINVAL; fi -if test -z "$HOST"; then err "HOST cannot be nil"; exit $EINVAL; fi -if test -z "$PORT"; then err "PORT cannot be nil"; exit $EINVAL; fi -if test -z "$LOCAL_PORT"; then err "LOCAL_PORT cannot be nil"; exit $EINVAL; fi -if test -z "$UUID"; then err "UUID cannot be nil"; exit $EINVAL; fi -if test -z "$SEQNO"; then err "SEQNO cannot be nil"; exit $EINVAL; fi +if test -z "$WSREP_SST_OPT_USER"; then err "USER cannot be nil"; exit $EINVAL; fi +if test -z "$WSREP_SST_OPT_HOST"; then err "HOST cannot be nil"; exit $EINVAL; fi +if test -z "$WSREP_SST_OPT_PORT"; then err "PORT cannot be nil"; exit $EINVAL; fi +if test -z "$WSREP_SST_OPT_LPORT"; then err "LPORT cannot be nil"; exit $EINVAL; fi +if test -z "$WSREP_SST_OPT_SOCKET";then err "SOCKET cannot be nil";exit $EINVAL; fi +if test -z "$WSREP_SST_OPT_GTID"; then err "GTID cannot be nil"; exit $EINVAL; fi -if local_ip $HOST && [ "$PORT" = "$LOCAL_PORT" ] +if local_ip $WSREP_SST_OPT_HOST && \ + [ "$WSREP_SST_OPT_PORT" = "$WSREP_SST_OPT_LPORT" ] then - err "destination address '$HOST:$PORT' matches source address." + wsrep_log_error \ + "destination address '$WSREP_SST_OPT_HOST:$WSREP_SST_OPT_PORT' matches source address." exit $EINVAL fi @@ -68,18 +57,17 @@ fi if ! mysql --version | grep 'Distrib 5.5' >/dev/null then mysql --version >&2 - err "this procedure requires MySQL client version 5.5.x" + err "this operation requires MySQL client version 5.5.x" exit $EINVAL fi -AUTH="-u$USER" -if test -n "$PSWD"; then AUTH="$AUTH -p$PSWD"; fi +AUTH="-u$WSREP_SST_OPT_USER" +if test -n "$WSREP_SST_OPT_PSWD"; then AUTH="$AUTH -p$WSREP_SST_OPT_PSWD"; fi STOP_WSREP="SET wsrep_on=OFF;" # NOTE: we don't use --routines here because we're dumping mysql.proc table -#MYSQLDUMP="@bindir@/mysqldump $AUTH -h$LOCAL_HOST -P$LOCAL_PORT \ -MYSQLDUMP="mysqldump $AUTH -h$LOCAL_HOST -P$LOCAL_PORT \ +MYSQLDUMP="mysqldump $AUTH -S$WSREP_SST_OPT_SOCKET \ --add-drop-database --add-drop-table --skip-add-locks --create-options \ --disable-keys --extended-insert --skip-lock-tables --quick --set-charset \ --skip-comments --flush-privileges --all-databases" @@ -102,10 +90,10 @@ PREPARE stmt FROM @str; EXECUTE stmt; DROP PREPARE stmt;" -SET_START_POSITION="SET GLOBAL wsrep_start_position='$UUID:$SEQNO';" +SET_START_POSITION="SET GLOBAL wsrep_start_position='$WSREP_SST_OPT_GTID';" -#MYSQL="@bindir@/mysql -u'$USER' -p'$PSWD' -h'$HOST' -P'$PORT'" -MYSQL="mysql $AUTH -h$HOST -P$PORT --disable-reconnect --connect_timeout=10" +MYSQL="mysql $AUTH -h$WSREP_SST_OPT_HOST -P$WSREP_SST_OPT_PORT "\ +"--disable-reconnect --connect_timeout=10" # need to disable logging when loading the dump # reason is that dump contains ALTER TABLE for log tables, and @@ -119,14 +107,14 @@ $MYSQL -e"$STOP_WSREP SET GLOBAL SLOW_QUERY_LOG=OFF" RESTORE_GENERAL_LOG="SET GLOBAL GENERAL_LOG=$GENERAL_LOG_OPT;" RESTORE_SLOW_QUERY_LOG="SET GLOBAL SLOW_QUERY_LOG=$SLOW_LOG_OPT;" -if [ $BYPASS -eq 0 ] +if [ $WSREP_SST_OPT_BYPASS -eq 0 ] then (echo $STOP_WSREP && $MYSQLDUMP && echo $CSV_TABLES_FIX \ && echo $RESTORE_GENERAL_LOG && echo $RESTORE_SLOW_QUERY_LOG \ && echo $SET_START_POSITION \ || echo "SST failed to complete;") | $MYSQL else - echo "Bypassing state dump." >&2 + wsrep_log_info "Bypassing state dump." echo $SET_START_POSITION | $MYSQL fi diff --git a/scripts/wsrep_sst_rsync.sh b/scripts/wsrep_sst_rsync.sh index ef3dda8231f..d346eb240f2 100644 --- a/scripts/wsrep_sst_rsync.sh +++ b/scripts/wsrep_sst_rsync.sh @@ -21,9 +21,11 @@ RSYNC_PID= RSYNC_CONF= +. $(dirname $0)/wsrep_sst_common + cleanup_joiner() { - echo -n "Joiner rsync SST cleanup..." >&2 + wsrep_log_info "Joiner cleanup." local PID=$(cat "$RSYNC_PID" 2>/dev/null || echo 0) [ "0" != "$PID" ] && kill $PID && sleep 0.5 && kill -9 $PID >/dev/null 2>&1 \ || : @@ -50,25 +52,16 @@ check_pid_and_port() grep LISTEN | grep \:$rsync_port | grep $rsync_pid/rsync >/dev/null } -ROLE=$1 -ADDR=$2 -AUTH=$3 -DATA=$4 -CONF=$5 - -MAGIC_FILE="$DATA/rsync_sst_complete" +MAGIC_FILE="$WSREP_SST_OPT_DATA/rsync_sst_complete" rm -rf "$MAGIC_FILE" -if [ "$ROLE" = "donor" ] +if [ "$WSREP_SST_OPT_ROLE" = "donor" ] then - UUID=$6 - SEQNO=$7 - BYPASS=$8 - if [ $BYPASS -eq 0 ] + if [ $WSREP_SST_OPT_BYPASS -eq 0 ] then - FLUSHED="$DATA/tables_flushed" + FLUSHED="$WSREP_SST_OPT_DATA/tables_flushed" rm -rf "$FLUSHED" # Use deltaxfer only for WAN @@ -100,7 +93,8 @@ then RC=0 rsync --archive --no-times --ignore-times --inplace --delete --quiet \ - $WHOLE_FILE_OPT "${FILTER[@]}" "$DATA" rsync://$ADDR || RC=$? + $WHOLE_FILE_OPT "${FILTER[@]}" "$WSREP_SST_OPT_DATA" \ + rsync://$WSREP_SST_OPT_ADDR || RC=$? [ $RC -ne 0 ] && echo "rsync returned code $RC:" >> /dev/stderr @@ -108,9 +102,9 @@ then 0) RC=0 # Success ;; 12) RC=71 # EPROTO - echo "rsync server on the other end has incompatible protocol. " \ - "Make sure you have the same version of rsync on all nodes."\ - >> /dev/stderr + wsrep_log_error \ + "rsync server on the other end has incompatible protocol. " \ + "Make sure you have the same version of rsync on all nodes." ;; 22) RC=12 # ENOMEM ;; @@ -121,23 +115,24 @@ then [ $RC -ne 0 ] && exit $RC else # BYPASS - STATE="$UUID:$SEQNO" + wsrep_log_info "Bypassing state dump." + STATE="$WSREP_SST_OPT_GTID" fi echo "continue" # now server can resume updating data echo "$STATE" > "$MAGIC_FILE" - rsync -aqc "$MAGIC_FILE" rsync://$ADDR + rsync -aqc "$MAGIC_FILE" rsync://$WSREP_SST_OPT_ADDR echo "done $STATE" -elif [ "$ROLE" = "joiner" ] +elif [ "$WSREP_SST_OPT_ROLE" = "joiner" ] then - MYSQLD_PID=$6 + MYSQLD_PID=$WSREP_SST_OPT_PARENT MODULE="rsync_sst" - RSYNC_PID="$DATA/$MODULE.pid" + RSYNC_PID="$WSREP_SST_OPT_DATA/$MODULE.pid" if check_pid $RSYNC_PID then @@ -146,6 +141,7 @@ then fi rm -rf "$RSYNC_PID" + ADDR=$WSREP_SST_OPT_ADDR RSYNC_PORT=$(echo $ADDR | awk -F ':' '{ print $2 }') if [ -z "$RSYNC_PORT" ] then @@ -159,13 +155,13 @@ then MYUID=$(id -u) MYGID=$(id -g) - RSYNC_CONF="$DATA/$MODULE.conf" + RSYNC_CONF="$WSREP_SST_OPT_DATA/$MODULE.conf" cat << EOF > "$RSYNC_CONF" pid file = $RSYNC_PID use chroot = no [$MODULE] - path = $DATA + path = $WSREP_SST_OPT_DATA read only = no timeout = 300 uid = $MYUID @@ -207,7 +203,7 @@ EOF # cleanup_joiner else - echo "Unrecognized role: $ROLE" + echo "Unrecognized role: '$WSREP_SST_OPT_ROLE'" exit 22 # EINVAL fi diff --git a/scripts/wsrep_sst_xtrabackup.sh b/scripts/wsrep_sst_xtrabackup.sh new file mode 100644 index 00000000000..5dad320e0f8 --- /dev/null +++ b/scripts/wsrep_sst_xtrabackup.sh @@ -0,0 +1,222 @@ +#!/bin/bash -ue + +# Copyright (C) 2011 Percona Inc +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; see the file COPYING. If not, write to the +# Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston +# MA 02110-1301 USA. + +# This is a reference script for Percona XtraBackup-based state snapshot tansfer + +TMPDIR="/tmp" + +. $(dirname $0)/wsrep_sst_common + +cleanup_joiner() +{ +#set -x + local PID=$(ps -aef |grep nc| grep $NC_PORT | awk '{ print $2 }') + wsrep_log_info "Killing nc pid $PID" + [ -n "$PID" -a "0" != "$PID" ] && kill $PID && (kill $PID && kill -9 $PID) || : + rm -f "$MAGIC_FILE" +#set +x +} + +check_pid() +{ + local pid_file=$1 + [ -r $pid_file ] && ps -p $(cat $pid_file) >/dev/null 2>&1 +} + +kill_xtrabackup() +{ +#set -x + local PID=$(cat $XTRABACKUP_PID) + [ -n "$PID" -a "0" != "$PID" ] && kill $PID && (kill $PID && kill -9 $PID) || : + rm -f "$XTRABACKUP_PID" +#set +x +} + +# waits ~10 seconds for nc to open the port and then reports ready +# (regardless of timeout) +wait_for_nc() +{ + local PORT=$1 + local ADDR=$2 + local MODULE=$3 + for i in $(seq 1 50) + do + netstat -nptl 2>/dev/null | grep '/nc\s*$' | awk '{ print $4 }' | \ + sed 's/.*://' | grep \^${PORT}\$ >/dev/null && break + sleep 0.2 + done + echo "ready ${ADDR}/${MODULE}" +} + +INNOBACKUPEX_BIN=innobackupex +INNOBACKUPEX_ARGS="" +NC_BIN=nc + +for TOOL_BIN in INNOBACKUPEX_BIN NC_BIN ; do + which ${!TOOL_BIN} > /dev/null 2>&1 + if [ $? -ne 0 ]; then + echo "Can't find ${!TOOL_BIN} in the path" + exit 22 # EINVAL + fi +done + +#ROLE=$1 +#ADDR=$2 +readonly AUTH=(${WSREP_SST_OPT_AUTH//:/ }) +readonly DATA="${WSREP_SST_OPT_DATA}" +#CONF=$5 + +INFO_FILE="xtrabackup_galera_info" +IST_FILE="xtrabackup_ist" + +MAGIC_FILE="${DATA}/${INFO_FILE}" +rm -f "${MAGIC_FILE}" + +if [ "$WSREP_SST_OPT_ROLE" = "donor" ] +then + +# UUID=$6 +# SEQNO=$7 +# BYPASS=$8 + + NC_PORT=$(echo $WSREP_SST_OPT_ADDR | awk -F '[:/]' '{ print $2 }') + REMOTEIP=$(echo $WSREP_SST_OPT_ADDR | awk -F ':' '{ print $1 }') + + if [ $WSREP_SST_OPT_BYPASS -eq 0 ] + then + + INNOBACKUPEX_ARGS="--galera-info --tmpdir=${TMPDIR} --stream=tar + --defaults-file=${WSREP_SST_OPT_CONF} + --socket=${WSREP_SST_OPT_SOCKET}" + + if [ "${AUTH[0]}" != "(null)" ]; then + INNOBACKUPEX_ARGS="${INNOBACKUPEX_ARGS} --user=${AUTH[0]}" + fi + + if [ ${#AUTH[*]} -eq 2 ]; then + INNOBACKUPEX_ARGS="${INNOBACKUPEX_ARGS} --password=${AUTH[1]}" + fi + + set +e + + # This file and variable seems to have no effect and probably should be deleted + XTRABACKUP_PID=$(mktemp --tmpdir wsrep_sst_xtrabackupXXXX.pid) + + ${INNOBACKUPEX_BIN} ${INNOBACKUPEX_ARGS} ${TMPDIR} \ + 2> ${DATA}/innobackup.backup.log | \ + ${NC_BIN} ${REMOTEIP} ${NC_PORT} + + RC=( "${PIPESTATUS[@]}" ) + set -e + + if [ ${RC[0]} -ne 0 ]; then + wsrep_log_error "${INNOBACKUPEX_BIN} finished with error: ${RC[0]}. " \ + "Check ${DATA}/innobackup.backup.log" + exit 22 + elif [ ${RC[1]} -ne 0 ]; then + wsrep_log_error "${NC_BIN} finished with error: ${RC[1]}" + exit 22 + fi + + if check_pid ${XTRABACKUP_PID} + then + wsrep_log_error "xtrabackup process is still running. Killing... " + kill_xtrabackup + exit 22 + fi + + rm -f ${XTRABACKUP_PID} + + else # BYPASS + STATE="${WSREP_SST_OPT_GTID}" + echo "continue" # now server can resume updating data + echo "${STATE}" > "${MAGIC_FILE}" + echo "1" > "${DATA}/${IST_FILE}" + (cd ${DATA}; tar cf - ${INFO_FILE} ${IST_FILE}) | ${NC_BIN} ${REMOTEIP} ${NC_PORT} + rm -f ${DATA}/${IST_FILE} + fi + + echo "done ${WSREP_SST_OPT_GTID}" + +elif [ "${WSREP_SST_OPT_ROLE}" = "joiner" ] +then + MODULE="xtrabackup_sst" + + rm -f ${DATA}/xtrabackup_* + + ADDR=${WSREP_SST_OPT_ADDR} + NC_PORT=$(echo ${ADDR} | awk -F ':' '{ print $2 }') + if [ -z "${NC_PORT}" ] + then + NC_PORT=4444 + ADDR="$(echo ${ADDR} | awk -F ':' '{ print $1 }'):${NC_PORT}" + fi + + wait_for_nc ${NC_PORT} ${ADDR} ${MODULE} & + +# trap "exit 32" HUP PIPE +# trap "exit 3" INT TERM + trap cleanup_joiner HUP PIPE INT TERM + + set +e + ${NC_BIN} -dl ${NC_PORT} | tar xfi - -C ${DATA} 1>&2 + RC=( "${PIPESTATUS[@]}" ) + set -e + + wait %% # join wait_for_nc thread + + if [ ${RC[0]} -ne 0 -o ${RC[1]} -ne 0 ]; + then + wsrep_log_error "Error while getting st data from donor node: " \ + "${RC[0]}, ${RC[1]}" + exit 32 + fi + + if [ ! -r "${MAGIC_FILE}" ] + then + # this message should cause joiner to abort + wsrep_log_error "xtrabackup process ended without creating '${MAGIC_FILE}'" + exit 32 + fi + + if ! ps -p ${WSREP_SST_OPT_PARENT} >/dev/null + then + wsrep_log_error "Parent mysqld process (PID:${WSREP_SST_OPT_PARENT}) terminated unexpectedly." >&2 + exit 32 + fi + + if [ ! -r "${IST_FILE}" ] + then + rm -f ${DATA}/ib_logfile* + ${INNOBACKUPEX_BIN} --defaults-file=${WSREP_SST_OPT_CONF} --apply-log \ + --ibbackup=xtrabackup ${DATA} 1>&2 2> ${DATA}/innobackup.prepare.log + if [ $? -ne 0 ]; + then + wsrep_log_error "${INNOBACKUPEX_BIN} finished with errors. Check ${DATA}/innobackup.prepare.log" >&2 + exit 22 + fi + fi + + cat "${MAGIC_FILE}" # output UUID:seqno + +else + wsrep_log_error "Unrecognized role: ${WSREP_SST_OPT_ROLE}" + exit 22 # EINVAL +fi + +exit 0 diff --git a/sql/handler.cc b/sql/handler.cc index cb220783e33..b10deec6862 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -5233,7 +5233,9 @@ void signal_log_not_needed(struct handlerton, char *log_file) int ha_wsrep_abort_transaction(THD *bf_thd, THD *victim_thd, my_bool signal) { DBUG_ENTER("ha_wsrep_abort_transaction"); - if (!WSREP(bf_thd)) { + if (!WSREP(bf_thd) && + !(wsrep_OSU_method_options == WSREP_OSU_RSU && + bf_thd->wsrep_exec_mode == TOTAL_ORDER)) { DBUG_RETURN(0); } diff --git a/sql/mysqld.cc b/sql/mysqld.cc index c4ed7c1e31b..a04dd01e943 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -4884,6 +4884,17 @@ static inline bool is_replaying_connection(THD *thd) return ret; } +static inline bool is_committing_connection(THD *thd) +{ + bool ret; + + mysql_mutex_lock(&thd->LOCK_wsrep_thd); + ret= (thd->wsrep_query_state == QUERY_COMMITTING) ? true : false; + mysql_mutex_unlock(&thd->LOCK_wsrep_thd); + + return ret; +} + static bool have_client_connections() { THD *tmp; @@ -4937,6 +4948,44 @@ static void wsrep_close_thread(THD *thd) } } +static my_bool have_committing_connections() +{ + THD *tmp; + mysql_mutex_lock(&LOCK_thread_count); // For unlink from list + + I_List_iterator<THD> it(threads); + while ((tmp=it++)) + { + if (!is_client_connection(tmp)) + continue; + + if (is_committing_connection(tmp)) + { + mysql_mutex_unlock(&LOCK_thread_count); + return TRUE; + } + } + mysql_mutex_unlock(&LOCK_thread_count); + return FALSE; +} + +int wsrep_wait_committing_connections_close(int wait_time) +{ + int sleep_time= 100; + + while (have_committing_connections() && wait_time > 0) + { + WSREP_DEBUG("wait for committing transaction to close: %d", wait_time); + my_sleep(sleep_time); + wait_time -= sleep_time; + } + if (have_committing_connections()) + { + return 1; + } + return 0; +} + void wsrep_close_client_connections(my_bool wait_to_end) { /* diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index fbb47b5c03b..4e582a238e1 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -8436,8 +8436,10 @@ int wsrep_abort_thd(void *bf_thd_ptr, void *victim_thd_ptr, my_bool signal) THD *bf_thd = (THD *) bf_thd_ptr; DBUG_ENTER("wsrep_abort_thd"); - if ( (WSREP(bf_thd) || - (WSREP_ON && bf_thd->wsrep_exec_mode == TOTAL_ORDER)) && victim_thd) + if ( (WSREP(bf_thd) || + ( (WSREP_ON || wsrep_OSU_method_options == WSREP_OSU_RSU) && + bf_thd->wsrep_exec_mode == TOTAL_ORDER) ) && + victim_thd) { WSREP_DEBUG("wsrep_abort_thd, by: %llu, victim: %llu", (bf_thd) ? (long long)bf_thd->real_id : 0, (long long)victim_thd->real_id); diff --git a/sql/wsrep_mysqld.cc b/sql/wsrep_mysqld.cc index 8627d0ff53b..eedd763b784 100644 --- a/sql/wsrep_mysqld.cc +++ b/sql/wsrep_mysqld.cc @@ -581,6 +581,15 @@ void wsrep_deinit() void wsrep_recover() { + if (!memcmp(&local_uuid, &WSREP_UUID_UNDEFINED, sizeof(wsrep_uuid_t)) && + local_seqno == -2) + { + char uuid_str[40]; + wsrep_uuid_print(&local_uuid, uuid_str, sizeof(uuid_str)); + WSREP_INFO("Position %s:%lld given at startup, skipping position recovery", + uuid_str, (long long)local_seqno); + return; + } XID xid; memset(&xid, 0, sizeof(xid)); xid.formatID= -1; @@ -1103,9 +1112,31 @@ static int wsrep_RSU_begin(THD *thd, char *db_, char *table_) ret = wsrep->desync(wsrep); if (ret != WSREP_OK) { - WSREP_WARN("desync failed %d for %s", ret, thd->query()); + WSREP_WARN("RSU desync failed %d for %s", ret, thd->query()); + my_error(ER_LOCK_DEADLOCK, MYF(0)); return(ret); } + mysql_mutex_lock(&LOCK_wsrep_replaying); + wsrep_replaying++; + mysql_mutex_unlock(&LOCK_wsrep_replaying); + + if (wsrep_wait_committing_connections_close(5000)) + { + /* no can do, bail out from DDL */ + WSREP_WARN("RSU failed due to pending transactions, %s", thd->query()); + mysql_mutex_lock(&LOCK_wsrep_replaying); + wsrep_replaying--; + mysql_mutex_unlock(&LOCK_wsrep_replaying); + + ret = wsrep->resync(wsrep); + if (ret != WSREP_OK) + { + WSREP_WARN("resync failed %d for %s", ret, thd->query()); + } + my_error(ER_LOCK_DEADLOCK, MYF(0)); + return(1); + } + wsrep_seqno_t seqno = wsrep->pause(wsrep); if (seqno == WSREP_SEQNO_UNDEFINED) { @@ -1123,6 +1154,11 @@ static void wsrep_RSU_end(THD *thd) WSREP_DEBUG("RSU END: %lld, %d : %s", (long long)thd->wsrep_trx_seqno, thd->wsrep_exec_mode, thd->query() ); + + mysql_mutex_lock(&LOCK_wsrep_replaying); + wsrep_replaying--; + mysql_mutex_unlock(&LOCK_wsrep_replaying); + ret = wsrep->resume(wsrep); if (ret != WSREP_OK) { diff --git a/sql/wsrep_mysqld.h b/sql/wsrep_mysqld.h index ce74cef4e64..af6c66609d4 100644 --- a/sql/wsrep_mysqld.h +++ b/sql/wsrep_mysqld.h @@ -143,6 +143,7 @@ extern void wsrep_recover(); extern void wsrep_init_startup(bool first); extern void wsrep_close_client_connections(my_bool wait_to_end); +extern int wsrep_wait_committing_connections_close(int wait_time); extern void wsrep_close_applier(THD *thd); extern void wsrep_wait_appliers_close(THD *thd); extern void wsrep_close_applier_threads(int count); @@ -176,7 +177,7 @@ extern wsrep_seqno_t wsrep_locked_seqno; // This is a workaround. It also prefixes all messages with "WSREP" #define WSREP_LOG(fun, ...) \ { \ - char msg[256] = {'\0'}; \ + char msg[1024] = {'\0'}; \ snprintf(msg, sizeof(msg) - 1, ## __VA_ARGS__); \ fun("WSREP: %s", msg); \ } diff --git a/sql/wsrep_sst.cc b/sql/wsrep_sst.cc index 8beadeb2ff2..53e3bbcfc79 100644 --- a/sql/wsrep_sst.cc +++ b/sql/wsrep_sst.cc @@ -25,6 +25,25 @@ extern const char wsrep_defaults_file[]; +#define WSREP_SST_OPT_ROLE "--role" +#define WSREP_SST_OPT_ADDR "--address" +#define WSREP_SST_OPT_AUTH "--auth" +#define WSREP_SST_OPT_DATA "--datadir" +#define WSREP_SST_OPT_CONF "--defaults-file" +#define WSREP_SST_OPT_PARENT "--parent" + +// mysqldump-specific options +#define WSREP_SST_OPT_USER "--user" +#define WSREP_SST_OPT_PSWD "--password" +#define WSREP_SST_OPT_HOST "--host" +#define WSREP_SST_OPT_PORT "--port" +#define WSREP_SST_OPT_LPORT "--local-port" + +// donor-specific +#define WSREP_SST_OPT_SOCKET "--socket" +#define WSREP_SST_OPT_GTID "--gtid" +#define WSREP_SST_OPT_BYPASS "--bypass" + #define WSREP_SST_MYSQLDUMP "mysqldump" #define WSREP_SST_SKIP "skip" #define WSREP_SST_DEFAULT WSREP_SST_MYSQLDUMP @@ -41,31 +60,19 @@ static const char* sst_auth_real = NULL; my_bool wsrep_sst_donor_rejects_queries = FALSE; -static const char *sst_methods[] = { - "mysqldump", - "rsync", - "rsync_wan", - "xtrabackup", - NULL -}; - bool wsrep_sst_method_check (sys_var *self, THD* thd, set_var* var) { char buff[FN_REFLEN]; String str(buff, sizeof(buff), system_charset_info), *res; const char* c_str = NULL; - if ((res = var->value->val_str(&str))) { - c_str = res->c_ptr(); - int i = 0; + if ((res = var->value->val_str(&str)) && + (c_str = res->c_ptr()) && + strlen(c_str) > 0) + return 0; - while (sst_methods[i] && strcasecmp(sst_methods[i], c_str)) i++; - if (!sst_methods[i]) { - my_error(ER_WRONG_VALUE_FOR_VAR, MYF(0), "wsrep_sst_method", c_str ? c_str : "NULL"); - return 1; - } - } - return 0; + my_error(ER_WRONG_VALUE_FOR_VAR, MYF(0), "wsrep_sst_method", c_str ? c_str : "NULL"); + return 1; } bool wsrep_sst_method_update (sys_var *self, THD* thd, enum_var_type type) @@ -381,7 +388,13 @@ static ssize_t sst_prepare_other (const char* method, const char* sst_dir= mysql_real_data_home; int ret= snprintf (cmd_str, cmd_len, - "wsrep_sst_%s 'joiner' '%s' '%s' '%s' '%s' '%d'", + "wsrep_sst_%s " + WSREP_SST_OPT_ROLE" 'joiner' " + WSREP_SST_OPT_ADDR" '%s' " + WSREP_SST_OPT_AUTH" '%s' " + WSREP_SST_OPT_DATA" '%s' " + WSREP_SST_OPT_CONF" '%s' " + WSREP_SST_OPT_PARENT" '%d'", method, addr_in, (sst_auth_real) ? sst_auth_real : "", sst_dir, wsrep_defaults_file, (int)getpid()); @@ -394,7 +407,13 @@ static ssize_t sst_prepare_other (const char* method, pthread_t tmp; sst_thread_arg arg(cmd_str); mysql_mutex_lock (&arg.lock); - pthread_create (&tmp, NULL, sst_joiner_thread, &arg); + ret = pthread_create (&tmp, NULL, sst_joiner_thread, &arg); + if (ret) + { + WSREP_ERROR("sst_prepare_other(): pthread_create() failed: %d (%s)", + ret, strerror(ret)); + return ret; + } mysql_cond_wait (&arg.cond, &arg.lock); *addr_out= arg.ret_str; @@ -665,9 +684,17 @@ static int sst_donate_mysqldump (const char* addr, if (!bypass && wsrep_sst_donor_rejects_queries) sst_reject_queries(TRUE); snprintf (cmd_str, cmd_len, - "wsrep_sst_mysqldump '%s' '%s' '%s' '%s' '%u' '%s' '%lld' '%d'", - user, pswd, host, port, mysqld_port, uuid_str, (long long)seqno, - bypass); + "wsrep_sst_mysqldump " + WSREP_SST_OPT_USER" '%s' " + WSREP_SST_OPT_PSWD" '%s' " + WSREP_SST_OPT_HOST" '%s' " + WSREP_SST_OPT_PORT" '%s' " + WSREP_SST_OPT_LPORT" '%u' " + WSREP_SST_OPT_SOCKET" '%s' " + WSREP_SST_OPT_GTID" '%s:%lld'" + "%s", + user, pswd, host, port, mysqld_port, mysqld_unix_port, uuid_str, + (long long)seqno, bypass ? " "WSREP_SST_OPT_BYPASS : ""); WSREP_DEBUG("Running: '%s'", cmd_str); @@ -880,10 +907,19 @@ static int sst_donate_other (const char* method, char cmd_str[cmd_len]; int ret= snprintf (cmd_str, cmd_len, - "wsrep_sst_%s 'donor' '%s' '%s' '%s' '%s' '%s' '%lld' '%d'" - , - method, addr, sst_auth_real, mysql_real_data_home, - wsrep_defaults_file, uuid, (long long) seqno, bypass); + "wsrep_sst_%s " + WSREP_SST_OPT_ROLE" 'donor' " + WSREP_SST_OPT_ADDR" '%s' " + WSREP_SST_OPT_AUTH" '%s' " + WSREP_SST_OPT_SOCKET" '%s' " + WSREP_SST_OPT_DATA" '%s' " + WSREP_SST_OPT_CONF" '%s' " + WSREP_SST_OPT_GTID" '%s:%lld'" + "%s", + method, addr, sst_auth_real, mysqld_unix_port, + mysql_real_data_home, wsrep_defaults_file, + uuid, (long long) seqno, + bypass ? " "WSREP_SST_OPT_BYPASS : ""); if (ret < 0 || ret >= cmd_len) { @@ -896,7 +932,13 @@ static int sst_donate_other (const char* method, pthread_t tmp; sst_thread_arg arg(cmd_str); mysql_mutex_lock (&arg.lock); - pthread_create (&tmp, NULL, sst_donor_thread, &arg); + ret = pthread_create (&tmp, NULL, sst_donor_thread, &arg); + if (ret) + { + WSREP_ERROR("sst_donate_other(): pthread_create() failed: %d (%s)", + ret, strerror(ret)); + return ret; + } mysql_cond_wait (&arg.cond, &arg.lock); WSREP_INFO("sst_donor_thread signaled with %d", arg.err); diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 0b3ea633829..22782dad174 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -5581,7 +5581,10 @@ ha_innobase::write_row( || sql_command == SQLCOM_DROP_INDEX) && num_write_row >= 10000) { #ifdef WITH_WSREP - WSREP_DEBUG("forced commit: %s", wsrep_thd_query(user_thd)); + if (wsrep_on(user_thd) && sql_command == SQLCOM_LOAD) { + WSREP_DEBUG("forced trx split for LOAD: %s", + wsrep_thd_query(user_thd)); + } #endif /* WITH_WSREP */ /* ALTER TABLE is COMMITted at every 10000 copied rows. The IX table lock for the original table has to be re-issued. diff --git a/support-files/mysql.spec.sh b/support-files/mysql.spec.sh index 93222c1d3ac..f52fee5de61 100644 --- a/support-files/mysql.spec.sh +++ b/support-files/mysql.spec.sh @@ -30,7 +30,7 @@ %define mysqld_group mysql %define mysqldatadir /var/lib/mysql -%define release 2 +%define release 1 # # Macros we use which are not available in all supported versions of RPM @@ -73,9 +73,6 @@ %if %{defined with_wsrep} %define mysql_version @VERSION@_wsrep_@WSREP_API_VERSION@.@WSREP_PATCH_VERSION@ %define wsrep_version @WSREP_VERSION@ -%define wsrep_comment , wsrep_%{wsrep_version} -%else -%define wsrep_comment %{nil} %endif # ---------------------------------------------------------------------------- @@ -104,10 +101,10 @@ # Server comment strings # ---------------------------------------------------------------------------- %if %{undefined compilation_comment_debug} -%define compilation_comment_debug MySQL Community Server - Debug (GPL)%{wsrep_comment} +%define compilation_comment_debug MySQL Community Server - Debug (GPL) %endif %if %{undefined compilation_comment_release} -%define compilation_comment_release MySQL Community Server (GPL)%{wsrep_comment} +%define compilation_comment_release MySQL Community Server (GPL) %endif # ---------------------------------------------------------------------------- @@ -1098,9 +1095,11 @@ echo "=====" >> $STATUS_HISTORY %attr(755, root, root) %{_bindir}/resolve_stack_dump %attr(755, root, root) %{_bindir}/resolveip %if %{defined with_wsrep} +%attr(755, root, root) %{_bindir}/wsrep_sst_common %attr(755, root, root) %{_bindir}/wsrep_sst_mysqldump %attr(755, root, root) %{_bindir}/wsrep_sst_rsync %attr(755, root, root) %{_bindir}/wsrep_sst_rsync_wan +%attr(755, root, root) %{_bindir}/wsrep_sst_xtrabackup %endif %attr(755, root, root) %{_sbindir}/mysqld diff --git a/support-files/wsrep.cnf.sh b/support-files/wsrep.cnf.sh index c04e5ddaf43..507f83324b9 100644 --- a/support-files/wsrep.cnf.sh +++ b/support-files/wsrep.cnf.sh @@ -110,7 +110,8 @@ wsrep_notify_cmd= # State Snapshot Transfer method wsrep_sst_method=mysqldump -# Address on THIS node to receive SST at. DON'T SET IT TO DONOR ADDRESS!!! +# Address which donor should send State Snapshot to. +# Should be the address of THIS node. DON'T SET IT TO DONOR ADDRESS!!! # (SST method dependent. Defaults to the first IP of the first interface) #wsrep_sst_receive_address= -- 2.30.9