References lp:1051808 - merged with lp:codership-mysql 5.5.27 based trunk

patched with: bzr diff lp:codership-mysql/5.5 -r3779..3790

.bzrignore

@@ -969,6 +969,8 @@ support-files/mysql.server
 support-files/mysql.spec
 support-files/mysqld_multi.server
 support-files/ndb-config-2-node.ini
+support-files/wsrep.cnf
+support-files/wsrep_notify
 TAGS
 test/ndbapi/bank/bankCreator
 test/ndbapi/bank/bankMakeGL
@@ -1092,6 +1094,9 @@ sql/share/slovak
 sql/share/spanish
 sql/share/swedish
 sql/share/ukrainian
+scripts/wsrep_sst_mysqldump
+scripts/wsrep_sst_rsync
+scripts/wsrep_sst_xtrabackup
 CPackConfig.cmake
 CPackSourceConfig.cmake
 Docs/INFO_BIN

Docs/README-wsrep

@@ -137,6 +137,7 @@ Additional packages to consider (if not yet installed):
    * galera (multi-master replication provider, https://launchpad.net/galera)
    * MySQL-client-community (for connecting to server and mysqldump-based SST)
    * rsync (for rsync-based SST)
+   * xtrabackup and nc (for xtrabackup-based SST)
 
 2.2 Upgrade system tables.
 
@@ -379,9 +380,20 @@ to join or start a cluster.
 wsrep_sst_method=mysqldump
    What method to use to copy database state to a newly joined node. Supported
    methods:
-   - mysqldump: slow (except for small datasets) but most tested.
-   - rsync:     much faster on large datasets.
-   - rsync_wan: same as rsync but with deltaxfer to minimize network traffic.
+   - mysqldump:  slow (except for small datasets) but most tested.
+   - rsync:      much faster on large datasets.
+   - rsync_wan:  same as rsync but with deltaxfer to minimize network traffic.
+   - xtrabackup: very fast and practically non-blocking SST method based on
+                 Percona's xtrabackup tool.
+
+   (for xtrabackup to work the following settings must be present in my.cnf
+    on all nodes:
+      [mysqld]
+      wsrep_sst_auth=root:<root password>
+      datadir=<path to data dir>
+      [client]
+      socket=<path to socket>
+   )
 
 wsrep_sst_receive_address=
    Address (hostname:port) at which this node wants to receive state snapshot.

scripts/CMakeLists.txt

@@ -311,7 +311,7 @@ IF(WIN32)
   ENDFOREACH()
 ELSE()
   IF(WITH_WSREP)
-    SET(WSREP_BINARIES wsrep_sst_mysqldump wsrep_sst_rsync)
+    SET(WSREP_BINARIES wsrep_sst_common wsrep_sst_mysqldump wsrep_sst_rsync wsrep_sst_xtrabackup)
   ENDIF()
   # On Unix, most of the files end up in the bin directory
   SET(mysql_config_COMPONENT COMPONENT Development)

scripts/mysqld_safe.sh

@@ -190,6 +190,31 @@ wsrep_pick_url() {
   echo $url
 }
 
+# Run mysqld with --wsrep-recover and parse recovered position from log.
+# Position will be stored in wsrep_start_position_opt global.
+wsrep_recovery() {
+  cmd="$@"
+  wr_logfile=$(mktemp)
+  log_notice "WSREP: Running position recovery"
+  $cmd --log_error=$wr_logfile --wsrep-recover
+  rp=$(grep "WSREP: Recovered position:" $wr_logfile)
+  if [ -z "$rp" ]; then
+    skipped=$(grep WSREP $wr_logfile | grep "skipping position recovery")
+    if [ -z "$skipped" ]; then
+      log_error "WSREP: Failed to recover position: " \
+          `cat $wr_logfile`;
+    else
+      log_notice "WSREP: Position recovery skipped"
+    fi
+  else
+    start_pos=$(echo $rp | sed 's/.*WSREP\:\ Recovered\ position://' \
+        | sed 's/^[ \t]*//')
+    wsrep_start_position_opt="--wsrep_start_position=$start_pos"
+    log_notice "WSREP: Recovered position $start_pos"
+  fi
+  rm $wr_logfile
+}
+
 parse_arguments() {
   # We only need to pass arguments through to the server if we don't
   # handle them here.  So, we collect unrecognized options (passed on
@@ -787,7 +812,8 @@ do
 done
 cmd="$cmd $args"
 # Avoid 'nohup: ignoring input' warning
-test -n "$NOHUP_NICENESS" && cmd="$cmd < /dev/null"
+nohup_redir=""
+test -n "$NOHUP_NICENESS" && nohup_redir=" < /dev/null"
 
 log_notice "Starting $MYSQLD daemon with databases from $DATADIR"
 
@@ -808,9 +834,11 @@ do
 
   if [ -z "$url" ]
   then
-    eval_log_error "$cmd"
+    wsrep_recovery "$cmd"
+    eval_log_error "$cmd $wsrep_start_position_opt $nohup_redir"
   else
-    eval_log_error "$cmd --wsrep_cluster_address=$url"
+    wsrep_recovery "$cmd"
+    eval_log_error "$cmd $wsrep_start_position_opt --wsrep_cluster_address=$url $nohup_redir"
   fi
 
   end_time=`date +%M%S`

scripts/wsrep_sst_common.sh

+# Copyright (C) 2010 Codership Oy
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; see the file COPYING. If not, write to the
+# Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston
+# MA  02110-1301  USA.
+
+# This is a common command line parser to be sourced by other SST scripts
+
+set -u
+
+WSREP_SST_OPT_BYPASS=0
+
+while [ $# -gt 0 ]; do
+case "$1" in
+    '--address')
+        readonly WSREP_SST_OPT_ADDR="$2"
+        shift
+        ;;
+    '--auth')
+        readonly WSREP_SST_OPT_AUTH="$2"
+        shift
+        ;;
+    '--bypass')
+        WSREP_SST_OPT_BYPASS=1
+        ;;
+    '--datadir')
+        readonly WSREP_SST_OPT_DATA="$2"
+        shift
+        ;;
+    '--defaults-file')
+        readonly WSREP_SST_OPT_CONF="$2"
+        shift
+        ;;
+    '--host')
+        readonly WSREP_SST_OPT_HOST="$2"
+        shift
+        ;;
+    '--local-port')
+        readonly WSREP_SST_OPT_LPORT="$2"
+        shift
+        ;;
+    '--parent')
+        readonly WSREP_SST_OPT_PARENT="$2"
+        shift
+        ;;
+    '--password')
+        readonly WSREP_SST_OPT_PSWD="$2"
+        shift
+        ;;
+    '--port')
+        readonly WSREP_SST_OPT_PORT="$2"
+        shift
+        ;;
+    '--role')
+        readonly WSREP_SST_OPT_ROLE="$2"
+        shift
+        ;;
+    '--socket')
+        readonly WSREP_SST_OPT_SOCKET="$2"
+        shift
+        ;;
+    '--user')
+        readonly WSREP_SST_OPT_USER="$2"
+        shift
+        ;;
+    '--gtid')
+        readonly WSREP_SST_OPT_GTID="$2"
+        shift
+        ;;
+    *) # must be command
+       # usage
+       # exit 1
+       ;;
+esac
+shift
+done
+readonly WSREP_SST_OPT_BYPASS
+
+wsrep_log()
+{
+    # echo everything to stderr so that it gets into common error log
+    # deliberately made to look different from the rest of the log
+    local readonly tst="$(date +%Y%m%d\ %H:%M:%S.%N | cut -b -21)"
+    echo "WSREP_SST: $* ($tst)" >/dev/stderr
+}
+
+wsrep_log_error()
+{
+    wsrep_log "[ERROR] $*"
+}
+
+wsrep_log_info()
+{
+    wsrep_log "[INFO] $*"
+}
+

scripts/wsrep_sst_mysqldump.sh

@@ -17,23 +17,10 @@
 
 # This is a reference script for mysqldump-based state snapshot tansfer
 
-USER=$1
-PSWD=$2
-HOST=$3
-PORT=$4
-LOCAL_HOST="127.0.0.1"
-LOCAL_PORT=$5
-UUID=$6
-SEQNO=$7
-BYPASS=$8
+. $(dirname $0)/wsrep_sst_common
 
 EINVAL=22
 
-err()
-{
-    echo "SST error: $*" >&2
-}
-
 local_ip()
 {
     PATH=$PATH:/usr/sbin:/usr/bin:/sbin:/bin
@@ -51,16 +38,18 @@ local_ip()
     return 1
 }
 
-if test -z "$USER";  then err "USER cannot be nil"; exit $EINVAL; fi
-if test -z "$HOST";  then err "HOST cannot be nil"; exit $EINVAL; fi
-if test -z "$PORT";  then err "PORT cannot be nil"; exit $EINVAL; fi
-if test -z "$LOCAL_PORT"; then err "LOCAL_PORT cannot be nil"; exit $EINVAL; fi
-if test -z "$UUID";  then err "UUID cannot be nil"; exit $EINVAL; fi
-if test -z "$SEQNO"; then err "SEQNO cannot be nil"; exit $EINVAL; fi
+if test -z "$WSREP_SST_OPT_USER";  then err "USER cannot be nil";  exit $EINVAL; fi
+if test -z "$WSREP_SST_OPT_HOST";  then err "HOST cannot be nil";  exit $EINVAL; fi
+if test -z "$WSREP_SST_OPT_PORT";  then err "PORT cannot be nil";  exit $EINVAL; fi
+if test -z "$WSREP_SST_OPT_LPORT"; then err "LPORT cannot be nil"; exit $EINVAL; fi
+if test -z "$WSREP_SST_OPT_SOCKET";then err "SOCKET cannot be nil";exit $EINVAL; fi
+if test -z "$WSREP_SST_OPT_GTID";  then err "GTID cannot be nil";  exit $EINVAL; fi
 
-if local_ip $HOST && [ "$PORT" = "$LOCAL_PORT" ]
+if local_ip $WSREP_SST_OPT_HOST && \
+   [ "$WSREP_SST_OPT_PORT" = "$WSREP_SST_OPT_LPORT" ]
 then
-    err "destination address '$HOST:$PORT' matches source address."
+    wsrep_log_error \
+    "destination address '$WSREP_SST_OPT_HOST:$WSREP_SST_OPT_PORT' matches source address."
     exit $EINVAL
 fi
 
@@ -68,18 +57,17 @@ fi
 if ! mysql --version | grep 'Distrib 5.5' >/dev/null
 then
     mysql --version >&2
-    err "this procedure requires MySQL client version 5.5.x"
+    err "this operation requires MySQL client version 5.5.x"
     exit $EINVAL
 fi
 
-AUTH="-u$USER"
-if test -n "$PSWD"; then AUTH="$AUTH -p$PSWD"; fi
+AUTH="-u$WSREP_SST_OPT_USER"
+if test -n "$WSREP_SST_OPT_PSWD"; then AUTH="$AUTH -p$WSREP_SST_OPT_PSWD"; fi
 
 STOP_WSREP="SET wsrep_on=OFF;"
 
 # NOTE: we don't use --routines here because we're dumping mysql.proc table
-#MYSQLDUMP="@bindir@/mysqldump $AUTH -h$LOCAL_HOST -P$LOCAL_PORT \
-MYSQLDUMP="mysqldump $AUTH -h$LOCAL_HOST -P$LOCAL_PORT \
+MYSQLDUMP="mysqldump $AUTH -S$WSREP_SST_OPT_SOCKET \
 --add-drop-database --add-drop-table --skip-add-locks --create-options \
 --disable-keys --extended-insert --skip-lock-tables --quick --set-charset \
 --skip-comments --flush-privileges --all-databases"
@@ -102,10 +90,10 @@ PREPARE stmt FROM @str;
 EXECUTE stmt;
 DROP PREPARE stmt;"
 
-SET_START_POSITION="SET GLOBAL wsrep_start_position='$UUID:$SEQNO';"
+SET_START_POSITION="SET GLOBAL wsrep_start_position='$WSREP_SST_OPT_GTID';"
 
-#MYSQL="@bindir@/mysql -u'$USER' -p'$PSWD' -h'$HOST' -P'$PORT'"
-MYSQL="mysql $AUTH -h$HOST -P$PORT --disable-reconnect --connect_timeout=10"
+MYSQL="mysql $AUTH -h$WSREP_SST_OPT_HOST -P$WSREP_SST_OPT_PORT "\
+"--disable-reconnect --connect_timeout=10"
 
 # need to disable logging when loading the dump
 # reason is that dump contains ALTER TABLE for log tables, and
@@ -119,14 +107,14 @@ $MYSQL -e"$STOP_WSREP SET GLOBAL SLOW_QUERY_LOG=OFF"
 RESTORE_GENERAL_LOG="SET GLOBAL GENERAL_LOG=$GENERAL_LOG_OPT;"
 RESTORE_SLOW_QUERY_LOG="SET GLOBAL SLOW_QUERY_LOG=$SLOW_LOG_OPT;"
 
-if [ $BYPASS -eq 0 ]
+if [ $WSREP_SST_OPT_BYPASS -eq 0 ]
 then
     (echo $STOP_WSREP && $MYSQLDUMP && echo $CSV_TABLES_FIX \
     && echo $RESTORE_GENERAL_LOG && echo $RESTORE_SLOW_QUERY_LOG \
     && echo $SET_START_POSITION \
     || echo "SST failed to complete;") | $MYSQL
 else
-    echo "Bypassing state dump." >&2
+    wsrep_log_info "Bypassing state dump."
     echo $SET_START_POSITION | $MYSQL
 fi
 

scripts/wsrep_sst_rsync.sh

@@ -21,9 +21,11 @@
 RSYNC_PID=
 RSYNC_CONF=
 
+. $(dirname $0)/wsrep_sst_common
+
 cleanup_joiner()
 {
-    echo -n "Joiner rsync SST cleanup..." >&2
+    wsrep_log_info "Joiner cleanup."
     local PID=$(cat "$RSYNC_PID" 2>/dev/null || echo 0)
     [ "0" != "$PID" ] && kill $PID && sleep 0.5 && kill -9 $PID >/dev/null 2>&1 \
     || :
@@ -50,25 +52,16 @@ check_pid_and_port()
     grep LISTEN | grep \:$rsync_port | grep $rsync_pid/rsync >/dev/null
 }
 
-ROLE=$1
-ADDR=$2
-AUTH=$3
-DATA=$4
-CONF=$5
-
-MAGIC_FILE="$DATA/rsync_sst_complete"
+MAGIC_FILE="$WSREP_SST_OPT_DATA/rsync_sst_complete"
 rm -rf "$MAGIC_FILE"
 
-if [ "$ROLE" = "donor" ]
+if [ "$WSREP_SST_OPT_ROLE" = "donor" ]
 then
-    UUID=$6
-    SEQNO=$7
-    BYPASS=$8
 
-    if [ $BYPASS -eq 0 ]
+    if [ $WSREP_SST_OPT_BYPASS -eq 0 ]
     then
 
-        FLUSHED="$DATA/tables_flushed"
+        FLUSHED="$WSREP_SST_OPT_DATA/tables_flushed"
         rm -rf "$FLUSHED"
 
         # Use deltaxfer only for WAN
@@ -100,7 +93,8 @@ then
 
         RC=0
         rsync --archive --no-times --ignore-times --inplace --delete --quiet \
-              $WHOLE_FILE_OPT "${FILTER[@]}" "$DATA" rsync://$ADDR || RC=$?
+              $WHOLE_FILE_OPT "${FILTER[@]}" "$WSREP_SST_OPT_DATA" \
+              rsync://$WSREP_SST_OPT_ADDR || RC=$?
 
         [ $RC -ne 0 ] && echo "rsync returned code $RC:" >> /dev/stderr
 
@@ -108,9 +102,9 @@ then
         0)  RC=0   # Success
             ;;
         12) RC=71  # EPROTO
-            echo "rsync server on the other end has incompatible protocol. " \
-                 "Make sure you have the same version of rsync on all nodes."\
-                 >> /dev/stderr
+            wsrep_log_error \
+                 "rsync server on the other end has incompatible protocol. " \
+                 "Make sure you have the same version of rsync on all nodes."
             ;;
         22) RC=12  # ENOMEM
             ;;
@@ -121,23 +115,24 @@ then
         [ $RC -ne 0 ] && exit $RC
 
     else # BYPASS
-        STATE="$UUID:$SEQNO"
+        wsrep_log_info "Bypassing state dump."
+        STATE="$WSREP_SST_OPT_GTID"
     fi
 
     echo "continue" # now server can resume updating data
 
     echo "$STATE" > "$MAGIC_FILE"
-    rsync -aqc "$MAGIC_FILE" rsync://$ADDR
+    rsync -aqc "$MAGIC_FILE" rsync://$WSREP_SST_OPT_ADDR
 
     echo "done $STATE"
 
-elif [ "$ROLE" = "joiner" ]
+elif [ "$WSREP_SST_OPT_ROLE" = "joiner" ]
 then
-    MYSQLD_PID=$6
+    MYSQLD_PID=$WSREP_SST_OPT_PARENT
 
     MODULE="rsync_sst"
 
-    RSYNC_PID="$DATA/$MODULE.pid"
+    RSYNC_PID="$WSREP_SST_OPT_DATA/$MODULE.pid"
 
     if check_pid $RSYNC_PID
     then
@@ -146,6 +141,7 @@ then
     fi
     rm -rf "$RSYNC_PID"
 
+    ADDR=$WSREP_SST_OPT_ADDR
     RSYNC_PORT=$(echo $ADDR | awk -F ':' '{ print $2 }')
     if [ -z "$RSYNC_PORT" ]
     then
@@ -159,13 +155,13 @@ then
 
     MYUID=$(id -u)
     MYGID=$(id -g)
-    RSYNC_CONF="$DATA/$MODULE.conf"
+    RSYNC_CONF="$WSREP_SST_OPT_DATA/$MODULE.conf"
 
 cat << EOF > "$RSYNC_CONF"
 pid file = $RSYNC_PID
 use chroot = no
 [$MODULE]
-	path = $DATA
+	path = $WSREP_SST_OPT_DATA
 	read only = no
 	timeout = 300
 	uid = $MYUID
@@ -207,7 +203,7 @@ EOF
 
 #    cleanup_joiner
 else
-    echo "Unrecognized role: $ROLE"
+    echo "Unrecognized role: '$WSREP_SST_OPT_ROLE'"
     exit 22 # EINVAL
 fi
 

scripts/wsrep_sst_xtrabackup.sh

+#!/bin/bash -ue
+
+# Copyright (C) 2011 Percona Inc
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; see the file COPYING. If not, write to the
+# Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston
+# MA  02110-1301  USA.
+
+# This is a reference script for Percona XtraBackup-based state snapshot tansfer
+
+TMPDIR="/tmp"
+
+. $(dirname $0)/wsrep_sst_common
+
+cleanup_joiner()
+{
+#set -x
+    local PID=$(ps -aef |grep nc| grep $NC_PORT  | awk '{ print $2 }')
+    wsrep_log_info "Killing nc pid $PID"
+    [ -n "$PID" -a "0" != "$PID" ] && kill $PID && (kill $PID && kill -9 $PID) || :
+    rm -f "$MAGIC_FILE"
+#set +x
+}
+
+check_pid()
+{
+    local pid_file=$1
+    [ -r $pid_file ] && ps -p $(cat $pid_file) >/dev/null 2>&1
+}
+
+kill_xtrabackup()
+{
+#set -x
+    local PID=$(cat $XTRABACKUP_PID)
+    [ -n "$PID" -a "0" != "$PID" ] && kill $PID && (kill $PID && kill -9 $PID) || :
+    rm -f "$XTRABACKUP_PID"
+#set +x
+}
+
+# waits ~10 seconds for nc to open the port and then reports ready
+# (regardless of timeout)
+wait_for_nc()
+{
+    local PORT=$1
+    local ADDR=$2
+    local MODULE=$3
+    for i in $(seq 1 50)
+    do
+        netstat -nptl 2>/dev/null | grep '/nc\s*$' | awk '{ print $4 }' | \
+        sed 's/.*://' | grep \^${PORT}\$ >/dev/null && break
+        sleep 0.2
+    done
+    echo "ready ${ADDR}/${MODULE}"
+}
+
+INNOBACKUPEX_BIN=innobackupex
+INNOBACKUPEX_ARGS=""
+NC_BIN=nc
+
+for TOOL_BIN in INNOBACKUPEX_BIN NC_BIN ; do 
+  which ${!TOOL_BIN} > /dev/null 2>&1
+  if [ $? -ne 0 ]; then 
+     echo "Can't find ${!TOOL_BIN} in the path"
+     exit 22 # EINVAL
+  fi
+done
+
+#ROLE=$1
+#ADDR=$2
+readonly AUTH=(${WSREP_SST_OPT_AUTH//:/ })
+readonly DATA="${WSREP_SST_OPT_DATA}"
+#CONF=$5
+
+INFO_FILE="xtrabackup_galera_info"
+IST_FILE="xtrabackup_ist"
+
+MAGIC_FILE="${DATA}/${INFO_FILE}"
+rm -f "${MAGIC_FILE}"
+
+if [ "$WSREP_SST_OPT_ROLE" = "donor" ]
+then
+
+#    UUID=$6
+#    SEQNO=$7
+#    BYPASS=$8
+
+    NC_PORT=$(echo $WSREP_SST_OPT_ADDR | awk -F '[:/]' '{ print $2 }')
+    REMOTEIP=$(echo $WSREP_SST_OPT_ADDR | awk -F ':' '{ print $1 }')
+
+    if [ $WSREP_SST_OPT_BYPASS -eq 0 ]
+    then
+
+        INNOBACKUPEX_ARGS="--galera-info --tmpdir=${TMPDIR} --stream=tar
+                           --defaults-file=${WSREP_SST_OPT_CONF}
+                           --socket=${WSREP_SST_OPT_SOCKET}"
+
+        if [ "${AUTH[0]}" != "(null)" ]; then
+           INNOBACKUPEX_ARGS="${INNOBACKUPEX_ARGS} --user=${AUTH[0]}"
+        fi
+
+        if [ ${#AUTH[*]} -eq 2 ]; then
+           INNOBACKUPEX_ARGS="${INNOBACKUPEX_ARGS} --password=${AUTH[1]}"
+        fi
+
+        set +e
+
+        # This file and variable seems to have no effect and probably should be deleted
+        XTRABACKUP_PID=$(mktemp --tmpdir wsrep_sst_xtrabackupXXXX.pid)
+
+        ${INNOBACKUPEX_BIN} ${INNOBACKUPEX_ARGS} ${TMPDIR} \
+        2> ${DATA}/innobackup.backup.log | \
+        ${NC_BIN} ${REMOTEIP} ${NC_PORT}
+
+        RC=( "${PIPESTATUS[@]}" )
+        set -e
+
+        if [ ${RC[0]} -ne 0 ]; then
+          wsrep_log_error "${INNOBACKUPEX_BIN} finished with error: ${RC[0]}. " \
+                          "Check ${DATA}/innobackup.backup.log"
+          exit 22
+        elif [  ${RC[1]} -ne 0 ]; then
+          wsrep_log_error "${NC_BIN} finished with error: ${RC[1]}"
+          exit 22
+        fi
+
+        if check_pid ${XTRABACKUP_PID}
+        then
+            wsrep_log_error "xtrabackup process is still running. Killing... "
+            kill_xtrabackup
+            exit 22
+        fi
+
+        rm -f ${XTRABACKUP_PID}
+
+    else # BYPASS
+        STATE="${WSREP_SST_OPT_GTID}"
+        echo "continue" # now server can resume updating data
+        echo "${STATE}" > "${MAGIC_FILE}"
+        echo "1" > "${DATA}/${IST_FILE}"
+        (cd ${DATA}; tar cf - ${INFO_FILE} ${IST_FILE}) | ${NC_BIN} ${REMOTEIP} ${NC_PORT}
+        rm -f ${DATA}/${IST_FILE}
+    fi
+
+    echo "done ${WSREP_SST_OPT_GTID}"
+
+elif [ "${WSREP_SST_OPT_ROLE}" = "joiner" ]
+then
+    MODULE="xtrabackup_sst"
+
+    rm -f ${DATA}/xtrabackup_*
+
+    ADDR=${WSREP_SST_OPT_ADDR}
+    NC_PORT=$(echo ${ADDR} | awk -F ':' '{ print $2 }')
+    if [ -z "${NC_PORT}" ]
+    then
+        NC_PORT=4444
+        ADDR="$(echo ${ADDR} | awk -F ':' '{ print $1 }'):${NC_PORT}"
+    fi
+
+    wait_for_nc ${NC_PORT} ${ADDR} ${MODULE} &
+
+#    trap "exit 32" HUP PIPE
+#    trap "exit 3"  INT TERM
+    trap cleanup_joiner HUP PIPE INT TERM
+
+    set +e
+    ${NC_BIN} -dl ${NC_PORT}  | tar xfi  - -C ${DATA}  1>&2
+    RC=( "${PIPESTATUS[@]}" )
+    set -e
+
+    wait %% # join wait_for_nc thread
+
+    if [ ${RC[0]} -ne 0 -o ${RC[1]} -ne 0 ];
+    then
+        wsrep_log_error "Error while getting st data from donor node: " \
+                        "${RC[0]}, ${RC[1]}"
+        exit 32
+    fi
+
+    if [ ! -r "${MAGIC_FILE}" ]
+    then
+        # this message should cause joiner to abort
+        wsrep_log_error "xtrabackup process ended without creating '${MAGIC_FILE}'"
+        exit 32
+    fi
+
+    if ! ps -p ${WSREP_SST_OPT_PARENT} >/dev/null
+    then
+        wsrep_log_error "Parent mysqld process (PID:${WSREP_SST_OPT_PARENT}) terminated unexpectedly." >&2
+        exit 32
+    fi
+
+    if [ ! -r "${IST_FILE}" ]
+    then
+        rm -f ${DATA}/ib_logfile*
+        ${INNOBACKUPEX_BIN} --defaults-file=${WSREP_SST_OPT_CONF} --apply-log \
+        --ibbackup=xtrabackup ${DATA} 1>&2 2> ${DATA}/innobackup.prepare.log
+        if [ $? -ne 0 ];
+        then
+            wsrep_log_error "${INNOBACKUPEX_BIN} finished with errors. Check ${DATA}/innobackup.prepare.log" >&2
+            exit 22
+        fi
+    fi
+
+    cat "${MAGIC_FILE}" # output UUID:seqno
+
+else
+    wsrep_log_error "Unrecognized role: ${WSREP_SST_OPT_ROLE}"
+    exit 22 # EINVAL
+fi
+
+exit 0

sql/handler.cc

@@ -5233,7 +5233,9 @@ void signal_log_not_needed(struct handlerton, char *log_file)
 int ha_wsrep_abort_transaction(THD *bf_thd, THD *victim_thd, my_bool signal)
 {
   DBUG_ENTER("ha_wsrep_abort_transaction");
-  if (!WSREP(bf_thd)) {
+  if (!WSREP(bf_thd) &&  
+      !(wsrep_OSU_method_options == WSREP_OSU_RSU &&
+        bf_thd->wsrep_exec_mode == TOTAL_ORDER)) {
     DBUG_RETURN(0);
   }
 

sql/mysqld.cc

@@ -4884,6 +4884,17 @@ static inline bool is_replaying_connection(THD *thd)
   return ret;
 }
 
+static inline bool is_committing_connection(THD *thd)
+{
+  bool ret;
+
+  mysql_mutex_lock(&thd->LOCK_wsrep_thd);
+  ret=  (thd->wsrep_query_state == QUERY_COMMITTING) ? true : false;
+  mysql_mutex_unlock(&thd->LOCK_wsrep_thd);
+
+  return ret;
+}
+
 static bool have_client_connections()
 {
   THD *tmp;
@@ -4937,6 +4948,44 @@ static void wsrep_close_thread(THD *thd)
   }
 }
 
+static my_bool have_committing_connections()
+{
+  THD *tmp;
+  mysql_mutex_lock(&LOCK_thread_count); // For unlink from list
+
+  I_List_iterator<THD> it(threads);
+  while ((tmp=it++))
+  {
+    if (!is_client_connection(tmp))
+      continue;
+
+    if (is_committing_connection(tmp))
+    {
+      mysql_mutex_unlock(&LOCK_thread_count);
+      return TRUE;
+    }
+  }
+  mysql_mutex_unlock(&LOCK_thread_count);
+  return FALSE;
+}
+
+int wsrep_wait_committing_connections_close(int wait_time)
+{
+  int sleep_time= 100;
+
+  while (have_committing_connections() && wait_time > 0)
+  {
+    WSREP_DEBUG("wait for committing transaction to close: %d", wait_time);
+    my_sleep(sleep_time);
+    wait_time -= sleep_time;
+  }
+  if (have_committing_connections())
+  {
+    return 1;
+  }
+  return 0;
+}
+
 void wsrep_close_client_connections(my_bool wait_to_end) 
 {
   /*

sql/sql_parse.cc

@@ -8436,8 +8436,10 @@ int wsrep_abort_thd(void *bf_thd_ptr, void *victim_thd_ptr, my_bool signal)
   THD *bf_thd     = (THD *) bf_thd_ptr;
   DBUG_ENTER("wsrep_abort_thd");
 
-  if ( (WSREP(bf_thd) || 
-	(WSREP_ON && bf_thd->wsrep_exec_mode == TOTAL_ORDER)) && victim_thd)
+  if ( (WSREP(bf_thd) ||
+         ( (WSREP_ON || wsrep_OSU_method_options == WSREP_OSU_RSU) &&  
+           bf_thd->wsrep_exec_mode == TOTAL_ORDER) )               &&
+       victim_thd)
   {
     WSREP_DEBUG("wsrep_abort_thd, by: %llu, victim: %llu", (bf_thd) ?
                 (long long)bf_thd->real_id : 0, (long long)victim_thd->real_id);

sql/wsrep_mysqld.cc

@@ -581,6 +581,15 @@ void wsrep_deinit()
 
 void wsrep_recover()
 {
+  if (!memcmp(&local_uuid, &WSREP_UUID_UNDEFINED, sizeof(wsrep_uuid_t)) &&
+      local_seqno == -2)
+  {
+    char uuid_str[40];
+    wsrep_uuid_print(&local_uuid, uuid_str, sizeof(uuid_str));
+    WSREP_INFO("Position %s:%lld given at startup, skipping position recovery",
+               uuid_str, (long long)local_seqno);
+    return;
+  }
   XID xid;
   memset(&xid, 0, sizeof(xid));
   xid.formatID= -1;
@@ -1103,9 +1112,31 @@ static int wsrep_RSU_begin(THD *thd, char *db_, char *table_)
   ret = wsrep->desync(wsrep);
   if (ret != WSREP_OK)
   {
-    WSREP_WARN("desync failed %d for %s", ret, thd->query());
+    WSREP_WARN("RSU desync failed %d for %s", ret, thd->query());
+    my_error(ER_LOCK_DEADLOCK, MYF(0));
     return(ret);
   }
+  mysql_mutex_lock(&LOCK_wsrep_replaying);
+  wsrep_replaying++;
+  mysql_mutex_unlock(&LOCK_wsrep_replaying);
+
+  if (wsrep_wait_committing_connections_close(5000))
+  {
+    /* no can do, bail out from DDL */
+    WSREP_WARN("RSU failed due to pending transactions, %s", thd->query());
+    mysql_mutex_lock(&LOCK_wsrep_replaying);
+    wsrep_replaying--;
+    mysql_mutex_unlock(&LOCK_wsrep_replaying);
+
+    ret = wsrep->resync(wsrep);
+    if (ret != WSREP_OK)
+    {
+      WSREP_WARN("resync failed %d for %s", ret, thd->query());
+    }
+    my_error(ER_LOCK_DEADLOCK, MYF(0));
+    return(1);
+  }
+
   wsrep_seqno_t seqno = wsrep->pause(wsrep);
   if (seqno == WSREP_SEQNO_UNDEFINED)
   {
@@ -1123,6 +1154,11 @@ static void wsrep_RSU_end(THD *thd)
   WSREP_DEBUG("RSU END: %lld, %d : %s", (long long)thd->wsrep_trx_seqno,
                thd->wsrep_exec_mode, thd->query() );
 
+
+  mysql_mutex_lock(&LOCK_wsrep_replaying);
+  wsrep_replaying--;
+  mysql_mutex_unlock(&LOCK_wsrep_replaying);
+
   ret = wsrep->resume(wsrep);
   if (ret != WSREP_OK)
   {

sql/wsrep_mysqld.h

@@ -143,6 +143,7 @@ extern void  wsrep_recover();
 extern void wsrep_init_startup(bool first);
 
 extern void wsrep_close_client_connections(my_bool wait_to_end);
+extern int  wsrep_wait_committing_connections_close(int wait_time);
 extern void wsrep_close_applier(THD *thd);
 extern void wsrep_wait_appliers_close(THD *thd);
 extern void wsrep_close_applier_threads(int count);
@@ -176,7 +177,7 @@ extern wsrep_seqno_t wsrep_locked_seqno;
 // This is a workaround. It also prefixes all messages with "WSREP"
 #define WSREP_LOG(fun, ...)                                       \
     {                                                             \
-        char msg[256] = {'\0'};                                   \
+        char msg[1024] = {'\0'};                                  \
         snprintf(msg, sizeof(msg) - 1, ## __VA_ARGS__);           \
         fun("WSREP: %s", msg);                                    \
     }

sql/wsrep_sst.cc

@@ -25,6 +25,25 @@
 
 extern const char wsrep_defaults_file[];
 
+#define WSREP_SST_OPT_ROLE     "--role"
+#define WSREP_SST_OPT_ADDR     "--address"
+#define WSREP_SST_OPT_AUTH     "--auth"
+#define WSREP_SST_OPT_DATA     "--datadir"
+#define WSREP_SST_OPT_CONF     "--defaults-file"
+#define WSREP_SST_OPT_PARENT   "--parent"
+
+// mysqldump-specific options
+#define WSREP_SST_OPT_USER     "--user"
+#define WSREP_SST_OPT_PSWD     "--password"
+#define WSREP_SST_OPT_HOST     "--host"
+#define WSREP_SST_OPT_PORT     "--port"
+#define WSREP_SST_OPT_LPORT    "--local-port"
+
+// donor-specific
+#define WSREP_SST_OPT_SOCKET   "--socket"
+#define WSREP_SST_OPT_GTID     "--gtid"
+#define WSREP_SST_OPT_BYPASS   "--bypass"
+
 #define WSREP_SST_MYSQLDUMP    "mysqldump"
 #define WSREP_SST_SKIP         "skip"
 #define WSREP_SST_DEFAULT      WSREP_SST_MYSQLDUMP
@@ -41,31 +60,19 @@ static const char* sst_auth_real      = NULL;
 
 my_bool wsrep_sst_donor_rejects_queries = FALSE;
 
-static const char *sst_methods[] = {
-  "mysqldump",
-  "rsync",
-  "rsync_wan",
-  "xtrabackup",
-  NULL
-};
-
 bool wsrep_sst_method_check (sys_var *self, THD* thd, set_var* var)
 {
     char   buff[FN_REFLEN];
     String str(buff, sizeof(buff), system_charset_info), *res;
     const char* c_str = NULL;
 
-    if ((res = var->value->val_str(&str))) {
-      c_str = res->c_ptr();
-      int i = 0;
+    if ((res   = var->value->val_str(&str)) &&
+        (c_str = res->c_ptr()) &&
+        strlen(c_str) > 0)
+        return 0;
 
-      while (sst_methods[i] && strcasecmp(sst_methods[i], c_str)) i++;
-      if (!sst_methods[i]) {
-        my_error(ER_WRONG_VALUE_FOR_VAR, MYF(0), "wsrep_sst_method", c_str ? c_str : "NULL");
-        return 1;
-      }
-    }
-    return 0;
+    my_error(ER_WRONG_VALUE_FOR_VAR, MYF(0), "wsrep_sst_method", c_str ? c_str : "NULL");
+    return 1;
 }
 
 bool wsrep_sst_method_update (sys_var *self, THD* thd, enum_var_type type)
@@ -381,7 +388,13 @@ static ssize_t sst_prepare_other (const char*  method,
   const char* sst_dir= mysql_real_data_home;
 
   int ret= snprintf (cmd_str, cmd_len,
-                     "wsrep_sst_%s 'joiner' '%s' '%s' '%s' '%s' '%d'",
+                     "wsrep_sst_%s "
+                     WSREP_SST_OPT_ROLE" 'joiner' "
+                     WSREP_SST_OPT_ADDR" '%s' "
+                     WSREP_SST_OPT_AUTH" '%s' "
+                     WSREP_SST_OPT_DATA" '%s' "
+                     WSREP_SST_OPT_CONF" '%s' "
+                     WSREP_SST_OPT_PARENT" '%d'",
                      method, addr_in, (sst_auth_real) ? sst_auth_real : "",
                      sst_dir, wsrep_defaults_file, (int)getpid());
 
@@ -394,7 +407,13 @@ static ssize_t sst_prepare_other (const char*  method,
   pthread_t tmp;
   sst_thread_arg arg(cmd_str);
   mysql_mutex_lock (&arg.lock);
-  pthread_create (&tmp, NULL, sst_joiner_thread, &arg);
+  ret = pthread_create (&tmp, NULL, sst_joiner_thread, &arg);
+  if (ret)
+  {
+    WSREP_ERROR("sst_prepare_other(): pthread_create() failed: %d (%s)",
+                ret, strerror(ret));
+    return ret;
+  }
   mysql_cond_wait (&arg.cond, &arg.lock);
 
   *addr_out= arg.ret_str;
@@ -665,9 +684,17 @@ static int sst_donate_mysqldump (const char*         addr,
     if (!bypass && wsrep_sst_donor_rejects_queries) sst_reject_queries(TRUE);
 
     snprintf (cmd_str, cmd_len,
-              "wsrep_sst_mysqldump '%s' '%s' '%s' '%s' '%u' '%s' '%lld' '%d'",
-              user, pswd, host, port, mysqld_port, uuid_str, (long long)seqno,
-              bypass);
+              "wsrep_sst_mysqldump "
+              WSREP_SST_OPT_USER" '%s' "
+              WSREP_SST_OPT_PSWD" '%s' "
+              WSREP_SST_OPT_HOST" '%s' "
+              WSREP_SST_OPT_PORT" '%s' "
+              WSREP_SST_OPT_LPORT" '%u' "
+              WSREP_SST_OPT_SOCKET" '%s' "
+              WSREP_SST_OPT_GTID" '%s:%lld'"
+              "%s",
+              user, pswd, host, port, mysqld_port, mysqld_unix_port, uuid_str,
+              (long long)seqno, bypass ? " "WSREP_SST_OPT_BYPASS : "");
 
     WSREP_DEBUG("Running: '%s'", cmd_str);
 
@@ -880,10 +907,19 @@ static int sst_donate_other (const char*   method,
   char    cmd_str[cmd_len];
 
   int ret= snprintf (cmd_str, cmd_len,
-                     "wsrep_sst_%s 'donor' '%s' '%s' '%s' '%s' '%s' '%lld' '%d'"
-                     ,
-                     method, addr, sst_auth_real, mysql_real_data_home,
-                     wsrep_defaults_file, uuid, (long long) seqno, bypass);
+                     "wsrep_sst_%s "
+                     WSREP_SST_OPT_ROLE" 'donor' "
+                     WSREP_SST_OPT_ADDR" '%s' "
+                     WSREP_SST_OPT_AUTH" '%s' "
+                     WSREP_SST_OPT_SOCKET" '%s' "
+                     WSREP_SST_OPT_DATA" '%s' "
+                     WSREP_SST_OPT_CONF" '%s' "
+                     WSREP_SST_OPT_GTID" '%s:%lld'"
+                     "%s",
+                     method, addr, sst_auth_real, mysqld_unix_port,
+                     mysql_real_data_home, wsrep_defaults_file,
+                     uuid, (long long) seqno,
+                     bypass ? " "WSREP_SST_OPT_BYPASS : "");
 
   if (ret < 0 || ret >= cmd_len)
   {
@@ -896,7 +932,13 @@ static int sst_donate_other (const char*   method,
   pthread_t tmp;
   sst_thread_arg arg(cmd_str);
   mysql_mutex_lock (&arg.lock);
-  pthread_create (&tmp, NULL, sst_donor_thread, &arg);
+  ret = pthread_create (&tmp, NULL, sst_donor_thread, &arg);
+  if (ret)
+  {
+    WSREP_ERROR("sst_donate_other(): pthread_create() failed: %d (%s)",
+                ret, strerror(ret));
+    return ret;
+  }
   mysql_cond_wait (&arg.cond, &arg.lock);
 
   WSREP_INFO("sst_donor_thread signaled with %d", arg.err);

storage/innobase/handler/ha_innodb.cc

@@ -5581,7 +5581,10 @@ ha_innobase::write_row(
 	     || sql_command == SQLCOM_DROP_INDEX)
 	    && num_write_row >= 10000) {
 #ifdef WITH_WSREP
-		WSREP_DEBUG("forced commit: %s", wsrep_thd_query(user_thd));
+		if (wsrep_on(user_thd) && sql_command == SQLCOM_LOAD) {
+			WSREP_DEBUG("forced trx split for LOAD: %s", 
+				    wsrep_thd_query(user_thd));
+		}
 #endif /* WITH_WSREP */
 		/* ALTER TABLE is COMMITted at every 10000 copied rows.
 		The IX table lock for the original table has to be re-issued.

support-files/mysql.spec.sh

@@ -30,7 +30,7 @@
 %define mysqld_group    mysql
 %define mysqldatadir    /var/lib/mysql
 
-%define release         2
+%define release         1
 
 #
 # Macros we use which are not available in all supported versions of RPM
@@ -73,9 +73,6 @@
 %if %{defined with_wsrep}
 %define mysql_version @VERSION@_wsrep_@WSREP_API_VERSION@.@WSREP_PATCH_VERSION@
 %define wsrep_version @WSREP_VERSION@
-%define wsrep_comment , wsrep_%{wsrep_version}
-%else
-%define wsrep_comment %{nil}
 %endif
 
 # ----------------------------------------------------------------------------
@@ -104,10 +101,10 @@
 # Server comment strings
 # ----------------------------------------------------------------------------
 %if %{undefined compilation_comment_debug}
-%define compilation_comment_debug       MySQL Community Server - Debug (GPL)%{wsrep_comment}
+%define compilation_comment_debug       MySQL Community Server - Debug (GPL)
 %endif
 %if %{undefined compilation_comment_release}
-%define compilation_comment_release     MySQL Community Server (GPL)%{wsrep_comment}
+%define compilation_comment_release     MySQL Community Server (GPL)
 %endif
 
 # ----------------------------------------------------------------------------
@@ -1098,9 +1095,11 @@ echo "====="                                     >> $STATUS_HISTORY
 %attr(755, root, root) %{_bindir}/resolve_stack_dump
 %attr(755, root, root) %{_bindir}/resolveip
 %if %{defined with_wsrep}
+%attr(755, root, root) %{_bindir}/wsrep_sst_common
 %attr(755, root, root) %{_bindir}/wsrep_sst_mysqldump
 %attr(755, root, root) %{_bindir}/wsrep_sst_rsync
 %attr(755, root, root) %{_bindir}/wsrep_sst_rsync_wan
+%attr(755, root, root) %{_bindir}/wsrep_sst_xtrabackup
 %endif
 
 %attr(755, root, root) %{_sbindir}/mysqld

support-files/wsrep.cnf.sh

@@ -110,7 +110,8 @@ wsrep_notify_cmd=
 # State Snapshot Transfer method
 wsrep_sst_method=mysqldump
 
-# Address on THIS node to receive SST at. DON'T SET IT TO DONOR ADDRESS!!!
+# Address which donor should send State Snapshot to.
+# Should be the address of THIS node. DON'T SET IT TO DONOR ADDRESS!!!
 # (SST method dependent. Defaults to the first IP of the first interface)
 #wsrep_sst_receive_address=