MDEV-21910 : KIlling thread on Galera could cause mutex deadlock

Whenever Galera BF (brute force) transaction decides to abort conflicting
transaction it will kill that thread using thd::awake()

User KILL [QUERY|CONNECTION] ... for a thread it will also call thd::awake()

Whenever one of these actions is executed we will hold number of InnoDB
internal mutexes and thd mutexes. Sometimes these mutexes are taken in
different order causing mutex deadlock.

Lets call BF kill as bf_thread and user KILL-query as kill_thread.

bf_thread takes mutexes in order:
(1) lock_sys->mutex (lock0lock.cc lock_rec_other_has_conflicting)
(2) victim_trx->mutex (lock0lock.cc lock_rec_other_has_conflicting)
(3) victim_thread->LOCK_thd_data (handler.cc wsrep_innobase_kill_one_trx)

kill_thread takes mutexes in order:
(1) victim_thread->LOCK_thd_data (sql_parse.cc find_thread_by_id)
(2) lock_sys->mutex (ha_innodb.cc innobase_kill_query)
(3) victim_trx->mutex (ha_innodb.cc innobase_kill_query)

Mutex deadlock is result of taking victim_thread->LOCK_thd_data
in different order.

In this patch we will fix Galera BF victim thread kill so that
it will not try to lock victim_thread->LOCK_thd_data mutex
while we hold InnoDB mutexes. Instead victim is inserted a list
for later kill processing.

A new background thread will pick victim thread from this new list and uses
thd::awake() with no InnoDB mutexes. Idea is similar to replication background
kill. This fix enforces that we take mutexes in same order:
(1) victim_thread->LOCK_thd_data
(2) lock_sys->mutex ->
(3) victim_trx->mutex

wsrep_mysqld.cc
	Here we introduce a list where victim threads are stored,
	condition variable to be used to wake up background thread
	and mutex to protect list.

wsrep_thd.cc
	Create a new background thread to handle victim thread
	abort. We may take victim_thread->LOCK_thd_data mutex
	here but not any InnoDB mutexes.

wsrep_innobase_kill_one_trx
	Remove all the wsrep code that was moved to wsrep_thd.cc
	We just enqueue required information to background kill
	list and cancel victim trx lock wait if there is such.
	Here we have InnoDB lock_sys->mutex and victim_trx->mutex
	so here we can't take victim_thread->LOCK_thd_data mutex.

wsrep_abort_transaction
	Cleanup only.

include/mysql/service_wsrep.h

@@ -70,6 +70,7 @@ struct xid_t;
 struct wsrep;
 struct wsrep_ws_handle;
 struct wsrep_buf;
+typedef struct wsrep_kill wsrep_kill_t;
 
 extern struct wsrep_service_st {
   struct wsrep *              (*get_wsrep_func)();
@@ -114,7 +115,8 @@ extern struct wsrep_service_st {
   int                         (*wsrep_trx_order_before_func)(MYSQL_THD, MYSQL_THD);
   void                        (*wsrep_unlock_rollback_func)();
   void                        (*wsrep_set_data_home_dir_func)(const char *data_dir);
-  my_bool                     (*wsrep_thd_is_applier_func)(MYSQL_THD);
+  my_bool                     (*wsrep_thd_is_applier_func)(MYSQL_THD thd);
+  bool                        (*wsrep_enqueue_background_kill_func)(wsrep_kill_t);
 } *wsrep_service;
 
 #ifdef MYSQL_DYNAMIC_PLUGIN
@@ -161,6 +163,7 @@ extern struct wsrep_service_st {
 #define wsrep_unlock_rollback() wsrep_service->wsrep_unlock_rollback_func()
 #define wsrep_set_data_home_dir(A) wsrep_service->wsrep_set_data_home_dir_func(A)
 #define wsrep_thd_is_applier(T) wsrep_service->wsrep_thd_is_applier_func(T)
+#define wsrep_enqueue_background_kill(T) wsrep_service->wsrep_enqueue_background_kill_func(T);
 
 #define wsrep_debug get_wsrep_debug()
 #define wsrep_log_conflicts get_wsrep_log_conflicts()
@@ -223,6 +226,7 @@ bool wsrep_thd_ignore_table(THD *thd);
 void wsrep_unlock_rollback();
 void wsrep_set_data_home_dir(const char *data_dir);
 my_bool wsrep_thd_is_applier(MYSQL_THD thd);
+bool wsrep_enqueue_background_kill(wsrep_kill_t);
 #endif
 
 #ifdef __cplusplus

mysql-test/suite/galera/disabled.def

@@ -10,7 +10,6 @@
 #
 ##############################################################################
 
-MW-286 : MDEV-18464 Killing thread can cause mutex deadlock if done concurrently with Galera/replication victim kill
 MW-328A : MDEV-22666 galera.MW-328A MTR failed: "Semaphore wait has lasted > 600 seconds" and do not release port 16002
 MW-328B : MDEV-22666 galera.MW-328A MTR failed: "Semaphore wait has lasted > 600 seconds" and do not release port 16002
 MW-329 : MDEV-19962 Galera test failure on MW-329

mysql-test/suite/galera/r/galera_bf_kill.result

+connection node_2;
+CREATE TABLE t1(a int not null primary key auto_increment,b int) engine=InnoDB;
+insert into t1 values (NULL,1);
+connect node_2a, 127.0.0.1, root, , test, $NODE_MYPORT_2;
+connection node_2a;
+begin;
+update t1 set a = 5;
+connection node_2;
+select * from t1;
+a	b
+2	1
+disconnect node_2a;
+connect node_2a, 127.0.0.1, root, , test, $NODE_MYPORT_2;
+connection node_2a;
+begin;
+update t1 set a =5;
+connection node_2;
+select * from t1;
+a	b
+2	1
+disconnect node_2a;
+connect node_2a, 127.0.0.1, root, , test, $NODE_MYPORT_2;
+connection node_2a;
+begin;
+update t1 set a =5, b=2;
+connection node_1;
+ALTER TABLE t1 ADD UNIQUE KEY bcc(b);
+select * from t1;
+a	b
+2	1
+ALTER TABLE t1 DROP KEY bcc;
+disconnect node_2a;
+connect node_2a, 127.0.0.1, root, , test, $NODE_MYPORT_2;
+connection node_2a;
+begin;
+update t1 set a =5, b=2;
+connect node_2b, 127.0.0.1, root, , test, $NODE_MYPORT_2;
+connection node_2b;
+begin;
+update t1 set a =6, b=7;
+connection node_1;
+ALTER TABLE t1 ADD UNIQUE KEY bcc(b);
+select * from t1;
+a	b
+2	1
+disconnect node_2a;
+disconnect node_2b;
+connect node_2a, 127.0.0.1, root, , test, $NODE_MYPORT_2;
+connection node_2a;
+set session wsrep_on=OFF;
+begin;
+update t1 set a =5, b=2;
+connection node_1;
+ALTER TABLE t1 DROP KEY bcc;
+select * from t1;
+a	b
+2	1
+drop table t1;
+disconnect node_2a;

mysql-test/suite/galera/r/galera_insert_multi.result

@@ -4,13 +4,13 @@ INSERT INTO t1 VALUES (1),(2);
 connection node_2;
 INSERT INTO t1 VALUES (3),(4);
 connection node_1;
-SELECT COUNT(*) = 4 FROM t1;
-COUNT(*) = 4
-1
+SELECT COUNT(*) AS EXPECT_4 FROM t1;
+EXPECT_4
+4
 connection node_2;
-SELECT COUNT(*) = 4 FROM t1;
-COUNT(*) = 4
-1
+SELECT COUNT(*) AS EXPECT_4 FROM t1;
+EXPECT_4
+4
 DROP TABLE t1;
 connection node_2;
 CREATE TABLE t1 (f1 INTEGER, KEY (f1)) ENGINE=InnoDB;
@@ -18,25 +18,25 @@ INSERT INTO t1 VALUES (1),(1);
 connection node_1;
 INSERT INTO t1 VALUES (2),(2);
 connection node_2;
-SELECT COUNT(*) = 4 FROM t1;
-COUNT(*) = 4
-1
+SELECT COUNT(*) AS EXPECT_4 FROM t1;
+EXPECT_4
+4
 connection node_1;
-SELECT COUNT(*) = 4 FROM t1;
-COUNT(*) = 4
-1
+SELECT COUNT(*) AS EXPECT_4 FROM t1;
+EXPECT_4
+4
 DROP TABLE t1;
 connection node_1;
 CREATE TABLE t1 (f1 INTEGER PRIMARY KEY) ENGINE=InnoDB;
 INSERT INTO t1 VALUES (1), (2), (3), (4), (5), (6), (7), (8), (9), (1);
 ERROR 23000: Duplicate entry '1' for key 'PRIMARY'
-SELECT COUNT(*) = 0 FROM t1;
-COUNT(*) = 0
-1
+SELECT COUNT(*) AS EXPECT_0 FROM t1;
+EXPECT_0
+0
 connection node_2;
-SELECT COUNT(*) = 0 FROM t1;
-COUNT(*) = 0
-1
+SELECT COUNT(*) AS EXPECT_0 FROM t1;
+EXPECT_0
+0
 DROP TABLE t1;
 connection node_1;
 CREATE TABLE t1 (f1 INTEGER PRIMARY KEY) ENGINE=InnoDB;
@@ -67,11 +67,11 @@ connection node_1;
 ROLLBACK;
 connection node_2;
 COMMIT;
-SELECT COUNT(*) = 2 FROM t1;
-COUNT(*) = 2
-1
+SELECT COUNT(*) AS EXPECT_2 FROM t1;
+EXPECT_2
+2
 connection node_1;
-SELECT COUNT(*) = 2 FROM t1;
-COUNT(*) = 2
-1
+SELECT COUNT(*) AS EXPECT_2 FROM t1;
+EXPECT_2
+2
 DROP TABLE t1;

mysql-test/suite/galera/r/galera_performance_schema.result

@@ -4,6 +4,7 @@ FROM threads
 WHERE name LIKE 'thread/sql/wsrep%'
 ORDER BY name;
 name	thread/sql/wsrep_applier_thread
+name	thread/sql/wsrep_killer_thread
 name	thread/sql/wsrep_rollbacker_thread
 use test;
 create table t1 (a int not null primary key) engine=innodb;
@@ -12,6 +13,7 @@ use performance_schema;
 select name from mutex_instances where name like 'wait/synch/mutex/sql/LOCK_wsrep%' order by name;
 name	wait/synch/mutex/sql/LOCK_wsrep_config_state
 name	wait/synch/mutex/sql/LOCK_wsrep_desync
+name	wait/synch/mutex/sql/LOCK_wsrep_kill
 name	wait/synch/mutex/sql/LOCK_wsrep_ready
 name	wait/synch/mutex/sql/LOCK_wsrep_replaying
 name	wait/synch/mutex/sql/LOCK_wsrep_rollback
@@ -19,6 +21,7 @@ name	wait/synch/mutex/sql/LOCK_wsrep_slave_threads
 name	wait/synch/mutex/sql/LOCK_wsrep_sst
 name	wait/synch/mutex/sql/LOCK_wsrep_sst_init
 select name from cond_instances where name like 'wait/synch/cond/sql/COND_wsrep%' order by name;
+name	wait/synch/cond/sql/COND_wsrep_kill
 name	wait/synch/cond/sql/COND_wsrep_ready
 name	wait/synch/cond/sql/COND_wsrep_replaying
 name	wait/synch/cond/sql/COND_wsrep_rollback

mysql-test/suite/galera/r/galera_serializable.result

@@ -7,6 +7,8 @@ SELECT * FROM t1;
 id	f2
 connection node_2;
 INSERT INTO t1 VALUES (1,1);
+connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1;
+connection node_1a;
 connection node_1;
 SELECT * FROM t1;
 ERROR 40001: Deadlock: wsrep aborted transaction
@@ -20,6 +22,7 @@ id	f2
 1	1
 connection node_2;
 UPDATE t1 SET f2 = 2;
+connection node_1a;
 connection node_1;
 UPDATE t1 SET f2 = 3;
 ERROR 40001: Deadlock: wsrep aborted transaction
@@ -27,11 +30,18 @@ ROLLBACK;
 DELETE FROM t1;
 connection node_1;
 START TRANSACTION;
-connection node_1;
 INSERT INTO t1 VALUES (1,1);
 connection node_2;
 INSERT INTO t1 VALUES (1,2);
 connection node_1;
 COMMIT;
 ERROR 40001: Deadlock: wsrep aborted transaction
+SELECT * from t1;
+id	f2
+1	2
+connection node_2;
+SELECT * from t1;
+id	f2
+1	2
 DROP TABLE t1;
+disconnect node_1a;

mysql-test/suite/galera/t/MW-286.cnf

+!include ../galera_2nodes.cnf
+
+[mysqld.1]
+wsrep-debug=ON
+wsrep-log-conflicts=ON
+
+[mysqld.2]
+wsrep-debug=ON
+wsrep-log-conflicts=ON
+

mysql-test/suite/galera/t/MW-286.test

@@ -3,8 +3,6 @@
 #
 
 --source include/galera_cluster.inc
---source include/have_innodb.inc
---source include/big_test.inc
 
 --connection node_1
 CREATE TABLE ten (f1 INTEGER) Engine=InnoDB;

mysql-test/suite/galera/t/galera_as_slave_gtid_myisam.test

@@ -69,9 +69,13 @@ DROP TABLE t1;
 reset master;
 
 --connection node_2
+--let $wait_condition = SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1';
+--source include/wait_condition.inc
 STOP SLAVE;
 RESET SLAVE ALL;
 reset master;
 
 --connection node_3
+--let $wait_condition = SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1';
+--source include/wait_condition.inc
 reset master;

mysql-test/suite/galera/t/galera_bf_abort_sleep.test

@@ -8,8 +8,8 @@
 CREATE TABLE t1 (f1 INTEGER PRIMARY KEY) ENGINE=InnoDB;
 
 --connection node_2
-SET AUTOCOMMIT=OFF;
 --let $wsrep_local_bf_aborts_before = `SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_bf_aborts'`
+SET AUTOCOMMIT=OFF;
 INSERT INTO t1 VALUES (1);
 --send SELECT SLEEP(1000);
 

mysql-test/suite/galera/t/galera_bf_kill.cnf

+!include ../galera_2nodes.cnf
+
+[mysqld.1]
+wsrep-debug=ON
+wsrep-log-conflicts=ON
+
+[mysqld.2]
+wsrep-debug=ON
+wsrep-log-conflicts=ON
+

mysql-test/suite/galera/t/galera_bf_kill.test

+--source include/galera_cluster.inc
+
+#
+# Test case 1: Start a transaction on node_2a and kill it
+# from other connection on same node
+#
+--connection node_1
+CREATE TABLE t1(a int not null primary key auto_increment,b int) engine=InnoDB;
+insert into t1 values (NULL,1);
+
+--connect node_2a, 127.0.0.1, root, , test, $NODE_MYPORT_2
+--connection node_2a
+begin;
+update t1 set a = 5;
+
+--connection node_2
+
+--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'root' AND COMMAND = 'Sleep' LIMIT 1
+--source include/wait_condition.inc
+
+--let $k_thread = `SELECT ID FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'root' AND COMMAND = 'Sleep' LIMIT 1`
+
+--disable_query_log
+--eval KILL $k_thread
+--enable_query_log
+
+select * from t1;
+--disconnect node_2a
+#
+# Test case 2: Start a transaction on node_2a and use
+# kill query from other connection on same node
+#
+--connect node_2a, 127.0.0.1, root, , test, $NODE_MYPORT_2
+--connection node_2a
+begin;
+update t1 set a =5;
+
+--connection node_2
+--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'root' AND COMMAND = 'Sleep' LIMIT 1
+--source include/wait_condition.inc
+
+--let $k_thread = `SELECT ID FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'root' AND COMMAND = 'Sleep' LIMIT 1`
+
+--disable_query_log
+--eval KILL QUERY $k_thread
+--enable_query_log
+
+select * from t1;
+--disconnect node_2a
+#
+# Test case 3: Start a transaction on node_2a and start a DDL on other node
+# that will then abort node_2a transaction
+#
+--connect node_2a, 127.0.0.1, root, , test, $NODE_MYPORT_2
+--connection node_2a
+begin;
+update t1 set a =5, b=2;
+
+--connection node_1
+ALTER TABLE t1 ADD UNIQUE KEY bcc(b);
+show create table t1;
+
+select * from t1;
+
+ALTER TABLE t1 DROP KEY bcc;
+show create table t1;
+
+--disconnect node_2a
+#
+# Test case 4: Start a transaction on node_2a and start a DDL on other transaction
+# on same node that will then abort node_2a transaction
+#
+--connect node_2a, 127.0.0.1, root, , test, $NODE_MYPORT_2
+--connection node_2a
+begin;
+update t1 set a =5, b=2;
+
+--connection node_2
+ALTER TABLE t1 ADD UNIQUE KEY bcc(b);
+
+select * from t1;
+show create table t1;
+
+ALTER TABLE t1 DROP KEY bcc;
+show create table t1;
+
+--disconnect node_2a
+#
+# Test case 5: Start a transaction on node_2a and conflicting transaction on node_2b
+# and start a DDL on other node that will then abort node_2a and node_2b
+# transactions
+#
+
+--connect node_2a, 127.0.0.1, root, , test, $NODE_MYPORT_2
+--connection node_2a
+begin;
+update t1 set a =5, b=2;
+
+--connect node_2b, 127.0.0.1, root, , test, $NODE_MYPORT_2
+--connection node_2b
+begin;
+send update t1 set a =6, b=7;
+
+--connection node_1
+ALTER TABLE t1 ADD UNIQUE KEY bcc(b);
+
+select * from t1;
+
+--disconnect node_2a
+--disconnect node_2b
+#
+# Test case 6: Start a transaction on node_2a and conflicting transaction on node_2b
+# and start a DDL on node_2 that will then abort node_2a and node_2b
+# transactions
+#
+
+--connect node_2a, 127.0.0.1, root, , test, $NODE_MYPORT_2
+--connection node_2a
+begin;
+update t1 set a =5, b=2;
+
+--connect node_2b, 127.0.0.1, root, , test, $NODE_MYPORT_2
+--connection node_2b
+begin;
+send update t1 set a =6, b=7;
+
+--connection node_2
+ALTER TABLE t1 DROP KEY bcc;
+
+select * from t1;
+show create table t1;
+
+--disconnect node_2a
+--disconnect node_2b
+#
+# Test case 7: Start a transaction on node_2a with WSREP_ON=OFF and
+# start a DDL on node_2 that will then bf abort node_2a
+#
+--connect node_2a, 127.0.0.1, root, , test, $NODE_MYPORT_2
+--connection node_2a
+set session wsrep_on=OFF;
+begin;
+update t1 set a =5, b=2;
+
+--connection node_2
+ALTER TABLE t1 ADD UNIQUE KEY bcc(b);
+
+select * from t1;
+
+drop table t1;
+
+--disconnect node_2a
+#
+# Test case 8: Start a transaction on node_2a with WSREP_ON=OFF and
+# start a DDL on node_1 that will then bf abort node_2a
+#
+--connect node_2a, 127.0.0.1, root, , test, $NODE_MYPORT_2
+--connection node_2a
+set session wsrep_on=OFF;
+begin;
+update t1 set a =5, b=2;
+
+--connection node_1
+ALTER TABLE t1 DROP KEY bcc;
+
+select * from t1;
+show create table t1;
+
+drop table t1;
+
+--disconnect node_2a

mysql-test/suite/galera/t/galera_drop_database.cnf

+!include ../galera_2nodes.cnf
+
+[mysqld.1]
+wsrep-debug=ON
+wsrep-log-conflicts=ON
+
+[mysqld.2]
+wsrep-debug=ON
+wsrep-log-conflicts=ON
+

mysql-test/suite/galera/t/galera_insert_multi.test

@@ -13,10 +13,16 @@ INSERT INTO t1 VALUES (1),(2);
 INSERT INTO t1 VALUES (3),(4);
 
 --connection node_1
-SELECT COUNT(*) = 4 FROM t1;
+--let $wait_condition = SELECT COUNT(*) = 4 FROM t1;
+--source include/wait_condition.inc
+
+SELECT COUNT(*) AS EXPECT_4 FROM t1;
 
 --connection node_2
-SELECT COUNT(*) = 4 FROM t1;
+--let $wait_condition = SELECT COUNT(*) = 4 FROM t1;
+--source include/wait_condition.inc
+
+SELECT COUNT(*) AS EXPECT_4 FROM t1;
 
 DROP TABLE t1;
 
@@ -32,10 +38,16 @@ INSERT INTO t1 VALUES (1),(1);
 INSERT INTO t1 VALUES (2),(2);
 
 --connection node_2
-SELECT COUNT(*) = 4 FROM t1;
+--let $wait_condition = SELECT COUNT(*) = 4 FROM t1;
+--source include/wait_condition.inc
+
+SELECT COUNT(*) AS EXPECT_4 FROM t1;
 
 --connection node_1
-SELECT COUNT(*) = 4 FROM t1;
+--let $wait_condition = SELECT COUNT(*) = 4 FROM t1;
+--source include/wait_condition.inc
+
+SELECT COUNT(*) AS EXPECT_4 FROM t1;
 
 DROP TABLE t1;
 
@@ -49,10 +61,10 @@ CREATE TABLE t1 (f1 INTEGER PRIMARY KEY) ENGINE=InnoDB;
 --error ER_DUP_ENTRY
 INSERT INTO t1 VALUES (1), (2), (3), (4), (5), (6), (7), (8), (9), (1);
 
-SELECT COUNT(*) = 0 FROM t1;
+SELECT COUNT(*) AS EXPECT_0 FROM t1;
 
 --connection node_2
-SELECT COUNT(*) = 0 FROM t1;
+SELECT COUNT(*) AS EXPECT_0 FROM t1;
 
 DROP TABLE t1;
 
@@ -107,10 +119,13 @@ ROLLBACK;
 
 --connection node_2
 COMMIT;
-SELECT COUNT(*) = 2 FROM t1;
+SELECT COUNT(*) AS EXPECT_2 FROM t1;
 
 --connection node_1
-SELECT COUNT(*) = 2 FROM t1;
+--let $wait_condition = SELECT COUNT(*) = 2 FROM t1;
+--source include/wait_condition.inc
+
+SELECT COUNT(*) AS EXPECT_2 FROM t1;
 
 DROP TABLE t1;
 

mysql-test/suite/galera/t/galera_serializable.test

@@ -7,7 +7,6 @@
 #
 
 --source include/galera_cluster.inc
---source include/have_innodb.inc
 
 --connection node_1
 
@@ -26,7 +25,12 @@ SELECT * FROM t1;
 --connection node_2
 INSERT INTO t1 VALUES (1,1);
 
---sleep 2
+--connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1
+--connection node_1a
+--let $wait_condition = SELECT COUNT(*) = 1 FROM t1;
+--error 0,ER_LOCK_DEADLOCK
+--source include/wait_condition.inc
+
 --connection node_1
 --error ER_LOCK_DEADLOCK
 SELECT * FROM t1;
@@ -46,7 +50,11 @@ SELECT * FROM t1;
 --connection node_2
 UPDATE t1 SET f2 = 2;
 
---sleep 2
+--connection node_1a
+--let $wait_condition = SELECT COUNT(*) = 1 FROM t1;
+--error 0,ER_LOCK_DEADLOCK
+--source include/wait_condition.inc
+
 --connection node_1
 --error ER_LOCK_DEADLOCK
 UPDATE t1 SET f2 = 3;
@@ -62,8 +70,6 @@ DELETE FROM t1;
 
 --connection node_1
 START TRANSACTION;
-
---connection node_1
 INSERT INTO t1 VALUES (1,1);
 
 --connection node_2
@@ -73,4 +79,18 @@ INSERT INTO t1 VALUES (1,2);
 --error ER_LOCK_DEADLOCK
 COMMIT;
 
+--let $wait_condition = SELECT COUNT(*) = 1 FROM t1;
+--source include/wait_condition.inc
+
+SELECT * from t1;
+
+--connection node_2
+
+--let $wait_condition = SELECT COUNT(*) = 1 FROM t1;
+--source include/wait_condition.inc
+
+SELECT * from t1;
+
 DROP TABLE t1;
+
+--disconnect node_1a

mysql-test/suite/galera/t/galera_unicode_pk.test

@@ -12,6 +12,9 @@ CREATE TABLE t1 (
 INSERT INTO t1 VALUES ('текст');
 
 --connection node_2
+--let $wait_condition = SELECT COUNT(*) = 1 FROM t1;
+--source include/wait_condition.inc
+
 SELECT f1 = 'текст' FROM t1;
 
 #

mysql-test/suite/galera/t/jan.test

+--source include/galera_cluster.inc
+
+--connection node_1
+CREATE TABLE t1(a int not null primary key auto_increment,b int) engine=InnoDB;
+insert into t1 values (NULL,1);
+
+--connect node_2a, 127.0.0.1, root, , test, $NODE_MYPORT_2
+--connection node_2a
+set session wsrep_on=OFF;
+begin;
+update t1 set a =5, b=2;
+
+--connection node_2
+ALTER TABLE t1 ADD UNIQUE KEY bcc(b);
+
+select * from t1;
+
+drop table t1;
+
+--disconnect node_2a

mysql-test/suite/wsrep/r/variables.result

@@ -149,7 +149,7 @@ VARIABLE_VALUE
 1
 SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_thread_count';
 VARIABLE_VALUE
-2
+3
 SELECT @@global.wsrep_provider;
 @@global.wsrep_provider
 libgalera_smm.so
@@ -164,7 +164,7 @@ Variable_name	Value
 Threads_connected	1
 SHOW STATUS LIKE 'wsrep_thread_count';
 Variable_name	Value
-wsrep_thread_count	2
+wsrep_thread_count	3
 
 SET @wsrep_slave_threads_saved= @@global.wsrep_slave_threads;
 SET GLOBAL wsrep_slave_threads= 10;
@@ -177,7 +177,7 @@ VARIABLE_VALUE
 1
 SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_thread_count';
 VARIABLE_VALUE
-11
+12
 SHOW STATUS LIKE 'threads_connected';
 Variable_name	Value
 Threads_connected	1

sql/sql_class.cc

@@ -652,7 +652,8 @@ THD::THD(my_thread_id id, bool is_wsrep_applier)
    wsrep_po_handle(WSREP_PO_INITIALIZER),
    wsrep_po_cnt(0),
    wsrep_apply_format(0),
-   wsrep_ignore_table(false)
+   wsrep_ignore_table(false),
+   wsrep_aborter(0)
 #endif
 {
   ulong tmp;
@@ -790,6 +791,7 @@ THD::THD(my_thread_id id, bool is_wsrep_applier)
   wsrep_replicate_GTID    = false;
   wsrep_skip_wsrep_GTID   = false;
   wsrep_split_flag        = false;
+  wsrep_aborter           = 0;
 #endif
   /* Call to init() below requires fully initialized Open_tables_state. */
   reset_open_tables_state(this);

sql/sql_class.h

@@ -3753,13 +3753,23 @@ class THD :public Statement,
       Resetting killed has to be done under a mutex to ensure
       its not done during an awake() call.
     */
+#ifdef WITH_WSREP
+    mysql_mutex_assert_not_owner(&LOCK_thd_kill);
+    mysql_mutex_assert_not_owner(&LOCK_thd_data);
+    mysql_mutex_lock(&LOCK_thd_data);
+#endif
+    mysql_mutex_lock(&LOCK_thd_kill);
     if (killed != NOT_KILLED)
     {
-      mysql_mutex_lock(&LOCK_thd_kill);
       killed= NOT_KILLED;
       killed_err= 0;
-      mysql_mutex_unlock(&LOCK_thd_kill);
     }
+
+#ifdef WITH_WSREP
+    wsrep_aborter= 0;
+    mysql_mutex_unlock(&LOCK_thd_data);
+#endif
+    mysql_mutex_unlock(&LOCK_thd_kill);
   }
   inline void reset_kill_query()
   {
@@ -4421,6 +4431,8 @@ class THD :public Statement,
 
 #ifdef WITH_WSREP
   const bool                wsrep_applier; /* dedicated slave applier thread */
+  bool                      wsrep_killer; /* dedicated background
+					  kill thread */
   bool                      wsrep_applier_closing; /* applier marked to close */
   bool                      wsrep_client_thread; /* to identify client threads*/
   bool                      wsrep_PA_safe;
@@ -4470,6 +4482,8 @@ class THD :public Statement,
   registered again, but replication of last chunk of rows is skipped
   by the innodb engine: */
   bool                      wsrep_split_flag;
+  /* thread who has started kill for this THD protected by LOCK_thd_data*/
+  my_thread_id              wsrep_aborter;
 #endif /* WITH_WSREP */
 
   /* Handling of timeouts for commands */

sql/sql_parse.cc

@@ -8896,8 +8896,15 @@ kill_one_thread(THD *thd, longlong id, killed_state kill_signal, killed_type typ
         thd->security_ctx->user_matches(tmp->security_ctx)) &&
 	!wsrep_thd_is_BF(tmp, false))
     {
-      tmp->awake(kill_signal);
-      error=0;
+#ifdef WITH_WSREP
+      /* We allow kill to continue only if there is no concurrent
+         kill in processing. */
+      if (wsrep_thd_set_wsrep_aborter(tmp, thd))
+#endif
+      {
+        tmp->awake(kill_signal);
+        error=0;
+      }
     }
     else
       error= (type == KILL_TYPE_QUERY ? ER_KILL_QUERY_DENIED_ERROR :

sql/sql_plugin_services.ic

@@ -184,7 +184,8 @@ static struct wsrep_service_st wsrep_handler = {
   wsrep_trx_order_before,
   wsrep_unlock_rollback,
   wsrep_set_data_home_dir,
-  wsrep_thd_is_applier
+  wsrep_thd_is_applier,
+  wsrep_enqueue_background_kill
 };
 
 static struct thd_specifics_service_st thd_specifics_handler=

sql/wsrep_dummy.cc

@@ -16,6 +16,7 @@
 #include <my_global.h>
 #include <sql_class.h>
 #include <mysql/service_wsrep.h>
+#include "wsrep_mysqld.h"
 
 my_bool wsrep_thd_is_BF(THD *, my_bool)
 { return 0; }
@@ -154,3 +155,9 @@ void wsrep_log(void (*)(const char *, ...), const char *, ...)
 
 my_bool wsrep_thd_is_applier(MYSQL_THD thd)
 { return false; }
+
+bool wsrep_enqueue_background_kill(wsrep_kill_t item)
+{ return false;}
+
+bool wsrep_thd_set_wsrep_aborter(THD *victim_thd, THD *bf_thd)
+{ return true; }

sql/wsrep_hton.cc

@@ -140,10 +140,10 @@ void wsrep_post_commit(THD* thd, bool all)
         => cleanup
       */
       if (thd->wsrep_conflict_state != MUST_REPLAY)
-     {
-       WSREP_DEBUG("cleanup transaction for LOCAL_STATE: %s",
-                   WSREP_QUERY(thd));
-     }
+      {
+        WSREP_DEBUG("cleanup transaction for LOCAL_STATE: %s",
+      	            wsrep_thd_query(thd));
+      }
       /*
         Run post-rollback hook to clean up in the case if
         some keys were populated for the transaction in provider

sql/wsrep_mysqld.cc

@@ -38,7 +38,8 @@
 #include <cstdlib>
 #include "log_event.h"
 #include "sql_plugin.h"                         /* wsrep_plugins_pre_init() */
-#include <vector>
+#include <list>
+#include <algorithm>
 
 wsrep_t *wsrep                  = NULL;
 /*
@@ -133,6 +134,8 @@ mysql_cond_t  COND_wsrep_replaying;
 mysql_mutex_t LOCK_wsrep_slave_threads;
 mysql_mutex_t LOCK_wsrep_desync;
 mysql_mutex_t LOCK_wsrep_config_state;
+mysql_mutex_t LOCK_wsrep_kill;
+mysql_cond_t  COND_wsrep_kill;
 
 int wsrep_replaying= 0;
 ulong  wsrep_running_threads = 0; // # of currently running wsrep
@@ -140,6 +143,7 @@ ulong  wsrep_running_threads = 0; // # of currently running wsrep
 ulong  wsrep_running_applier_threads = 0; // # of running applier threads
 ulong  wsrep_running_rollbacker_threads = 0; // # of running
 					     // # rollbacker threads
+ulong  wsrep_running_killer_threads = 0;
 ulong  my_bind_addr;
 
 #ifdef HAVE_PSI_INTERFACE
@@ -147,11 +151,13 @@ PSI_mutex_key key_LOCK_wsrep_rollback,
   key_LOCK_wsrep_replaying, key_LOCK_wsrep_ready, key_LOCK_wsrep_sst,
   key_LOCK_wsrep_sst_thread, key_LOCK_wsrep_sst_init,
   key_LOCK_wsrep_slave_threads, key_LOCK_wsrep_desync,
-  key_LOCK_wsrep_config_state;
+  key_LOCK_wsrep_config_state,
+  key_LOCK_wsrep_kill;
 
 PSI_cond_key key_COND_wsrep_rollback,
   key_COND_wsrep_replaying, key_COND_wsrep_ready, key_COND_wsrep_sst,
-  key_COND_wsrep_sst_init, key_COND_wsrep_sst_thread;
+  key_COND_wsrep_sst_init, key_COND_wsrep_sst_thread,
+  key_COND_wsrep_kill;
 
 PSI_file_key key_file_wsrep_gra_log;
 
@@ -166,7 +172,8 @@ static PSI_mutex_info wsrep_mutexes[]=
   { &key_LOCK_wsrep_replaying, "LOCK_wsrep_replaying", PSI_FLAG_GLOBAL},
   { &key_LOCK_wsrep_slave_threads, "LOCK_wsrep_slave_threads", PSI_FLAG_GLOBAL},
   { &key_LOCK_wsrep_desync, "LOCK_wsrep_desync", PSI_FLAG_GLOBAL},
-  { &key_LOCK_wsrep_config_state, "LOCK_wsrep_config_state", PSI_FLAG_GLOBAL}
+  { &key_LOCK_wsrep_config_state, "LOCK_wsrep_config_state", PSI_FLAG_GLOBAL},
+  { &key_LOCK_wsrep_kill, "LOCK_wsrep_kill", PSI_FLAG_GLOBAL}
 };
 
 static PSI_cond_info wsrep_conds[]=
@@ -176,7 +183,8 @@ static PSI_cond_info wsrep_conds[]=
   { &key_COND_wsrep_sst_init, "COND_wsrep_sst_init", PSI_FLAG_GLOBAL},
   { &key_COND_wsrep_sst_thread, "wsrep_sst_thread", 0},
   { &key_COND_wsrep_rollback, "COND_wsrep_rollback", PSI_FLAG_GLOBAL},
-  { &key_COND_wsrep_replaying, "COND_wsrep_replaying", PSI_FLAG_GLOBAL}
+  { &key_COND_wsrep_replaying, "COND_wsrep_replaying", PSI_FLAG_GLOBAL},
+  { &key_COND_wsrep_kill, "COND_wsrep_kill", PSI_FLAG_GLOBAL}
 };
 
 static PSI_file_info wsrep_files[]=
@@ -185,14 +193,15 @@ static PSI_file_info wsrep_files[]=
 };
 
 PSI_thread_key key_wsrep_sst_joiner, key_wsrep_sst_donor,
-  key_wsrep_rollbacker, key_wsrep_applier;
+  key_wsrep_rollbacker, key_wsrep_applier, key_wsrep_killer;
 
 static PSI_thread_info wsrep_threads[]=
 {
  {&key_wsrep_sst_joiner, "wsrep_sst_joiner_thread", PSI_FLAG_GLOBAL},
  {&key_wsrep_sst_donor, "wsrep_sst_donor_thread", PSI_FLAG_GLOBAL},
  {&key_wsrep_rollbacker, "wsrep_rollbacker_thread", PSI_FLAG_GLOBAL},
- {&key_wsrep_applier, "wsrep_applier_thread", PSI_FLAG_GLOBAL}
+ {&key_wsrep_applier, "wsrep_applier_thread", PSI_FLAG_GLOBAL},
+ {&key_wsrep_killer, "wsrep_killer_thread", PSI_FLAG_GLOBAL}
 };
 
 #endif /* HAVE_PSI_INTERFACE */
@@ -239,6 +248,7 @@ wsp::Config_state *wsrep_config_state;
 // if there was no state gap on receiving first view event.
 static my_bool   wsrep_startup = TRUE;
 
+std::list< wsrep_kill_t > wsrep_kill_list;
 
 static void wsrep_log_cb(wsrep_log_level_t level, const char *msg) {
   switch (level) {
@@ -831,6 +841,8 @@ void wsrep_thr_init()
   mysql_mutex_init(key_LOCK_wsrep_slave_threads, &LOCK_wsrep_slave_threads, MY_MUTEX_INIT_FAST);
   mysql_mutex_init(key_LOCK_wsrep_desync, &LOCK_wsrep_desync, MY_MUTEX_INIT_FAST);
   mysql_mutex_init(key_LOCK_wsrep_config_state, &LOCK_wsrep_config_state, MY_MUTEX_INIT_FAST);
+  mysql_mutex_init(key_LOCK_wsrep_kill, &LOCK_wsrep_kill, MY_MUTEX_INIT_FAST);
+  mysql_cond_init(key_COND_wsrep_kill, &COND_wsrep_kill, NULL);
 
   DBUG_VOID_RETURN;
 }
@@ -867,6 +879,7 @@ void wsrep_init_startup (bool first)
   if (!wsrep_start_replication()) unireg_abort(1);
 
   wsrep_create_rollbacker();
+  wsrep_create_killer();
   wsrep_create_appliers(1);
 
   if (first && !wsrep_sst_wait()) unireg_abort(1);// wait until SST is completed
@@ -908,6 +921,8 @@ void wsrep_thr_deinit()
   mysql_mutex_destroy(&LOCK_wsrep_slave_threads);
   mysql_mutex_destroy(&LOCK_wsrep_desync);
   mysql_mutex_destroy(&LOCK_wsrep_config_state);
+  mysql_mutex_destroy(&LOCK_wsrep_kill);
+  mysql_cond_destroy(&COND_wsrep_kill);
   delete wsrep_config_state;
   wsrep_config_state= 0;                        // Safety
 }
@@ -1666,7 +1681,7 @@ static int wsrep_TOI_begin(THD *thd, char *db_, char *table_,
 
   if (wsrep_can_run_in_toi(thd, db_, table_, table_list) == false)
   {
-    WSREP_DEBUG("No TOI for %s", WSREP_QUERY(thd));
+    WSREP_DEBUG("No TOI for %s", wsrep_thd_query(thd));
     return 1;
   }
 
@@ -2162,9 +2177,13 @@ pthread_handler_t start_wsrep_THD(void *arg)
     case WSREP_ROLLBACKER_THREAD:
       wsrep_running_rollbacker_threads++;
       break;
+    case WSREP_KILLER_THREAD:
+      wsrep_running_killer_threads++;
+      thd->wsrep_killer= true;
+      break;
     default:
       WSREP_ERROR("Incorrect wsrep thread type: %d", args->thread_type);
-      break;
+      assert(0);
   }
 
   mysql_cond_broadcast(&COND_thread_count);
@@ -2187,9 +2206,13 @@ pthread_handler_t start_wsrep_THD(void *arg)
       DBUG_ASSERT(wsrep_running_rollbacker_threads > 0);
       wsrep_running_rollbacker_threads--;
       break;
+    case WSREP_KILLER_THREAD:
+      DBUG_ASSERT(wsrep_running_killer_threads > 0);
+      wsrep_running_killer_threads--;
+      break;
     default:
       WSREP_ERROR("Incorrect wsrep thread type: %d", args->thread_type);
-      break;
+      assert(0);
   }
 
   my_free(args);
@@ -2436,7 +2459,11 @@ void wsrep_close_client_connections(my_bool wait_to_end, THD *except_caller_thd)
   }
 
   DBUG_PRINT("quit",("Waiting for threads to die (count=%u)",thread_count));
-  WSREP_DEBUG("waiting for client connections to close: %u", thread_count);
+  WSREP_DEBUG("Waiting for client connections to close: %u", thread_count);
+  WSREP_DEBUG("Waiting for rollbacker threads to close: %lu", wsrep_running_rollbacker_threads);
+  WSREP_DEBUG("Waiting for applier threads to close: %lu", wsrep_running_applier_threads);
+  WSREP_DEBUG("Waiting for killer threads to close: %lu", wsrep_running_killer_threads);
+  WSREP_DEBUG("Waiting for wsrep threads to close: %lu", wsrep_running_threads);
 
   while (wait_to_end && have_client_connections())
   {
@@ -2470,7 +2497,7 @@ void wsrep_close_threads(THD *thd)
     DBUG_PRINT("quit",("Informing thread %lld that it's time to die",
                        (longlong) tmp->thread_id));
     /* We skip slave threads & scheduler on this first loop through. */
-    if (tmp->wsrep_applier && tmp != thd)
+    if ((tmp->wsrep_applier || tmp->wsrep_killer) && tmp != thd)
     {
       WSREP_DEBUG("closing wsrep thread %lld", (longlong) tmp->thread_id);
       wsrep_close_thread (tmp);
@@ -2484,7 +2511,7 @@ void wsrep_wait_appliers_close(THD *thd)
 {
   /* Wait for wsrep appliers to gracefully exit */
   mysql_mutex_lock(&LOCK_thread_count);
-  while (wsrep_running_threads > 1)
+  while (wsrep_running_threads > 2)
   // 1 is for rollbacker thread which needs to be killed explicitly.
   // This gotta be fixed in a more elegant manner if we gonna have arbitrary
   // number of non-applier wsrep threads.
@@ -2760,11 +2787,11 @@ extern "C" void wsrep_thd_set_wsrep_last_query_id(THD *thd, query_id_t id)
 
 extern "C" void wsrep_thd_awake(THD *thd, my_bool signal)
 {
+  mysql_mutex_assert_owner(&thd->LOCK_thd_data);
+
   if (signal)
   {
-    mysql_mutex_lock(&thd->LOCK_thd_data);
     thd->awake(KILL_QUERY);
-    mysql_mutex_unlock(&thd->LOCK_thd_data);
   }
   else
   {
@@ -2772,6 +2799,8 @@ extern "C" void wsrep_thd_awake(THD *thd, my_bool signal)
     mysql_cond_broadcast(&COND_wsrep_replaying);
     mysql_mutex_unlock(&LOCK_wsrep_replaying);
   }
+
+  mysql_mutex_unlock(&thd->LOCK_thd_data);
 }
 
 
@@ -3029,3 +3058,47 @@ bool wsrep_node_is_synced()
 {
   return (WSREP_ON) ? (wsrep_config_state->get_status() == 4) : false;
 }
+
+bool wsrep_enqueue_background_kill(wsrep_kill_t item)
+{
+  std::list< wsrep_kill_t >::iterator it;
+  bool inserted= false;
+
+  mysql_mutex_lock(&LOCK_wsrep_kill);
+
+  for (it = wsrep_kill_list.begin(); it != wsrep_kill_list.end(); it++)
+  {
+    if ((*it).victim_thd_id == item.victim_thd_id)
+      break;
+  }
+
+  if(it != wsrep_kill_list.end())
+  {
+    WSREP_DEBUG("Thread: %lu already on kill list", item.victim_thd_id);
+  }
+  else
+  {
+    wsrep_kill_list.push_back(item);
+    mysql_cond_signal(&COND_wsrep_kill);
+    inserted= true;
+  }
+
+  mysql_mutex_unlock(&LOCK_wsrep_kill);
+  return inserted;
+}
+
+bool wsrep_thd_set_wsrep_aborter(THD *victim_thd, THD *bf_thd)
+{
+  WSREP_DEBUG("wsrep_thd_set_wsrep_aborter called victim: %llu bf: %llu",
+	       victim_thd->thread_id, bf_thd->thread_id);
+  mysql_mutex_assert_owner(&victim_thd->LOCK_thd_data);
+
+  if (victim_thd->wsrep_aborter && victim_thd->wsrep_aborter != bf_thd->thread_id)
+  {
+    return false;
+  }
+
+  victim_thd->wsrep_aborter = bf_thd->thread_id;
+
+  return true;
+}

sql/wsrep_mysqld.h

@@ -18,9 +18,20 @@
 #ifndef WSREP_MYSQLD_H
 #define WSREP_MYSQLD_H
 
+#include <my_config.h>
+#include <stdint.h>
 #include <mysql/plugin.h>
 #include <mysql/service_wsrep.h>
 
+typedef struct wsrep_kill {
+  unsigned long        victim_thd_id;
+  unsigned long        bf_thd_id;
+  uint64_t             victim_trx_id;
+  uint64_t             bf_trx_id;
+  bool                 signal;
+  bool                 wait_lock;
+} wsrep_kill_t;
+
 #ifdef WITH_WSREP
 
 typedef struct st_mysql_show_var SHOW_VAR;
@@ -92,6 +103,7 @@ extern my_bool     wsrep_slave_UK_checks;
 extern ulong       wsrep_running_threads;
 extern ulong       wsrep_running_applier_threads;
 extern ulong       wsrep_running_rollbacker_threads;
+extern ulong       wsrep_running_killer_threads;
 extern bool        wsrep_new_cluster;
 extern bool        wsrep_gtid_mode;
 extern uint32      wsrep_gtid_domain_id;
@@ -223,8 +235,6 @@ void wsrep_log(void (*fun)(const char *, ...), const char *format, ...);
 #define WSREP_PROVIDER_EXISTS                                                  \
   (wsrep_provider && strncasecmp(wsrep_provider, WSREP_NONE, FN_REFLEN))
 
-#define WSREP_QUERY(thd) (thd->query())
-
 extern my_bool wsrep_ready_get();
 extern void wsrep_ready_wait();
 
@@ -254,6 +264,8 @@ extern mysql_cond_t  COND_wsrep_replaying;
 extern mysql_mutex_t LOCK_wsrep_slave_threads;
 extern mysql_mutex_t LOCK_wsrep_desync;
 extern mysql_mutex_t LOCK_wsrep_config_state;
+extern mysql_mutex_t LOCK_wsrep_kill;
+extern mysql_cond_t  COND_wsrep_kill;
 extern wsrep_aborting_thd_t wsrep_aborting_thd;
 extern my_bool       wsrep_emulate_bin_log;
 extern int           wsrep_to_isolation;
@@ -278,6 +290,8 @@ extern PSI_mutex_key key_LOCK_wsrep_replaying;
 extern PSI_cond_key  key_COND_wsrep_replaying;
 extern PSI_mutex_key key_LOCK_wsrep_slave_threads;
 extern PSI_mutex_key key_LOCK_wsrep_desync;
+extern PSI_mutex_key key_LOCK_wsrep_kill;
+extern PSI_cond_key  key_COND_wsrep_kill;
 
 extern PSI_file_key key_file_wsrep_gra_log;
 
@@ -285,6 +299,7 @@ extern PSI_thread_key key_wsrep_sst_joiner;
 extern PSI_thread_key key_wsrep_sst_donor;
 extern PSI_thread_key key_wsrep_rollbacker;
 extern PSI_thread_key key_wsrep_applier;
+extern PSI_thread_key key_wsrep_killer;
 #endif /* HAVE_PSI_INTERFACE */
 
 
@@ -311,7 +326,8 @@ void thd_binlog_trx_reset(THD * thd);
 
 enum wsrep_thread_type {
   WSREP_APPLIER_THREAD=1,
-  WSREP_ROLLBACKER_THREAD=2
+  WSREP_ROLLBACKER_THREAD=2,
+  WSREP_KILLER_THREAD=3
 };
 
 typedef void (*wsrep_thd_processor_fun)(THD *);
@@ -358,6 +374,8 @@ void wsrep_keys_free(wsrep_key_arr_t* key_arr);
    ((wsrep_forced_binlog_format != BINLOG_FORMAT_UNSPEC) ?     \
    wsrep_forced_binlog_format : my_format)
 
+bool wsrep_thd_set_wsrep_aborter(THD *victim_thd, THD *bf_thd);
+
 #else /* WITH_WSREP */
 
 #define WSREP(T)  (0)

sql/wsrep_thd.h

@@ -29,6 +29,7 @@ void wsrep_replay_sp_transaction(THD* thd);
 void wsrep_replay_transaction(THD *thd);
 bool wsrep_create_appliers(long threads, bool thread_count_lock=false);
 void wsrep_create_rollbacker();
+void wsrep_create_killer();
 
 int  wsrep_abort_thd(void *bf_thd_ptr, void *victim_thd_ptr,
                                 my_bool signal);

sql/wsrep_var.cc

@@ -500,6 +500,7 @@ bool wsrep_cluster_address_update (sys_var *self, THD* thd, enum_var_type type)
   if (wsrep_start_replication())
   {
     wsrep_create_rollbacker();
+    wsrep_create_killer();
     WSREP_DEBUG("Cluster address update creating %ld applier threads running %lu",
 	    wsrep_slave_threads, wsrep_running_applier_threads);
     wsrep_create_appliers(wsrep_slave_threads);

storage/innobase/include/ha_prototypes.h

@@ -235,10 +235,9 @@ innobase_casedn_str(
 #ifdef WITH_WSREP
 UNIV_INTERN
 int
-wsrep_innobase_kill_one_trx(void * const thd_ptr,
-                            const trx_t * const bf_trx,
+wsrep_innobase_kill_one_trx(THD* bf_thd,
                             trx_t *victim_trx,
-                            ibool signal);
+                            my_bool signal);
 int wsrep_innobase_mysql_sort(int mysql_type, uint charset_number,
                              unsigned char* str, unsigned int str_length,
                              unsigned int buf_length);

storage/innobase/include/trx0trx.h

@@ -34,6 +34,7 @@ Created 3/26/1996 Heikki Tuuri
 #include "trx0xa.h"
 #include "ut0vec.h"
 #include "fts0fts.h"
+#include "read0read.h"
 
 #include <vector>
 #include <set>
@@ -543,28 +544,11 @@ Check transaction state */
 	ut_error;							\
 } while (0)
 
-/** Check if transaction is free so that it can be re-initialized.
-@param t transaction handle */
-#define	assert_trx_is_free(t)	do {					\
-	ut_ad(trx_state_eq((t), TRX_STATE_NOT_STARTED));		\
-	ut_ad(!(t)->id);						\
-	ut_ad(!(t)->has_logged());					\
-	ut_ad(!(t)->is_referenced());					\
-	ut_ad(!(t)->is_wsrep());					\
-	ut_ad(!MVCC::is_view_active((t)->read_view));			\
-	ut_ad((t)->lock.wait_thr == NULL);				\
-	ut_ad(UT_LIST_GET_LEN((t)->lock.trx_locks) == 0);		\
-	ut_ad((t)->lock.table_locks.empty());				\
-	ut_ad(!(t)->autoinc_locks					\
-	      || ib_vector_is_empty((t)->autoinc_locks));		\
-	ut_ad((t)->dict_operation == TRX_DICT_OP_NONE);			\
-} while(0)
-
 /** Check if transaction is in-active so that it can be freed and put back to
 transaction pool.
 @param t transaction handle */
 #define assert_trx_is_inactive(t) do {					\
-	assert_trx_is_free((t));					\
+	t->assert_freed();						\
 	ut_ad((t)->dict_operation_lock_mode == 0);			\
 } while(0)
 
@@ -656,6 +640,11 @@ struct trx_lock_t {
 					lock_sys->mutex. Otherwise, this may
 					only be modified by the thread that is
 					serving the running transaction. */
+#ifdef WITH_WSREP
+	bool		was_chosen_as_wsrep_victim;
+					/*!< high priority wsrep thread has
+					marked this trx to abort */
+#endif /* WITH_WSREP */
 
 	/** Pre-allocated record locks */
 	struct {
@@ -1205,7 +1194,25 @@ struct trx_t {
   /** Free the memory to trx_pools */
   inline void free();
 
-
+  /** Check if transaction is free so that it can be re-initialized. */
+  void assert_freed()
+  {
+	ut_ad(state == TRX_STATE_NOT_STARTED);
+	ut_ad(!id);
+	ut_ad(!has_logged());
+	ut_ad(!is_referenced());
+	ut_ad(!is_wsrep());
+#ifdef WITH_WSREP
+	ut_ad(!lock.was_chosen_as_wsrep_victim);
+#endif
+	ut_ad(!MVCC::is_view_active(read_view));
+	ut_ad(lock.wait_thr == NULL);
+	ut_ad(UT_LIST_GET_LEN(lock.trx_locks) == 0);
+	ut_ad(lock.table_locks.empty());
+	ut_ad(!autoinc_locks
+	      || ib_vector_is_empty(autoinc_locks));
+	ut_ad(dict_operation == TRX_DICT_OP_NONE);
+  }
 private:
 	/** Assign a rollback segment for modifying temporary tables.
 	@return the assigned rollback segment */

storage/innobase/lock/lock0lock.cc

@@ -1149,7 +1149,7 @@ wsrep_kill_victim(
 			}
 
 			wsrep_innobase_kill_one_trx(trx->mysql_thd,
-						    trx, lock->trx, TRUE);
+						    lock->trx, true);
 		}
 	}
 }

storage/innobase/trx/trx0roll.cc

@@ -191,6 +191,7 @@ dberr_t trx_rollback_for_mysql(trx_t* trx)
 		ut_ad(trx->in_mysql_trx_list);
 #ifdef WITH_WSREP
 		trx->wsrep = false;
+		trx->lock.was_chosen_as_wsrep_victim= false;
 #endif
 		return(DB_SUCCESS);
 
@@ -412,9 +413,7 @@ trx_rollback_to_savepoint_for_mysql_low(
 	trx->op_info = "";
 
 #ifdef WITH_WSREP
-	if (trx->is_wsrep()) {
-		trx->lock.was_chosen_as_deadlock_victim = false;
-	}
+	trx->lock.was_chosen_as_wsrep_victim= false;
 #endif
 	return(err);
 }

storage/innobase/trx/trx0trx.cc

@@ -330,8 +330,7 @@ trx_t *trx_allocate_for_background()
 	MEM_CHECK_DEFINED() in trx_t::free(). */
 	MEM_MAKE_DEFINED(trx, sizeof *trx);
 #endif
-
-	assert_trx_is_free(trx);
+	trx->assert_freed();
 
 	mem_heap_t*	heap;
 	ib_alloc_t*	alloc;
@@ -1857,15 +1856,14 @@ trx_commit_in_memory(
 	trx->state = TRX_STATE_NOT_STARTED;
 #ifdef WITH_WSREP
 	trx->wsrep = false;
+	trx->lock.was_chosen_as_wsrep_victim= false;
 #endif
 
 	/* trx->in_mysql_trx_list would hold between
 	trx_allocate_for_mysql() and trx_free_for_mysql(). It does not
 	hold for recovered transactions or system transactions. */
-	assert_trx_is_free(trx);
-
+	trx->assert_freed();
 	trx_init(trx);
-
 	trx_mutex_exit(trx);
 
 	ut_a(trx->error_state == DB_SUCCESS);