MDEV-19746: Galera test failures because of wsrep_slave_threads identification

Problem was that tests select INFORMATION_SCHEMA.PROCESSLIST processes
from user system user and certain state. However, state could be unexpected.

Added new status variables that store current amount of applier threads
(wsrep_applier_thread_count) and rollbacker threads
(wsrep_rollbacker_thread_count). This will make clear how many slave threads
of certain type there is. Additionally, added THD state "wsrep applier idle"
when applier slave thread is waiting for work.

Modified tests to use wsrep_applier_thread_count when waiting slave threads
to start.

Changes to be committed:
	modified:   mysql-test/suite/galera/disabled.def
	modified:   mysql-test/suite/galera/r/MW-336.result
	modified:   mysql-test/suite/galera/r/galera_var_slave_threads.result
	modified:   mysql-test/suite/galera/t/MW-336.test
	modified:   mysql-test/suite/galera/t/galera_kill_applier.test
	modified:   mysql-test/suite/galera/t/galera_parallel_autoinc_largetrx.test
	modified:   mysql-test/suite/galera/t/galera_parallel_autoinc_manytrx.test
	modified:   mysql-test/suite/galera/t/galera_var_slave_threads.test
	modified:   mysql-test/suite/wsrep/disabled.def
	modified:   mysql-test/suite/wsrep/r/variables.result
	modified:   mysql-test/suite/wsrep/t/variables.test
	modified:   sql/wsrep_mysqld.cc
	modified:   sql/wsrep_mysqld.h
	modified:   sql/wsrep_thd.cc
	modified:   sql/wsrep_var.cc

mysql-test/suite/galera/disabled.def

@@ -15,7 +15,6 @@ MW-328A : MDEV-17847 Galera test failure on MW-328[A|B|C]
 MW-328B : MDEV-17847 Galera test failure on MW-328[A|B|C]
 MW-328C : MDEV-17847 Galera test failure on MW-328[A|B|C]
 MW-329 : MDEV-19962 Galera test failure on MW-329
-MW-336 : MDEV-19746 Galera test failures because of wsrep_slave_threads identification
 MW-388: MDEV-19803 Long semaphore wait error on galera.MW-388	
 galera_account_management : MariaDB 10.0 does not support ALTER USER
 galera_as_master_gtid : Requires MySQL GTID

mysql-test/suite/galera/r/MW-336.result

-CREATE TABLE t1 (f1 INTEGER) Engine=InnoDB;
-INSERT INTO t1 values(0);
+CREATE TABLE t1 (f1 INTEGER NOT NULL AUTO_INCREMENT PRIMARY KEY) Engine=InnoDB;
+INSERT INTO t1 values(NULL);
 connection node_1;
 SET GLOBAL wsrep_slave_threads = 10;
 SET GLOBAL wsrep_slave_threads = 1;
 # Wait 10 slave threads to start 1
+SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_applier_thread_count';
+VARIABLE_VALUE
+10
+SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_rollbacker_thread_count';
+VARIABLE_VALUE
+1
+SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_thread_count';
+VARIABLE_VALUE
+11
 connection node_2;
 SET SESSION wsrep_sync_wait=15;
+SELECT COUNT(*) = 1 FROM t1;
+COUNT(*) = 1
+1
 # Generate 100 replication events
 connection node_1;
 SET SESSION wsrep_sync_wait=15;
@@ -13,10 +25,28 @@ SELECT COUNT(*) FROM t1;
 COUNT(*)
 101
 # Wait 9 slave threads to exit 1
+SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_applier_thread_count';
+VARIABLE_VALUE
+1
+SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_rollbacker_thread_count';
+VARIABLE_VALUE
+1
+SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_thread_count';
+VARIABLE_VALUE
+2
 SET GLOBAL wsrep_slave_threads = 10;
 # Wait 10 slave threads to start 2
 SET GLOBAL wsrep_slave_threads = 20;
 # Wait 20 slave threads to start 3
+SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_applier_thread_count';
+VARIABLE_VALUE
+20
+SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_rollbacker_thread_count';
+VARIABLE_VALUE
+1
+SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_thread_count';
+VARIABLE_VALUE
+21
 SET GLOBAL wsrep_slave_threads = 1;
 connection node_2;
 # Generate 100 replication events
@@ -25,6 +55,15 @@ SELECT COUNT(*) FROM t1;
 COUNT(*)
 201
 # Wait 10 slave threads to exit 3
+SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_applier_thread_count';
+VARIABLE_VALUE
+1
+SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_rollbacker_thread_count';
+VARIABLE_VALUE
+1
+SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_thread_count';
+VARIABLE_VALUE
+2
 SET GLOBAL wsrep_slave_threads = 10;
 SET GLOBAL wsrep_slave_threads = 1;
 # Wait 10 slave threads to start 3
@@ -35,5 +74,13 @@ SELECT COUNT(*) FROM t1;
 COUNT(*)
 301
 # Wait 10 slave threads to exit 4
-connection node_1;
+SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_applier_thread_count';
+VARIABLE_VALUE
+1
+SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_rollbacker_thread_count';
+VARIABLE_VALUE
+1
+SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_thread_count';
+VARIABLE_VALUE
+2
 DROP TABLE t1;

mysql-test/suite/galera/r/galera_var_slave_threads.result

@@ -16,22 +16,19 @@ SET GLOBAL wsrep_slave_threads = 1;
 SELECT COUNT(*) FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND STATE LIKE '%wsrep aborter%';
 COUNT(*)
 1
-SET GLOBAL wsrep_slave_threads = 64;
 connection node_1;
 INSERT INTO t1 VALUES (1);
 connection node_2;
-SELECT COUNT(*) = 1 FROM t1;
-COUNT(*) = 1
+SELECT COUNT(*) FROM t1;
+COUNT(*)
 1
+SET GLOBAL wsrep_slave_threads = 64;
 SET GLOBAL wsrep_slave_threads = 1;
 connection node_1;
 connection node_2;
 SELECT COUNT(*) FROM t2;
 COUNT(*)
-64
-SELECT COUNT(*) FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND STATE LIKE '%wsrep aborter%';
-COUNT(*)
-1
+70
 SET GLOBAL wsrep_slave_threads = 1;
 DROP TABLE t1;
 DROP TABLE t2;
@@ -47,11 +44,12 @@ connection node_1;
 INSERT INTO t1 VALUES (DEFAULT);
 INSERT INTO t1 VALUES (DEFAULT);
 INSERT INTO t1 VALUES (DEFAULT);
+INSERT INTO t1 VALUES (DEFAULT);
+INSERT INTO t1 VALUES (DEFAULT);
+INSERT INTO t1 VALUES (DEFAULT);
+INSERT INTO t1 VALUES (DEFAULT);
+INSERT INTO t1 VALUES (DEFAULT);
 DROP TABLE t1;
 connection node_2;
 SELECT NAME FROM INFORMATION_SCHEMA.INNODB_SYS_TABLES WHERE NAME LIKE 'test/t%';
 NAME
-SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND STATE LIKE '%wsrep aborter%';
-COUNT(*) = 1
-1
-# End of tests

mysql-test/suite/galera/t/MW-336.test

@@ -5,8 +5,8 @@
 --source include/galera_cluster.inc
 --source include/have_innodb.inc
 
-CREATE TABLE t1 (f1 INTEGER) Engine=InnoDB;
-INSERT INTO t1 values(0);
+CREATE TABLE t1 (f1 INTEGER NOT NULL AUTO_INCREMENT PRIMARY KEY) Engine=InnoDB;
+INSERT INTO t1 values(NULL);
 
 --connection node_1
 
@@ -15,21 +15,24 @@ SET GLOBAL wsrep_slave_threads = 1;
 
 --echo # Wait 10 slave threads to start 1
 --let $wait_timeout=600
---let $wait_condition = SELECT COUNT(*) = 11 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND (STATE IS NULL OR STATE NOT LIKE 'InnoDB%');
+--let $wait_condition = SELECT VARIABLE_VALUE = 10 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_applier_thread_count';
 --source include/wait_condition.inc
 
+SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_applier_thread_count';
+SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_rollbacker_thread_count';
+SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_thread_count';
+
 --connection node_2
 SET SESSION wsrep_sync_wait=15;
-# Wait until inserts are replicated
---let $wait_condition = SELECT COUNT(*) = 1 FROM t1;
---source include/wait_condition.inc
+SELECT COUNT(*) = 1 FROM t1;
+
 --echo # Generate 100 replication events
 --disable_query_log
 --disable_result_log
 --let $count = 100
 while ($count)
 {
-  INSERT INTO t1 VALUES (1);
+  INSERT INTO t1 VALUES (NULL);
   --dec $count
 }
 --enable_result_log
@@ -42,21 +45,29 @@ SELECT COUNT(*) FROM t1;
 
 --echo # Wait 9 slave threads to exit 1
 # Wait until appliers exit
---let $wait_condition = SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND (STATE IS NULL OR STATE NOT LIKE 'InnoDB%');
+--let $wait_condition = SELECT VARIABLE_VALUE = 1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_applier_thread_count';
 --source include/wait_condition.inc
 
+SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_applier_thread_count';
+SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_rollbacker_thread_count';
+SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_thread_count';
+
 SET GLOBAL wsrep_slave_threads = 10;
 
 --echo # Wait 10 slave threads to start 2
---let $wait_condition = SELECT COUNT(*) = 11 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND (STATE IS NULL OR STATE NOT LIKE 'InnoDB%');
+--let $wait_condition = SELECT VARIABLE_VALUE = 10 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_applier_thread_count';
 --source include/wait_condition.inc
 
 SET GLOBAL wsrep_slave_threads = 20;
 
 --echo # Wait 20 slave threads to start 3
---let $wait_condition = SELECT COUNT(*) = 21 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND (STATE IS NULL OR STATE NOT LIKE 'InnoDB%');
+--let $wait_condition = SELECT VARIABLE_VALUE = 20 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_applier_thread_count';
 --source include/wait_condition.inc
 
+SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_applier_thread_count';
+SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_rollbacker_thread_count';
+SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_thread_count';
+
 SET GLOBAL wsrep_slave_threads = 1;
 
 --connection node_2
@@ -66,7 +77,7 @@ SET GLOBAL wsrep_slave_threads = 1;
 --let $count = 100
 while ($count)
 {
-  INSERT INTO t1 VALUES (1);
+  INSERT INTO t1 VALUES (NULL);
   --dec $count
 }
 --enable_query_log
@@ -77,15 +88,19 @@ SELECT COUNT(*) FROM t1;
 
 --echo # Wait 10 slave threads to exit 3
 # Wait until appliers exit
---let $wait_condition = SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND (STATE IS NULL OR STATE NOT LIKE 'InnoDB%');
+--let $wait_condition = SELECT VARIABLE_VALUE = 1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_applier_thread_count';
 --source include/wait_condition.inc
 
+SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_applier_thread_count';
+SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_rollbacker_thread_count';
+SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_thread_count';
+
 SET GLOBAL wsrep_slave_threads = 10;
 SET GLOBAL wsrep_slave_threads = 1;
 
 --echo # Wait 10 slave threads to start 3
 --let $wait_timeout=600
---let $wait_condition = SELECT COUNT(*) = 11 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND (STATE IS NULL OR STATE NOT LIKE 'InnoDB%');
+--let $wait_condition = SELECT VARIABLE_VALUE = 10 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_applier_thread_count';
 --source include/wait_condition.inc
 
 --connection node_2
@@ -95,7 +110,7 @@ SET GLOBAL wsrep_slave_threads = 1;
 --let $count = 100
 while ($count)
 {
-  INSERT INTO t1 VALUES (1);
+  INSERT INTO t1 VALUES (NULL);
   --dec $count
 }
 --enable_result_log
@@ -106,8 +121,11 @@ SELECT COUNT(*) FROM t1;
 
 --echo # Wait 10 slave threads to exit 4
 # Wait until appliers exit
---let $wait_condition = SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND (STATE IS NULL OR STATE NOT LIKE 'InnoDB%');
+--let $wait_condition = SELECT VARIABLE_VALUE = 1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_applier_thread_count';
 --source include/wait_condition.inc
 
---connection node_1
+SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_applier_thread_count';
+SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_rollbacker_thread_count';
+SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_thread_count';
+
 DROP TABLE t1;

mysql-test/suite/galera/t/galera_kill_applier.test

@@ -6,7 +6,11 @@
 --source include/have_innodb.inc
 
 --connection node_1
---let $applier_thread = `SELECT ID FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND STATE IS NULL LIMIT 1`
+--let $wait_timeout=600
+--let $wait_condition = SELECT VARIABLE_VALUE = 1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_applier_thread_count';
+--source include/wait_condition.inc
+
+--let $applier_thread = `SELECT ID FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND STATE = 'wsrep applier idle' LIMIT 1`
 
 --disable_query_log
 --error ER_KILL_DENIED_ERROR,ER_KILL_DENIED_ERROR

mysql-test/suite/galera/t/galera_parallel_autoinc_largetrx.test

@@ -20,11 +20,11 @@ CREATE TABLE t1 (f1 INTEGER AUTO_INCREMENT PRIMARY KEY, f2 INTEGER) Engine=InnoD
 --connection node_2
 set session wsrep_sync_wait=15;
 --let $wsrep_slave_threads_orig = `SELECT @@wsrep_slave_threads`
+
 SET GLOBAL wsrep_slave_threads = 4;
---let $wait_condition = SELECT COUNT(*) = @@wsrep_slave_threads + 1 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND (STATE IS NULL OR STATE NOT LIKE 'InnoDB%');
+--let $wait_condition = SELECT VARIABLE_VALUE = 4 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_applier_thread_count';
 --source include/wait_condition.inc
 
-
 --connection node_1
 --send INSERT INTO t1 (f2) SELECT 1 FROM ten AS a1, ten AS a2, ten AS a3, ten AS a4;
 

mysql-test/suite/galera/t/galera_parallel_autoinc_manytrx.test

@@ -24,8 +24,9 @@ CREATE TABLE t1 (f1 INTEGER AUTO_INCREMENT PRIMARY KEY, f2 INTEGER) Engine=InnoD
 --connection node_2
 set session wsrep_sync_wait=15;
 --let $wsrep_slave_threads_orig = `SELECT @@wsrep_slave_threads`
+
 SET GLOBAL wsrep_slave_threads = 4;
---let $wait_condition = SELECT COUNT(*) = @@wsrep_slave_threads + 1 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND (STATE IS NULL OR STATE NOT LIKE 'InnoDB%');
+--let $wait_condition = SELECT VARIABLE_VALUE = 4 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_applier_thread_count';
 --source include/wait_condition.inc
 
 --connection node_1

mysql-test/suite/galera/t/galera_var_slave_threads.test

@@ -6,9 +6,9 @@
 
 --source include/galera_cluster.inc
 --source include/have_innodb.inc
---let $wsrep_slave_threads_orig = `SELECT @@wsrep_slave_threads`
 
 --connection node_1
+--let $wsrep_slave_threads_orig = `SELECT @@wsrep_slave_threads`
 CREATE TABLE t1 (f1 INT PRIMARY KEY) Engine=InnoDB;
 CREATE TABLE t2 (f1 INT AUTO_INCREMENT PRIMARY KEY) Engine=InnoDB;
 
@@ -23,23 +23,23 @@ SET GLOBAL wsrep_slave_threads = 1;
 # There is a separate wsrep_aborter thread at all times
 SELECT COUNT(*) FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND STATE LIKE '%wsrep aborter%';
 
-#
-# Increase the number of slave threads. The change takes effect immediately
-#
-
-SET GLOBAL wsrep_slave_threads = 64;
-
 --connection node_1
 INSERT INTO t1 VALUES (1);
 
 --connection node_2
+
 --let $wait_timeout=600
 --let $wait_condition = SELECT COUNT(*) = 1 FROM t1;
 --source include/wait_condition.inc
 
-SELECT COUNT(*) = 1 FROM t1;
+SELECT COUNT(*) FROM t1;
+
+#
+# Increase the number of slave threads. The change takes effect immediately
+#
+SET GLOBAL wsrep_slave_threads = 64;
 
---let $wait_condition = SELECT COUNT(*) = @@wsrep_slave_threads + 1 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND (STATE IS NULL OR STATE NOT LIKE 'InnoDB%');
+--let $wait_condition = SELECT VARIABLE_VALUE = 64 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_applier_thread_count';
 --source include/wait_condition.inc
 
 #
@@ -52,8 +52,8 @@ SET GLOBAL wsrep_slave_threads = 1;
 
 --disable_result_log
 --disable_query_log
-# Generate 64 replication events
---let $count = 64
+# Generate 70 replication events
+--let $count = 70
 while ($count)
 {
   INSERT INTO t2 VALUES (DEFAULT);
@@ -65,10 +65,8 @@ while ($count)
 --connection node_2
 SELECT COUNT(*) FROM t2;
 
---let $wait_condition = SELECT COUNT(*) = @@wsrep_slave_threads + 1 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND (STATE IS NULL OR STATE NOT LIKE 'InnoDB%')
+--let $wait_condition = SELECT VARIABLE_VALUE = 1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_applier_thread_count';
 --source include/wait_condition.inc
-SELECT COUNT(*) FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND STATE LIKE '%wsrep aborter%';
-
 
 --eval SET GLOBAL wsrep_slave_threads = $wsrep_slave_threads_orig
 
@@ -84,7 +82,7 @@ CREATE TABLE t1 (i INT AUTO_INCREMENT PRIMARY KEY) ENGINE=INNODB;
 
 --connection node_2
 SET GLOBAL wsrep_slave_threads = 4;
---let $wait_condition = SELECT COUNT(*) = @@wsrep_slave_threads + 1 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND (STATE IS NULL OR STATE NOT LIKE 'InnoDB%')
+--let $wait_condition = SELECT VARIABLE_VALUE = 4 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_applier_thread_count';
 --source include/wait_condition.inc
 
 SET GLOBAL wsrep_slave_threads = 1;
@@ -93,21 +91,20 @@ SET GLOBAL wsrep_slave_threads = 1;
 INSERT INTO t1 VALUES (DEFAULT);
 INSERT INTO t1 VALUES (DEFAULT);
 INSERT INTO t1 VALUES (DEFAULT);
+INSERT INTO t1 VALUES (DEFAULT);
+INSERT INTO t1 VALUES (DEFAULT);
+INSERT INTO t1 VALUES (DEFAULT);
+INSERT INTO t1 VALUES (DEFAULT);
+INSERT INTO t1 VALUES (DEFAULT);
 DROP TABLE t1;
 
 --connection node_2
 
 # Wait until above DDL is replicated
---let $wait_condition = SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.INNODB_SYS_TABLES WHERE NAME LIKE 'test/t%';
---source include/wait_condition.inc
-
-SELECT NAME FROM INFORMATION_SCHEMA.INNODB_SYS_TABLES WHERE NAME LIKE 'test/t%';
-
 #
 # make sure that we are left with exactly one applier thread before we leaving the test
 #
---let $wait_condition = SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND (STATE IS NULL OR STATE NOT LIKE 'InnoDB%')
+--let $wait_condition = SELECT VARIABLE_VALUE = 1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_applier_thread_count';
 --source include/wait_condition.inc
-SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND STATE LIKE '%wsrep aborter%';
 
---echo # End of tests
+SELECT NAME FROM INFORMATION_SCHEMA.INNODB_SYS_TABLES WHERE NAME LIKE 'test/t%';

mysql-test/suite/wsrep/disabled.def

@@ -9,5 +9,3 @@
 #  Do not use any TAB characters for whitespace.
 #
 ##############################################################################
-
-variables : MDEV-19746 Galera test failures because of wsrep_slave_threads identification

mysql-test/suite/wsrep/r/variables.result

@@ -22,6 +22,7 @@ CALL mtr.add_suppression("WSREP: Could not open saved state file for reading.*")
 # wsrep
 SHOW GLOBAL STATUS LIKE 'wsrep%';
 Variable_name	Value
+wsrep_applier_thread_count	#
 wsrep_apply_oooe	#
 wsrep_apply_oool	#
 wsrep_apply_window	#
@@ -75,6 +76,7 @@ wsrep_repl_keys_bytes	#
 wsrep_repl_other_bytes	#
 wsrep_replicated	#
 wsrep_replicated_bytes	#
+wsrep_rollbacker_thread_count	#
 wsrep_thread_count	#
 SHOW GLOBAL STATUS LIKE 'wsrep_local_state_comment';
 Variable_name	Value
@@ -133,6 +135,15 @@ wsrep_thread_count	0
 # applier/rollbacker threads.
 SET GLOBAL wsrep_cluster_address= 'gcomm://';
 # Wait for applier threads to get created.
+SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_applier_thread_count';
+VARIABLE_VALUE
+1
+SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_rollbacker_thread_count';
+VARIABLE_VALUE
+1
+SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_thread_count';
+VARIABLE_VALUE
+2
 SELECT @@global.wsrep_provider;
 @@global.wsrep_provider
 libgalera_smm.so
@@ -149,12 +160,18 @@ wsrep_thread_count	2
 SET @wsrep_slave_threads_saved= @@global.wsrep_slave_threads;
 SET GLOBAL wsrep_slave_threads= 10;
 # Wait for applier threads to get created.
+SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_applier_thread_count';
+VARIABLE_VALUE
+10
+SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_rollbacker_thread_count';
+VARIABLE_VALUE
+1
+SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_thread_count';
+VARIABLE_VALUE
+11
 SHOW STATUS LIKE 'threads_connected';
 Variable_name	Value
 Threads_connected	1
-SHOW STATUS LIKE 'wsrep_thread_count';
-Variable_name	Value
-wsrep_thread_count	11
 set wsrep_on=0;
 set wsrep_on=1;
 create user test@localhost;

mysql-test/suite/wsrep/t/variables.test

@@ -103,10 +103,15 @@ SHOW STATUS LIKE 'wsrep_thread_count';
 SET GLOBAL wsrep_cluster_address= 'gcomm://';
 
 --echo # Wait for applier threads to get created.
+
 --let $wait_timeout=600
---let $wait_condition = SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND (STATE IS NULL OR STATE NOT LIKE 'InnoDB%');
+--let $wait_condition = SELECT VARIABLE_VALUE = 1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_applier_thread_count';
 --source include/wait_condition.inc
 
+SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_applier_thread_count';
+SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_rollbacker_thread_count';
+SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_thread_count';
+
 --replace_regex /.*libgalera_smm.*/libgalera_smm.so/
 SELECT @@global.wsrep_provider;
 SELECT @@global.wsrep_cluster_address;
@@ -118,11 +123,14 @@ SET @wsrep_slave_threads_saved= @@global.wsrep_slave_threads;
 SET GLOBAL wsrep_slave_threads= 10;
 
 --echo # Wait for applier threads to get created.
---let $wait_condition = SELECT COUNT(*) = 11 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND (STATE IS NULL OR STATE NOT LIKE 'InnoDB%');
+--let $wait_condition = SELECT VARIABLE_VALUE = 10 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_applier_thread_count';
 --source include/wait_condition.inc
 
+SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_applier_thread_count';
+SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_rollbacker_thread_count';
+SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_thread_count';
+
 SHOW STATUS LIKE 'threads_connected';
-SHOW STATUS LIKE 'wsrep_thread_count';
 
 #
 # privileges for wsrep_on

sql/wsrep_mysqld.cc

@@ -39,6 +39,7 @@
 #include "log_event.h"
 #include <slave.h>
 #include "sql_plugin.h"                         /* wsrep_plugins_pre_init() */
+#include <vector>
 
 wsrep_t *wsrep                  = NULL;
 /*
@@ -135,7 +136,11 @@ mysql_mutex_t LOCK_wsrep_desync;
 mysql_mutex_t LOCK_wsrep_config_state;
 
 int wsrep_replaying= 0;
-ulong  wsrep_running_threads = 0; // # of currently running wsrep threads
+ulong  wsrep_running_threads = 0; // # of currently running wsrep
+				  // # threads
+ulong  wsrep_running_applier_threads = 0; // # of running applier threads
+ulong  wsrep_running_rollbacker_threads = 0; // # of running
+					     // # rollbacker threads
 ulong  my_bind_addr;
 
 #ifdef HAVE_PSI_INTERFACE
@@ -2021,7 +2026,8 @@ bool wsrep_grant_mdl_exception(MDL_context *requestor_ctx,
 pthread_handler_t start_wsrep_THD(void *arg)
 {
   THD *thd;
-  wsrep_thd_processor_fun processor= (wsrep_thd_processor_fun)arg;
+  wsrep_thread_args* args= (wsrep_thread_args*)arg;
+  wsrep_thd_processor_fun processor= args->processor;
 
   if (my_thread_init() || (!(thd= new THD(next_thread_id(), true))))
   {
@@ -2099,6 +2105,19 @@ pthread_handler_t start_wsrep_THD(void *arg)
 
   mysql_mutex_lock(&LOCK_thread_count);
   wsrep_running_threads++;
+
+  switch (args->thread_type) {
+    case WSREP_APPLIER_THREAD:
+      wsrep_running_applier_threads++;
+      break;
+    case WSREP_ROLLBACKER_THREAD:
+      wsrep_running_rollbacker_threads++;
+      break;
+    default:
+      WSREP_ERROR("Incorrect wsrep thread type: %d", args->thread_type);
+      break;
+  }
+
   mysql_cond_broadcast(&COND_thread_count);
   mysql_mutex_unlock(&LOCK_thread_count);
 
@@ -2107,7 +2126,25 @@ pthread_handler_t start_wsrep_THD(void *arg)
   close_connection(thd, 0);
 
   mysql_mutex_lock(&LOCK_thread_count);
+  DBUG_ASSERT(wsrep_running_threads > 0);
   wsrep_running_threads--;
+
+  switch (args->thread_type) {
+    case WSREP_APPLIER_THREAD:
+      DBUG_ASSERT(wsrep_running_applier_threads > 0);
+      wsrep_running_applier_threads--;
+      break;
+    case WSREP_ROLLBACKER_THREAD:
+      DBUG_ASSERT(wsrep_running_rollbacker_threads > 0);
+      wsrep_running_rollbacker_threads--;
+      break;
+    default:
+      WSREP_ERROR("Incorrect wsrep thread type: %d", args->thread_type);
+      break;
+  }
+
+  my_free(args);
+
   WSREP_DEBUG("wsrep running threads now: %lu", wsrep_running_threads);
   mysql_cond_broadcast(&COND_thread_count);
   mysql_mutex_unlock(&LOCK_thread_count);
@@ -2141,6 +2178,8 @@ pthread_handler_t start_wsrep_THD(void *arg)
 error:
   WSREP_ERROR("Failed to create/initialize system thread");
 
+  my_free(args);
+
   /* Abort if its the first applier/rollbacker thread. */
   if (!mysqld_server_initialized)
     unireg_abort(1);

sql/wsrep_mysqld.h

@@ -31,6 +31,7 @@ typedef struct st_mysql_show_var SHOW_VAR;
 #include "mysqld.h"
 #include "sql_table.h"
 #include "wsrep_mysqld_c.h"
+#include <vector>
 
 #define WSREP_UNDEFINED_TRX_ID ULONGLONG_MAX
 
@@ -89,6 +90,8 @@ extern my_bool     wsrep_restart_slave_activated;
 extern my_bool     wsrep_slave_FK_checks;
 extern my_bool     wsrep_slave_UK_checks;
 extern ulong       wsrep_running_threads;
+extern ulong       wsrep_running_applier_threads;
+extern ulong       wsrep_running_rollbacker_threads;
 extern bool        wsrep_new_cluster;
 extern bool        wsrep_gtid_mode;
 extern uint32      wsrep_gtid_domain_id;
@@ -310,7 +313,21 @@ void thd_binlog_flush_pending_rows_event(THD *thd, bool stmt_end);
 void thd_binlog_rollback_stmt(THD * thd);
 void thd_binlog_trx_reset(THD * thd);
 
+enum wsrep_thread_type {
+  WSREP_APPLIER_THREAD=1,
+  WSREP_ROLLBACKER_THREAD=2
+};
+
 typedef void (*wsrep_thd_processor_fun)(THD *);
+
+typedef struct {
+	pthread_t thread_id;
+	wsrep_thd_processor_fun processor;
+	enum wsrep_thread_type thread_type;
+} wsrep_thread_args;
+
+extern std::vector<wsrep_thread_args*> wsrep_thread_arg;
+
 pthread_handler_t start_wsrep_THD(void *arg);
 int wsrep_wait_committing_connections_close(int wait_time);
 extern void wsrep_close_client_connections(my_bool wait_to_end,

sql/wsrep_thd.cc

@@ -360,6 +360,7 @@ static void wsrep_replication_process(THD *thd)
   thd->variables.option_bits|= OPTION_BEGIN;
   thd->server_status|= SERVER_STATUS_IN_TRANS;
 
+  thd_proc_info(thd, "wsrep applier idle");
   rcode = wsrep->recv(wsrep, (void *)thd);
   DBUG_PRINT("wsrep",("wsrep_repl returned: %d", rcode));
 
@@ -415,13 +416,12 @@ static void wsrep_replication_process(THD *thd)
   DBUG_VOID_RETURN;
 }
 
-static bool create_wsrep_THD(wsrep_thd_processor_fun processor)
+static bool create_wsrep_THD(wsrep_thread_args* args)
 {
   ulong old_wsrep_running_threads= wsrep_running_threads;
-  pthread_t unused;
   mysql_mutex_lock(&LOCK_thread_count);
-  bool res= pthread_create(&unused, &connection_attrib, start_wsrep_THD,
-                           (void*)processor);
+  bool res= pthread_create(&args->thread_id, &connection_attrib, start_wsrep_THD,
+                           (void*)args);
   /*
     if starting a thread on server startup, wait until the this thread's THD
     is fully initialized (otherwise a THD initialization code might
@@ -451,8 +451,20 @@ void wsrep_create_appliers(long threads)
 
   long wsrep_threads=0;
   while (wsrep_threads++ < threads) {
-    if (create_wsrep_THD(wsrep_replication_process))
+    wsrep_thread_args* arg;
+    if((arg = (wsrep_thread_args*)my_malloc(sizeof(wsrep_thread_args), MYF(0))) == NULL) {
+      WSREP_ERROR("Can't allocate memory for wsrep replication thread %ld\n", wsrep_threads);
+      assert(0);
+    }
+
+    arg->thread_type = WSREP_APPLIER_THREAD;
+    arg->processor = wsrep_replication_process;
+
+    if (create_wsrep_THD(arg)) {
       WSREP_WARN("Can't create thread to manage wsrep replication");
+      my_free(arg);
+      return;
+    }
   }
 }
 
@@ -539,9 +551,21 @@ void wsrep_create_rollbacker()
 {
   if (wsrep_provider && strcasecmp(wsrep_provider, "none"))
   {
+    wsrep_thread_args* arg;
+    if((arg = (wsrep_thread_args*)my_malloc(sizeof(wsrep_thread_args), MYF(0))) == NULL) {
+      WSREP_ERROR("Can't allocate memory for wsrep rollbacker thread\n");
+      assert(0);
+    }
+
+    arg->thread_type = WSREP_ROLLBACKER_THREAD;
+    arg->processor = wsrep_rollback_process;
+
     /* create rollbacker */
-    if (create_wsrep_THD(wsrep_rollback_process))
+    if (create_wsrep_THD(arg)) {
       WSREP_WARN("Can't create thread to manage wsrep rollback");
+      my_free(arg);
+      return;
+    }
   }
 }
 

sql/wsrep_var.cc

@@ -708,7 +708,9 @@ static SHOW_VAR wsrep_status_vars[]=
   {"provider_name",     (char*) &wsrep_provider_name,     SHOW_CHAR_PTR},
   {"provider_version",  (char*) &wsrep_provider_version,  SHOW_CHAR_PTR},
   {"provider_vendor",   (char*) &wsrep_provider_vendor,   SHOW_CHAR_PTR},
-  {"thread_count",      (char*) &wsrep_running_threads,   SHOW_LONG_NOFLUSH}
+  {"thread_count",      (char*) &wsrep_running_threads,   SHOW_LONG_NOFLUSH},
+  {"applier_thread_count", (char*)&wsrep_running_applier_threads, SHOW_LONG_NOFLUSH},
+  {"rollbacker_thread_count", (char *)&wsrep_running_rollbacker_threads, SHOW_LONG_NOFLUSH},
 };
 
 static int show_var_cmp(const void *var1, const void *var2)