MDEV-33036 : Galera test case galera_3nodes.galera_ist_gcache_rollover has warning

Correct used configuration and force server restarts before test
case. Add wait condition instead of sleep to verify that
all expected nodes are back to cluster.
Signed-off-by: Julius Goryavsky <julius.goryavsky@mariadb.com>

mysql-test/suite/galera_3nodes/r/galera_ist_gcache_rollover.result

@@ -9,6 +9,7 @@ INSERT INTO t1 VALUES (01), (02), (03), (04), (05);
 connection node_2;
 Unloading wsrep provider ...
 SET GLOBAL wsrep_cluster_address = '';
+connection node_1;
 connection node_3;
 Unloading wsrep provider ...
 SET GLOBAL wsrep_cluster_address = '';
@@ -33,14 +34,16 @@ SET GLOBAL wsrep_provider_options = 'dbug=';
 SET GLOBAL wsrep_provider_options = 'signal=ist_sender_send_after_get_buffers';
 INSERT INTO t1 VALUES (51), (52), (53), (54), (55);
 connection node_2;
+connection node_1;
 connection node_3;
+connection node_1;
 connection node_2;
-SELECT COUNT(*) = 30 FROM t1;
-COUNT(*) = 30
-1
-SELECT COUNT(*) = 3 FROM t2;
-COUNT(*) = 3
-1
+SELECT COUNT(*) AS EXPECT_30 FROM t1;
+EXPECT_30
+30
+SELECT COUNT(*) AS EXPECT_3 FROM t2;
+EXPECT_3
+3
 SELECT LENGTH(f1) = 512 * 1024 FROM t2;
 LENGTH(f1) = 512 * 1024
 1
@@ -48,12 +51,12 @@ LENGTH(f1) = 512 * 1024
 1
 CALL mtr.add_suppression("WSREP: Unsupported protocol downgrade: incremental data collection disabled");
 connection node_3;
-SELECT COUNT(*) = 30 FROM t1;
-COUNT(*) = 30
-1
-SELECT COUNT(*) = 3 FROM t2;
-COUNT(*) = 3
-1
+SELECT COUNT(*) AS EXPECT_30 FROM t1;
+EXPECT_30
+30
+SELECT COUNT(*) AS EXPECT_3 FROM t2;
+EXPECT_3
+3
 SELECT LENGTH(f1) = 512 * 1024 FROM t2;
 LENGTH(f1) = 512 * 1024
 1

mysql-test/suite/galera_3nodes/r/galera_join_with_cc_B.result

 connection node_2;
 connection node_1;
 connection node_1;
+connection node_2;
+connection node_3;
+connection node_1;
 CREATE TABLE t1 (pk INT PRIMARY KEY, node INT) ENGINE=innodb;
 INSERT INTO t1 VALUES (1, 1);
 connection node_2;
@@ -89,3 +92,7 @@ connection node_2;
 call mtr.add_suppression("WSREP: Rejecting JOIN message from \(.*\): new State Transfer required.");
 connection node_3;
 call mtr.add_suppression("WSREP: Rejecting JOIN message from \(.*\): new State Transfer required.");
+disconnect node_1a;
+disconnect node_3;
+disconnect node_2;
+disconnect node_1;

mysql-test/suite/galera_3nodes/t/galera_ist_gcache_rollover.cnf

@@ -2,10 +2,22 @@
 
 [mysqld.1]
 wsrep_provider_options='repl.causal_read_timeout=PT90S;base_port=@mysqld.1.#galera_port;evs.suspect_timeout=PT10S;evs.inactive_timeout=PT30S;evs.install_timeout=PT15S;pc.ignore_sb=true;gcache.size=1M'
+auto_increment_increment=1
+auto_increment_offset=1
+# this will force server restarts before this test
+loose-galera-ist-gcache-rollover=1
+wsrep-debug=1
 
 [mysqld.2]
 wsrep_provider_options='repl.causal_read_timeout=PT90S;base_port=@mysqld.2.#galera_port;evs.suspect_timeout=PT10S;evs.inactive_timeout=PT30S;evs.install_timeout=PT15S;pc.ignore_sb=true;gcache.size=1M'
+auto_increment_increment=2
+auto_increment_offset=2
+loose-galera-ist-gcache-rollover=2
+wsrep-debug=1
 
 [mysqld.3]
 wsrep_provider_options='repl.causal_read_timeout=PT90S;base_port=@mysqld.3.#galera_port;evs.suspect_timeout=PT10S;evs.inactive_timeout=PT30S;evs.install_timeout=PT15S;pc.ignore_sb=true;gcache.size=1M'
-
+auto_increment_increment=3
+auto_increment_offset=3
+loose-galera-ist-gcache-rollover=3
+wsrep-debug=1

mysql-test/suite/galera_3nodes/t/galera_ist_gcache_rollover.test

@@ -12,6 +12,7 @@
 --source include/have_innodb.inc
 --source include/have_debug_sync.inc
 --source include/galera_have_debug_sync.inc
+--source include/force_restart.inc
 
 --let $galera_connection_name = node_3
 --let $galera_server_number = 3
@@ -24,6 +25,9 @@
 --source ../galera/include/auto_increment_offset_save.inc
 
 --connection node_1
+--let $wait_condition = SELECT VARIABLE_VALUE = 3 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'
+--let $wait_condition_on_error_output = SELECT * FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME LIKE 'wsrep%'; show processlist
+--source include/wait_condition_with_debug.inc
 CREATE TABLE t1 (f1 INTEGER PRIMARY KEY);
 INSERT INTO t1 VALUES (01), (02), (03), (04), (05);
 
@@ -32,12 +36,15 @@ INSERT INTO t1 VALUES (01), (02), (03), (04), (05);
 --let $wsrep_cluster_address_orig2 = `select @@wsrep_cluster_address`
 --source suite/galera/include/galera_stop_replication.inc
 
+--connection node_1
+--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'
+--source include/wait_condition.inc
+
 --connection node_3
 --let $wsrep_cluster_address_orig3 = `select @@wsrep_cluster_address`
 --source suite/galera/include/galera_stop_replication.inc
 
 --connection node_1
---source include/wait_until_connected_again.inc
 INSERT INTO t1 VALUES (11), (12), (13), (14), (15);
 
 # Wait until nodes #2 and #3 have left
@@ -88,29 +95,39 @@ INSERT INTO t1 VALUES (51), (52), (53), (54), (55);
 --connection node_2
 --source include/wait_until_connected_again.inc
 
+--connection node_1
+--let $wait_condition = SELECT VARIABLE_VALUE = 2 OR VARIABLE_VALUE = 3 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'
+--source include/wait_condition.inc
+
 --connection node_3
 --source include/wait_until_connected_again.inc
 
-sleep 5;
+--connection node_1
+--let $wait_condition = SELECT VARIABLE_VALUE = 3 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'
+--source include/wait_condition.inc
 
 # Final checks 
 --connection node_2
-SELECT COUNT(*) = 30 FROM t1;
-SELECT COUNT(*) = 3 FROM t2;
+--let $wait_condition = SELECT COUNT(*) = 30 FROM t1
+--source include/wait_condition.inc
+
+SELECT COUNT(*) AS EXPECT_30 FROM t1;
+SELECT COUNT(*) AS EXPECT_3 FROM t2;
 SELECT LENGTH(f1) = 512 * 1024 FROM t2;
 CALL mtr.add_suppression("WSREP: Unsupported protocol downgrade: incremental data collection disabled");
 
 # Final checks
 --connection node_3
-SELECT COUNT(*) = 30 FROM t1;
-SELECT COUNT(*) = 3 FROM t2;
+--let $wait_condition = SELECT COUNT(*) = 30 FROM t1
+--source include/wait_condition.inc
+SELECT COUNT(*) AS EXPECT_30 FROM t1;
+SELECT COUNT(*) AS EXPECT_3 FROM t2;
 SELECT LENGTH(f1) = 512 * 1024 FROM t2;
 CALL mtr.add_suppression("WSREP: Unsupported protocol downgrade: incremental data collection disabled");
 
 DROP TABLE t1, t2;
 
 # Restore original auto_increment_offset values.
---source ../galera/include/auto_increment_offset_restore.inc
-
 --let $galera_cluster_size=3
+--source ../galera/include/auto_increment_offset_restore.inc
 --source include/galera_end.inc

mysql-test/suite/galera_3nodes/t/galera_join_with_cc_B.test

@@ -15,6 +15,12 @@
 --let $galera_server_number = 3
 --source include/galera_connect.inc
 
+# Save original auto_increment_offset values.
+--let $node_1=node_1
+--let $node_2=node_2
+--let $node_3=node_3
+--source ../galera/include/auto_increment_offset_save.inc
+
 --connection node_1
 --let $wait_condition = SELECT VARIABLE_VALUE = 3 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size';
 --source include/wait_condition.inc
@@ -271,3 +277,10 @@ call mtr.add_suppression("WSREP: Rejecting JOIN message from \(.*\): new State T
 
 --connection node_3
 call mtr.add_suppression("WSREP: Rejecting JOIN message from \(.*\): new State Transfer required.");
+
+--disconnect node_1a
+
+# Restore original auto_increment_offset values.
+--let $galera_cluster_size=3
+--source ../galera/include/auto_increment_offset_restore.inc
+--source include/galera_end.inc