MDEV-33897 : Galera test failure on galera_3nodes.galera_gtid_consistency

Based on logs SST was started before donor reached
Primaty state. Add wait_conditions to make sure that
nodes reach Primary state before starting next node.
Signed-off-by: Julius Goryavsky <julius.goryavsky@mariadb.com>

mysql-test/suite/galera_3nodes/r/galera_gtid_consistency.result

 connection node_2;
 connection node_1;
 connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3;
+connection node_1;
+connection node_2;
+connection node_3;
 connect node_2b, 127.0.0.1, root, , test, $NODE_MYPORT_2;
 set wsrep_sync_wait=0;
 connect node_1b, 127.0.0.1, root, , test, $NODE_MYPORT_1;
@@ -44,9 +47,9 @@ connection node_1b;
 connection node_1;
 connection node_3;
 connection node_1;
-CALL insert_row('node1', 500);
+CALL insert_row('node1', 100);
 connection node_3;
-CALL insert_row('node3', 500);
+CALL insert_row('node3', 100);
 CREATE TABLE t2(i int primary key) engine=innodb;
 connection node_2;
 # Restart node_2
@@ -60,7 +63,7 @@ Variable_name	Value
 wsrep_gtid_domain_id	1111
 show variables like '%gtid_binlog_pos%';
 Variable_name	Value
-gtid_binlog_pos	1111-1-2503
+gtid_binlog_pos	1111-1-1703
 connection node_2;
 # GTID in node2
 show variables like 'wsrep_gtid_domain_id';
@@ -68,7 +71,7 @@ Variable_name	Value
 wsrep_gtid_domain_id	1111
 show variables like '%gtid_binlog_pos%';
 Variable_name	Value
-gtid_binlog_pos	1111-1-2503
+gtid_binlog_pos	1111-1-1703
 connection node_3;
 # GTID in node3
 show variables like 'wsrep_gtid_domain_id';
@@ -76,7 +79,7 @@ Variable_name	Value
 wsrep_gtid_domain_id	1111
 show variables like '%gtid_binlog_pos%';
 Variable_name	Value
-gtid_binlog_pos	1111-1-2503
+gtid_binlog_pos	1111-1-1703
 # Shutdown node_3
 connection node_3;
 SET GLOBAL wsrep_provider_options = 'gmcast.isolate = 1';
@@ -98,7 +101,7 @@ Variable_name	Value
 wsrep_gtid_domain_id	1111
 show variables like '%gtid_binlog_pos%';
 Variable_name	Value
-gtid_binlog_pos	1111-1-2554
+gtid_binlog_pos	1111-1-1754
 connection node_2;
 # GTID in node2
 show variables like 'wsrep_gtid_domain_id';
@@ -106,7 +109,7 @@ Variable_name	Value
 wsrep_gtid_domain_id	1111
 show variables like '%gtid_binlog_pos%';
 Variable_name	Value
-gtid_binlog_pos	1111-1-2554
+gtid_binlog_pos	1111-1-1754
 connection node_3;
 # GTID in node3
 show variables like 'wsrep_gtid_domain_id';
@@ -114,7 +117,7 @@ Variable_name	Value
 wsrep_gtid_domain_id	1111
 show variables like '%gtid_binlog_pos%';
 Variable_name	Value
-gtid_binlog_pos	1111-1-2554
+gtid_binlog_pos	1111-1-1754
 # One by one shutdown all nodes
 connection node_3;
 # shutdown node_3
@@ -132,7 +135,7 @@ Variable_name	Value
 wsrep_gtid_domain_id	1111
 show variables like '%gtid_binlog_pos%';
 Variable_name	Value
-gtid_binlog_pos	1111-1-2554
+gtid_binlog_pos	1111-1-1754
 ANALYZE TABLE t2;
 Table	Op	Msg_type	Msg_text
 test.t2	analyze	status	Engine-independent statistics collected
@@ -163,7 +166,7 @@ Variable_name	Value
 wsrep_gtid_domain_id	1111
 show variables like '%gtid_binlog_pos%';
 Variable_name	Value
-gtid_binlog_pos	1111-1-2756
+gtid_binlog_pos	1111-1-1956
 connection node_2;
 node2 GTID
 show variables like 'wsrep_gtid_domain_id';
@@ -171,7 +174,7 @@ Variable_name	Value
 wsrep_gtid_domain_id	1111
 show variables like '%gtid_binlog_pos%';
 Variable_name	Value
-gtid_binlog_pos	1111-1-2756
+gtid_binlog_pos	1111-1-1956
 connection node_3;
 node3 GTID
 show variables like 'wsrep_gtid_domain_id';
@@ -179,22 +182,22 @@ Variable_name	Value
 wsrep_gtid_domain_id	1111
 show variables like '%gtid_binlog_pos%';
 Variable_name	Value
-gtid_binlog_pos	1111-1-2756
+gtid_binlog_pos	1111-1-1956
 connection node_1;
 table size in node1
 SELECT COUNT(*) FROM t1;
 COUNT(*)
-2750
+1950
 connection node_2;
 table size in node2
 SELECT COUNT(*) FROM t1;
 COUNT(*)
-2750
+1950
 connection node_3;
 table size in node3
 SELECT COUNT(*) FROM t1;
 COUNT(*)
-2750
+1950
 connection node_2;
 call mtr.add_suppression("WSREP: Ignoring server id for non bootstrap node");
 call mtr.add_suppression("Sending JOIN failed: ");

mysql-test/suite/galera_3nodes/t/galera_gtid_consistency.cnf

 !include ../galera_3nodes.cnf
 
-[mysqld.1]
-wsrep-node-name="node1"
-wsrep_gtid_domain_id=1111
-gtid_domain_id=2
-server_id=10999
+[mysqld]
+loose-galera-gtid-consistency=1
 wsrep_sst_auth="root:"
 wsrep_sst_method=mariabackup
 log_slave_updates=ON
 log_bin=mariadb-bin-log
 binlog-format=row
 wsrep-gtid-mode=ON
+wsrep-debug=1
+gtid-strict-mode=1
+
+[mysqld.1]
+wsrep-node-name="node1"
+gtid_domain_id=2
+server_id=10999
+wsrep_gtid_domain_id=1111
 
 [mysqld.2]
 wsrep-node-name="node2"
-wsrep_gtid_domain_id=1112
 gtid_domain_id=3
-wsrep_sst_auth="root:"
-wsrep_sst_method=mariabackup
-log_slave_updates=ON
-log_bin=mariadb-bin-log
-binlog-format=row
-wsrep-gtid-mode=ON
+wsrep_gtid_domain_id=1112
 
 [mysqld.3]
 wsrep-node-name="node3"
-wsrep_gtid_domain_id=1113
 gtid_domain_id=4
-wsrep_sst_auth="root:"
-wsrep_sst_method=mariabackup
-log_slave_updates=ON
-log_bin=mariadb-bin-log
-binlog-format=row
-wsrep-gtid-mode=ON
+wsrep_gtid_domain_id=1113
 
 [sst]
 transferfmt=@ENV.MTR_GALERA_TFMT

mysql-test/suite/galera_3nodes/t/galera_gtid_consistency.test

@@ -13,6 +13,13 @@
 # from the bootstrap node (node_1), and use it
 #
 --connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3
+
+# Save original auto_increment_offset values.
+--let $node_1=node_1
+--let $node_2=node_2
+--let $node_3=node_3
+--source ../galera/include/auto_increment_offset_save.inc
+
 --connect node_2b, 127.0.0.1, root, , test, $NODE_MYPORT_2
 set wsrep_sync_wait=0;
 --connect node_1b, 127.0.0.1, root, , test, $NODE_MYPORT_1
@@ -98,10 +105,10 @@ show variables like '%gtid_binlog_pos%';
 # while node 2 is absent
 #
 --connection node_1
-CALL insert_row('node1', 500);
+CALL insert_row('node1', 100);
 
 --connection node_3
-CALL insert_row('node3', 500);
+CALL insert_row('node3', 100);
 
 CREATE TABLE t2(i int primary key) engine=innodb;
 
@@ -225,12 +232,19 @@ show variables like '%gtid_binlog_pos%';
 # bootstap cluster in order node1 - node2 - node3
 # send some inserts and DDL after each node started
 #
---sleep 5
+
 --echo # Bootstrap from node_1
 --connection node_1
 --let $restart_parameters = --wsrep_new_cluster
 --source include/start_mysqld.inc
 
+--let $wait_condition = SELECT VARIABLE_VALUE = 'ON' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_ready';
+--source include/wait_condition.inc
+--let $wait_condition = SELECT VARIABLE_VALUE = 'Primary' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_status';
+--source include/wait_condition.inc
+--let $wait_condition = SELECT VARIABLE_VALUE = 'Synced' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_state_comment'
+--source include/wait_condition.inc
+
 show variables like 'wsrep_gtid_domain_id';
 show variables like '%gtid_binlog_pos%';
 
@@ -243,6 +257,13 @@ ANALYZE TABLE t2;
 --let $_expect_file_name= $MYSQLTEST_VARDIR/tmp/mysqld.2.expect
 --source include/start_mysqld.inc
 
+--let $wait_condition = SELECT VARIABLE_VALUE = 'ON' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_ready';
+--source include/wait_condition.inc
+--let $wait_condition = SELECT VARIABLE_VALUE = 'Primary' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_status';
+--source include/wait_condition.inc
+--let $wait_condition = SELECT VARIABLE_VALUE = 'Synced' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_state_comment'
+--source include/wait_condition.inc
+
 #
 # connection node_1b may not be functional anymore, after node was
 # shutdown, open node_1c for controlling node 1 state
@@ -265,6 +286,14 @@ ALTER TABLE t2 ADD COLUMN (k int);
 --let $_expect_file_name= $MYSQLTEST_VARDIR/tmp/mysqld.3.expect
 --source include/start_mysqld.inc
 
+--let $wait_condition = SELECT VARIABLE_VALUE = 'ON' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_ready';
+--source include/wait_condition.inc
+--let $wait_condition = SELECT VARIABLE_VALUE = 'Primary' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_status';
+--source include/wait_condition.inc
+--let $wait_condition = SELECT VARIABLE_VALUE = 'Synced' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_state_comment'
+--source include/wait_condition.inc
+
+
 --connection node_1c
 --echo # wait until all nodes are back in cluster
 --let $wait_condition = SELECT VARIABLE_VALUE = 3 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size';
@@ -335,13 +364,18 @@ DROP TABLE t2;
 DROP TABLE t3;
 
 --connection node_3
---let $wait_condition = SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't2'
+--let $wait_condition = SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't3'
 --source include/wait_condition.inc
 --connection node_2
---let $wait_condition = SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't2'
+--let $wait_condition = SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't3'
 --source include/wait_condition.inc
 
+# Restore original auto_increment_offset values.
+--let $galera_cluster_size=3
+--source ../galera/include/auto_increment_offset_restore.inc
+
 --disconnect node_3
 --disconnect node_2b
 --disconnect node_1b
 --disconnect node_1c
+