MDEV-28435: rpl.rpl_mysqlbinlog_slave_consistency fails intermittently on tables comparison

Problem:
========
The test logic checked for the wrong condition to validate that the
slave had caught up with the master. Specifically, it used the
thread stage of the IO and SQL thread to be in the “Waiting for
master to send event” and “Slave has read all relay log; waiting for
more updates” states, respectively. The problem exposed by this MDEV
is that, this state is also the initial slave state before reading
data from the primary (whereas the intended state was having already
read all available events from the primary and now waiting for new
events). This made the MTR test validate data that it had not yet
received, and thereby fail.

Solution:
========
Instead of using the IO/SQL thread states, use the existing helper
functions save_master_gtid.inc and sync_with_master_gtid.inc. Note
that the test result file also needed to be updated to reflect
this fix.

Special thanks to Angelique Sklavounos for pointing out that
--stop-position was not specified in any buildbot failures, as this
led to an IF block in the MTR test that was the source of the test
failure.

Reviewed By
============
Andrei Elkin <andrei.elkin@mariadb.com>

mysql-test/suite/rpl/include/mysqlbinlog_slave_consistency.inc

@@ -54,6 +54,7 @@ if ($strict_mode)
 --connection $con1
 --echo # Populating $con1 data
 --source $sql_input_file
+--source include/save_master_gtid.inc
 
 --let $MYSQLD_DATADIR=`select @@datadir`
 --let $MYSQLBINLOG_STDERR=$MYSQLD_DATADIR/mysqlbinlog_stderr.out
@@ -114,15 +115,7 @@ if (!$slave_sql_errno)
     # Stop position was not specified
     if (`SELECT strcmp("$stop_position","") = 0`)
     {
-        --source include/wait_for_slave_to_start.inc
-        --echo # Wait for $con2 IO thread to catch up
-        --let $wait_condition= SELECT STATE="Waiting for master to send event" from information_schema.PROCESSLIST where COMMAND="Slave_IO"
-        --source include/wait_condition.inc
-
-        --echo # Wait for $con2 SQL thread to catch up
-        --let $wait_condition= SELECT STATE="Slave has read all relay log; waiting for more updates" from information_schema.PROCESSLIST where COMMAND="Slave_SQL"
-        --source include/wait_condition.inc
-
+        --source include/sync_with_master_gtid.inc
     }
 
     # Stop position was specified

mysql-test/suite/rpl/r/rpl_mysqlbinlog_slave_consistency.result

@@ -25,8 +25,6 @@ connection slave;
 CHANGE MASTER TO MASTER_USE_GTID=slave_pos;
 START SLAVE ;
 # No slave error expecting - waiting for slave to catch up to master
-# Wait for slave IO thread to catch up
-# Wait for slave SQL thread to catch up
 # Stop slave so it stops receiving master events.
 connection master;
 DROP TABLE IF EXISTS t1, t2, t3, t4, t5;
@@ -85,8 +83,6 @@ connection slave;
 CHANGE MASTER TO MASTER_USE_GTID=slave_pos, DO_DOMAIN_IDS=(0);
 START SLAVE ;
 # No slave error expecting - waiting for slave to catch up to master
-# Wait for slave IO thread to catch up
-# Wait for slave SQL thread to catch up
 # Stop slave so it stops receiving master events.
 connection master;
 DROP TABLE IF EXISTS t1, t2, t3, t4, t5;
@@ -138,8 +134,6 @@ connection slave;
 CHANGE MASTER TO MASTER_USE_GTID=slave_pos, DO_DOMAIN_IDS=(0,1);
 START SLAVE ;
 # No slave error expecting - waiting for slave to catch up to master
-# Wait for slave IO thread to catch up
-# Wait for slave SQL thread to catch up
 # Stop slave so it stops receiving master events.
 connection master;
 DROP TABLE IF EXISTS t1, t2, t3, t4, t5;
@@ -196,8 +190,6 @@ connection slave;
 CHANGE MASTER TO MASTER_USE_GTID=slave_pos, IGNORE_DOMAIN_IDS=(0);
 START SLAVE ;
 # No slave error expecting - waiting for slave to catch up to master
-# Wait for slave IO thread to catch up
-# Wait for slave SQL thread to catch up
 # Stop slave so it stops receiving master events.
 connection master;
 DROP TABLE IF EXISTS t1, t2, t3, t4, t5;
@@ -252,8 +244,6 @@ connection slave;
 CHANGE MASTER TO MASTER_USE_GTID=slave_pos, IGNORE_DOMAIN_IDS=(0,1);
 START SLAVE ;
 # No slave error expecting - waiting for slave to catch up to master
-# Wait for slave IO thread to catch up
-# Wait for slave SQL thread to catch up
 # Stop slave so it stops receiving master events.
 connection master;
 DROP TABLE IF EXISTS t1, t2, t3, t4, t5;
@@ -301,8 +291,6 @@ connection slave;
 CHANGE MASTER TO MASTER_USE_GTID=slave_pos, IGNORE_SERVER_IDS=(1);
 START SLAVE ;
 # No slave error expecting - waiting for slave to catch up to master
-# Wait for slave IO thread to catch up
-# Wait for slave SQL thread to catch up
 # Stop slave so it stops receiving master events.
 connection master;
 DROP TABLE IF EXISTS t1, t2, t3, t4, t5;
@@ -354,8 +342,6 @@ connection slave;
 CHANGE MASTER TO MASTER_USE_GTID=slave_pos, IGNORE_SERVER_IDS=(0,1);
 START SLAVE ;
 # No slave error expecting - waiting for slave to catch up to master
-# Wait for slave IO thread to catch up
-# Wait for slave SQL thread to catch up
 # Stop slave so it stops receiving master events.
 connection master;
 DROP TABLE IF EXISTS t1, t2, t3, t4, t5;
@@ -458,8 +444,6 @@ set global gtid_slave_pos="1-4-2";
 CHANGE MASTER TO MASTER_USE_GTID=slave_pos;
 START SLAVE ;
 # No slave error expecting - waiting for slave to catch up to master
-# Wait for slave IO thread to catch up
-# Wait for slave SQL thread to catch up
 # Stop slave so it stops receiving master events.
 connection master;
 DROP TABLE IF EXISTS t1, t2, t3, t4, t5;
@@ -517,8 +501,6 @@ connection slave;
 CHANGE MASTER TO MASTER_USE_GTID=slave_pos, DO_DOMAIN_IDS=(0), IGNORE_SERVER_IDS=(1);
 START SLAVE ;
 # No slave error expecting - waiting for slave to catch up to master
-# Wait for slave IO thread to catch up
-# Wait for slave SQL thread to catch up
 # Stop slave so it stops receiving master events.
 connection master;
 DROP TABLE IF EXISTS t1, t2, t3, t4, t5;
@@ -567,8 +549,6 @@ connection slave;
 CHANGE MASTER TO MASTER_USE_GTID=slave_pos, IGNORE_DOMAIN_IDS=(1,2), IGNORE_SERVER_IDS=(1);
 START SLAVE ;
 # No slave error expecting - waiting for slave to catch up to master
-# Wait for slave IO thread to catch up
-# Wait for slave SQL thread to catch up
 # Stop slave so it stops receiving master events.
 connection master;
 DROP TABLE IF EXISTS t1, t2, t3, t4, t5;
@@ -774,8 +754,6 @@ set global gtid_slave_pos="1-1-2";
 CHANGE MASTER TO MASTER_USE_GTID=slave_pos, DO_DOMAIN_IDS=(2);
 START SLAVE ;
 # No slave error expecting - waiting for slave to catch up to master
-# Wait for slave IO thread to catch up
-# Wait for slave SQL thread to catch up
 # Stop slave so it stops receiving master events.
 connection master;
 DROP TABLE IF EXISTS t1, t2, t3, t4, t5;
@@ -825,8 +803,6 @@ set global gtid_slave_pos="0-1-1";
 CHANGE MASTER TO MASTER_USE_GTID=slave_pos, IGNORE_DOMAIN_IDS=(0);
 START SLAVE ;
 # No slave error expecting - waiting for slave to catch up to master
-# Wait for slave IO thread to catch up
-# Wait for slave SQL thread to catch up
 # Stop slave so it stops receiving master events.
 connection master;
 DROP TABLE IF EXISTS t1, t2, t3, t4, t5;
@@ -882,8 +858,6 @@ set global gtid_slave_pos="0-1-1";
 CHANGE MASTER TO MASTER_USE_GTID=slave_pos, IGNORE_SERVER_IDS=(1);
 START SLAVE ;
 # No slave error expecting - waiting for slave to catch up to master
-# Wait for slave IO thread to catch up
-# Wait for slave SQL thread to catch up
 # Stop slave so it stops receiving master events.
 connection master;
 DROP TABLE IF EXISTS t1, t2, t3, t4, t5;
@@ -1202,8 +1176,6 @@ connection slave;
 CHANGE MASTER TO MASTER_USE_GTID=slave_pos, DO_DOMAIN_IDS=(0), IGNORE_SERVER_IDS=(3);
 START SLAVE ;
 # No slave error expecting - waiting for slave to catch up to master
-# Wait for slave IO thread to catch up
-# Wait for slave SQL thread to catch up
 # Stop slave so it stops receiving master events.
 connection master;
 DROP TABLE IF EXISTS t1, t2, t3, t4, t5;