MDEV-4488: When master is on the list of ignore_server_ids, GTID position on slave is not updated

The ignored events are not written to the relay log, but instead a fake
Rotate event is generated to handle update of position.

Extend this for Gtid so we similarly generate a fake Gtid_list event
to update the GTID position.

Also fix an unrelated test issue that got triggered by the added test cases.

mysql-test/suite/rpl/r/rpl_gtid_ignored.result

+include/rpl_init.inc [topology=1->2]
+ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB;
+SET @old_gtid_strict_mode= @@GLOBAL.gtid_strict_mode;
+SET GLOBAL gtid_strict_mode= 1;
+include/stop_slave.inc
+SET @old_gtid_strict_mode= @@GLOBAL.gtid_strict_mode;
+SET GLOBAL gtid_strict_mode=1;
+CHANGE MASTER TO master_use_gtid=slave_pos;
+include/start_slave.inc
+CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE=InnoDB;
+INSERT INTO t1 VALUES (1);
+**** MDEV-4488: GTID position should be updated for events that are ignored due to server id ***
+include/stop_slave.inc
+CHANGE MASTER TO ignore_server_ids=(1);
+include/start_slave.inc
+INSERT INTO t1 VALUES (2);
+INSERT INTO t1 VALUES (3);
+RESULT
+OK
+SELECT * FROM t1 ORDER BY a;
+a
+1
+include/stop_slave.inc
+CHANGE MASTER TO ignore_server_ids=();
+include/start_slave.inc
+RESULT
+OK
+SELECT * FROM t1 ORDER BY a;
+a
+1
+INSERT INTO t1 VALUES (4);
+INSERT INTO t1 VALUES (5);
+RESULT
+OK
+a
+1
+4
+5
+*** Test the same thing when IO thread exits before SQL thread reaches end of log. ***
+include/stop_slave.inc
+SET @old_dbug= @@GLOBAL.debug_dbug;
+SET GLOBAL debug_dbug= "+d,inject_slave_sql_before_apply_event";
+CHANGE MASTER TO ignore_server_ids=(1);
+include/start_slave.inc
+INSERT INTO t1 VALUES (6);
+INSERT INTO t1 VALUES (7);
+include/wait_for_slave_param.inc [Read_Master_Log_Pos]
+STOP SLAVE IO_THREAD;
+SET debug_sync = "now SIGNAL continue";
+RESULT
+OK
+RESULT
+OK
+include/stop_slave.inc
+CHANGE MASTER TO ignore_server_ids=();
+SET GLOBAL debug_dbug= @old_dbug;
+include/start_slave.inc
+INSERT INTO t1 VALUES (8);
+INSERT INTO t1 VALUES (9);
+SELECT * FROM t1 ORDER BY a;
+a
+1
+4
+5
+8
+9
+DROP TABLE t1;
+SET GLOBAL gtid_strict_mode= @old_gtid_strict_mode;
+SET GLOBAL gtid_strict_mode= @old_gtid_strict_mode;
+include/rpl_end.inc

mysql-test/suite/rpl/r/rpl_gtid_mdev4484.result

@@ -7,10 +7,11 @@ INSERT INTO t1 VALUES (1);
 include/stop_slave.inc
 SET @old_dbug= @@GLOBAL.debug_dbug;
 SET GLOBAL debug_dbug="+d,gtid_slave_pos_simulate_failed_delete";
-include/start_slave.inc
 SET sql_log_bin= 0;
 CALL mtr.add_suppression("Can't find file");
+ALTER TABLE mysql.gtid_slave_pos ENGINE=MyISAM;
 SET sql_log_bin= 1;
+include/start_slave.inc
 INSERT INTO t1 VALUES (2);
 include/wait_for_slave_sql_error.inc [errno=1942]
 STOP SLAVE IO_THREAD;

mysql-test/suite/rpl/t/rpl_gtid_ignored.test

+--source include/have_innodb.inc
+--source include/have_debug.inc
+--source include/have_debug_sync.inc
+
+--let $rpl_topology=1->2
+--source include/rpl_init.inc
+
+--connection server_1
+ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB;
+
+SET @old_gtid_strict_mode= @@GLOBAL.gtid_strict_mode;
+SET GLOBAL gtid_strict_mode= 1;
+
+--connection server_2
+--source include/stop_slave.inc
+SET @old_gtid_strict_mode= @@GLOBAL.gtid_strict_mode;
+SET GLOBAL gtid_strict_mode=1;
+CHANGE MASTER TO master_use_gtid=slave_pos;
+--source include/start_slave.inc
+
+--connection server_1
+CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE=InnoDB;
+INSERT INTO t1 VALUES (1);
+--save_master_pos
+
+--connection server_2
+--sync_with_master
+
+--echo **** MDEV-4488: GTID position should be updated for events that are ignored due to server id ***
+--source include/stop_slave.inc
+CHANGE MASTER TO ignore_server_ids=(1);
+--source include/start_slave.inc
+
+--connection server_1
+# These inserts should be ignored (not applied) on the slave, but the
+# gtid_slave_pos should still be updated.
+INSERT INTO t1 VALUES (2);
+INSERT INTO t1 VALUES (3);
+--save_master_pos
+--let gtid_pos=`SELECT @@GLOBAL.gtid_binlog_pos`
+
+--connection server_2
+--sync_with_master
+--let $wait_condition= SELECT @@GLOBAL.gtid_slave_pos = '$gtid_pos'
+--source include/wait_condition.inc
+--disable_query_log
+eval SELECT IF(@@GLOBAL.gtid_slave_pos = '$gtid_pos', 'OK', CONCAT("ERROR: Expected $gtid_pos got ", @@GLOBAL.gtid_slave_pos)) AS RESULT;
+--enable_query_log
+
+SELECT * FROM t1 ORDER BY a;
+
+--source include/stop_slave.inc
+CHANGE MASTER TO ignore_server_ids=();
+--source include/start_slave.inc
+--sync_with_master
+--disable_query_log
+eval SELECT IF(@@GLOBAL.gtid_slave_pos = '$gtid_pos', 'OK', CONCAT("ERROR: Expected $gtid_pos got ", @@GLOBAL.gtid_slave_pos)) AS RESULT;
+--enable_query_log
+
+SELECT * FROM t1 ORDER BY a;
+
+--connection server_1
+INSERT INTO t1 VALUES (4);
+INSERT INTO t1 VALUES (5);
+--let gtid_pos=`SELECT @@GLOBAL.gtid_binlog_pos`
+--save_master_pos
+
+--connection server_2
+--sync_with_master
+--disable_query_log
+eval SELECT IF(@@GLOBAL.gtid_slave_pos = '$gtid_pos', 'OK', CONCAT("ERROR: Expected $gtid_pos got ", @@GLOBAL.gtid_slave_pos)) AS RESULT;
+SELECT * FROM t1 ORDER BY a;
+--enable_query_log
+
+
+--echo *** Test the same thing when IO thread exits before SQL thread reaches end of log. ***
+--connection server_2
+--source include/stop_slave.inc
+SET @old_dbug= @@GLOBAL.debug_dbug;
+SET GLOBAL debug_dbug= "+d,inject_slave_sql_before_apply_event";
+CHANGE MASTER TO ignore_server_ids=(1);
+--source include/start_slave.inc
+
+--connection server_1
+INSERT INTO t1 VALUES (6);
+INSERT INTO t1 VALUES (7);
+--let $master_pos= query_get_value(SHOW MASTER STATUS, Position, 1)
+--let gtid_pos=`SELECT @@GLOBAL.gtid_binlog_pos`
+--save_master_pos
+
+--connection server_2
+# Wait for IO thread to have read all events from master, and for SQL thread to
+# sit in the debug_sync point.
+
+--let $slave_param= Read_Master_Log_Pos
+--let $slave_param_value= $master_pos
+--source include/wait_for_slave_param.inc
+
+# Now stop the IO thread, and let the SQL thread continue. The IO thread
+# should write a Gtid_list event that the SQL thread can use to update the
+# gtid_slave_pos with the GTIDs of the skipped events.
+STOP SLAVE IO_THREAD;
+SET debug_sync = "now SIGNAL continue";
+
+--sync_with_master
+--let $wait_condition= SELECT @@GLOBAL.gtid_slave_pos = '$gtid_pos'
+--source include/wait_condition.inc
+--disable_query_log
+eval SELECT IF(@@GLOBAL.gtid_slave_pos = '$gtid_pos', 'OK', CONCAT("ERROR: Expected $gtid_pos got ", @@GLOBAL.gtid_slave_pos)) AS RESULT;
+--let $slave_pos= query_get_value(SHOW SLAVE STATUS, Exec_Master_Log_Pos, 1)
+eval SELECT IF('$slave_pos' = '$master_pos', 'OK', "ERROR: Expected $master_pos got $slave_pos") AS RESULT;
+--enable_query_log
+
+
+--source include/stop_slave.inc
+CHANGE MASTER TO ignore_server_ids=();
+SET GLOBAL debug_dbug= @old_dbug;
+--source include/start_slave.inc
+
+--connection server_1
+INSERT INTO t1 VALUES (8);
+INSERT INTO t1 VALUES (9);
+--save_master_pos
+
+--connection server_2
+--sync_with_master
+SELECT * FROM t1 ORDER BY a;
+
+# Clean up.
+--connection server_1
+DROP TABLE t1;
+SET GLOBAL gtid_strict_mode= @old_gtid_strict_mode;
+--connection server_2
+SET GLOBAL gtid_strict_mode= @old_gtid_strict_mode;
+
+--source include/rpl_end.inc

mysql-test/suite/rpl/t/rpl_gtid_mdev4484.test

@@ -18,10 +18,14 @@ INSERT INTO t1 VALUES (1);
 --source include/stop_slave.inc
 SET @old_dbug= @@GLOBAL.debug_dbug;
 SET GLOBAL debug_dbug="+d,gtid_slave_pos_simulate_failed_delete";
---source include/start_slave.inc
 SET sql_log_bin= 0;
 CALL mtr.add_suppression("Can't find file");
+# Since we inject an error updating mysql.gtid_slave_pos, we will get different
+# output depending on whether it is InnoDB or MyISAM (roll back or no roll
+# back). So fix it to make sure we are consistent.
+ALTER TABLE mysql.gtid_slave_pos ENGINE=MyISAM;
 SET sql_log_bin= 1;
+--source include/start_slave.inc
 
 --connection master
 INSERT INTO t1 VALUES (2);

sql/log_event.cc

@@ -6334,7 +6334,7 @@ Gtid_log_event::print(FILE *file, PRINT_EVENT_INFO *print_event_info)
 
 Gtid_list_log_event::Gtid_list_log_event(const char *buf, uint event_len,
                const Format_description_log_event *description_event)
-  : Log_event(buf, description_event), count(0), list(0)
+  : Log_event(buf, description_event), count(0), list(0), sub_id_list(0)
 {
   uint32 i;
   uint32 val;
@@ -6363,6 +6363,31 @@ Gtid_list_log_event::Gtid_list_log_event(const char *buf, uint event_len,
     list[i].seq_no= uint8korr(buf);
     buf+= 8;
   }
+
+#ifdef MYSQL_SERVER
+  if ((gl_flags & FLAG_IGN_GTIDS))
+  {
+    uint32 i;
+    if (!(sub_id_list= (uint64 *)my_malloc(count*sizeof(uint64), MYF(MY_WME))))
+    {
+      my_free(list);
+      list= NULL;
+      return;
+    }
+    for (i= 0; i < count; ++i)
+    {
+      if (!(sub_id_list[i]=
+            rpl_global_gtid_slave_state.next_sub_id(list[i].domain_id)))
+      {
+        my_free(list);
+        my_free(sub_id_list);
+        list= NULL;
+        sub_id_list= NULL;
+        return;
+      }
+    }
+  }
+#endif
 }
 
 
@@ -6370,7 +6395,7 @@ Gtid_list_log_event::Gtid_list_log_event(const char *buf, uint event_len,
 
 Gtid_list_log_event::Gtid_list_log_event(rpl_binlog_state *gtid_set,
                                          uint32 gl_flags_)
-  : count(gtid_set->count()), gl_flags(gl_flags_), list(0)
+  : count(gtid_set->count()), gl_flags(gl_flags_), list(0), sub_id_list(0)
 {
   cache_type= EVENT_NO_CACHE;
   /* Failure to allocate memory will be caught by is_valid() returning false. */
@@ -6381,6 +6406,45 @@ Gtid_list_log_event::Gtid_list_log_event(rpl_binlog_state *gtid_set,
 }
 
 
+Gtid_list_log_event::Gtid_list_log_event(slave_connection_state *gtid_set,
+                                         uint32 gl_flags_)
+  : count(gtid_set->count()), gl_flags(gl_flags_), list(0), sub_id_list(0)
+{
+  cache_type= EVENT_NO_CACHE;
+  /* Failure to allocate memory will be caught by is_valid() returning false. */
+  if (count < (1<<28) &&
+      (list = (rpl_gtid *)my_malloc(count * sizeof(*list) + (count == 0),
+                                    MYF(MY_WME))))
+  {
+    gtid_set->get_gtid_list(list, count);
+    if (gl_flags & FLAG_IGN_GTIDS)
+    {
+      uint32 i;
+
+      if (!(sub_id_list= (uint64 *)my_malloc(count * sizeof(uint64),
+                                             MYF(MY_WME))))
+      {
+        my_free(list);
+        list= NULL;
+        return;
+      }
+      for (i= 0; i < count; ++i)
+      {
+        if (!(sub_id_list[i]=
+              rpl_global_gtid_slave_state.next_sub_id(list[i].domain_id)))
+        {
+          my_free(list);
+          my_free(sub_id_list);
+          list= NULL;
+          sub_id_list= NULL;
+          return;
+        }
+      }
+    }
+  }
+}
+
+
 #if defined(HAVE_REPLICATION) && !defined(MYSQL_CLIENT)
 bool
 Gtid_list_log_event::to_packet(String *packet)
@@ -6432,7 +6496,20 @@ Gtid_list_log_event::write(IO_CACHE *file)
 int
 Gtid_list_log_event::do_apply_event(Relay_log_info const *rli)
 {
-  int ret= Log_event::do_apply_event(rli);
+  int ret;
+  if (gl_flags & FLAG_IGN_GTIDS)
+  {
+    uint32 i;
+    for (i= 0; i < count; ++i)
+    {
+      if ((ret= rpl_global_gtid_slave_state.record_gtid(thd, &list[i],
+                                                        sub_id_list[i],
+                                                        false, false)))
+        return ret;
+      rpl_global_gtid_slave_state.update_state_hash(sub_id_list[i], &list[i]);
+    }
+  }
+  ret= Log_event::do_apply_event(rli);
   if (rli->until_condition == Relay_log_info::UNTIL_GTID &&
       (gl_flags & FLAG_UNTIL_REACHED))
   {

sql/log_event.h

@@ -3197,12 +3197,15 @@ class Gtid_list_log_event: public Log_event
   uint32 count;
   uint32 gl_flags;
   struct rpl_gtid *list;
+  uint64 *sub_id_list;
 
   static const uint element_size= 4+4+8;
   static const uint32 FLAG_UNTIL_REACHED= (1<<28);
+  static const uint32 FLAG_IGN_GTIDS= (1<<29);
 
 #ifdef MYSQL_SERVER
   Gtid_list_log_event(rpl_binlog_state *gtid_set, uint32 gl_flags);
+  Gtid_list_log_event(slave_connection_state *gtid_set, uint32 gl_flags);
 #ifdef HAVE_REPLICATION
   void pack_info(THD *thd, Protocol *protocol);
 #endif
@@ -3211,7 +3214,7 @@ class Gtid_list_log_event: public Log_event
 #endif
   Gtid_list_log_event(const char *buf, uint event_len,
                       const Format_description_log_event *description_event);
-  ~Gtid_list_log_event() { my_free(list); }
+  ~Gtid_list_log_event() { my_free(list); my_free(sub_id_list); }
   Log_event_type get_type_code() { return GTID_LIST_EVENT; }
   int get_data_size() {
     /*

sql/rpl_gtid.cc

@@ -1419,6 +1419,15 @@ slave_connection_state::remove(const rpl_gtid *in_gtid)
 }
 
 
+void
+slave_connection_state::remove_if_present(const rpl_gtid *in_gtid)
+{
+  uchar *rec= my_hash_search(&hash, (const uchar *)(&in_gtid->domain_id), 0);
+  if (rec)
+    my_hash_delete(&hash, rec);
+}
+
+
 int
 slave_connection_state::to_string(String *out_str)
 {
@@ -1442,3 +1451,22 @@ slave_connection_state::append_to_string(String *out_str)
   }
   return 0;
 }
+
+
+int
+slave_connection_state::get_gtid_list(rpl_gtid *gtid_list, uint32 list_size)
+{
+  uint32 i, pos;
+
+  pos= 0;
+  for (i= 0; i < hash.records; ++i)
+  {
+    entry *e;
+    if (pos >= list_size)
+      return 1;
+    e= (entry *)my_hash_element(&hash, i);
+    memcpy(&gtid_list[pos++], &e->gtid, sizeof(e->gtid));
+  }
+
+  return 0;
+}

sql/rpl_gtid.h

@@ -195,9 +195,11 @@ struct slave_connection_state
   entry *find_entry(uint32 domain_id);
   int update(const rpl_gtid *in_gtid);
   void remove(const rpl_gtid *gtid);
+  void remove_if_present(const rpl_gtid *in_gtid);
   ulong count() const { return hash.records; }
   int to_string(String *out_str);
   int append_to_string(String *out_str);
+  int get_gtid_list(rpl_gtid *gtid_list, uint32 list_size);
 };
 
 extern bool rpl_slave_state_tostring_helper(String *dest, const rpl_gtid *gtid,

sql/rpl_rli.h

@@ -303,6 +303,8 @@ class Relay_log_info : public Slave_reporting_capability
   */
   char ign_master_log_name_end[FN_REFLEN];
   ulonglong ign_master_log_pos_end;
+  /* Similar for ignored GTID events. */
+  slave_connection_state ign_gtids;
 
   /* 
     Indentifies where the SQL Thread should create temporary files for the