MDEV-26473: mysqld got exception 0xc0000005 (rpl_slave_state/rpl_load_gtid_slave_state)

Problem:
========
During mysqld initialization, if the number of GTIDs added since
that last purge of the mysql.gtid_slave_pos tables is greater than
or equal to the –-gtid-cleanup-batch-size value, a race condition
can occur. Specifically, the binlog background thread will submit
the bg_gtid_delete_pending job to the mysql handle manager; however,
the mysql handle manager may not be initialized, leading to crashes.

Solution:
========
Force the mysql handle manager to initialize/start before the binlog
background thread is created.

Reviewed By:
============
Andrei Elkin <andrei.elkin@mariadb.com>

mysql-test/suite/rpl/r/rpl_mysql_manager_race_condition.result

+include/master-slave.inc
+[connection master]
+connection master;
+# Create a GTID event so the binlog background thread will submit a
+# mysql handler job the next time mysqld is restarted.
+create table t1 (a int);
+include/save_master_gtid.inc
+connection slave;
+include/sync_with_master_gtid.inc
+# Set a debug point that forces the main mysqld thread to sleep before
+# anything is initialized for the mysql handle manager
+# Restart the slave mysqld instance so it re-initializes with the
+# binlog background thread submitting a mysql handler job and the
+# mysql handler initialization suspending for a second. Without the fix
+# associated with this test/patch, the following restart will error
+# with a failed assertion.
+include/rpl_restart_server.inc [server_number=2 parameters: --debug_dbug="+d,delay_start_handle_manager"]
+include/start_slave.inc
+#
+# Cleanup
+#
+connection master;
+drop table t1;
+include/save_master_gtid.inc
+connection slave;
+include/sync_with_master_gtid.inc
+include/rpl_end.inc

mysql-test/suite/rpl/t/rpl_mysql_manager_race_condition-slave.opt

+--gtid-cleanup-batch-size=1

mysql-test/suite/rpl/t/rpl_mysql_manager_race_condition.test

+#
+# Purpose:
+#   This test ensures that, during mysqld initialization, the mysql handle
+# manager starts before the binlog background thread. This is because the
+# binlog background thread uses the mysql handle manager, and if the background
+# thread tries to submit a job to the handle manager before it is
+# initialized/started, mysqld can crash (the actual behavior is undefined).
+# This race condition lead to the problem described in MDEV-26473.
+#
+# Methodology:
+#   This test ensures that the binlog background thread cannot be started
+# before the mysql manager is started. Specifically, it forces a path in
+# the binlog background thread to call mysql_manager_submit() by reducing
+# --gtid-cleanup-batch-size to be 1 (which submits a job to delete unused rows
+# from the mysql.gtid_slave_pos* tables). With this path forced, the main
+# mysqld thread is suspended just before its handle manager initialization to
+# allow time for the binlog thread to call mysql_manager_submit. The fix
+# associated with this test should enforce that the binlog background thread is
+# not created before the handle manager is initialized.
+#
+# References:
+#  MDEV-26473 mysqld got exception 0xc0000005 (rpl_slave_state/rpl_load_gtid_slave_state)
+#
+
+--source include/have_debug.inc
+--source include/master-slave.inc
+
+# The race condition discovered from MDEV-26473 is binlog format independent.
+# We use ROW format though because it was used by the reporter.
+--source include/have_binlog_format_row.inc
+
+--connection master
+
+--echo # Create a GTID event so the binlog background thread will submit a
+--echo # mysql handler job the next time mysqld is restarted.
+create table t1 (a int);
+--source include/save_master_gtid.inc
+
+--connection slave
+--source include/sync_with_master_gtid.inc
+
+--echo # Set a debug point that forces the main mysqld thread to sleep before
+--echo # anything is initialized for the mysql handle manager
+--let $rpl_server_parameters=--debug_dbug="+d,delay_start_handle_manager"
+
+
+--echo # Restart the slave mysqld instance so it re-initializes with the
+--echo # binlog background thread submitting a mysql handler job and the
+--echo # mysql handler initialization suspending for a second. Without the fix
+--echo # associated with this test/patch, the following restart will error
+--echo # with a failed assertion.
+--source include/rpl_restart_server.inc
+--source include/start_slave.inc
+
+
+--echo #
+--echo # Cleanup
+--echo #
+
+--connection master
+drop table t1;
+--source include/save_master_gtid.inc
+
+--connection slave
+--source include/sync_with_master_gtid.inc
+
+--source include/rpl_end.inc

sql/mysqld.cc

@@ -5405,6 +5405,7 @@ static int init_server_components()
     unireg_abort(1);
   }
 
+  start_handle_manager();
   if (opt_bin_log)
   {
     int error;
@@ -5864,7 +5865,6 @@ int mysqld_main(int argc, char **argv)
   }
 
   create_shutdown_event();
-  start_handle_manager();
 
   /* Copy default global rpl_filter to global_rpl_filter */
   copy_filter_setting(global_rpl_filter, get_or_create_rpl_filter("", 0));

sql/sql_manager.cc

@@ -133,6 +133,7 @@ void start_handle_manager()
   {
     pthread_t hThread;
     int err;
+    DBUG_EXECUTE_IF("delay_start_handle_manager", my_sleep(1000););
     manager_thread_in_use = 1;
     mysql_cond_init(key_COND_manager, &COND_manager,NULL);
     mysql_mutex_init(key_LOCK_manager, &LOCK_manager, NULL);