Merge 10.2 into 10.3

mysql-test/collections/smoke_test

+archive.archive
+archive.archive_gis
+archive.partition_archive
+binlog.binlog_base64_flag
+binlog.binlog_database
+binlog.binlog_innodb
+binlog.binlog_parallel_replication_marks_stm_mix
+binlog.binlog_row_mix_innodb_myisam
+binlog.flashback
+binlog_encryption.encrypted_slave
+connect.bin
+connect.bson
+connect.dbf
+connect.dir
+connect.endian
+connect.general
+connect.json
+connect.mysql
+connect.secure_file_priv
+connect.tbl
+csv.csv
+disks.disks
+encryption.encryption_force
+encryption.innodb_encryption_tables
+encryption.tempfiles_encrypted
+federated.federated
+federated.federatedx
+gcol.gcol_select_innodb
+handler.interface
+heap.heap
+innodb.innodb
+innodb.autoinc_persist
+innodb.innodb_defrag_binlog
+innodb.innodb_mysql
+innodb.monitor
+innodb.purge
+innodb.table_flags
+innodb.xa_recovery
+innodb_fts.fulltext
+innodb_gis.geometry
+innodb_gis.rtree
+innodb_zip.innodb-zip
+innodb_zip.page_size
+json.json_no_table
+main.blackhole
+main.bootstrap
+main.compress
+main.connect
+main.ctype_collate
+main.ctype_utf8
+main.default
+main.dyncol
+main.fulltext
+main.function_defaults
+main.gis
+main.grant
+main.handlersocket
+main.information_schema
+main.innodb_ext_key
+main.log_tables
+main.lowercase_fs_off
+main.myisam
+main.mysql_client_test
+main.mysql_protocols
+main.mysql_upgrade
+main.mysqladmin
+main.mysqlbinlog
+main.mysqlcheck
+main.mysqld--defaults-file
+main.mysqldump
+main.mysqlhotcopy_myisam
+main.mysqlshow
+main.mysqlslap
+main.mysqltest
+main.parser
+main.partition
+main.perror
+main.plugin
+main.plugin_auth
+main.pool_of_threads
+main.ps
+main.repair
+main.shutdown
+main.sp
+main.ssl
+main.ssl_compress
+main.stat_tables
+main.statistics
+main.subselect
+main.symlink
+main.temp_table
+main.timezone
+main.type_timestamp_hires
+main.user_var
+main.userstat
+main.variables
+main.view
+main.win
+main.xa
+maria.maria
+mariabackup.full_backup
+metadata_lock_info.table_metadata_lock
+mroonga/storage.variable_version
+mroonga/wrapper.count_star
+multi_source.multisource
+oqgraph.general-innodb
+parts.rpl_partition
+perfschema.selects
+plugins.auth_ed25519
+plugins.cracklib_password_check
+plugins.dialog
+plugins.fulltext_plugin
+plugins.locales
+plugins.pam_cleartext
+plugins.processlist
+plugins.qc_info
+plugins.server_audit
+plugins.simple_password_check
+plugins.sql_error_log
+plugins.two_password_validations
+plugins.unix_socket
+query_response_time.basic
+rocksdb.rocksdb
+roles.definer
+rpl.rpl_gtid_basic
+rpl.rpl_relayrotate
+rpl.rpl_row_blob_innodb
+rpl.rpl_semi_sync_event
+rpl.rpl_sp
+rpl.rpl_stm_binlog_max_cache_size
+rpl.rpl_switch_stm_row_mixed
+sequence.simple
+spider.basic_sql
+spider.ha
+sql_discovery.simple
+sys_vars.sysvars_aria
+sys_vars.sysvars_server_notembedded
+wsrep.variables
+compat/oracle.binlog_ptr_mysqlbinlog
+compat/oracle.sp-package
+compat/maxdb.rpl_mariadb_timestamp
+sql_sequence.mysqldump
+versioning.simple
+versioning.trx_id

mysql-test/suite/binlog/r/binlog_checkpoint_flush_logs.result

+SET GLOBAL innodb_flush_log_at_trx_commit= 1;
+RESET MASTER;
+CREATE TABLE t1 (a INT PRIMARY KEY, b MEDIUMTEXT) ENGINE=Innodb;
+*** Test that FLUSH LOGS waits if a transaction ordered commit is in progress.
+connect con1,localhost,root,,;
+SET DEBUG_SYNC= "commit_before_get_LOCK_commit_ordered SIGNAL con1_ready WAIT_FOR con1_go";
+INSERT INTO t1 VALUES (1, REPEAT("x", 1));
+connection default;
+SET DEBUG_SYNC= "now WAIT_FOR con1_ready";
+SET DEBUG_SYNC= "rotate_after_rotate SIGNAL con_flush_ready WAIT_FOR default_go";
+FLUSH LOGS;
+connect con2,localhost,root,,;
+Trx_1 is not yet committed:
+SELECT count(*) as 'ZERO' from t1;
+ZERO
+0
+Wait for Trx_2 has rotated binlog:
+SET DEBUG_SYNC= "now WAIT_FOR con_flush_ready";
+SET DEBUG_SYNC= "now SIGNAL default_go";
+connection default;
+Must be tree logs in the list:
+show binary logs;
+Log_name	File_size
+master-bin.000001	#
+master-bin.000002	#
+master-bin.000003	#
+include/show_binlog_events.inc
+Log_name	Pos	Event_type	Server_id	End_log_pos	Info
+master-bin.000001	#	Format_desc	#	#	SERVER_VERSION, BINLOG_VERSION
+master-bin.000001	#	Gtid_list	#	#	[]
+master-bin.000001	#	Binlog_checkpoint	#	#	master-bin.000001
+master-bin.000001	#	Gtid	#	#	GTID #-#-#
+master-bin.000001	#	Query	#	#	use `test`; CREATE TABLE t1 (a INT PRIMARY KEY, b MEDIUMTEXT) ENGINE=Innodb
+master-bin.000001	#	Gtid	#	#	BEGIN GTID #-#-#
+master-bin.000001	#	Annotate_rows	#	#	INSERT INTO t1 VALUES (1, REPEAT("x", 1))
+master-bin.000001	#	Table_map	#	#	table_id: # (test.t1)
+master-bin.000001	#	Write_rows_v1	#	#	table_id: # flags: STMT_END_F
+master-bin.000001	#	Xid	#	#	COMMIT /* XID */
+master-bin.000001	#	Rotate	#	#	master-bin.000002;pos=POS
+Only one Binlog checkpoint must exist and point to master-bin.000001
+include/show_binlog_events.inc
+Log_name	Pos	Event_type	Server_id	End_log_pos	Info
+master-bin.000002	#	Format_desc	#	#	SERVER_VERSION, BINLOG_VERSION
+master-bin.000002	#	Gtid_list	#	#	[#-#-#]
+master-bin.000002	#	Binlog_checkpoint	#	#	master-bin.000001
+SELECT count(*) as 'ONE' from t1;
+ONE
+1
+connection default;
+DROP TABLE t1;
+SET debug_sync = 'reset';

mysql-test/suite/binlog/t/binlog_checkpoint_flush_logs.test

+--source include/have_innodb.inc
+--source include/have_debug.inc
+--source include/have_debug_sync.inc
+--source include/have_binlog_format_row.inc
+
+# References:
+#
+# MDEV-24526 binlog rotate via FLUSH LOGS may obsolate binlog file too eary
+#
+# The test for MDEV-24526 proves the fixes correct observed race condition
+# between a commiting transaction and FLUSH-LOGS.
+# The plot.
+# Trx_1 (con1) transaction binlogs first
+# to yield its turn acquiring LOCK_commit_ordered to Trx_2 and stand
+# still waiting of a signal that will never arrive.
+# Trx_2 can't acquire it in the fixed version even though
+# Trx_3 makes sure Trx_2 has reached a post-rotation execution point
+# to signal it to proceed.
+# Then the server gets crashed and Trx_1 must recover unlike
+# in the OLD buggy version.
+#
+SET GLOBAL innodb_flush_log_at_trx_commit= 1;
+RESET MASTER;
+
+CREATE TABLE t1 (a INT PRIMARY KEY, b MEDIUMTEXT) ENGINE=Innodb;
+
+--echo *** Test that FLUSH LOGS waits if a transaction ordered commit is in progress.
+
+connect(con1,localhost,root,,);  # Trx_1
+# hang before doing acquiring Commit Ordered mutex
+SET DEBUG_SYNC= "commit_before_get_LOCK_commit_ordered SIGNAL con1_ready WAIT_FOR con1_go";
+
+--send INSERT INTO t1 VALUES (1, REPEAT("x", 1))
+
+connection default;              # Trx_2
+
+SET DEBUG_SYNC= "now WAIT_FOR con1_ready";
+SET DEBUG_SYNC= "rotate_after_rotate SIGNAL con_flush_ready WAIT_FOR default_go";
+--send FLUSH LOGS
+
+connect(con2,localhost,root,,);  # Trx_3
+--echo Trx_1 is not yet committed:
+SELECT count(*) as 'ZERO' from t1;
+
+--echo Wait for Trx_2 has rotated binlog:
+SET DEBUG_SYNC= "now WAIT_FOR con_flush_ready";
+# Useless signal to prove Trx_2 cannot race Trx_1's commit
+# even though Trx_1 never received the being waited 'con1_go'.
+SET DEBUG_SYNC= "now SIGNAL default_go";
+
+--let $shutdown_timeout=0
+--source include/restart_mysqld.inc
+
+connection default;
+--enable_reconnect
+--error 0,2013
+--reap
+
+--echo Must be tree logs in the list:
+--source include/show_binary_logs.inc
+--let $binlog_file= master-bin.000001
+--let $binlog_start= 4
+--source include/show_binlog_events.inc
+
+--echo Only one Binlog checkpoint must exist and point to master-bin.000001
+--let $binlog_file= master-bin.000002
+--let $binlog_start= 4
+--source include/show_binlog_events.inc
+
+
+# In the buggy server version the following select may have
+# resulted with ZERO:
+SELECT count(*) as 'ONE' from t1;
+
+# Clean up.
+connection default;
+
+DROP TABLE t1;
+SET debug_sync = 'reset';

sql/event_data_objects.cc

@@ -178,6 +178,7 @@ Event_queue_element_for_exec::init(const LEX_CSTRING *db, const LEX_CSTRING *n)
   if (!(name.str= my_strndup(n->str, name.length= n->length, MYF(MY_WME))))
   {
     my_free(const_cast<char*>(dbname.str));
+    dbname.str= NULL;
     return TRUE;
   }
   return FALSE;

sql/event_data_objects.h

@@ -33,7 +33,11 @@ struct TABLE;
 class Event_queue_element_for_exec
 {
 public:
-  Event_queue_element_for_exec(){};
+  Event_queue_element_for_exec()
+  {
+    dbname.str= NULL; dbname.length= 0;
+    name.str= NULL; name.length= 0;
+  };
   ~Event_queue_element_for_exec();
 
   bool

sql/event_queue.cc

@@ -639,6 +639,7 @@ Event_queue::get_top_for_execution_if_time(THD *thd,
     if (!(*event_name= new Event_queue_element_for_exec()) ||
         (*event_name)->init(&top->dbname, &top->name))
     {
+      delete *event_name;
       ret= TRUE;
       break;
     }

sql/log.cc

@@ -6908,6 +6908,9 @@ int MYSQL_BIN_LOG::rotate_and_purge(bool force_rotate,
   bool check_purge= false;
 
   mysql_mutex_lock(&LOCK_log);
+
+  DEBUG_SYNC(current_thd, "rotate_after_acquire_LOCK_log");
+
   prev_binlog_id= current_binlog_id;
 
   if ((err_gtid= do_delete_gtid_domain(domain_drop_lex)))
@@ -6918,11 +6921,22 @@ int MYSQL_BIN_LOG::rotate_and_purge(bool force_rotate,
   }
   else if (unlikely((error= rotate(force_rotate, &check_purge))))
     check_purge= false;
+
+  DEBUG_SYNC(current_thd, "rotate_after_rotate");
+
   /*
     NOTE: Run purge_logs wo/ holding LOCK_log because it does not need
           the mutex. Otherwise causes various deadlocks.
+          Explicit binlog rotation must be synchronized with a concurrent
+          binlog ordered commit, in particular not let binlog
+          checkpoint notification request until early binlogged
+          concurrent commits have has been completed.
   */
+  mysql_mutex_lock(&LOCK_after_binlog_sync);
   mysql_mutex_unlock(&LOCK_log);
+  mysql_mutex_lock(&LOCK_commit_ordered);
+  mysql_mutex_unlock(&LOCK_after_binlog_sync);
+  mysql_mutex_unlock(&LOCK_commit_ordered);
 
   if (check_purge)
     checkpoint_and_purge(prev_binlog_id);
@@ -8079,7 +8093,12 @@ MYSQL_BIN_LOG::trx_group_commit_leader(group_commit_entry *leader)
   }
 
   DEBUG_SYNC(leader->thd, "commit_before_get_LOCK_commit_ordered");
+
   mysql_mutex_lock(&LOCK_commit_ordered);
+  DBUG_EXECUTE_IF("crash_before_engine_commit",
+      {
+        DBUG_SUICIDE();
+      });
   last_commit_pos_offset= commit_offset;
 
   /*

storage/innobase/handler/ha_innodb.cc

@@ -6193,9 +6193,10 @@ ha_innobase::open(const char* name, int, uint)
 
 	innobase_copy_frm_flags_from_table_share(ib_table, table->s);
 
+	const bool bk_thread = THDVAR(thd, background_thread);
 	/* No point to init any statistics if tablespace is still encrypted
 	or if table is being opened by background thread */
-	if (THDVAR(thd, background_thread)) {
+	if (bk_thread) {
 	} else if (ib_table->is_readable()) {
 		dict_stats_init(ib_table);
 	} else {
@@ -6394,11 +6395,14 @@ ha_innobase::open(const char* name, int, uint)
 		}
 	}
 
-	if (table && m_prebuilt->table) {
-		ut_ad(table->versioned() == m_prebuilt->table->versioned());
+	ut_ad(!m_prebuilt->table
+	      || table->versioned() == m_prebuilt->table->versioned());
+
+	if (!bk_thread) {
+		info(HA_STATUS_NO_LOCK | HA_STATUS_VARIABLE | HA_STATUS_CONST
+		     | HA_STATUS_OPEN);
 	}
 
-	info(HA_STATUS_NO_LOCK | HA_STATUS_VARIABLE | HA_STATUS_CONST | HA_STATUS_OPEN);
 	DBUG_RETURN(0);
 }