MDEV-23511 shutdown_server 10 times out, causing server kill at shutdown

Shutdown of mtr tests may be too impatient, esp on CI environment where
10 seconds of `arg` of `shutdown_server arg` may not be enough for the clean
shutdown to complete.

This is fixed to remove explicit non-zero timeout argument to
`shutdown_server` from all mtr tests. mysqltest computes 60 seconds default
value for the timeout for the argless `shutdown_server` command.
This policy is additionally ensured with a compile time assert.

client/mysqltest.cc

@@ -185,7 +185,7 @@ static uint opt_tail_lines= 0;
 
 static uint opt_connect_timeout= 0;
 static uint opt_wait_for_pos_timeout= 0;
-
+static const  uint default_wait_for_pos_timeout= 300;
 static char delimiter[MAX_DELIMITER_LENGTH]= ";";
 static uint delimiter_length= 1;
 
@@ -5074,6 +5074,8 @@ void do_shutdown_server(struct st_command *command)
   };
   DBUG_ENTER("do_shutdown_server");
 
+  /* the wait-for-pos' default based value of 'timeout' must fit to MDEV-23511 */
+  compile_time_assert(default_wait_for_pos_timeout / 5 >= 60);
   check_command_args(command, command->first_argument, shutdown_args,
                      sizeof(shutdown_args)/sizeof(struct command_arg),
                      ' ');
@@ -7058,7 +7060,7 @@ static struct my_option my_long_options[] =
   {"wait_for_pos_timeout", 0,
    "Number of seconds to wait for master_pos_wait",
    &opt_wait_for_pos_timeout, &opt_wait_for_pos_timeout, 0, GET_UINT,
-   REQUIRED_ARG, 300, 0, 3600 * 12, 0, 0, 0},
+   REQUIRED_ARG, default_wait_for_pos_timeout, 0, 3600 * 12, 0, 0, 0},
   {"plugin_dir", 0, "Directory for client-side plugins.",
     &opt_plugin_dir, &opt_plugin_dir, 0,
    GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},

mysql-test/include/restart_mysqld.inc

 # ==== Usage ====
 #
-# [--let $shutdown_timeout= 30]
+# [--let $shutdown_timeout= 60]
 # [--let $allow_rpl_inited= 1]
 # --source include/restart_mysqld.inc
 

mysql-test/include/rpl_stop_server.inc

@@ -47,8 +47,8 @@ if ($rpl_debug)
 --exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.$rpl_server_number.expect
 
 # Send shutdown to the connected server and give
-# it 60 seconds to die before zapping it
-shutdown_server 60;
+# it 60 seconds (of mysqltest's default) to die before zapping it
+shutdown_server;
 
 --source include/wait_until_disconnected.inc
 

mysql-test/include/search_pattern_in_file.inc

@@ -36,7 +36,7 @@
 #    # Stop the server
 #    let $restart_file= $MYSQLTEST_VARDIR/tmp/mysqld.1.expect;
 #    --exec echo "wait" > $restart_file
-#    --shutdown_server 10
+#    --shutdown_server
 #    --source include/wait_until_disconnected.inc
 #
 #    --error 1

mysql-test/include/shutdown_mysqld.inc

 # ==== Usage ====
 #
-# [--let $shutdown_timeout= 30]
+# [--let $shutdown_timeout= 60]
 # [--let $allow_rpl_inited= 1]
 # --source include/shutdown_mysqld.inc
 

mysql-test/suite/binlog/t/binlog_max_extension.test

@@ -41,7 +41,7 @@ RESET MASTER;
 
 # 1. Stop master server
 -- exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
--- shutdown_server 10
+-- shutdown_server
 -- source include/wait_until_disconnected.inc
 
 # 2. Prepare log and index file
@@ -70,7 +70,7 @@ FLUSH LOGS;
 
 # 1. Stop the server
 -- exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
--- shutdown_server 10
+-- shutdown_server
 -- source include/wait_until_disconnected.inc
 
 # 2. Undo changes to index and log files

mysql-test/suite/binlog_encryption/restart_server.inc

@@ -22,7 +22,7 @@
 --enable_reconnect
 --exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.$rpl_server_number.expect
 
-shutdown_server 10;
+shutdown_server;
 
 --source include/wait_until_disconnected.inc
 

mysql-test/suite/encryption/t/innochecksum.test

@@ -63,7 +63,7 @@ let MYSQLD_DATADIR=`select @@datadir`;
 --echo # We give 30 seconds to do a clean shutdown because we do not want
 --echo # to redo apply the pages of t1.ibd at the time of recovery.
 --echo # We want SQL to initiate the first access to t1.ibd.
-shutdown_server 30;
+shutdown_server;
 
 --echo # Wait until disconnected.
 --source include/wait_until_disconnected.inc

mysql-test/suite/innodb/t/innodb-corrupted-table.test

@@ -24,7 +24,7 @@ alter table t1 add primary key (pk);
 --echo # Stop the server, replace the frm with the old one and restart the server
 
 --exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
---shutdown_server 10
+--shutdown_server
 --source include/wait_until_disconnected.inc
 
 --remove_file $datadir/test/t1.frm

mysql-test/suite/innodb/t/innodb_bug60196.test

@@ -50,7 +50,7 @@ SELECT * FROM bug_60196;
 -- exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
 
 # Send a shutdown request to the server
--- shutdown_server 10
+-- shutdown_server
 
 # Call script that will poll the server waiting for it to disapear
 -- source include/wait_until_disconnected.inc
@@ -124,7 +124,7 @@ SELECT * FROM Bug_60309;
 -- exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
 
 # Send a shutdown request to the server
--- shutdown_server 10
+-- shutdown_server
 
 # Call script that will poll the server waiting for it to disapear
 -- source include/wait_until_disconnected.inc

mysql-test/suite/multi_source/info_logs.test

@@ -150,7 +150,7 @@ show all slaves status;
 --append_file $MYSQLTEST_VARDIR/tmp/mysqld.3.expect
 restart
 EOF
---shutdown_server 60
+--shutdown_server
 --source include/wait_until_connected_again.inc
 --source include/wait_for_slave_to_start.inc
 set default_master_connection = 'MASTER 2.2';

mysql-test/suite/percona/slow_extended.patch/percona_slow_extended-slave_innodb_stats.test

@@ -23,7 +23,7 @@ STOP SLAVE;
 --write_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect
 wait
 EOF
---shutdown_server 10
+--shutdown_server
 --source include/wait_until_disconnected.inc
 --append_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect
 restart

mysql-test/suite/roles/acl_load_mutex-5170.test

@@ -10,7 +10,7 @@ flush tables;
 --append_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
 wait
 EOF
---shutdown_server 60
+--shutdown_server
 --source include/wait_until_disconnected.inc
 --enable_reconnect
 --append_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect

mysql-test/suite/rpl/t/rpl_gtid_crash.test

@@ -624,7 +624,7 @@ SELECT * FROM t1 WHERE a >= 30 ORDER BY a;
 --write_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
 wait
 EOF
-shutdown_server 10;
+shutdown_server;
 --source include/wait_until_disconnected.inc
 
 --remove_file $datadir/master-bin.state

mysql-test/suite/rpl/t/rpl_gtid_stop_start.test

@@ -29,7 +29,7 @@ CHANGE MASTER TO master_use_gtid=current_pos;
 wait
 EOF
 FLUSH LOGS;
---shutdown_server 30
+--shutdown_server
 --source include/wait_until_disconnected.inc
 
 --connection server_1
@@ -70,7 +70,7 @@ SHOW BINLOG EVENTS IN 'master-bin.000004' LIMIT 1,1;
 --write_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
 wait
 EOF
---shutdown_server 30
+--shutdown_server
 --source include/wait_until_disconnected.inc
 
 --append_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
@@ -103,7 +103,7 @@ SELECT * FROM t1 ORDER BY a;
 --write_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect
 wait
 EOF
---shutdown_server 30
+--shutdown_server
 --source include/wait_until_disconnected.inc
 
 --append_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect
@@ -148,7 +148,7 @@ SELECT * FROM t1 ORDER BY a;
 --write_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect
 wait
 EOF
---shutdown_server 30
+--shutdown_server
 --source include/wait_until_disconnected.inc
 
 --append_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect
@@ -191,7 +191,7 @@ SET sql_log_bin= 1;
 --write_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect
 wait
 EOF
---shutdown_server 30
+--shutdown_server
 --source include/wait_until_disconnected.inc
 
 --append_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect
@@ -221,7 +221,7 @@ SELECT * FROM t1 ORDER BY a;
 --write_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect
 wait
 EOF
---shutdown_server 30
+--shutdown_server
 --source include/wait_until_disconnected.inc
 
 --append_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect
@@ -259,7 +259,7 @@ SELECT domain_id, COUNT(*) FROM mysql.gtid_slave_pos GROUP BY domain_id;
 --write_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect
 wait
 EOF
---shutdown_server 30
+--shutdown_server
 --source include/wait_until_disconnected.inc
 
 --append_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect
@@ -291,7 +291,7 @@ SET sql_log_bin=1;
 --write_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect
 wait
 EOF
---shutdown_server 30
+--shutdown_server
 --source include/wait_until_disconnected.inc
 
 # Let the slave mysqld server start again.

mysql-test/suite/rpl/t/rpl_mdev382.test

@@ -212,7 +212,7 @@ SELECT * FROM `db1``; select 'oops!'`.`t``1` ORDER BY 1;
 wait-rpl_mdev382.test
 EOF
 
---shutdown_server 30
+--shutdown_server
 
 --append_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
 restart-rpl_mdev382.test

mysql-test/suite/rpl/t/rpl_parallel_partition.test

@@ -42,7 +42,7 @@ ALTER TABLE `E` REMOVE PARTITIONING;
 --write_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
 wait
 EOF
---shutdown_server 30
+--shutdown_server
 --source include/wait_until_disconnected.inc
 --connection default
 --source include/wait_until_disconnected.inc

mysql-test/suite/storage_engine/alter_tablespace.test

@@ -46,7 +46,7 @@ wait
 EOF
 
   --enable_reconnect
-  --shutdown_server 60
+  --shutdown_server
 
   --source include/wait_until_disconnected.inc
 

mysql-test/suite/storage_engine/trx/xa_recovery.test

@@ -12,7 +12,7 @@
 --append_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
 restart
 EOF
---shutdown_server 60
+--shutdown_server
 --source include/wait_until_connected_again.inc
 
 

mysql-test/t/empty_server_name-8224.test

@@ -4,7 +4,7 @@
 --source include/not_embedded.inc
 create server '' foreign data wrapper w2 options (host '127.0.0.1');
 --exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
---shutdown_server 10
+--shutdown_server
 --source include/wait_until_disconnected.inc
 --exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
 

mysql-test/t/host_cache_size_functionality.test

@@ -44,7 +44,7 @@ select @@global.Host_Cache_Size > 0;
 
 let $restart_file= $MYSQLTEST_VARDIR/tmp/mysqld.1.expect;
 --exec echo "wait" > $restart_file
---shutdown_server 10
+--shutdown_server
 --source include/wait_until_disconnected.inc
 -- exec echo "restart:--host_cache_size=1  " > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
 -- enable_reconnect
@@ -143,7 +143,7 @@ SELECT Host_Cache_Size = @@SESSION.Host_Cache_Size;
 
 #let $restart_file= $MYSQLTEST_VARDIR/tmp/mysqld.1.expect;
 #--exec echo "wait" > $restart_file
-#--shutdown_server 10
+#--shutdown_server
 #--source include/wait_until_disconnected.inc
 #-- exec echo "restart:--bind-address=$bind_ip  " > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
 #-- enable_reconnect

mysql-test/t/init_file_set_password-7656.test

@@ -15,7 +15,7 @@ EOF
 
 --exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
 
---shutdown_server 10
+--shutdown_server
 --source include/wait_until_disconnected.inc
 
 --exec echo "restart:--init-file=$MYSQLTEST_VARDIR/init.file  " > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect

mysql-test/t/log_errchk.test

@@ -31,7 +31,7 @@
 --echo #         and slow query log file.
 # Restart server with fifo file as general log file.
 --exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
---shutdown_server 60
+--shutdown_server
 --source include/wait_until_disconnected.inc
 --enable_reconnect
 # Write file to make mysql-test-run.pl start up the server again

storage/rocksdb/mysql-test/rocksdb/include/restart_mysqld_with_option.inc

@@ -15,7 +15,7 @@ if ($rpl_inited)
 
 # Send shutdown to the connected server and give
 # it 10 seconds to die before zapping it
-shutdown_server 10;
+shutdown_server;
 
 # Write file to make mysql-test-run.pl start up the server again
 --exec echo "restart:$_mysqld_option" > $_expect_file_name

storage/rocksdb/mysql-test/rocksdb/t/check_ignore_unknown_options.test

@@ -41,7 +41,7 @@ perl;
 EOF
 
 --exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
---shutdown_server 10
+--shutdown_server
 
 --error 1
 --exec $MYSQLD_CMD --plugin_load=$HA_ROCKSDB_SO --rocksdb_ignore_unknown_options=0 --log-error=$error_log

storage/rocksdb/mysql-test/rocksdb/t/insert_optimized_config.test

@@ -8,7 +8,7 @@ DROP TABLE IF EXISTS t1;
 # reload with load optimized config
 let $restart_file= $MYSQLTEST_VARDIR/tmp/mysqld.1.expect;
 --exec echo "wait" > $restart_file
---shutdown_server 10
+--shutdown_server
 --source include/wait_until_disconnected.inc
 -- exec echo "restart:--rocksdb_write_disable_wal=1 --rocksdb_flush_log_at_trx_commit=0 --rocksdb_default_cf_options=write_buffer_size=16k;target_file_size_base=16k;level0_file_num_compaction_trigger=4;level0_slowdown_writes_trigger=256;level0_stop_writes_trigger=256;max_write_buffer_number=16;compression_per_level=kNoCompression;memtable=vector:1024 --rocksdb_override_cf_options=__system__={memtable=skip_list:16} --rocksdb_compaction_sequential_deletes=0 --rocksdb_compaction_sequential_deletes_window=0 --rocksdb_allow_concurrent_memtable_write=0" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
 -- enable_reconnect
@@ -39,7 +39,7 @@ select count(*), sum(id), sum(i1), sum(i2) from t1;
 # reload without load optimized config
 let $restart_file= $MYSQLTEST_VARDIR/tmp/mysqld.1.expect;
 --exec echo "wait" > $restart_file
---shutdown_server 10
+--shutdown_server
 --source include/wait_until_disconnected.inc
 -- exec echo "restart:--rocksdb_write_disable_wal=0 --rocksdb_default_cf_options=write_buffer_size=64k;target_file_size_base=64k;max_bytes_for_level_base=1m;compression_per_level=kNoCompression;" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
 -- enable_reconnect

storage/rocksdb/mysql-test/rocksdb/t/mysqldump2.test

@@ -21,7 +21,7 @@ optimize table t1;
 #wiping block cache
 let $restart_file= $MYSQLTEST_VARDIR/tmp/mysqld.1.expect;
 --exec echo "wait" > $restart_file
---shutdown_server 10
+--shutdown_server
 --source include/wait_until_disconnected.inc
 -- exec echo "restart:--rocksdb_default_cf_options=write_buffer_size=64k;target_file_size_base=64k;max_bytes_for_level_base=1m;compression_per_level=kNoCompression;disable_auto_compactions=true;level0_stop_writes_trigger=1000  " > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
 -- enable_reconnect

storage/rocksdb/mysql-test/rocksdb/t/optimize_table.test

@@ -32,7 +32,7 @@ while ($t <= 6) {
 # Disable auto compaction so that effects of optimize table are stable
 let $restart_file= $MYSQLTEST_VARDIR/tmp/mysqld.1.expect;
 --exec echo "wait" > $restart_file
---shutdown_server 10
+--shutdown_server
 --source include/wait_until_disconnected.inc
 -- exec echo "restart:--rocksdb_default_cf_options=write_buffer_size=64k;target_file_size_base=64k;max_bytes_for_level_base=1m;compression_per_level=kNoCompression;disable_auto_compactions=true;level0_stop_writes_trigger=1000  " > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
 -- enable_reconnect

storage/rocksdb/mysql-test/rocksdb/t/persistent_cache.test

@@ -10,7 +10,7 @@ DROP TABLE IF EXISTS t1;
 --exec echo "wait" >$_expect_file_name
 
 # restart server with correct parameters
-shutdown_server 10;
+shutdown_server;
 --exec echo "restart:--rocksdb_persistent_cache_path=$_cache_file_name --rocksdb_persistent_cache_size_mb=100" >$_expect_file_name
 --sleep 5
 --enable_reconnect
@@ -28,7 +28,7 @@ select * from t1 where a = 1;
 
 # restart server to re-read cache
 --exec echo "wait" >$_expect_file_name
-shutdown_server 10;
+shutdown_server;
 --exec echo "restart:--rocksdb_persistent_cache_path=$_cache_file_name --rocksdb_persistent_cache_size_mb=100" >$_expect_file_name
 --sleep 5
 --enable_reconnect

storage/rocksdb/mysql-test/rocksdb/t/shutdown.test

@@ -23,7 +23,7 @@ while ($i <= $max) {
 # Restart the server
 let $restart_file= $MYSQLTEST_VARDIR/tmp/mysqld.1.expect;
 --exec echo "wait" > $restart_file
---shutdown_server 10
+--shutdown_server
 --source include/wait_until_disconnected.inc
 -- exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
 -- enable_reconnect

storage/rocksdb/mysql-test/rocksdb/t/validate_datadic.test

@@ -25,7 +25,7 @@ CREATE TABLE t2 (pk int primary key) ENGINE=ROCKSDB PARTITION BY KEY(pk) PARTITI
 
 # Send shutdown to the connected server and give it 10 seconds to die before
 # zapping it
-shutdown_server 10;
+shutdown_server;
 
 # Write file to make mysql-test-run.pl start up the server again
 --exec echo "restart" >$_expect_file_name
@@ -42,7 +42,7 @@ shutdown_server 10;
 
 # Now shut down again and rename one of the .frm files
 --exec echo "wait" >$_expect_file_name
-shutdown_server 10;
+shutdown_server;
 
 # Rename the file
 --move_file $MYSQLTEST_VARDIR/mysqld.1/data/test/t1.frm $MYSQLTEST_VARDIR/mysqld.1/data/test/t1.frm.tmp
@@ -70,7 +70,7 @@ shutdown_server 10;
 
 # Now shut down again and rename one the .frm file back and make a copy of it
 --exec echo "wait" >$_expect_file_name
-shutdown_server 10;
+shutdown_server;
 --remove_file $LOG
 # Rename the file
 --move_file $MYSQLTEST_VARDIR/mysqld.1/data/test/t1.frm.tmp $MYSQLTEST_VARDIR/mysqld.1/data/test/t1.frm
@@ -92,7 +92,7 @@ shutdown_server 10;
 
 # Shut down an clean up
 --exec echo "wait" >$_expect_file_name
-shutdown_server 10;
+shutdown_server;
 --remove_file $MYSQLTEST_VARDIR/mysqld.1/data/test/t1_dummy.frm
 --exec echo "restart" >$_expect_file_name
 --enable_reconnect

storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_rate_limiter_bytes_per_sec_basic.test

@@ -13,7 +13,7 @@ SET @@global.rocksdb_rate_limiter_bytes_per_sec = 10000;
 
 # Send shutdown to the connected server and give it 10 seconds to die before
 # zapping it
-shutdown_server 10;
+shutdown_server;
 
 # Attempt to restart the server with the rate limiter on
 --exec echo "restart:--rocksdb_rate_limiter_bytes_per_sec=10000" >$_expect_file_name
@@ -53,7 +53,7 @@ SET @@global.rocksdb_rate_limiter_bytes_per_sec = -1;
 
 # Restart the server without the rate limiter
 --exec echo "wait" >$_expect_file_name
-shutdown_server 10;
+shutdown_server;
 --exec echo "restart" >$_expect_file_name
 --sleep 5