Speed up `mtr --parallel=<lots>` by scheduling some slow tests earlier.

The patch also fixes a race in rpl_stop_slave.test.

On machines with lots of CPU and memory, something like `mtr --parallel=10`
can speed up the test suite enormously. However, we have a few test cases
that run for long (several minutes), and if we are unlucky and happen to
schedule those towards the end of the test suite, we end up with most
workers idle while waiting for the last slow test to end, significantly
delaying the finish of the entire suite.

Improve this by marking the offending tests as taking "long", and trying
to schedule those tests early. This reduces the time towards the end of
the test suite run where some workers are waiting with nothing to do for
the remaining workers each to finish their last test.

Also, the rpl_stop_slave test had a race which could cause it to take
a 300 seconds debug_sync timeout; this is fixed.

Testing on a 4-core 8GB machine, this patch speeds up the test suite with
around 30% for --parallel=10 (debug build), allowing to run the entire
suite in 5 minutes.

mysql-test/extra/rpl_tests/rpl_stop_slave.test

@@ -42,6 +42,7 @@ send STOP SLAVE SQL_THREAD;
 connection slave1;
 --echo # To resume slave SQL thread
 SET DEBUG_SYNC= 'now SIGNAL signal.continue';
+SET DEBUG_SYNC= 'now WAIT_FOR signal.continued';
 SET DEBUG_SYNC= 'RESET';
 
 --echo

mysql-test/include/long_test.inc

+# We use this --source include to mark a test as taking long to run.
+# We can use this to schedule such test early (to not be left with
+# only one or two long tests running, and rests of works idle), or to
+# run a quick test skipping long-running test cases.

mysql-test/lib/mtr_cases.pm

@@ -89,6 +89,20 @@ sub init_pattern {
 }
 
 
+sub testcase_sort_order {
+  my ($a, $b, $sort_criteria)= @_;
+  my $a_sort_criteria= $sort_criteria->{$a->fullname()};
+  my $b_sort_criteria= $sort_criteria->{$b->fullname()};
+  my $res= $a_sort_criteria cmp $b_sort_criteria;
+  return $res if $res;
+  # Run slow tests first, trying to avoid getting stuck at the end
+  # with a slow test in one worker and the other workers idle.
+  return -1 if $a->{'long_test'} && !$b->{'long_test'};
+  return 1 if !$a->{'long_test'} && $b->{'long_test'};
+
+  return $a->fullname() cmp $b->fullname();
+}
+
 ##############################################################################
 #
 #  Collect information about test cases to be run
@@ -177,9 +191,7 @@ sub collect_test_cases ($$$) {
       $sort_criteria{$tinfo->fullname()} = join(" ", @criteria);
     }
 
-    @$cases = sort {
-      $sort_criteria{$a->fullname()} . $a->fullname() cmp
-	$sort_criteria{$b->fullname()} . $b->fullname() } @$cases;
+    @$cases = sort { testcase_sort_order($a, $b, \%sort_criteria) } @$cases;
 
     # For debugging the sort-order
     # foreach my $tinfo (@$cases)
@@ -1054,6 +1066,7 @@ my @tags=
  ["include/not_valgrind.inc", "not_valgrind", 1],
  ["include/have_example_plugin.inc", "example_plugin_test", 1],
  ["include/have_ssl.inc", "need_ssl", 1],
+ ["include/long_test.inc", "long_test", 1],
 );
 
 

mysql-test/mysql-test-run.pl

@@ -729,9 +729,11 @@ sub run_test_server ($$$) {
 	    last;
 	  }
 
-	  # Second best choice is the first that does not fulfill
-	  # any of the above conditions
-	  if (!defined $second_best){
+	  # From secondary choices, we prefer to pick a 'long-running' test if
+          # possible; this helps avoid getting stuck with a few of those at the
+          # end of high --parallel runs, with most workers being idle.
+	  if (!defined $second_best ||
+              ($t->{'long_test'} && !($tests->[$second_best]{'long_test'}))){
 	    #mtr_report("Setting second_best to $i");
 	    $second_best= $i;
 	  }

mysql-test/suite/federated/federated_debug.test

 --source include/have_debug.inc
+--source include/long_test.inc
 --source federated.inc
 
 --echo #

mysql-test/suite/maria/t/maria-recovery-rtree-ft.test

@@ -6,6 +6,7 @@
 # Binary must be compiled with debug for crash to occur
 --source include/have_debug.inc
 --source include/have_maria.inc
+--source include/long_test.inc
 
 set global maria_log_file_size=4294967295;
 let $MARIA_LOG=.;

mysql-test/suite/parts/t/partition_alter2_1_myisam.test

@@ -22,6 +22,8 @@
 # any of the variables.
 #
 
+--source include/long_test.inc
+
 #------------------------------------------------------------------------------#
 # General not engine specific settings and requirements
 

mysql-test/suite/parts/t/partition_alter2_2_myisam.test

@@ -22,6 +22,8 @@
 # any of the variables.
 #
 
+--source include/long_test.inc
+
 #------------------------------------------------------------------------------#
 # General not engine specific settings and requirements
 

mysql-test/suite/parts/t/partition_basic_innodb.test

@@ -22,6 +22,8 @@
 # any of the variables.
 #
 
+--source include/long_test.inc
+
 #------------------------------------------------------------------------------#
 # General not engine specific settings and requirements
 

mysql-test/suite/parts/t/partition_decimal_myisam.test

@@ -22,6 +22,8 @@
 # any of the variables.
 #
 
+--source include/long_test.inc
+
 #------------------------------------------------------------------------------#
 # General not engine specific settings and requirements
 

mysql-test/suite/parts/t/partition_float_myisam.test

@@ -22,6 +22,8 @@
 # any of the variables.
 #
 
+--source include/long_test.inc
+
 #------------------------------------------------------------------------------#
 # General not engine specific settings and requirements
 

mysql-test/suite/parts/t/partition_int_myisam.test

@@ -22,6 +22,8 @@
 # any of the variables.
 #
 
+--source include/long_test.inc
+
 #------------------------------------------------------------------------------#
 # General not engine specific settings and requirements
 

mysql-test/suite/rpl/r/rpl_stop_slave.result

@@ -38,6 +38,7 @@ STOP SLAVE SQL_THREAD;
 [ On Slave1 ]
 # To resume slave SQL thread
 SET DEBUG_SYNC= 'now SIGNAL signal.continue';
+SET DEBUG_SYNC= 'now WAIT_FOR signal.continued';
 SET DEBUG_SYNC= 'RESET';
 
 [ On Slave ]
@@ -63,6 +64,7 @@ STOP SLAVE SQL_THREAD;
 [ On Slave1 ]
 # To resume slave SQL thread
 SET DEBUG_SYNC= 'now SIGNAL signal.continue';
+SET DEBUG_SYNC= 'now WAIT_FOR signal.continued';
 SET DEBUG_SYNC= 'RESET';
 
 [ On Slave ]
@@ -89,6 +91,7 @@ STOP SLAVE SQL_THREAD;
 [ On Slave1 ]
 # To resume slave SQL thread
 SET DEBUG_SYNC= 'now SIGNAL signal.continue';
+SET DEBUG_SYNC= 'now WAIT_FOR signal.continued';
 SET DEBUG_SYNC= 'RESET';
 
 [ On Slave ]
@@ -115,6 +118,7 @@ STOP SLAVE SQL_THREAD;
 [ On Slave1 ]
 # To resume slave SQL thread
 SET DEBUG_SYNC= 'now SIGNAL signal.continue';
+SET DEBUG_SYNC= 'now WAIT_FOR signal.continued';
 SET DEBUG_SYNC= 'RESET';
 
 [ On Slave ]

mysql-test/suite/rpl/t/rpl_deadlock_innodb.test

@@ -7,5 +7,6 @@
 ########################################################
 -- source include/not_ndb_default.inc
 -- source include/have_innodb.inc
+-- source include/long_test.inc
 let $engine_type=innodb;
 -- source extra/rpl_tests/rpl_deadlock.test

mysql-test/suite/rpl/t/rpl_row_sp003.test

@@ -10,6 +10,7 @@
 -- source include/have_binlog_format_row.inc
 # Slow test, don't run during staging part
 -- source include/not_staging.inc
+--source include/long_test.inc
 -- source include/master-slave.inc
 
 let $engine_type=INNODB;

mysql-test/t/join_outer.test

+--source include/long_test.inc
+
 #
 # test of left outer join
 #

mysql-test/t/multi_update2.test

+--source include/long_test.inc
+
 #
 # Test of update statement that uses many tables.
 #

mysql-test/t/pool_of_threads.test

@@ -4,6 +4,7 @@
 -- source include/have_pool_of_threads.inc
 # Slow test, don't run during staging part
 -- source include/not_staging.inc
+-- source include/long_test.inc
 -- source include/common-tests.inc
 
 

mysql-test/t/query_cache.test

 -- source include/have_query_cache.inc
+-- source include/long_test.inc
 
 #
 # Tests with query cache

sql/sql_parse.cc

@@ -3286,12 +3286,17 @@ end_with_restore_list:
 
     DBUG_EXECUTE_IF("after_mysql_insert",
                     {
-                      const char act[]=
+                      const char act1[]=
                         "now "
                         "wait_for signal.continue";
+                      const char act2[]=
+                        "now "
+                        "signal signal.continued";
                       DBUG_ASSERT(opt_debug_sync_timeout > 0);
-                      DBUG_ASSERT(!debug_sync_set_action(current_thd,
-                                                         STRING_WITH_LEN(act)));
+                      DBUG_ASSERT(!debug_sync_set_action(thd,
+                                                         STRING_WITH_LEN(act1)));
+                      DBUG_ASSERT(!debug_sync_set_action(thd,
+                                                         STRING_WITH_LEN(act2)));
                     };);
     break;
   }