merge from 5.1-mtr

mysql-test/include/mtr_check.sql

@@ -57,3 +57,13 @@ BEGIN
     mysql.user;
 
 END||
+
+--
+-- Procedure used by test case used to force all
+-- servers to restart after testcase and thus skipping
+-- check test case after test
+--
+CREATE DEFINER=root@localhost PROCEDURE force_restart()
+BEGIN
+  SELECT 1 INTO OUTFILE 'force_restart';
+END||

mysql-test/lib/My/CoreDump.pm

@@ -22,6 +22,33 @@ use My::Platform;
 
 use File::Temp qw/ tempfile tempdir /;
 
+my $hint_mysqld;		# Last resort guess for executable path
+
+# If path in core file is 79 chars we assume it's been truncated
+# Looks like we can still find the full path using 'strings'
+# If that doesn't work, use the hint (mysqld path) as last resort.
+
+sub _verify_binpath {
+  my ($binary, $core_name)= @_;
+  my $binpath;
+
+  if (length $binary != 79) {
+    $binpath= $binary;
+    print "Core generated by '$binpath'\n";
+  } else {
+    # Last occurrence of path ending in /mysql*, cut from first /
+    if (`strings '$core_name' | grep "/mysql[^/. ]*\$" | tail -1` =~ /(\/.*)/) {
+      $binpath= $1;
+      print "Guessing that core was generated by '$binpath'\n";
+    } else {
+      return unless $hint_mysqld;
+      $binpath= $hint_mysqld;
+      print "Wild guess that core was generated by '$binpath'\n";
+    }
+  }
+  return $binpath;
+}
+
 sub _gdb {
   my ($core_name)= @_;
 
@@ -33,7 +60,8 @@ sub _gdb {
   `gdb -c '$core_name' --batch 2>&1` =~
     /Core was generated by `([^\s\'\`]+)/;
   my $binary= $1 or return;
-  print "Core generated by '$binary'\n";
+
+  $binary= _verify_binpath ($binary, $core_name) or return;
 
   # Create tempfile containing gdb commands
   my ($tmp, $tmp_name) = tempfile();
@@ -73,7 +101,8 @@ sub _dbx {
   `echo | dbx - '$core_name' 2>&1` =~
     /Corefile specified executable: "([^"]+)"/;
   my $binary= $1 or return;
-  print "Core generated by '$binary'\n";
+
+  $binary= _verify_binpath ($binary, $core_name) or return;
 
   # Find all threads
   my @thr_ids = `echo threads | dbx '$binary' '$core_name' 2>&1` =~ /t@\d+/g;
@@ -203,7 +232,7 @@ sub _cdb {
 
   my $cdb_cmd = "!sym prompts off; !analyze -v; .ecxr; !for_each_frame dv /t;!uniqstack -p;q";
   my $cdb_output=
-    `cdb -z $core_name -i "$image_path" -y "$symbol_path" -t 0 -lines -c "$cdb_cmd" 2>&1`;
+    `cdb -c "$cdb_cmd" -z $core_name -i "$image_path" -y "$symbol_path" -t 0 -lines 2>&1`;
   return if $? >> 8;
   return unless $cdb_output;
   
@@ -225,7 +254,8 @@ EOF
 
 
 sub show {
-  my ($class, $core_name)= @_;
+  my ($class, $core_name, $exe_mysqld)= @_;
+  $hint_mysqld= $exe_mysqld;
 
   # On Windows, rely on cdb to be there...
   if (IS_WINDOWS)

mysql-test/lib/My/File/Path.pm

@@ -164,6 +164,9 @@ sub copytree {
       copytree("$from_dir/$_", "$to_dir/$_");
       next;
     }
+
+    # Only copy plain files
+    next unless -f "$from_dir/$_";
     copy("$from_dir/$_", "$to_dir/$_");
   }
   closedir(DIR);

mysql-test/lib/My/SafeProcess.pm

@@ -536,7 +536,37 @@ sub wait_any {
   return $proc;
 }
 
+
+#
+# Wait for all processes to exit
 #
+sub wait_all {
+  while(keys %running)
+  {
+    wait_any();
+  }
+}
+
+
+#
+# Check if any process has exited, but don't wait.
+#
+# Returns a reference to the SafeProcess that
+# exited or undefined
+#
+sub check_any {
+  for my $proc (values %running){
+    if ( $proc->is_child($$) ) {
+      if (not $proc->wait_one(0)) {
+	_verbose ("Found exited $proc");
+	return $proc;
+      }
+    }
+  }
+  return undef;
+}
+
+
 # Overload string operator
 # and fallback to default functions if no
 # overloaded function is found

mysql-test/lib/My/SafeProcess/Base.pm

@@ -83,6 +83,13 @@ sub exit_status {
   };
 }
 
+# threads.pm may not exist everywhere, so use only on Windows.
+
+use if $^O eq "MSWin32", "threads";
+use if $^O eq "MSWin32", "threads::shared";
+
+my $win32_spawn_lock :shared;
+
 
 #
 # Create a new process
@@ -104,6 +111,8 @@ sub create_process {
 
   if ($^O eq "MSWin32"){
 
+    lock($win32_spawn_lock);
+
     #printf STDERR "stdin %d, stdout %d, stderr %d\n",
     #    fileno STDIN, fileno STDOUT, fileno STDERR;
 

mysql-test/lib/My/SafeProcess/safe_process_win.cc

@@ -259,22 +259,37 @@ int main(int argc, const char** argv )
     the JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE flag, making sure it will be
     terminated when the last handle to it is closed(which is owned by
     this process).
+
+    If breakaway from job fails on some reason, fallback is to create a
+    new process group. Process groups also allow to kill process and its 
+    descedants, subject to some restrictions (processes have to run within
+    the same console,and must not ignore CTRL_BREAK)
   */
-  if (CreateProcess(NULL, (LPSTR)child_args,
+  DWORD create_flags[]= {CREATE_BREAKAWAY_FROM_JOB, CREATE_NEW_PROCESS_GROUP, 0};
+  BOOL process_created= FALSE;
+  BOOL jobobject_assigned= FALSE;
+
+  for (int i=0; i < sizeof(create_flags)/sizeof(create_flags[0]); i++)
+  { 
+    process_created= CreateProcess(NULL, (LPSTR)child_args,
                     NULL,
                     NULL,
                     TRUE, /* inherit handles */
-                    CREATE_SUSPENDED | CREATE_BREAKAWAY_FROM_JOB,
+                    CREATE_SUSPENDED | create_flags[i],
                     NULL,
                     NULL,
                     &si,
-                    &process_info) == 0)
-    die("CreateProcess failed");
+                    &process_info);
+    if (process_created)
+    {
+     jobobject_assigned= AssignProcessToJobObject(job_handle, process_info.hProcess);
+     break;
+    }
+  }
 
-  if (AssignProcessToJobObject(job_handle, process_info.hProcess) == 0)
+  if (!process_created)
   {
-    TerminateProcess(process_info.hProcess, 200);
-    die("AssignProcessToJobObject failed");
+    die("CreateProcess failed");
   }
   ResumeThread(process_info.hThread);
   CloseHandle(process_info.hThread);
@@ -312,6 +327,13 @@ int main(int argc, const char** argv )
     message("TerminateJobObject failed");
   CloseHandle(job_handle);
   message("Job terminated and closed");
+
+  if (!jobobject_assigned)
+  {
+    GenerateConsoleCtrlEvent(CTRL_BREAK_EVENT, process_info.dwProcessId);
+    TerminateProcess(process_info.hProcess, 202);
+  }
+
   if (wait_res != WAIT_OBJECT_0 + CHILD)
   {
     /* The child has not yet returned, wait for it */

mysql-test/lib/mtr_cases.pm

@@ -33,7 +33,7 @@ our $print_testcases;
 our $skip_rpl;
 our $do_test;
 our $skip_test;
-our $opt_skip_combination;
+our $skip_combinations;
 our $binlog_format;
 our $enable_disabled;
 our $default_storage_engine;
@@ -119,11 +119,22 @@ sub collect_test_cases ($$) {
 	if ( $test->{name} =~ /.*\.$tname/ )
 	{
 	  $found= 1;
+	  last;
 	}
       }
       if ( not $found )
       {
-	mtr_error("Could not find '$tname' in '$suites' suite(s)");
+	mtr_error("Could not find '$tname' in '$suites' suite(s)") unless $sname;
+	# If suite was part of name, find it there
+	my ($this_case) = collect_one_suite($sname, [ $tname ]);
+	if ($this_case)
+        {
+	  push (@$cases, $this_case);
+	}
+	else
+	{
+	  mtr_error("Could not find '$tname' in '$sname' suite");
+        }
       }
     }
   }
@@ -375,7 +386,7 @@ sub collect_one_suite($)
   # Read combinations for this suite and build testcases x combinations
   # if any combinations exists
   # ----------------------------------------------------------------------
-  if ( ! $opt_skip_combination )
+  if ( ! $skip_combinations )
   {
     my @combinations;
     my $combination_file= "$suitedir/combinations";

mysql-test/lib/mtr_process.pl

@@ -21,6 +21,24 @@
 use strict;
 use Socket;
 use Errno;
+use My::Platform;
+use if IS_WINDOWS, "Net::Ping";
+
+# Ancient perl might not have port_number method for Net::Ping.
+# Check it and use fallback to connect() if it is not present.
+BEGIN 
+{
+  my $use_netping= 0;
+  if (IS_WINDOWS)
+  {
+    my $ping = Net::Ping->new();
+    if ($ping->can("port_number"))
+    {
+      $use_netping= 1;
+    }
+  }
+  eval 'sub USE_NETPING { $use_netping }';
+}
   
 sub sleep_until_file_created ($$$);
 sub mtr_ping_port ($);
@@ -30,6 +48,24 @@ sub mtr_ping_port ($) {
 
   mtr_verbose("mtr_ping_port: $port");
 
+  if (IS_WINDOWS && USE_NETPING)
+  {
+    # Under Windows, connect to a port that is not open is slow
+    # It takes ~1sec. Net::Ping with small timeout is much faster.
+    my $ping = Net::Ping->new();
+    $ping->port_number($port);
+    if ($ping->ping("localhost",0.1))
+    {
+      mtr_verbose("USED");
+      return 1;
+    }
+    else
+    {
+      mtr_verbose("FREE");
+      return 0;
+    }
+  }
+  
   my $remote= "localhost";
   my $iaddr=  inet_aton($remote);
   if ( ! $iaddr )

mysql-test/lib/mtr_report.pm

@@ -30,6 +30,8 @@ our @EXPORT= qw(report_option mtr_print_line mtr_print_thick_line
 		mtr_report_test);
 
 use mtr_match;
+use My::Platform;
+use POSIX qw[ _exit ];
 require "mtr_io.pl";
 
 my $tot_real_time= 0;
@@ -257,6 +259,17 @@ sub mtr_report_stats ($) {
       $tot_restarts++;
     }
 
+    # Add counts for repeated runs, if any.
+    # Note that the last run has already been counted above.
+    my $num_repeat = $tinfo->{'repeat'} - 1;
+    if ( $num_repeat > 0 )
+    {
+      $tot_tests += $num_repeat;
+      my $rep_failed = $tinfo->{'rep_failures'} || 0;
+      $tot_failed += $rep_failed;
+      $tot_passed += $num_repeat - $rep_failed;
+    }
+
     # Look for warnings produced by mysqltest
     my $base_file= mtr_match_extension($tinfo->{'result_file'},
 				       "result"); # Trim extension
@@ -336,7 +349,7 @@ sub mtr_report_stats ($) {
     foreach my $tinfo (@$tests)
     {
       my $tname= $tinfo->{'name'};
-      if ( $tinfo->{failures} and ! $seen{$tname})
+      if ( ($tinfo->{failures} || $tinfo->{rep_failures}) and ! $seen{$tname})
       {
         print " $tname";
 	$seen{$tname}= 1;
@@ -459,7 +472,14 @@ sub mtr_warning (@) {
 sub mtr_error (@) {
   print STDERR _name(), _timestamp(),
     "mysql-test-run: *** ERROR: ", join(" ", @_), "\n";
+  if (IS_WINDOWS)
+  {
+    POSIX::_exit(1);
+  }
+  else
+  {
     exit(1);
+  }
 }
 
 

mysql-test/lib/mtr_unique.pm

@@ -28,32 +28,36 @@ sub msg {
  # print "### unique($$) - ", join(" ", @_), "\n";
 }
 
-my $file;
+my $dir;
 
 if(!IS_WINDOWS)
 {
-  $file= "/tmp/mysql-test-ports";
+  $dir= "/tmp/mysql-unique-ids";
 }
 else
 {
-  $file= $ENV{'TEMP'}."/mysql-test-ports";
+  # Try to use machine-wide directory location for unique IDs,
+  # $ALLUSERSPROFILE . IF it is not available, fallback to $TEMP
+  # which is typically a per-user temporary directory
+  if (exists $ENV{'ALLUSERSPROFILE'} && -w $ENV{'ALLUSERSPROFILE'})
+  {
+    $dir= $ENV{'ALLUSERSPROFILE'}."/mysql-unique-ids";
+  }
+  else
+  {
+    $dir= $ENV{'TEMP'}."/mysql-unique-ids";
+  }
 }
 
+my $mtr_unique_fh = undef;
 
-my %mtr_unique_ids;
-
-END {
-  my $allocated_id= $mtr_unique_ids{$$};
-  if (defined $allocated_id)
-  {
-    mtr_release_unique_id($allocated_id);
-  }
-  delete $mtr_unique_ids{$$};
+END
+{
+  mtr_release_unique_id();
 }
 
 #
-# Get a unique, numerical ID, given a file name (where all
-# requested IDs are stored), a minimum and a maximum value.
+# Get a unique, numerical ID in a specified range.
 #
 # If no unique ID within the specified parameters can be
 # obtained, return undef.
@@ -61,135 +65,63 @@ END {
 sub mtr_get_unique_id($$) {
   my ($min, $max)= @_;;
 
-  msg("get, '$file', $min-$max");
+  msg("get $min-$max, $$");
 
-  die "Can only get one unique id per process!" if $mtr_unique_ids{$$};
+  die "Can only get one unique id per process!" if defined $mtr_unique_fh;
 
-  my $ret = undef;
-  my $changed = 0;
 
-  if(eval("readlink '$file'") || eval("readlink '$file.sem'")) {
-    die 'lock file is a symbolic link';
-  }
-
-  chmod 0777, "$file.sem";
-  open SEM, ">", "$file.sem" or die "can't write to $file.sem";
-  flock SEM, LOCK_EX or die "can't lock $file.sem";
-  if(! -e $file) {
-    open FILE, ">", $file or die "can't create $file";
-    close FILE;
+  # Make sure our ID directory exists
+  if (! -d $dir)
+  {
+    # If there is a file with the reserved
+    # directory name, just delete the file.
+    if (-e $dir)
+    {
+      unlink($dir);
     }
 
-  msg("HAVE THE LOCK");
-
-  if(eval("readlink '$file'") || eval("readlink '$file.sem'")) {
-    die 'lock file is a symbolic link';
-  }
+    mkdir $dir;
+    chmod 0777, $dir;
 
-  chmod 0777, $file;
-  open FILE, "+<", $file or die "can't open $file";
-  #select undef,undef,undef,0.2;
-  seek FILE, 0, 0;
-  my %taken = ();
-  while(<FILE>) {
-    chomp;
-    my ($id, $pid) = split / /;
-    $taken{$id} = $pid;
-    msg("taken: $id, $pid");
-    # Check if process with given pid is alive
-    if(!process_alive($pid)) {
-      print "Removing slot $id used by missing process $pid\n";
-      msg("Removing slot $id used by missing process $pid");
-      delete $taken{$id};
-      $changed++;
-    }
+    if(! -d $dir)
+    {
+      die "can't make directory $dir";
     }
-  for(my $i=$min; $i<=$max; ++$i) {
-    if(! exists $taken{$i}) {
-      $ret = $i;
-      $taken{$i} = $$;
-      $changed++;
-      # Remember the id this process got
-      $mtr_unique_ids{$$}= $i;
-      msg(" got $i"); 
-      last;
   }
+
+
+  my $fh;
+  for(my $id = $min; $id <= $max; $id++)
+  {
+    open( $fh, ">$dir/$id");
+    chmod 0666, "$dir/$id";
+    # Try to lock the file exclusively. If lock succeeds, we're done.
+    if (flock($fh, LOCK_EX|LOCK_NB))
+    {
+      # Store file handle - we would need it to release the ID (==unlock the file)
+      $mtr_unique_fh = $fh;
+      return $id;
     }
-  if($changed) {
-    seek FILE, 0, 0;
-    truncate FILE, 0 or die "can't truncate $file";
-    for my $k (keys %taken) {
-      print FILE $k . ' ' . $taken{$k} . "\n";
+    else
+    {
+      close $fh;
     }
   }
-  close FILE;
-
-  msg("RELEASING THE LOCK");
-  flock SEM, LOCK_UN or warn "can't unlock $file.sem";
-  close SEM;
-
-  return $ret;
+  return undef;
 }
 
 
 #
 # Release a unique ID.
 #
-sub mtr_release_unique_id($) {
-  my ($myid)= @_;
-
-  msg("release, $myid");
-
-
-  if(eval("readlink '$file'") || eval("readlink '$file.sem'")) {
-    die 'lock file is a symbolic link';
-  }
-
-  open SEM, ">", "$file.sem" or die "can't write to $file.sem";
-  flock SEM, LOCK_EX or die "can't lock $file.sem";
-
-  msg("HAVE THE LOCK");
-
-  if(eval("readlink '$file'") || eval("readlink '$file.sem'")) {
-    die 'lock file is a symbolic link';
-  }
-
-  if(! -e $file) {
-    open FILE, ">", $file or die "can't create $file";
-    close FILE;
-  }
-  open FILE, "+<", $file or die "can't open $file";
-  #select undef,undef,undef,0.2;
-  seek FILE, 0, 0;
-  my %taken = ();
-  while(<FILE>) {
-    chomp;
-    my ($id, $pid) = split / /;
-    msg(" taken, $id $pid");
-    $taken{$id} = $pid;
-  }
-
-  if ($taken{$myid} != $$)
+sub mtr_release_unique_id()
+{
+  msg("release $$");
+  if (defined $mtr_unique_fh)
   {
-    msg(" The unique id for this process does not match pid");
-  }
-
-
-  msg(" removing $myid");
-  delete $taken{$myid};
-  seek FILE, 0, 0;
-  truncate FILE, 0 or die "can't truncate $file";
-  for my $k (keys %taken) {
-    print FILE $k . ' ' . $taken{$k} . "\n";
+    close $mtr_unique_fh;
+    $mtr_unique_fh = undef;
   }
-  close FILE;
-
-  msg("RELEASE THE LOCK");
-
-  flock SEM, LOCK_UN or warn "can't unlock $file.sem";
-  close SEM;
-
-  delete $mtr_unique_ids{$$};
 }
 
 

mysql-test/mysql-test-run.pl

@@ -209,6 +209,7 @@ sub check_timeout { return $opt_testcase_timeout * 6; };
 
 my $opt_start;
 my $opt_start_dirty;
+my $opt_wait_all;
 my $opt_repeat= 1;
 my $opt_retry= 3;
 my $opt_retry_failure= 2;
@@ -429,6 +430,7 @@ sub run_test_server ($$$) {
   my $completed= [];
   my %running;
   my $result;
+  my $exe_mysqld= find_mysqld($basedir) || ""; # Used as hint to CoreDump
 
   my $suite_timeout_proc= My::SafeProcess->timer(suite_timeout());
 
@@ -500,7 +502,7 @@ sub run_test_server ($$$) {
 			   mtr_report(" - found '$core_name'",
 				      "($num_saved_cores/$opt_max_save_core)");
 
-			   My::CoreDump->show($core_file);
+			   My::CoreDump->show($core_file, $exe_mysqld);
 
 			   if ($num_saved_cores >= $opt_max_save_core) {
 			     mtr_report(" - deleting it, already saved",
@@ -515,6 +517,7 @@ sub run_test_server ($$$) {
 	      }
 	    }
 	    $num_saved_datadir++;
+	    $num_failed_test++ unless $result->{retries};
 
 	    if ( !$opt_force ) {
 	      # Test has failed, force is off
@@ -529,7 +532,6 @@ sub run_test_server ($$$) {
 			 "Terminating...");
 	      return undef;
 	    }
-	    $num_failed_test++;
 	  }
 
 	  # Retry test run after test failure
@@ -554,9 +556,11 @@ sub run_test_server ($$$) {
 
 	  # Repeat test $opt_repeat number of times
 	  my $repeat= $result->{repeat} || 1;
-	  if ($repeat < $opt_repeat)
+	  # Don't repeat if test was skipped
+	  if ($repeat < $opt_repeat && $result->{'result'} ne 'MTR_RES_SKIPPED')
 	  {
 	    $result->{retries}= 0;
+	    $result->{rep_failures}++ if $result->{failures};
 	    $result->{failures}= 0;
 	    delete($result->{result});
 	    $result->{repeat}= $repeat+1;
@@ -876,6 +880,7 @@ sub command_line_setup {
              'sleep=i'                  => \$opt_sleep,
              'start-dirty'              => \$opt_start_dirty,
              'start'                    => \$opt_start,
+             'wait-all'                 => \$opt_wait_all,
 	     'print-testcases'          => \&collect_option,
 	     'repeat=i'                 => \$opt_repeat,
 	     'retry=i'                  => \$opt_retry,
@@ -1233,6 +1238,15 @@ sub command_line_setup {
     }
   }
 
+  # --------------------------------------------------------------------------
+  # Check use of wait-all
+  # --------------------------------------------------------------------------
+
+  if ($opt_wait_all && ! ($opt_start_dirty || $opt_start))
+  {
+    mtr_error("--wait-all can only be used with --start or --start-dirty");
+  }
+
   # --------------------------------------------------------------------------
   # Check timeout arguments
   # --------------------------------------------------------------------------
@@ -1323,28 +1337,30 @@ sub set_build_thread_ports($) {
 
   if ( lc($opt_build_thread) eq 'auto' ) {
     my $found_free = 0;
-    $build_thread = 250;	# Start attempts from here
+    $build_thread = 300;	# Start attempts from here
     while (! $found_free)
     {
-      $build_thread= mtr_get_unique_id($build_thread, 299);
+      $build_thread= mtr_get_unique_id($build_thread, 349);
       if ( !defined $build_thread ) {
         mtr_error("Could not get a unique build thread id");
       }
       $found_free= check_ports_free($build_thread);
       # If not free, release and try from next number
-      mtr_release_unique_id($build_thread++) unless $found_free;
+      if (! $found_free) {
+        mtr_release_unique_id();
+        $build_thread++;
+      }
     }
   }
   else
   {
     $build_thread = $opt_build_thread + $thread - 1;
-  }
-  $ENV{MTR_BUILD_THREAD}= $build_thread;
-
     if (! check_ports_free($build_thread)) {
       # Some port was not free(which one has already been printed)
       mtr_error("Some port(s) was not free")
     }
+  }
+  $ENV{MTR_BUILD_THREAD}= $build_thread;
 
   # Calculate baseport
   $baseport= $build_thread * 10 + 10000;
@@ -3134,6 +3150,26 @@ sub find_analyze_request
 }
 
 
+# The test can leave a file in var/tmp/ to signal
+# that all servers should be restarted
+sub restart_forced_by_test
+{
+  my $restart = 0;
+  foreach my $mysqld ( mysqlds() )
+  {
+    my $datadir = $mysqld->value('datadir');
+    my $force_restart_file = "$datadir/mtr/force_restart";
+    if ( -f $force_restart_file )
+    {
+      mtr_verbose("Restart of servers forced by test");
+      $restart = 1;
+      last;
+    }
+  }
+  return $restart;
+}
+
+
 # Return timezone value of tinfo or default value
 sub timezone {
   my ($tinfo)= @_;
@@ -3175,7 +3211,7 @@ sub run_testcase ($) {
     {
 
       # Remove old datadirs
-      clean_datadir();
+      clean_datadir() unless $opt_start_dirty;
 
       # Restore old ENV
       while (my ($option, $value)= each( %old_env )) {
@@ -3242,11 +3278,20 @@ sub run_testcase ($) {
   # --------------------------------------------------------------------
   # If --start or --start-dirty given, stop here to let user manually
   # run tests
+  # If --wait-all is also given, do the same, but don't die if one
+  # server exits
   # ----------------------------------------------------------------------
+
   if ( $opt_start or $opt_start_dirty )
   {
     mtr_print("\nStarted", started(all_servers()));
     mtr_print("Waiting for server(s) to exit...");
+    if ( $opt_wait_all ) {
+      My::SafeProcess->wait_all();
+      mtr_print( "All servers exited" );
+      exit(1);
+    }
+    else {
       my $proc= My::SafeProcess->wait_any();
       if ( grep($proc eq $_, started(all_servers())) )
       {
@@ -3256,6 +3301,7 @@ sub run_testcase ($) {
       mtr_print("Unknown process $proc died");
       exit(1);
     }
+  }
 
   my $test_timeout_proc= My::SafeProcess->timer(testcase_timeout());
 
@@ -3272,10 +3318,38 @@ sub run_testcase ($) {
   }
 
   my $test= start_mysqltest($tinfo);
+  # Set only when we have to keep waiting after expectedly died server
+  my $keep_waiting_proc = 0;
 
   while (1)
   {
-    my $proc= My::SafeProcess->wait_any();
+    my $proc;
+    if ($keep_waiting_proc)
+    {
+      # Any other process exited?
+      $proc = My::SafeProcess->check_any();
+      if ($proc)
+      {
+	mtr_verbose ("Found exited process $proc");
+	# If that was the timeout, cancel waiting
+	if ( $proc eq $test_timeout_proc )
+	{
+	  $keep_waiting_proc = 0;
+	}
+      }
+      else
+      {
+	$proc = $keep_waiting_proc;
+      }
+    }
+    else
+    {
+      $proc= My::SafeProcess->wait_any();
+    }
+
+    # Will be restored if we need to keep waiting
+    $keep_waiting_proc = 0;
+
     unless ( defined $proc )
     {
       mtr_error("wait_any failed");
@@ -3302,7 +3376,11 @@ sub run_testcase ($) {
       if ( $res == 0 )
       {
 	my $check_res;
-	if ( $opt_check_testcases and
+	if ( restart_forced_by_test() )
+	{
+	  stop_all_servers();
+	}
+	elsif ( $opt_check_testcases and
 	     $check_res= check_testcase($tinfo, "after"))
 	{
 	  if ($check_res == 1) {
@@ -3367,8 +3445,12 @@ sub run_testcase ($) {
     # ----------------------------------------------------
     # Check if it was an expected crash
     # ----------------------------------------------------
-    if ( check_expected_crash_and_restart($proc) )
+    my $check_crash = check_expected_crash_and_restart($proc);
+    if ($check_crash)
     {
+      # Keep waiting if it returned 2, if 1 don't wait or stop waiting.
+      $keep_waiting_proc = 0 if $check_crash == 1;
+      $keep_waiting_proc = $proc if $check_crash == 2;
       next;
     }
 
@@ -3709,16 +3791,16 @@ sub check_expected_crash_and_restart {
     {
       mtr_verbose("Crash was expected, file '$expect_file' exists");
 
-      while (1){
-
+      for (my $waits = 0;  $waits < 50;  $waits++)
+      {
 	# If last line in expect file starts with "wait"
 	# sleep a little and try again, thus allowing the
 	# test script to control when the server should start
-	# up again
+	# up again. Keep trying for up to 5s at a time.
 	my $last_line= mtr_lastlinesfromfile($expect_file, 1);
 	if ($last_line =~ /^wait/ )
 	{
-	  mtr_verbose("Test says wait before restart");
+	  mtr_verbose("Test says wait before restart") if $waits == 0;
 	  mtr_milli_sleep(100);
 	  next;
 	}
@@ -3728,11 +3810,11 @@ sub check_expected_crash_and_restart {
 	# Start server with same settings as last time
 	mysqld_start($mysqld, $mysqld->{'started_opts'});
 
-	last;
+	return 1;
       }
+      # Loop ran through: we should keep waiting after a re-check
+      return 2;
     }
-
-    return 1;
   }
 
   # Not an expected crash
@@ -4430,6 +4512,8 @@ sub start_servers($) {
 
     my $mysqld_basedir= $mysqld->value('basedir');
     if ( $basedir eq $mysqld_basedir )
+    {
+      if (! $opt_start_dirty)	# If dirty, keep possibly grown system db
       {
 	# Copy datadir from installed system db
 	for my $path ( "$opt_vardir", "$opt_vardir/..") {
@@ -4440,6 +4524,7 @@ sub start_servers($) {
 	mtr_error("Failed to copy system db to '$datadir'")
 	  unless -d $datadir;
       }
+    }
     else
     {
       mysql_install_db($mysqld); # For versional testing
@@ -4978,10 +5063,13 @@ Options to control what engine/variation to run
   vs-config             Visual Studio configuration used to create executables
                         (default: MTR_VS_CONFIG environment variable)
 
-  config|defaults-file=<config template> Use fixed config template for all
+  defaults-file=<config template> Use fixed config template for all
                         tests
   defaults_extra_file=<config template> Extra config template to add to
                         all generated configs
+  combination=<opt>     Use at least twice to run tests with specified 
+                        options to mysqld
+  skip-combinations     Ignore combination file (or options)
 
 Options to control directories to use
   tmpdir=DIR            The directory where temporary files are stored
@@ -5004,7 +5092,6 @@ Options to control what test suites or cases to run
   force                 Continue to run the suite after failure
   with-ndbcluster-only  Run only tests that include "ndb" in the filename
   skip-ndb[cluster]     Skip all tests that need cluster
-  skip-ndb[cluster]-slave Skip all tests that need a slave cluster
   do-test=PREFIX or REGEX
                         Run test cases which name are prefixed with PREFIX
                         or fulfills REGEX
@@ -5019,6 +5106,9 @@ Options to control what test suites or cases to run
                         The default is: "$DEFAULT_SUITES"
   skip-rpl              Skip the replication test cases.
   big-test              Also run tests marked as "big"
+  enable-disabled       Run also tests marked as disabled
+  print_testcases       Don't run the tests but print details about all the
+                        selected tests, in the order they would be run.
 
 Options that specify ports
 
@@ -5087,7 +5177,7 @@ Options for valgrind
   valgrind-options=ARGS Deprecated, use --valgrind-option
   valgrind-option=ARGS  Option to give valgrind, replaces default option(s),
                         can be specified more then once
-  valgrind-path=[EXE]   Path to the valgrind executable
+  valgrind-path=<EXE>   Path to the valgrind executable
   callgrind             Instruct valgrind to use callgrind
 
 Misc options
@@ -5095,14 +5185,18 @@ Misc options
   comment=STR           Write STR to the output
   notimer               Don't show test case execution time
   verbose               More verbose output(use multiple times for even more)
+  verbose-restart       Write when and why servers are restarted
   start                 Only initialize and start the servers, using the
                         startup settings for the first specified test case
                         Example:
                          $0 --start alias &
   start-dirty           Only start the servers (without initialization) for
                         the first specified test case
+  wait-all              If --start or --start-dirty option is used, wait for all
+                        servers to exit before finishing the process
   fast                  Run as fast as possible, dont't wait for servers
                         to shutdown etc.
+  parallel=N            Run tests in N parallel threads (default=1)
   repeat=N              Run each test N number of times
   retry=N               Retry tests that fail N times, limit number of failures
                         to $opt_retry_failure
@@ -5120,6 +5214,12 @@ Misc options
   sleep=SECONDS         Passed to mysqltest, will be used as fixed sleep time
   gcov                  Collect coverage information after the test.
                         The result is a gcov file per source and header file.
+  experimental=<file>   Refer to list of tests considered experimental;
+                        failures will be marked exp-fail instead of fail.
+  report-features       First run a "test" that reports mysql features
+  timestamp             Print timestamp before each test report line
+  timediff              With --timestamp, also print time passed since
+                        *previous* test started
 
 HERE
   exit(1);

mysql-test/r/init_file.result

@@ -4,6 +4,7 @@ SELECT * INTO @Y FROM init_file.startup limit 1,1;
 SELECT YEAR(@X)-YEAR(@Y);
 YEAR(@X)-YEAR(@Y)
 0
+DROP DATABASE init_file;
 ok
 end of 4.1 tests
 select * from t1;
@@ -19,3 +20,5 @@ y
 3
 11
 13
+drop table t1, t2;
+call mtr.force_restart();

mysql-test/suite/funcs_1/datadict/is_routines.inc

@@ -96,10 +96,11 @@ CREATE FUNCTION function_for_routines() RETURNS INT RETURN 0;
 SELECT specific_name,routine_catalog,routine_schema,routine_name,routine_type,
        routine_body,external_name,external_language,parameter_style,sql_path
 FROM information_schema.routines
-WHERE routine_catalog   IS NOT NULL OR external_name   IS NOT NULL
+WHERE routine_schema = 'test' AND
+   (routine_catalog   IS NOT NULL OR external_name   IS NOT NULL
    OR external_language IS NOT NULL OR sql_path        IS NOT NULL
    OR routine_body      <> 'SQL'    OR parameter_style <> 'SQL'
-   OR specific_name     <> routine_name;
+   OR specific_name     <> routine_name);
 
 DROP PROCEDURE sp_for_routines;
 DROP FUNCTION  function_for_routines;

mysql-test/suite/funcs_1/r/is_routines.result

@@ -111,10 +111,11 @@ CREATE FUNCTION function_for_routines() RETURNS INT RETURN 0;
 SELECT specific_name,routine_catalog,routine_schema,routine_name,routine_type,
 routine_body,external_name,external_language,parameter_style,sql_path
 FROM information_schema.routines
-WHERE routine_catalog   IS NOT NULL OR external_name   IS NOT NULL
+WHERE routine_schema = 'test' AND
+(routine_catalog   IS NOT NULL OR external_name   IS NOT NULL
 OR external_language IS NOT NULL OR sql_path        IS NOT NULL
 OR routine_body      <> 'SQL'    OR parameter_style <> 'SQL'
-   OR specific_name     <> routine_name;
+   OR specific_name     <> routine_name);
 specific_name	routine_catalog	routine_schema	routine_name	routine_type	routine_body	external_name	external_language	parameter_style	sql_path
 DROP PROCEDURE sp_for_routines;
 DROP FUNCTION  function_for_routines;

mysql-test/t/init_file.test

@@ -14,7 +14,7 @@ SELECT * INTO @X FROM init_file.startup limit 0,1;
 SELECT * INTO @Y FROM init_file.startup limit 1,1;
 SELECT YEAR(@X)-YEAR(@Y);
 # Enable this DROP DATABASE only after resolving bug #42507
-# DROP DATABASE init_file;
+DROP DATABASE init_file;
 
 --echo ok
 --echo end of 4.1 tests
@@ -28,4 +28,9 @@ select * from t1;
 #   30, 3, 11, 13
 select * from t2;
 # Enable this DROP TABLE only after resolving bug #42507
-#drop table t1, t2;
+drop table t1, t2;
+
+# MTR will restart server anyway, but by forcing it we avoid being warned
+# about the apparent side effect
+
+call mtr.force_restart();