This is another patch for test suite to make IM tests work smoothly.

The patch affects only test suite and has the following changes:

1. On starting Instance Manager -- added code for waiting for guarded
mysqld instances to create PID files;
2. Polishing;
3. Move IM-related operations out of mtr_process.pl into new dedicated
for IM file -- mtr_im.pl


mysql-test/mysql-test-run.pl:
  1. Require new file with IM-related operations;
  2. Added timeout to wait for PID files of guarded instances;
  3. Polishing.
mysql-test/lib/mtr_process.pl:
  Move IM-related operations to new file.
mysql-test/lib/mtr_im.pl:
  File for IM-related operations.

BitKeeper/etc/collapsed

@@ -2,3 +2,5 @@
 44ec850ac2k4y2Omgr92GiWPBAVKGQ
 44edb86b1iE5knJ97MbliK_3lCiAXA
 44f33f3aj5KW5qweQeekY1LU0E9ZCg
+45001f7c3b2hhCXDKfUvzkX9TNe6VA
+45002051rHJfMEXAIMiAZV0clxvKSA

mysql-test/lib/mtr_im.pl

+# -*- cperl -*-
+
+# This is a library file used by the Perl version of mysql-test-run,
+# and is part of the translation of the Bourne shell script with the
+# same name.
+
+use strict;
+
+# Private IM-related operations.
+
+sub mtr_im_kill_process ($$$$);
+sub mtr_im_load_pids ($);
+sub mtr_im_terminate ($);
+sub mtr_im_check_alive ($);
+sub mtr_im_check_main_alive ($);
+sub mtr_im_check_angel_alive ($);
+sub mtr_im_check_mysqlds_alive ($);
+sub mtr_im_check_mysqld_alive ($);
+sub mtr_im_cleanup ($);
+sub mtr_im_rm_file ($);
+sub mtr_im_errlog ($);
+sub mtr_im_kill ($);
+sub mtr_im_wait_for_connection ($$$);
+sub mtr_im_wait_for_mysqld($$$);
+
+# Public IM-related operations.
+
+sub mtr_im_start ($$);
+sub mtr_im_stop ($);
+
+##############################################################################
+#
+#  Private operations.
+#
+##############################################################################
+
+sub mtr_im_kill_process ($$$$) {
+  my $pid_lst= shift;
+  my $signal= shift;
+  my $total_retries= shift;
+  my $timeout= shift;
+
+  my %pids;
+
+  foreach my $pid ( @{$pid_lst} )
+  {
+    $pids{$pid}= 1;
+  }
+
+  for ( my $cur_attempt= 1; $cur_attempt <= $total_retries; ++$cur_attempt )
+  {
+    foreach my $pid ( keys %pids )
+    {
+      mtr_debug("Sending $signal to $pid...");
+
+      kill($signal, $pid);
+
+      unless ( kill (0, $pid) )
+      {
+        mtr_debug("Process $pid died.");
+        delete $pids{$pid};
+      }
+    }
+
+    return if scalar keys %pids == 0;
+
+    mtr_debug("Sleeping $timeout second(s) waiting for processes to die...");
+
+    sleep($timeout);
+  }
+
+  mtr_debug("Process(es) " .
+            join(' ', keys %pids) .
+            " is still alive after $total_retries " .
+            "of sending signal $signal.");
+}
+
+###########################################################################
+
+sub mtr_im_load_pids($) {
+  my $im= shift;
+
+  mtr_debug("Loading PID files...");
+
+  # Obtain mysqld-process pids.
+
+  my $instances = $im->{'instances'};
+
+  for ( my $idx= 0; $idx < 2; ++$idx )
+  {
+    mtr_debug("IM-guarded mysqld[$idx] PID file: '" .
+              $instances->[$idx]->{'path_pid'} . "'.");
+
+    my $mysqld_pid;
+
+    if ( -r $instances->[$idx]->{'path_pid'} )
+    {
+      $mysqld_pid= mtr_get_pid_from_file($instances->[$idx]->{'path_pid'});
+      mtr_debug("IM-guarded mysqld[$idx] PID: $mysqld_pid.");
+    }
+    else
+    {
+      $mysqld_pid= undef;
+      mtr_debug("IM-guarded mysqld[$idx]: no PID file.");
+    }
+
+    $instances->[$idx]->{'pid'}= $mysqld_pid;
+  }
+
+  # Re-read Instance Manager PIDs from the file, since during tests Instance
+  # Manager could have been restarted, so its PIDs could have been changed.
+
+  #   - IM-main
+
+  mtr_debug("IM-main PID file: '$im->{path_pid}'.");
+
+  if ( -f $im->{'path_pid'} )
+  {
+    $im->{'pid'} =
+      mtr_get_pid_from_file($im->{'path_pid'});
+
+    mtr_debug("IM-main PID: $im->{pid}.");
+  }
+  else
+  {
+    mtr_debug("IM-main: no PID file.");
+    $im->{'pid'}= undef;
+  }
+
+  #   - IM-angel
+
+  mtr_debug("IM-angel PID file: '$im->{path_angel_pid}'.");
+
+  if ( -f $im->{'path_angel_pid'} )
+  {
+    $im->{'angel_pid'} =
+      mtr_get_pid_from_file($im->{'path_angel_pid'});
+
+    mtr_debug("IM-angel PID: $im->{'angel_pid'}.");
+  }
+  else
+  {
+    mtr_debug("IM-angel: no PID file.");
+    $im->{'angel_pid'} = undef;
+  }
+}
+
+###########################################################################
+
+sub mtr_im_terminate($) {
+  my $im= shift;
+
+  # Load pids from pid-files. We should do it first of all, because IM deletes
+  # them on shutdown.
+
+  mtr_im_load_pids($im);
+
+  mtr_debug("Shutting Instance Manager down...");
+
+  # Ignoring SIGCHLD so that all children could rest in peace.
+
+  start_reap_all();
+
+  # Send SIGTERM to IM-main.
+
+  if ( defined $im->{'pid'} )
+  {
+    mtr_debug("IM-main pid: $im->{pid}.");
+    mtr_debug("Stopping IM-main...");
+
+    mtr_im_kill_process([ $im->{'pid'} ], 'TERM', 10, 1);
+  }
+  else
+  {
+    mtr_debug("IM-main pid: n/a.");
+  }
+
+  # If IM-angel was alive, wait for it to die.
+
+  if ( defined $im->{'angel_pid'} )
+  {
+    mtr_debug("IM-angel pid: $im->{'angel_pid'}.");
+    mtr_debug("Waiting for IM-angel to die...");
+
+    my $total_attempts= 10;
+
+    for ( my $cur_attempt=1; $cur_attempt <= $total_attempts; ++$cur_attempt )
+    {
+      unless ( kill (0, $im->{'angel_pid'}) )
+      {
+        mtr_debug("IM-angel died.");
+        last;
+      }
+
+      sleep(1);
+    }
+  }
+  else
+  {
+    mtr_debug("IM-angel pid: n/a.");
+  }
+
+  stop_reap_all();
+
+  # Re-load PIDs.
+
+  mtr_im_load_pids($im);
+}
+
+###########################################################################
+
+sub mtr_im_check_alive($) {
+  my $im= shift;
+
+  mtr_debug("Checking whether IM-components are alive...");
+
+  return 1 if mtr_im_check_main_alive($im);
+
+  return 1 if mtr_im_check_angel_alive($im);
+
+  return 1 if mtr_im_check_mysqlds_alive($im);
+
+  return 0;
+}
+
+###########################################################################
+
+sub mtr_im_check_main_alive($) {
+  my $im= shift;
+
+  # Check that the process, that we know to be IM's, is dead.
+
+  if ( defined $im->{'pid'} )
+  {
+    if ( kill (0, $im->{'pid'}) )
+    {
+      mtr_debug("IM-main (PID: $im->{pid}) is alive.");
+      return 1;
+    }
+    else
+    {
+      mtr_debug("IM-main (PID: $im->{pid}) is dead.");
+    }
+  }
+  else
+  {
+    mtr_debug("No PID file for IM-main.");
+  }
+
+  # Check that IM does not accept client connections.
+
+  if ( mtr_ping_mysqld_server($im->{'port'}) )
+  {
+    mtr_debug("IM-main (port: $im->{port}) " .
+              "is accepting connections.");
+
+    mtr_im_errlog("IM-main is accepting connections on port " .
+                  "$im->{port}, but there is no " .
+                  "process information.");
+    return 1;
+  }
+  else
+  {
+    mtr_debug("IM-main (port: $im->{port}) " .
+              "does not accept connections.");
+    return 0;
+  }
+}
+
+###########################################################################
+
+sub mtr_im_check_angel_alive($) {
+  my $im= shift;
+
+  # Check that the process, that we know to be the Angel, is dead.
+
+  if ( defined $im->{'angel_pid'} )
+  {
+    if ( kill (0, $im->{'angel_pid'}) )
+    {
+      mtr_debug("IM-angel (PID: $im->{angel_pid}) is alive.");
+      return 1;
+    }
+    else
+    {
+      mtr_debug("IM-angel (PID: $im->{angel_pid}) is dead.");
+      return 0;
+    }
+  }
+  else
+  {
+    mtr_debug("No PID file for IM-angel.");
+    return 0;
+  }
+}
+
+###########################################################################
+
+sub mtr_im_check_mysqlds_alive($) {
+  my $im= shift;
+
+  mtr_debug("Checking for IM-guarded mysqld instances...");
+
+  my $instances = $im->{'instances'};
+
+  for ( my $idx= 0; $idx < 2; ++$idx )
+  {
+    mtr_debug("Checking mysqld[$idx]...");
+
+    return 1
+      if mtr_im_check_mysqld_alive($instances->[$idx]);
+  }
+}
+
+###########################################################################
+
+sub mtr_im_check_mysqld_alive($) {
+  my $mysqld_instance= shift;
+
+  # Check that the process is dead.
+
+  if ( defined $mysqld_instance->{'pid'} )
+  {
+    if ( kill (0, $mysqld_instance->{'pid'}) )
+    {
+      mtr_debug("Mysqld instance (PID: $mysqld_instance->{pid}) is alive.");
+      return 1;
+    }
+    else
+    {
+      mtr_debug("Mysqld instance (PID: $mysqld_instance->{pid}) is dead.");
+    }
+  }
+  else
+  {
+    mtr_debug("No PID file for mysqld instance.");
+  }
+
+  # Check that mysqld does not accept client connections.
+
+  if ( mtr_ping_mysqld_server($mysqld_instance->{'port'}) )
+  {
+    mtr_debug("Mysqld instance (port: $mysqld_instance->{port}) " .
+              "is accepting connections.");
+
+    mtr_im_errlog("Mysqld is accepting connections on port " .
+                  "$mysqld_instance->{port}, but there is no " .
+                  "process information.");
+    return 1;
+  }
+  else
+  {
+    mtr_debug("Mysqld instance (port: $mysqld_instance->{port}) " .
+              "does not accept connections.");
+    return 0;
+  }
+}
+
+###########################################################################
+
+sub mtr_im_cleanup($) {
+  my $im= shift;
+
+  mtr_im_rm_file($im->{'path_pid'});
+  mtr_im_rm_file($im->{'path_sock'});
+
+  mtr_im_rm_file($im->{'path_angel_pid'});
+
+  for ( my $idx= 0; $idx < 2; ++$idx )
+  {
+    mtr_im_rm_file($im->{'instances'}->[$idx]->{'path_pid'});
+    mtr_im_rm_file($im->{'instances'}->[$idx]->{'path_sock'});
+  }
+}
+
+###########################################################################
+
+sub mtr_im_rm_file($)
+{
+  my $file_path= shift;
+
+  if ( -f $file_path )
+  {
+    mtr_debug("Removing '$file_path'...");
+
+    unless ( unlink($file_path) )
+    {
+      mtr_warning("Can not remove '$file_path'.")
+    }
+  }
+  else
+  {
+    mtr_debug("File '$file_path' does not exist already.");
+  }
+}
+
+###########################################################################
+
+sub mtr_im_errlog($) {
+  my $msg= shift;
+
+  # Complain in error log so that a warning will be shown.
+  # 
+  # TODO: unless BUG#20761 is fixed, we will print the warning to stdout, so
+  # that it can be seen on console and does not produce pushbuild error.
+
+  # my $errlog= "$opt_vardir/log/mysql-test-run.pl.err";
+  # 
+  # open (ERRLOG, ">>$errlog") ||
+  #   mtr_error("Can not open error log ($errlog)");
+  # 
+  # my $ts= localtime();
+  # print ERRLOG
+  #   "Warning: [$ts] $msg\n";
+  # 
+  # close ERRLOG;
+
+  my $ts= localtime();
+  print "Warning: [$ts] $msg\n";
+}
+
+###########################################################################
+
+sub mtr_im_kill($) {
+  my $im= shift;
+
+  # Re-load PIDs. That can be useful because some processes could have been
+  # restarted.
+
+  mtr_im_load_pids($im);
+
+  # Ignoring SIGCHLD so that all children could rest in peace.
+
+  start_reap_all();
+
+  # Kill IM-angel first of all.
+
+  if ( defined $im->{'angel_pid'} )
+  {
+    mtr_debug("Killing IM-angel (PID: $im->{angel_pid})...");
+    mtr_im_kill_process([ $im->{'angel_pid'} ], 'KILL', 10, 1)
+  }
+  else
+  {
+    mtr_debug("IM-angel is dead.");
+  }
+
+  # Re-load PIDs again.
+
+  mtr_im_load_pids($im);
+
+  # Kill IM-main.
+  
+  if ( defined $im->{'pid'} )
+  {
+    mtr_debug("Killing IM-main (PID: $im->pid})...");
+    mtr_im_kill_process([ $im->{'pid'} ], 'KILL', 10, 1);
+  }
+  else
+  {
+    mtr_debug("IM-main is dead.");
+  }
+
+  # Re-load PIDs again.
+
+  mtr_im_load_pids($im);
+
+  # Kill guarded mysqld instances.
+
+  my @mysqld_pids;
+
+  mtr_debug("Collecting PIDs of mysqld instances to kill...");
+
+  for ( my $idx= 0; $idx < 2; ++$idx )
+  {
+    my $pid= $im->{'instances'}->[$idx]->{'pid'};
+
+    unless ( defined $pid )
+    {
+      next;
+    }
+
+    mtr_debug("  - IM-guarded mysqld[$idx] PID: $pid.");
+
+    push (@mysqld_pids, $pid);
+  }
+
+  if ( scalar @mysqld_pids > 0 )
+  {
+    mtr_debug("Killing IM-guarded mysqld instances...");
+    mtr_im_kill_process(\@mysqld_pids, 'KILL', 10, 1);
+  }
+
+  # That's all.
+
+  stop_reap_all();
+}
+
+##############################################################################
+
+sub mtr_im_wait_for_connection($$$) {
+  my $im= shift;
+  my $total_attempts= shift;
+  my $connect_timeout= shift;
+
+  mtr_debug("Waiting for IM on port $im->{port} " .
+            "to start accepting connections...");
+
+  for ( my $cur_attempt= 1; $cur_attempt <= $total_attempts; ++$cur_attempt )
+  {
+    mtr_debug("Trying to connect to IM ($cur_attempt of $total_attempts)...");
+
+    if ( mtr_ping_mysqld_server($im->{'port'}) )
+    {
+      mtr_debug("IM is accepting connections " .
+                "on port $im->{port}.");
+      return 1;
+    }
+
+    mtr_debug("Sleeping $connect_timeout...");
+    sleep($connect_timeout);
+  }
+
+  mtr_debug("IM does not accept connections " .
+            "on port $im->{port} after " .
+            ($total_attempts * $connect_timeout) . " seconds.");
+
+  return 0;
+}
+
+##############################################################################
+
+sub mtr_im_wait_for_mysqld($$$) {
+  my $mysqld= shift;
+  my $total_attempts= shift;
+  my $connect_timeout= shift;
+
+  mtr_debug("Waiting for IM-guarded mysqld on port $mysqld->{port} " .
+            "to start accepting connections...");
+
+  for ( my $cur_attempt= 1; $cur_attempt <= $total_attempts; ++$cur_attempt )
+  {
+    mtr_debug("Trying to connect to mysqld " .
+              "($cur_attempt of $total_attempts)...");
+
+    if ( mtr_ping_mysqld_server($mysqld->{'port'}) )
+    {
+      mtr_debug("Mysqld is accepting connections " .
+                "on port $mysqld->{port}.");
+      return 1;
+    }
+
+    mtr_debug("Sleeping $connect_timeout...");
+    sleep($connect_timeout);
+  }
+
+  mtr_debug("Mysqld does not accept connections " .
+            "on port $mysqld->{port} after " .
+            ($total_attempts * $connect_timeout) . " seconds.");
+
+  return 0;
+}
+
+##############################################################################
+#
+#  Public operations.
+#
+##############################################################################
+
+sub mtr_im_start($$) {
+  my $im = shift;
+  my $opts = shift;
+
+  mtr_debug("Starting Instance Manager...");
+
+  my $args;
+  mtr_init_args(\$args);
+  mtr_add_arg($args, "--defaults-file=%s", $im->{'defaults_file'});
+
+  foreach my $opt ( @{$opts} )
+  {
+    mtr_add_arg($args, $opt);
+  }
+
+  $im->{'pid'} =
+    mtr_spawn(
+      $::exe_im,                        # path to the executable
+      $args,                            # cmd-line args
+      '',                               # stdin
+      $im->{'path_log'},                # stdout
+      $im->{'path_err'},                # stderr
+      '',                               # pid file path (not used)
+      { append_log_file => 1 }          # append log files
+      );
+
+  unless ( $im->{'pid'} )
+  {
+    mtr_error('Could not start Instance Manager.')
+  }
+
+  # Instance Manager can be run in daemon mode. In this case, it creates
+  # several processes and the parent process, created by mtr_spawn(), exits just
+  # after start. So, we have to obtain Instance Manager PID from the PID file.
+
+  mtr_debug("Waiting for IM to create PID file (" .
+            "path: '$im->{path_pid}'; " .
+            "timeout: $im->{start_timeout})...");
+
+  unless ( sleep_until_file_created($im->{'path_pid'},
+                                    $im->{'start_timeout'},
+                                    -1) ) # real PID is still unknown
+  {
+    mtr_debug("IM has not created PID file in $im->{start_timeout} secs.");
+    mtr_debug("Aborting test suite...");
+
+    mtr_kill_leftovers();
+
+    mtr_report("IM has not created PID file in $im->{start_timeout} secs.");
+    return 0;
+  }
+
+  $im->{'pid'}= mtr_get_pid_from_file($im->{'path_pid'});
+
+  mtr_debug("Instance Manager started. PID: $im->{pid}.");
+
+  # Wait until we can connect to IM.
+
+  my $IM_CONNECT_TIMEOUT= 30;
+
+  unless ( mtr_im_wait_for_connection($im,
+                                      $IM_CONNECT_TIMEOUT, 1) )
+  {
+    mtr_debug("Can not connect to Instance Manager " .
+              "in $IM_CONNECT_TIMEOUT seconds after start.");
+    mtr_debug("Aborting test suite...");
+
+    mtr_kill_leftovers();
+
+    mtr_report("Can not connect to Instance Manager " .
+               "in $IM_CONNECT_TIMEOUT seconds after start.");
+    return 0;
+  }
+
+  # Wait for IM to start guarded instances:
+  #   - wait for PID files;
+
+  mtr_debug("Waiting for guarded mysqlds instances to create PID files...");
+
+  for ( my $idx= 0; $idx < 2; ++$idx )
+  {
+    my $mysqld= $im->{'instances'}->[$idx];
+
+    if ( exists $mysqld->{'nonguarded'} )
+    {
+      next;
+    }
+
+    mtr_debug("Waiting for mysqld[$idx] to create PID file (" .
+              "path: '$mysqld->{path_pid}'; " .
+              "timeout: $mysqld->{start_timeout})...");
+
+    unless ( sleep_until_file_created($mysqld->{'path_pid'},
+                                      $mysqld->{'start_timeout'},
+                                      -1) ) # real PID is still unknown
+    {
+      mtr_debug("mysqld[$idx] has not created PID file in " .
+                 "$mysqld->{start_timeout} secs.");
+      mtr_debug("Aborting test suite...");
+
+      mtr_kill_leftovers();
+
+      mtr_report("mysqld[$idx] has not created PID file in " .
+                 "$mysqld->{start_timeout} secs.");
+      return 0;
+    }
+
+    mtr_debug("PID file for mysqld[$idx] ($mysqld->{path_pid} created.");
+  }
+
+  # Wait until we can connect to guarded mysqld-instances
+  # (in other words -- wait for IM to start guarded instances).
+
+  mtr_debug("Waiting for guarded mysqlds to start accepting connections...");
+
+  for ( my $idx= 0; $idx < 2; ++$idx )
+  {
+    my $mysqld= $im->{'instances'}->[$idx];
+
+    if ( exists $mysqld->{'nonguarded'} )
+    {
+      next;
+    }
+
+    mtr_debug("Waiting for mysqld[$idx] to accept connection...");
+
+    unless ( mtr_im_wait_for_mysqld($mysqld, 30, 1) )
+    {
+      mtr_debug("Can not connect to mysqld[$idx] " .
+                "in $IM_CONNECT_TIMEOUT seconds after start.");
+      mtr_debug("Aborting test suite...");
+
+      mtr_kill_leftovers();
+
+      mtr_report("Can not connect to mysqld[$idx] " .
+                 "in $IM_CONNECT_TIMEOUT seconds after start.");
+      return 0;
+    }
+
+    mtr_debug("mysqld[$idx] started.");
+  }
+
+  mtr_debug("Instance Manager and its components are up and running.");
+
+  return 1;
+}
+
+##############################################################################
+
+sub mtr_im_stop($) {
+  my $im= shift;
+
+  mtr_debug("Stopping Instance Manager...");
+
+  # Try graceful shutdown.
+
+  mtr_im_terminate($im);
+
+  # Check that all processes died.
+
+  unless ( mtr_im_check_alive($im) )
+  {
+    mtr_debug("Instance Manager has been stopped successfully.");
+    mtr_im_cleanup($im);
+    return 1;
+  }
+
+  # Instance Manager don't want to die. We should kill it.
+
+  mtr_im_errlog("Instance Manager did not shutdown gracefully.");
+
+  mtr_im_kill($im);
+
+  # Check again that all IM-related processes have been killed.
+
+  my $im_is_alive= mtr_im_check_alive($im);
+
+  mtr_im_cleanup($im);
+
+  if ( $im_is_alive )
+  {
+    mtr_debug("Can not kill Instance Manager or its children.");
+    return 0;
+  }
+
+  mtr_debug("Instance Manager has been killed successfully.");
+  return 1;
+}
+
+###########################################################################
+
+1;

mysql-test/lib/mtr_process.pl

@@ -22,28 +22,6 @@ sub sleep_until_file_created ($$$);
 sub mtr_kill_processes ($);
 sub mtr_ping_mysqld_server ($);
 
-# Private IM-related operations.
-
-sub mtr_im_kill_process ($$$$);
-sub mtr_im_load_pids ($);
-sub mtr_im_terminate ($);
-sub mtr_im_check_alive ($);
-sub mtr_im_check_main_alive ($);
-sub mtr_im_check_angel_alive ($);
-sub mtr_im_check_mysqlds_alive ($);
-sub mtr_im_check_mysqld_alive ($);
-sub mtr_im_cleanup ($);
-sub mtr_im_rm_file ($);
-sub mtr_im_errlog ($);
-sub mtr_im_kill ($);
-sub mtr_im_wait_for_connection ($$$);
-sub mtr_im_wait_for_mysqld($$$);
-
-# Public IM-related operations.
-
-sub mtr_im_start ($$);
-sub mtr_im_stop ($);
-
 # static in C
 sub spawn_impl ($$$$$$$$);
 
@@ -381,9 +359,16 @@ sub mtr_process_exit_status {
 
 sub mtr_kill_leftovers () {
 
+  mtr_report("Killing Possible Leftover Processes");
   mtr_debug("mtr_kill_leftovers(): started.");
 
-  mtr_im_stop($::instance_manager);
+  mkpath("$::opt_vardir/log"); # Needed for mysqladmin log
+
+  # Stop or kill Instance Manager and all its children. If we failed to do
+  # that, we can only abort -- there is nothing left to do.
+
+  mtr_error("Failed to stop Instance Manager.")
+    unless mtr_im_stop($::instance_manager);
 
   # Kill mysqld servers (masters and slaves) that would conflict with this
   # run. Make sure to remove the PID file, if any.
@@ -1002,675 +987,6 @@ sub mtr_exit ($) {
   exit($code);
 }
 
-##############################################################################
-#
-#  Instance Manager management routines.
-#
-##############################################################################
-
-sub mtr_im_kill_process ($$$$) {
-  my $pid_lst= shift;
-  my $signal= shift;
-  my $total_retries= shift;
-  my $timeout= shift;
-
-  my %pids;
-
-  foreach my $pid (@{$pid_lst})
-  {
-    $pids{$pid}= 1;
-  }
-
-  for (my $cur_attempt= 1; $cur_attempt <= $total_retries; ++$cur_attempt)
-  {
-    foreach my $pid (keys %pids)
-    {
-      mtr_debug("Sending $signal to $pid...");
-
-      kill($signal, $pid);
-
-      unless (kill (0, $pid))
-      {
-        mtr_debug("Process $pid died.");
-        delete $pids{$pid};
-      }
-    }
-
-    return if scalar keys %pids == 0;
-
-    mtr_debug("Sleeping $timeout second(s) waiting for processes to die...");
-
-    sleep($timeout);
-  }
-
-  mtr_debug("Process(es) " .
-            join(' ', keys %pids) .
-            " is still alive after $total_retries " .
-            "of sending signal $signal.");
-}
-
-###########################################################################
-
-sub mtr_im_load_pids($) {
-  my $instance_manager= shift;
-
-  mtr_debug("Loading PID files...");
-
-  # Obtain mysqld-process pids.
-
-  my $instances = $instance_manager->{'instances'};
-
-  for (my $idx= 0; $idx < 2; ++$idx)
-  {
-    mtr_debug("IM-guarded mysqld[$idx] PID file: '" .
-              $instances->[$idx]->{'path_pid'} . "'.");
-
-    my $mysqld_pid;
-
-    if (-r $instances->[$idx]->{'path_pid'})
-    {
-      $mysqld_pid= mtr_get_pid_from_file($instances->[$idx]->{'path_pid'});
-      mtr_debug("IM-guarded mysqld[$idx] PID: $mysqld_pid.");
-    }
-    else
-    {
-      $mysqld_pid= undef;
-      mtr_debug("IM-guarded mysqld[$idx]: no PID file.");
-    }
-
-    $instances->[$idx]->{'pid'}= $mysqld_pid;
-  }
-
-  # Re-read Instance Manager PIDs from the file, since during tests Instance
-  # Manager could have been restarted, so its PIDs could have been changed.
-
-  #   - IM-main
-
-  mtr_debug("IM-main PID file: '$instance_manager->{path_pid}'.");
-
-  if (-f $instance_manager->{'path_pid'})
-  {
-    $instance_manager->{'pid'} =
-      mtr_get_pid_from_file($instance_manager->{'path_pid'});
-
-    mtr_debug("IM-main PID: $instance_manager->{pid}.");
-  }
-  else
-  {
-    mtr_debug("IM-main: no PID file.");
-    $instance_manager->{'pid'}= undef;
-  }
-
-  #   - IM-angel
-
-  mtr_debug("IM-angel PID file: '$instance_manager->{path_angel_pid}'.");
-
-  if (-f $instance_manager->{'path_angel_pid'})
-  {
-    $instance_manager->{'angel_pid'} =
-      mtr_get_pid_from_file($instance_manager->{'path_angel_pid'});
-
-    mtr_debug("IM-angel PID: $instance_manager->{'angel_pid'}.");
-  }
-  else
-  {
-    mtr_debug("IM-angel: no PID file.");
-    $instance_manager->{'angel_pid'} = undef;
-  }
-}
-
-###########################################################################
-
-sub mtr_im_terminate($) {
-  my $instance_manager= shift;
-
-  # Load pids from pid-files. We should do it first of all, because IM deletes
-  # them on shutdown.
-
-  mtr_im_load_pids($instance_manager);
-
-  mtr_debug("Shutting Instance Manager down...");
-
-  # Ignoring SIGCHLD so that all children could rest in peace.
-
-  start_reap_all();
-
-  # Send SIGTERM to IM-main.
-
-  if (defined $instance_manager->{'pid'})
-  {
-    mtr_debug("IM-main pid: $instance_manager->{pid}.");
-    mtr_debug("Stopping IM-main...");
-
-    mtr_im_kill_process([ $instance_manager->{'pid'} ], 'TERM', 10, 1);
-  }
-  else
-  {
-    mtr_debug("IM-main pid: n/a.");
-  }
-
-  # If IM-angel was alive, wait for it to die.
-
-  if (defined $instance_manager->{'angel_pid'})
-  {
-    mtr_debug("IM-angel pid: $instance_manager->{'angel_pid'}.");
-    mtr_debug("Waiting for IM-angel to die...");
-
-    my $total_attempts= 10;
-
-    for (my $cur_attempt=1; $cur_attempt <= $total_attempts; ++$cur_attempt)
-    {
-      unless (kill (0, $instance_manager->{'angel_pid'}))
-      {
-        mtr_debug("IM-angel died.");
-        last;
-      }
-
-      sleep(1);
-    }
-  }
-  else
-  {
-    mtr_debug("IM-angel pid: n/a.");
-  }
-
-  stop_reap_all();
-
-  # Re-load PIDs.
-
-  mtr_im_load_pids($instance_manager);
-}
-
-###########################################################################
-
-sub mtr_im_check_alive($) {
-  my $instance_manager= shift;
-
-  mtr_debug("Checking whether IM-components are alive...");
-
-  return 1 if mtr_im_check_main_alive($instance_manager);
-
-  return 1 if mtr_im_check_angel_alive($instance_manager);
-
-  return 1 if mtr_im_check_mysqlds_alive($instance_manager);
-
-  return 0;
-}
-
-###########################################################################
-
-sub mtr_im_check_main_alive($) {
-  my $instance_manager= shift;
-
-  # Check that the process, that we know to be IM's, is dead.
-
-  if (defined $instance_manager->{'pid'})
-  {
-    if (kill (0, $instance_manager->{'pid'}))
-    {
-      mtr_debug("IM-main (PID: $instance_manager->{pid}) is alive.");
-      return 1;
-    }
-    else
-    {
-      mtr_debug("IM-main (PID: $instance_manager->{pid}) is dead.");
-    }
-  }
-  else
-  {
-    mtr_debug("No PID file for IM-main.");
-  }
-
-  # Check that IM does not accept client connections.
-
-  if (mtr_ping_mysqld_server($instance_manager->{'port'}))
-  {
-    mtr_debug("IM-main (port: $instance_manager->{port}) " .
-              "is accepting connections.");
-
-    mtr_im_errlog("IM-main is accepting connections on port " .
-                  "$instance_manager->{port}, but there is no " .
-                  "process information.");
-    return 1;
-  }
-  else
-  {
-    mtr_debug("IM-main (port: $instance_manager->{port}) " .
-              "does not accept connections.");
-    return 0;
-  }
-}
-
-###########################################################################
-
-sub mtr_im_check_angel_alive($) {
-  my $instance_manager= shift;
-
-  # Check that the process, that we know to be the Angel, is dead.
-
-  if (defined $instance_manager->{'angel_pid'})
-  {
-    if (kill (0, $instance_manager->{'angel_pid'}))
-    {
-      mtr_debug("IM-angel (PID: $instance_manager->{angel_pid}) is alive.");
-      return 1;
-    }
-    else
-    {
-      mtr_debug("IM-angel (PID: $instance_manager->{angel_pid}) is dead.");
-      return 0;
-    }
-  }
-  else
-  {
-    mtr_debug("No PID file for IM-angel.");
-    return 0;
-  }
-}
-
-###########################################################################
-
-sub mtr_im_check_mysqlds_alive($) {
-  my $instance_manager= shift;
-
-  mtr_debug("Checking for IM-guarded mysqld instances...");
-
-  my $instances = $instance_manager->{'instances'};
-
-  for (my $idx= 0; $idx < 2; ++$idx)
-  {
-    mtr_debug("Checking mysqld[$idx]...");
-
-    return 1
-      if mtr_im_check_mysqld_alive($instances->[$idx]);
-  }
-}
-
-###########################################################################
-
-sub mtr_im_check_mysqld_alive($) {
-  my $mysqld_instance= shift;
-
-  # Check that the process is dead.
-
-  if (defined $mysqld_instance->{'pid'})
-  {
-    if (kill (0, $mysqld_instance->{'pid'}))
-    {
-      mtr_debug("Mysqld instance (PID: $mysqld_instance->{pid}) is alive.");
-      return 1;
-    }
-    else
-    {
-      mtr_debug("Mysqld instance (PID: $mysqld_instance->{pid}) is dead.");
-    }
-  }
-  else
-  {
-    mtr_debug("No PID file for mysqld instance.");
-  }
-
-  # Check that mysqld does not accept client connections.
-
-  if (mtr_ping_mysqld_server($mysqld_instance->{'port'}))
-  {
-    mtr_debug("Mysqld instance (port: $mysqld_instance->{port}) " .
-              "is accepting connections.");
-
-    mtr_im_errlog("Mysqld is accepting connections on port " .
-                  "$mysqld_instance->{port}, but there is no " .
-                  "process information.");
-    return 1;
-  }
-  else
-  {
-    mtr_debug("Mysqld instance (port: $mysqld_instance->{port}) " .
-              "does not accept connections.");
-    return 0;
-  }
-}
-
-###########################################################################
-
-sub mtr_im_cleanup($) {
-  my $instance_manager= shift;
-
-  mtr_im_rm_file($instance_manager->{'path_pid'});
-  mtr_im_rm_file($instance_manager->{'path_sock'});
-
-  mtr_im_rm_file($instance_manager->{'path_angel_pid'});
-
-  for (my $idx= 0; $idx < 2; ++$idx)
-  {
-    mtr_im_rm_file($instance_manager->{'instances'}->[$idx]->{'path_pid'});
-    mtr_im_rm_file($instance_manager->{'instances'}->[$idx]->{'path_sock'});
-  }
-}
-
-###########################################################################
-
-sub mtr_im_rm_file($)
-{
-  my $file_path= shift;
-
-  if (-f $file_path)
-  {
-    mtr_debug("Removing '$file_path'...");
-
-    mtr_warning("Can not remove '$file_path'.")
-      unless unlink($file_path);
-  }
-  else
-  {
-    mtr_debug("File '$file_path' does not exist already.");
-  }
-}
-
-###########################################################################
-
-sub mtr_im_errlog($) {
-  my $msg= shift;
-
-  # Complain in error log so that a warning will be shown.
-  # 
-  # TODO: unless BUG#20761 is fixed, we will print the warning to stdout, so
-  # that it can be seen on console and does not produce pushbuild error.
-
-  # my $errlog= "$opt_vardir/log/mysql-test-run.pl.err";
-  # 
-  # open (ERRLOG, ">>$errlog") ||
-  #   mtr_error("Can not open error log ($errlog)");
-  # 
-  # my $ts= localtime();
-  # print ERRLOG
-  #   "Warning: [$ts] $msg\n";
-  # 
-  # close ERRLOG;
-
-  my $ts= localtime();
-  print "Warning: [$ts] $msg\n";
-}
-
-###########################################################################
-
-sub mtr_im_kill($) {
-  my $instance_manager= shift;
-
-  # Re-load PIDs. That can be useful because some processes could have been
-  # restarted.
-
-  mtr_im_load_pids($instance_manager);
-
-  # Ignoring SIGCHLD so that all children could rest in peace.
-
-  start_reap_all();
-
-  # Kill IM-angel first of all.
-
-  if (defined $instance_manager->{'angel_pid'})
-  {
-    mtr_debug("Killing IM-angel (PID: $instance_manager->{angel_pid})...");
-    mtr_im_kill_process([ $instance_manager->{'angel_pid'} ], 'KILL', 10, 1)
-  }
-  else
-  {
-    mtr_debug("IM-angel is dead.");
-  }
-
-  # Re-load PIDs again.
-
-  mtr_im_load_pids($instance_manager);
-
-  # Kill IM-main.
-  
-  if (defined $instance_manager->{'pid'})
-  {
-    mtr_debug("Killing IM-main (PID: $instance_manager->pid})...");
-    mtr_im_kill_process([ $instance_manager->{'pid'} ], 'KILL', 10, 1);
-  }
-  else
-  {
-    mtr_debug("IM-main is dead.");
-  }
-
-  # Re-load PIDs again.
-
-  mtr_im_load_pids($instance_manager);
-
-  # Kill guarded mysqld instances.
-
-  my @mysqld_pids;
-
-  mtr_debug("Collecting PIDs of mysqld instances to kill...");
-
-  for (my $idx= 0; $idx < 2; ++$idx)
-  {
-    my $pid= $instance_manager->{'instances'}->[$idx]->{'pid'};
-
-    next unless defined $pid;
-
-    mtr_debug("  - IM-guarded mysqld[$idx] PID: $pid.");
-
-    push (@mysqld_pids, $pid);
-  }
-
-  if (scalar @mysqld_pids > 0)
-  {
-    mtr_debug("Killing IM-guarded mysqld instances...");
-    mtr_im_kill_process(\@mysqld_pids, 'KILL', 10, 1);
-  }
-
-  # That's all.
-
-  stop_reap_all();
-}
-
-##############################################################################
-
-sub mtr_im_wait_for_connection($$$) {
-  my $instance_manager= shift;
-  my $total_attempts= shift;
-  my $connect_timeout= shift;
-
-  mtr_debug("Waiting for IM on port $instance_manager->{port} " .
-            "to start accepting connections...");
-
-  for (my $cur_attempt= 1; $cur_attempt <= $total_attempts; ++$cur_attempt)
-  {
-    mtr_debug("Trying to connect to IM ($cur_attempt of $total_attempts)...");
-
-    if (mtr_ping_mysqld_server($instance_manager->{'port'}))
-    {
-      mtr_debug("IM is accepting connections " .
-                "on port $instance_manager->{port}.");
-      return 1;
-    }
-
-    mtr_debug("Sleeping $connect_timeout...");
-    sleep($connect_timeout);
-  }
-
-  mtr_debug("IM does not accept connections " .
-            "on port $instance_manager->{port} after " .
-            ($total_attempts * $connect_timeout) . " seconds.");
-
-  return 0;
-}
-
-##############################################################################
-
-sub mtr_im_wait_for_mysqld($$$) {
-  my $mysqld= shift;
-  my $total_attempts= shift;
-  my $connect_timeout= shift;
-
-  mtr_debug("Waiting for IM-guarded mysqld on port $mysqld->{port} " .
-            "to start accepting connections...");
-
-  for (my $cur_attempt= 1; $cur_attempt <= $total_attempts; ++$cur_attempt)
-  {
-    mtr_debug("Trying to connect to mysqld " .
-              "($cur_attempt of $total_attempts)...");
-
-    if (mtr_ping_mysqld_server($mysqld->{'port'}))
-    {
-      mtr_debug("Mysqld is accepting connections " .
-                "on port $mysqld->{port}.");
-      return 1;
-    }
-
-    mtr_debug("Sleeping $connect_timeout...");
-    sleep($connect_timeout);
-  }
-
-  mtr_debug("Mysqld does not accept connections " .
-            "on port $mysqld->{port} after " .
-            ($total_attempts * $connect_timeout) . " seconds.");
-
-  return 0;
-}
-
-##############################################################################
-
-sub mtr_im_start($$) {
-  my $instance_manager = shift;
-  my $opts = shift;
-
-  mtr_debug("Starting Instance Manager...");
-
-  my $args;
-  mtr_init_args(\$args);
-  mtr_add_arg($args, "--defaults-file=%s",
-              $instance_manager->{'defaults_file'});
-
-  foreach my $opt (@{$opts})
-  {
-    mtr_add_arg($args, $opt);
-  }
-
-  $instance_manager->{'pid'} =
-    mtr_spawn(
-      $::exe_im,                        # path to the executable
-      $args,                            # cmd-line args
-      '',                               # stdin
-      $instance_manager->{'path_log'},  # stdout
-      $instance_manager->{'path_err'},  # stderr
-      '',                               # pid file path (not used)
-      { append_log_file => 1 }          # append log files
-      );
-
-  if ( ! $instance_manager->{'pid'} )
-  {
-    mtr_report('Could not start Instance Manager');
-    return;
-  }
-
-  # Instance Manager can be run in daemon mode. In this case, it creates
-  # several processes and the parent process, created by mtr_spawn(), exits just
-  # after start. So, we have to obtain Instance Manager PID from the PID file.
-
-  if ( ! sleep_until_file_created(
-                                  $instance_manager->{'path_pid'},
-                                  $instance_manager->{'start_timeout'},
-                                  -1)) # real PID is still unknown
-  {
-    mtr_report("Instance Manager PID file is missing");
-    return;
-  }
-
-  $instance_manager->{'pid'} =
-    mtr_get_pid_from_file($instance_manager->{'path_pid'});
-
-  mtr_debug("Instance Manager started. PID: $instance_manager->{pid}.");
-
-  # Wait until we can connect to IM.
-
-  my $IM_CONNECT_TIMEOUT= 30;
-
-  unless (mtr_im_wait_for_connection($instance_manager,
-                                     $IM_CONNECT_TIMEOUT, 1))
-  {
-    mtr_debug("Can not connect to Instance Manager " .
-              "in $IM_CONNECT_TIMEOUT seconds after start.");
-    mtr_debug("Aborting test suite...");
-
-    mtr_kill_leftovers();
-
-    mtr_error("Can not connect to Instance Manager " .
-              "in $IM_CONNECT_TIMEOUT seconds after start.");
-  }
-
-  # Wait until we can connect to guarded mysqld-instances
-  # (in other words -- wait for IM to start guarded instances).
-
-  for (my $idx= 0; $idx < 2; ++$idx)
-  {
-    my $mysqld= $instance_manager->{'instances'}->[$idx];
-
-    next if exists $mysqld->{'nonguarded'};
-
-    mtr_debug("Waiting for mysqld[$idx] to start...");
-
-    unless (mtr_im_wait_for_mysqld($mysqld, 30, 1))
-    {
-      mtr_debug("Can not connect to mysqld[$idx] " .
-                "in $IM_CONNECT_TIMEOUT seconds after start.");
-      mtr_debug("Aborting test suite...");
-
-      mtr_kill_leftovers();
-
-      mtr_error("Can not connect to mysqld[$idx] " .
-                "in $IM_CONNECT_TIMEOUT seconds after start.");
-    }
-
-    mtr_debug("mysqld[$idx] started.");
-  }
-
-  mtr_debug("Instance Manager started.");
-}
-
-##############################################################################
-
-sub mtr_im_stop($) {
-  my $instance_manager= shift;
-
-  mtr_debug("Stopping Instance Manager...");
-
-  # Try graceful shutdown.
-
-  mtr_im_terminate($instance_manager);
-
-  # Check that all processes died.
-
-  unless (mtr_im_check_alive($instance_manager))
-  {
-    mtr_debug("Instance Manager has been stopped successfully.");
-    mtr_im_cleanup($instance_manager);
-    return 1;
-  }
-
-  # Instance Manager don't want to die. We should kill it.
-
-  mtr_im_errlog("Instance Manager did not shutdown gracefully.");
-
-  mtr_im_kill($instance_manager);
-
-  # Check again that all IM-related processes have been killed.
-
-  my $im_is_alive= mtr_im_check_alive($instance_manager);
-
-  mtr_im_cleanup($instance_manager);
-
-  if ($im_is_alive)
-  {
-    mtr_error("Can not kill Instance Manager or its children.");
-    return 0;
-  }
-
-  mtr_debug("Instance Manager has been killed successfully.");
-  return 1;
-}
-
 ###########################################################################
 
 1;

mysql-test/mysql-test-run.pl

@@ -88,6 +88,7 @@ use strict;
 #use diagnostics;
 
 require "lib/mtr_cases.pl";
+require "lib/mtr_im.pl";
 require "lib/mtr_process.pl";
 require "lib/mtr_timer.pl";
 require "lib/mtr_io.pl";
@@ -964,6 +965,7 @@ sub command_line_setup () {
    path_datadir => "$opt_vardir/im_mysqld_1.data",
    path_sock    => "$sockdir/mysqld_1.sock",
    path_pid     => "$opt_vardir/run/mysqld_1.pid",
+   start_timeout => 400, # enough time create innodb tables
   };
 
   $instance_manager->{'instances'}->[1]=
@@ -974,6 +976,7 @@ sub command_line_setup () {
    path_sock    => "$sockdir/mysqld_2.sock",
    path_pid     => "$opt_vardir/run/mysqld_2.pid",
    nonguarded   => 1,
+   start_timeout => 400, # enough time create innodb tables
   };
 
   if ( $opt_extern )
@@ -1316,9 +1319,6 @@ sub kill_running_server () {
     # started from ths run of the script, this is terminating
     # leftovers from previous runs.
 
-    mtr_report("Killing Possible Leftover Processes");
-    mkpath("$opt_vardir/log"); # Needed for mysqladmin log
-
     mtr_kill_leftovers();
 
     $using_ndbcluster_master= $opt_with_ndbcluster;
@@ -2112,7 +2112,10 @@ sub run_testcase ($) {
 
       im_create_defaults_file($instance_manager);
 
-      mtr_im_start($instance_manager, $tinfo->{im_opts});
+      unless ( mtr_im_start($instance_manager, $tinfo->{im_opts}) )
+      {
+        mtr_error("Failed to start Instance Manager.")
+      }
     }
 
     # ----------------------------------------------------------------------
@@ -2209,7 +2212,10 @@ sub run_testcase ($) {
 
   if ( ! $glob_use_running_server and $tinfo->{'component_id'} eq 'im' )
   {
-    mtr_im_stop($instance_manager);
+    unless ( mtr_im_stop($instance_manager) )
+    {
+      mtr_error("Failed to stop Instance Manager.")
+    }
   }
 }
 
@@ -2738,7 +2744,10 @@ sub stop_masters_slaves () {
   print  "Ending Tests\n";
 
   print  "Shutting-down Instance Manager\n";
-  mtr_im_stop($instance_manager);
+  unless ( mtr_im_stop($instance_manager) )
+  {
+    mtr_error("Failed to stop Instance Manager.")
+  }
 
   print  "Shutting-down MySQL daemon\n\n";
   stop_masters();