Improved handling of marking processes as dead

Run ndb_mgmd as deamon
Make extra attempt to check if processes are still alive


mysql-test/lib/mtr_process.pl:
  Add common function to mark processes as dead
  When all attempts to kil processes has failed make an extra attempt with ping to check if they really are still alive
mysql-test/mysql-test-run.pl:
  Run ndb_mgmd with --nodaemon

mysql-test/lib/mtr_process.pl

@@ -272,40 +272,17 @@ sub spawn_parent_impl {
           last;
         }
 
-        # If one of the mysqld processes died, we want to
+        # If one of the processes died, we want to
         # mark this, and kill the mysqltest process.
 
-        foreach my $idx (0..1)
-        {
-          if ( $::master->[$idx]->{'pid'} eq $ret_pid )
-          {
-            mtr_debug("child $ret_pid was master[$idx], " .
-                      "exit during mysqltest run");
-            $::master->[$idx]->{'pid'}= 0;
-            last;
-          }
-        }
-
-        foreach my $idx (0..2)
-        {
-          if ( $::slave->[$idx]->{'pid'} eq $ret_pid )
-          {
-            mtr_debug("child $ret_pid was slave[$idx], " .
-                      "exit during mysqltest run");
-            $::slave->[$idx]->{'pid'}= 0;
-            last;
-          }
-        }
-
-        mtr_debug("waitpid() caught exit of unknown child $ret_pid, " .
-                  "exit during mysqltest run");
+	mark_process_dead($ret_pid);
       }
 
       if ( $ret_pid != $pid )
       {
         # We terminated the waiting because a "mysqld" process died.
         # Kill the mysqltest process.
-
+	mtr_verbose("Kill mysqltest because another process died");
         kill(9,$pid);
 
         $ret_pid= waitpid($pid,0);
@@ -639,15 +616,21 @@ sub mtr_check_stop_servers ($) {
               mtr_warning("couldn't delete $file");
             }
           }
+	  $srv->{'pid'}= 0;
         }
       }
     }
     if ( $errors )
     {
-      # We are in trouble, just die....
+      # There where errors killing processes
+      # do one last attempt to ping the servers
+      # and if they can't be pinged, assume they are dead
+      if ( ! mtr_ping_with_timeout( \@$spec ) )
+      {
 	mtr_error("we could not kill or clean up all processes");
       }
     }
+  }
 
   # FIXME We just assume they are all dead, for Cygwin we are not
   # really sure
@@ -773,39 +756,23 @@ sub mtr_ping_with_timeout($) {
   return $res;
 }
 
-##############################################################################
+
 #
-#  The operating system will keep information about dead children, 
-#  we read this information here, and if we have records the process
-#  is alive, we mark it as dead.
+# Loop through our list of processes and look for and entry
+# with the provided pid
+# Set the pid of that process to 0 if found
 #
-##############################################################################
-
-sub mtr_record_dead_children () {
-
-  my $ret_pid;
-
-  # Wait without blockinng to see if any processes had died
-  # -1 or 0 means there are no more procesess to wait for
-  while ( ($ret_pid= waitpid(-1,&WNOHANG)) != 0 and $ret_pid != -1)
-  {
-    mtr_warning("waitpid() caught exit of child $ret_pid");
-    foreach my $idx (0..1)
-    {
-      if ( $::master->[$idx]->{'pid'} eq $ret_pid )
-      {
-        mtr_warning("child $ret_pid was master[$idx]");
-        $::master->[$idx]->{'pid'}= 0;
-      }
-    }
+sub mark_process_dead($)
+{
+  my $ret_pid= shift;
 
-    foreach my $idx (0..2)
+  foreach my $mysqld (@{$::master}, @{$::slave})
   {
-      if ( $::slave->[$idx]->{'pid'} eq $ret_pid )
+    if ( $mysqld->{'pid'} eq $ret_pid )
     {
-        mtr_warning("child $ret_pid was slave[$idx]");
-        $::slave->[$idx]->{'pid'}= 0;
-        last;
+      mtr_verbose("$mysqld->{'type'} $mysqld->{'idx'} exited, pid: $ret_pid");
+      $mysqld->{'pid'}= 0;
+      return;
     }
   }
 
@@ -813,21 +780,43 @@ sub mtr_record_dead_children () {
   {
     if ( $cluster->{'pid'} eq $ret_pid )
     {
-       mtr_warning("child $ret_pid was $cluster->{'name'} cluster ndb_mgmd");
+      mtr_verbose("$cluster->{'name'} cluster ndb_mgmd exited, pid: $ret_pid");
       $cluster->{'pid'}= 0;
-       last;
+      return;
     }
 
     foreach my $ndbd (@{$cluster->{'ndbds'}})
     {
       if ( $ndbd->{'pid'} eq $ret_pid )
       {
-	 mtr_warning("child $ret_pid was $cluster->{'name'} cluster ndbd");
+	mtr_verbose("$cluster->{'name'} cluster ndbd exited, pid: $ret_pid");
 	$ndbd->{'pid'}= 0;
-	 last;
+	return;
       }
     }
   }
+  mtr_warning("mark_process_dead couldn't find an entry for pid: $ret_pid");
+
+}
+
+##############################################################################
+#
+#  The operating system will keep information about dead children, 
+#  we read this information here, and if we have records the process
+#  is alive, we mark it as dead.
+#
+##############################################################################
+
+sub mtr_record_dead_children () {
+
+  my $ret_pid;
+
+  # Wait without blockinng to see if any processes had died
+  # -1 or 0 means there are no more procesess to wait for
+  while ( ($ret_pid= waitpid(-1,&WNOHANG)) != 0 and $ret_pid != -1)
+  {
+    mtr_warning("mtr_record_dead_children: $ret_pid");
+    mark_process_dead($ret_pid);
   }
 }
 
@@ -843,7 +832,8 @@ sub start_reap_all {
   my $pid;
   while(($pid= waitpid(-1, &WNOHANG)) != 0 and $pid != -1)
   {
-    print "start_reap_all: pid: $pid.\n";
+    mtr_warning("start_reap_all pid: $pid");
+    mark_process_dead($pid);
   };
 }
 
@@ -903,6 +893,7 @@ sub sleep_until_file_created ($$$) {
     # Check if it died after the fork() was successful
     if ( $pid != 0 && waitpid($pid,&WNOHANG) == $pid )
     {
+      mtr_warning("Process $pid died");
       return 0;
     }
 

mysql-test/mysql-test-run.pl

@@ -1687,6 +1687,7 @@ sub ndbcluster_wait_started($){
 }
 
 
+
 sub mysqld_wait_started($){
   my $mysqld= shift;
 
@@ -1706,6 +1707,7 @@ sub ndb_mgmd_start ($) {
   mtr_init_args(\$args);
   mtr_add_arg($args, "--no-defaults");
   mtr_add_arg($args, "--core");
+  mtr_add_arg($args, "--nodaemon");
   mtr_add_arg($args, "--config-file=%s", "$cluster->{'data_dir'}/config.ini");
 
 
@@ -1716,9 +1718,23 @@ sub ndb_mgmd_start ($) {
 		  "",
 		  { append_log_file => 1 });
 
+
+  # FIXME Should not be needed
+  # Unfortunately the cluster nodes will fail to start
+  # if ndb_mgmd has not started properly
+  sleep(1);
+
+ # if (!sleep_until_file_created($cluster->{'path_pid'},
+ #				30, # Seconds
+ #				$pid))
+ #  {
+ #    mtr_warning("Failed to start ndb_mgd for $cluster->{'name'} cluster");
+ #    return 1;
+ #  }
+
   # Remember pid of ndb_mgmd
   $cluster->{'pid'}= $pid;
-  mtr_verbose("ndb_mgmd_start, pid: $pid");
+
   return $pid;
 }
 
@@ -1774,19 +1790,6 @@ sub ndbcluster_start ($$) {
 
   my $pid= ndb_mgmd_start($cluster);
 
-  # FIXME Should not be needed
-  # Unfortunately cluster will fail
-  # if ndb_mgmd has not started properly
-  # Wait for the ndb_mgmd pid file to be created
-  if (!sleep_until_file_created($cluster->{'path_pid'},
-				60,
-				$pid))
-  {
-    mtr_warning("Failed to start ndb_mgmd for $cluster->{'name'} cluster");
-    return 1;
-  }
-
-
   for ( my $idx= 0; $idx < $cluster->{'nodes'}; $idx++ )
   {
     ndbd_start($cluster, $idx, $extra_args);