Bug#19535 mysql-test-run cannot handle crashing test cases

 - Make mysql-test-run.pl restart a crashed/stopped process if it was expected it should crash.
 - Added testcase for "crash_commit_before", which tests behaviour when server crashes just before commit.
  * The testcase first write a small var/tmp/master0.expect file indicating expected crash
  * Then sets "DEBUG" variable to for example "d,crash_commit_before" and then executes a commit.
  * The server will crash and be brought back up by mysql-test-run.pl
  * Test case will then wait for the process to come back online before continuing.


client/mysqltest.c:
  Add "die" command used to abort a failing test case
  Remove unused "exit" command
mysql-test/lib/mtr_process.pl:
  When a process crash during execution of mysqltest, check if it was expected and restart the process in that case.
mysql-test/mysql-test-run.pl:
  Remmeber options used when ndbd's and mysqld's was started, to be used by restart code
mysql-test/r/crash_commit_before.result:
  New BitKeeper file ``mysql-test/r/crash_commit_before.result''
mysql-test/include/wait_until_connected_again.inc:
  New BitKeeper file ``mysql-test/include/wait_until_connected_again.inc''
mysql-test/t/crash_commit_before.test:
  New BitKeeper file ``mysql-test/t/crash_commit_before.test''

client/mysqltest.c

@@ -338,11 +338,10 @@ Q_DISPLAY_VERTICAL_RESULTS, Q_DISPLAY_HORIZONTAL_RESULTS,
 Q_QUERY_VERTICAL, Q_QUERY_HORIZONTAL,
 Q_START_TIMER, Q_END_TIMER,
 Q_CHARACTER_SET, Q_DISABLE_PS_PROTOCOL, Q_ENABLE_PS_PROTOCOL,
-Q_EXIT,
 Q_DISABLE_RECONNECT, Q_ENABLE_RECONNECT,
 Q_IF,
 Q_DISABLE_PARSING, Q_ENABLE_PARSING,
-Q_REPLACE_REGEX,
+Q_REPLACE_REGEX, Q_DIE,
 
 Q_UNKNOWN,			       /* Unknown command.   */
 Q_COMMENT,			       /* Comments, ignored. */
@@ -424,13 +423,13 @@ const char *command_names[]=
   "character_set",
   "disable_ps_protocol",
   "enable_ps_protocol",
-  "exit",
   "disable_reconnect",
   "enable_reconnect",
   "if",
   "disable_parsing",
   "enable_parsing",
   "replace_regex",
+  "die",
   0
 };
 
@@ -5081,7 +5080,7 @@ static void init_var_hash(MYSQL *mysql)
   test run completes
 
 */
-static void mark_progress(struct st_query* q, int line)
+static void mark_progress(struct st_query* q __attribute__((unused)), int line)
 {
   char buf[32], *end;
   ulonglong timer= timer_now();
@@ -5116,7 +5115,7 @@ static void mark_progress(struct st_query* q, int line)
 int main(int argc, char **argv)
 {
   struct st_query *q;
-  my_bool require_file=0, q_send_flag=0, abort_flag= 0,
+  my_bool require_file=0, q_send_flag=0,
           query_executed= 0;
   char save_file[FN_REFLEN];
   MY_STAT res_info;
@@ -5219,7 +5218,7 @@ int main(int argc, char **argv)
     open_file(opt_include);
   }
 
-  while (!abort_flag && !read_query(&q))
+  while (!read_query(&q))
   {
     int current_line_inc = 1, processed = 0;
     if (q->type == Q_UNKNOWN || q->type == Q_COMMENT_WITH_COMMAND)
@@ -5434,8 +5433,8 @@ int main(int argc, char **argv)
           parsing_disabled--;
         break;
 
-      case Q_EXIT:
-        abort_flag= 1;
+      case Q_DIE:
+        die("%s", q->first_argument);
         break;
 
       default:

mysql-test/include/wait_until_connected_again.inc

+#
+# Include this script to wait until the connection to the
+# server has been restored or timeout occurs
+--disable_result_log
+--disable_query_log
+let $counter= 100;
+while ($mysql_errno)
+{
+  --error 0,2002,2006
+  show status;
+
+  dec $counter;
+  if (!$counter)
+  {
+    --die Server failed to restart
+  }
+  --sleep 0.1
+}
+--enable_query_log
+--enable_result_log

mysql-test/lib/mtr_process.pl

@@ -272,10 +272,10 @@ sub spawn_parent_impl {
           last;
         }
 
-        # If one of the processes died, we want to
-        # mark this, and kill the mysqltest process.
+        # One of the child processes died, unless this was expected
+	# mysqltest should be killed and test aborted
 
-	mark_process_dead($ret_pid);
+	check_expected_crash_and_restart($ret_pid);
       }
 
       if ( $ret_pid != $pid )
@@ -809,6 +809,81 @@ sub mark_process_dead($)
 
 }
 
+#
+# Loop through our list of processes and look for and entry
+# with the provided pid, if found check for the file indicating
+# expected crash and restart it.
+#
+sub check_expected_crash_and_restart($)
+{
+  my $ret_pid= shift;
+
+  foreach my $mysqld (@{$::master}, @{$::slave})
+  {
+    if ( $mysqld->{'pid'} eq $ret_pid )
+    {
+      mtr_verbose("$mysqld->{'type'} $mysqld->{'idx'} exited, pid: $ret_pid");
+      $mysqld->{'pid'}= 0;
+
+      # Check if crash expected and restart if it was
+      my $expect_file= "$::opt_vardir/tmp/" . "$mysqld->{'type'}" .
+	"$mysqld->{'idx'}" . ".expect";
+      if ( -f $expect_file )
+      {
+	mtr_verbose("Crash was expected, file $expect_file exists");
+	mysqld_start($mysqld, $mysqld->{'start_opts'},
+		     $mysqld->{'start_slave_master_info'});
+	unlink($expect_file);
+      }
+
+      return;
+    }
+  }
+
+  foreach my $cluster (@{$::clusters})
+  {
+    if ( $cluster->{'pid'} eq $ret_pid )
+    {
+      mtr_verbose("$cluster->{'name'} cluster ndb_mgmd exited, pid: $ret_pid");
+      $cluster->{'pid'}= 0;
+
+      # Check if crash expected and restart if it was
+      my $expect_file= "$::opt_vardir/tmp/ndb_mgmd_" . "$cluster->{'type'}" .
+	".expect";
+      if ( -f $expect_file )
+      {
+	mtr_verbose("Crash was expected, file $expect_file exists");
+	ndbmgmd_start($cluster);
+	unlink($expect_file);
+      }
+      return;
+    }
+
+    foreach my $ndbd (@{$cluster->{'ndbds'}})
+    {
+      if ( $ndbd->{'pid'} eq $ret_pid )
+      {
+	mtr_verbose("$cluster->{'name'} cluster ndbd exited, pid: $ret_pid");
+	$ndbd->{'pid'}= 0;
+
+	# Check if crash expected and restart if it was
+	my $expect_file= "$::opt_vardir/tmp/ndbd_" . "$cluster->{'type'}" .
+	  "$ndbd->{'idx'}" . ".expect";
+	if ( -f $expect_file )
+	{
+	  mtr_verbose("Crash was expected, file $expect_file exists");
+	  ndbd_start($cluster, $ndbd->{'idx'},
+		     $ndbd->{'start_extra_args'});
+	  unlink($expect_file);
+	}
+	return;
+      }
+    }
+  }
+  mtr_warning("check_expected_crash_and_restart couldn't find an entry for pid: $ret_pid");
+
+}
+
 ##############################################################################
 #
 #  The operating system will keep information about dead children, 

mysql-test/mysql-test-run.pl

@@ -1793,6 +1793,10 @@ sub ndbd_start ($$$) {
   # Add pid to list of pids for this cluster
   $cluster->{'ndbds'}->[$idx]->{'pid'}= $pid;
 
+  # Rememeber options used when starting
+  $cluster->{'ndbds'}->[$idx]->{'start_extra_args'}= $extra_args;
+  $cluster->{'ndbds'}->[$idx]->{'idx'}= $idx;
+
   mtr_verbose("ndbd_start, pid: $pid");
 
   return $pid;
@@ -2894,6 +2898,7 @@ sub mysqld_start ($$$) {
 
   # Remember options used when starting
   $mysqld->{'start_opts'}= $extra_opt;
+  $mysqld->{'start_slave_master_info'}= $slave_master_info;
 
   mtr_verbose("mysqld pid: $pid");
   return $pid;

mysql-test/r/crash_commit_before.result

+CREATE TABLE t1(a int) engine=innodb;
+START TRANSACTION;
+insert into t1 values(9);
+SET SESSION debug="d,crash_commit_before";
+COMMIT;
+ERROR HY000: Lost connection to MySQL server during query
+SHOW CREATE TABLE t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `a` int(11) DEFAULT NULL
+) ENGINE=InnoDB DEFAULT CHARSET=latin1
+SELECT * FROM t1;
+a

mysql-test/t/crash_commit_before.test

+--source include/have_debug.inc
+
+CREATE TABLE t1(a int) engine=innodb;
+START TRANSACTION;
+insert into t1 values(9);
+
+# Setup the mysqld to crash at certain point
+SET SESSION debug="d,crash_commit_before";
+
+# Write file to make mysql-test-run.pl expect crash and restart
+--exec echo "restart" > $MYSQLTEST_VARDIR/tmp/master0.expect
+
+# Run the crashing query
+--error 2013
+COMMIT;
+
+# Call script that will poll the server waiting for it to be back online again
+--source include/wait_until_connected_again.inc
+
+SHOW CREATE TABLE t1;
+
+SELECT * FROM t1;