Save datadir in case of test failure, but limit the number of times

datadir can be saved in case of constantly failing servers
Remove datadirs each time all servers have been stopped thus limiting
the size of vardir.
The datadirs of failed tests have been saved and that should be
enough to diagnose any failures

mysql-test/mysql-test-run.pl

@@ -193,9 +193,12 @@ our %mysqld_variables;
 
 my $source_dist= 0;
 
-our $opt_max_save_core= 5;
+my $opt_max_save_core= $ENV{MTR_MAX_SAVE_CORE} || 5;
 my $num_saved_cores= 0;  # Number of core files saved in vardir/log/ so far.
 
+my $opt_max_save_datadir= $ENV{MTR_MAX_SAVE_DATADIR} || 20;
+my $num_saved_datadir= 0;  # Number of datadirs saved in vardir/log/ so far.
+
 select(STDOUT);
 $| = 1; # Automatically flush STDOUT
 
@@ -351,6 +354,7 @@ sub command_line_setup {
 	     'client-debugger=s'        => \$opt_client_debugger,
              'strace-client:s'          => \$opt_strace_client,
              'max-save-core=i'          => \$opt_max_save_core,
+             'max-save-datadir=i'       => \$opt_max_save_datadir,
 
              # Coverage, profiling etc
              'gcov'                     => \$opt_gcov,
@@ -2325,6 +2329,10 @@ sub run_testcase ($) {
 
     if ( started(all_servers()) == 0 )
     {
+
+      # Remove old datadirs
+      clean_datadir();
+
       # Restore old ENV
       while (my ($option, $value)= each( %old_env )) {
 	if (defined $value){
@@ -2448,15 +2456,12 @@ sub run_testcase ($) {
 	  mtr_report_test_passed($tinfo, $opt_timer);
 	}
 
-	if ( $opt_check_testcases )
+	if ( $opt_check_testcases and check_testcase($tinfo, "after"))
 	{
-	  if (check_testcase($tinfo, "after"))
-	  {
-	    # Stop all servers that are known to be running
-	    stop_all_servers();
-	    after_test_failure($tinfo->{'name'});
-	    mtr_report("Resuming tests...\n");
-	  }
+	  # Stop all servers that are known to be running
+	  stop_all_servers();
+	  clean_datadir();
+	  mtr_report("Resuming tests...\n");
 	}
       }
       elsif ( $res == 62 )
@@ -2523,7 +2528,7 @@ sub run_testcase ($) {
     {
       # Server failed, probably crashed
       $tinfo->{comment}=
-	"Server failed during test run";
+	"Server $proc failed during test run";
 
       report_failure_and_restart($tinfo);
       return 1;
@@ -2667,67 +2672,119 @@ sub check_expected_crash_and_restart {
 }
 
 
+sub clean_datadir {
+
+  mtr_verbose("Cleaning datadirs...");
+
+  foreach my $cluster ( clusters() )
+  {
+    my $cluster_dir= "$opt_vardir/".$cluster->{name};
+    mtr_verbose(" - removing '$cluster_dir'");
+    rmtree($cluster_dir);
+
+  }
+
+  foreach my $mysqld ( mysqlds() )
+  {
+    my $mysqld_dir= dirname($mysqld->value('datadir'));
+    if (-d $mysqld_dir ) {
+      mtr_verbose(" - removing '$mysqld_dir'");
+      rmtree($mysqld_dir);
+    }
+  }
+
+  # Remove all files in tmp and var/tmp
+  rmtree("$opt_vardir/tmp");
+  mkpath("$opt_vardir/tmp");
+  if ($opt_tmpdir ne "$opt_vardir/tmp"){
+    rmtree($opt_tmpdir);
+    mkpath($opt_tmpdir);
+  }
+}
+
+
 #
-# Save any interesting files in the data_dir
-# before the data dir is removed.
+# Limit number of core files saved
 #
-sub save_files_after_test_failure($$) {
-  my $test_name= shift;
-  my $data_dir= shift;
-  my $save_name= "$opt_vardir/log/$test_name";
+sub limit_cores_after_failure ($) {
+  my ($datadir)= @_;
 
   # Look for core files
-  foreach my $core_file ( glob("$data_dir/core*") )
+  foreach my $core_file ( glob("$datadir/core*") )
   {
-    last if $opt_max_save_core > 0 && $num_saved_cores >= $opt_max_save_core;
     my $core_name= basename($core_file);
-    mtr_report(" - saving '$core_name'");
-    mkpath($save_name) if ! -d $save_name;
-    rename("$core_file", "$save_name/$core_name");
+    if ($opt_max_save_core > 0 && $num_saved_cores >= $opt_max_save_core) {
+      # Delete file to avoid saving it  when the datadir is later saved
+      mtr_report(" - deleting '$core_name'",
+		 "($num_saved_cores/$opt_max_save_core)");
+      unlink("$core_file");
+    }
+    else {
+      mtr_report(" - found '$core_name'",
+		 "($num_saved_cores/$opt_max_save_core)");
+    }
     ++$num_saved_cores;
   }
 }
 
+#
+# Save datadir before it's removed
+#
+sub save_datadir_after_failure($$) {
+  my ($dir, $savedir)= @_;
 
-sub after_test_failure ($) {
-  my $test_name= shift;
-
-  mtr_report("Cleaning datadirs...");
-
-  foreach my $mysqld ( mysqlds() )
+  if ($opt_max_save_datadir > 0 &&
+      $num_saved_datadir >= $opt_max_save_datadir)
   {
-    my $data_dir= $mysqld->value('datadir');
-    my $name= basename($data_dir);
-    save_files_after_test_failure($test_name, $data_dir);
-    mtr_debug("Removing '$data_dir'");
-    rmtree($data_dir);
+    mtr_report(" - skipping '$dir'");
+  }
+  else {
+    mtr_report(" - saving '$dir'");
+    my $dir_name= basename($dir);
+    rename("$dir", "$savedir/$dir_name");
   }
+}
 
-  # Remove the ndb_*_fs dirs for all ndbd nodes
-  # forcing a clean start of ndb next time
-  foreach my $cluster ( clusters() )
-  {
-    foreach my $ndbd ( ndbds($cluster) )
-    {
-      my $data_dir= $ndbd->value('DataDir');
-      foreach my $fs_dir ( glob("$data_dir/ndb_*_fs") ) {
-	rmtree($fs_dir);
-	mtr_debug("Removing '$fs_dir'");
+
+sub after_failure ($) {
+  my ($tinfo)= @_;
+
+  mtr_report("Saving datadirs...");
+
+  my $save_dir= "$opt_vardir/log/";
+  $save_dir.= $tinfo->{name};
+  # Add combination name if any
+  $save_dir.= "_$tinfo->{combination}"
+    if defined $tinfo->{combination};
+
+  mkpath($save_dir) if ! -d $save_dir;
+
+  # Save the used my.cnf file
+  copy($path_config_file, $save_dir);
+
+  if ( clusters() ) {
+    foreach my $cluster ( clusters() ) {
+
+      foreach my $server ( ndbds($cluster), ndb_mgmds($cluster) ) {
+	my $data_dir= $server->value('DataDir');
+	limit_cores_after_failure($data_dir);
       }
 
-      my $backup_dir= $ndbd->value('BackupDataDir');
-      rmtree("$backup_dir/BACKUP");
-      mtr_debug("Removing '$backup_dir'");
+      my $cluster_dir= "$opt_vardir/".$cluster->{name};
+      save_datadir_after_failure($cluster_dir, $save_dir);
     }
   }
-
-  # Remove all files in tmp and var/tmp
-  rmtree("$opt_vardir/tmp");
-  mkpath("$opt_vardir/tmp");
-  if ($opt_tmpdir ne "$opt_vardir/tmp"){
-    rmtree($opt_tmpdir);
-    mkpath($opt_tmpdir);
+  else {
+    foreach my $mysqld ( mysqlds() ) {
+      my $data_dir= $mysqld->value('datadir');
+      limit_cores_after_failure($data_dir);
+      save_datadir_after_failure(dirname($data_dir), $save_dir);
+    }
   }
+
+  $num_saved_datadir++;
+
+  clean_datadir();
 }
 
 
@@ -2741,7 +2798,7 @@ sub report_failure_and_restart ($) {
     # Stop all servers that are known to be running
     stop_all_servers();
 
-    after_test_failure($tinfo->{'name'});
+    after_failure($tinfo);
     mtr_report("Resuming tests...\n");
     return;
   }
@@ -3849,7 +3906,12 @@ Options for debugging the product
                         Example: $0 --strace-client=ktrace
   max-save-core         Limit the number of core files saved (to avoid filling
                         up disks for heavily crashing server). Defaults to
-                        $opt_max_save_core, set to 0 for no limit.
+                        $opt_max_save_core, set to 0 for no limit. Set
+                        it's default with MTR_MAX_SAVE_CORE
+  max-save-datadir      Limit the number of datadir saved (to avoid filling
+                        up disks for heavily crashing server). Defaults to
+                        $opt_max_save_datadir, set to 0 for no limit. Set
+                        it's default with MTR_MAX_SAVE_DATDIR
 
 Options for valgrind