merge 42888 to 5.1-mtr

mysql-test/Makefile.am

@@ -75,7 +75,7 @@ EXTRA_DIST =	README \
 
 # List of directories containing test + result files and the
 # related test data files that should be copied
-TEST_DIRS = t r include std_data std_data/parts \
+TEST_DIRS = t r include std_data std_data/parts collections \
 	std_data/ndb_backup50 std_data/ndb_backup51 \
 	std_data/ndb_backup51_data_be std_data/ndb_backup51_data_le \
 	std_data/funcs_1 \

mysql-test/collections/README

+This directory contains collections of test runs that we run during our
+integration and release testing. Each file contains zero or more lines,
+with one invocation of mysql-test-run.pl on each. These invocations are
+written so that, with the assumption that perl is in your search path,
+any collection can run as a shell script or a batch file, with the parent
+mysql-test directory being the current working directory.
+
+During integration testing, we choose the collection to run by following
+these steps:
+
+1) We choose the extension to look for, based on these rules:
+   - If we're running a per-push test, we choose ".push" as the extension.
+   - If we're running a daily test, we choose ".daily" as the extension.
+   - If we're running a weekly test, we choose ".weekly" as the extension.
+
+2) If there is a collection that has the same name as the branch we're
+   testing plus the extension as determined in step 1, we choose that
+   collection.
+
+3) If the branch is unknown or we have removed all characters from it
+   and still not found a matching collection, we choose the name "default"
+   plus the extension determined in step 1. If there is no such file,
+   we give up and don't test anything at all.
+
+4) If we haven't found a collection yet, we remove the last character from 
+   the branch name and go back to step 2.
+
+5) The commands from the collection are run line by line via execv() or
+   similar system calls. They are not run as a shell script. Shell 
+   expansions are not guaranteed to work and most likely won't.

mysql-test/collections/README.experimental

+The .experimental files in this directory contain names of test cases that
+are still in development and whose failures should be considered expected,
+instead of regressions.
+
+These files are to be used with the --experimental option of 
+mysql-test-run.pl. Please look at its help screen for usage information.
+
+The syntax is as follows:
+
+1) One line per test case.
+
+2) Empty lines and lines starting with a hash (#) are ignored.
+
+3) If any other line contains a blank followed by a hash (#), the hash 
+   and any subsequent characters are ignored.
+
+4) The full test case name including the suite and execution mode
+   must be specified, for example:
+        main.alias 'row'     # bug#00000
+
+5) As an exception to item 4, the last character of the test case
+   specification may be an asterisk (*). In that case, all test cases that
+   start with the same characters up to the last letter before the asterisk
+   are considered experimental:
+        main.a*  # get rid of main.alias, main.alibaba and main.agliolio

mysql-test/collections/default.daily

+perl mysql-test-run.pl --timer --force --comment=rpl_ndb_row --suite=rpl_ndb,ndb --mysqld=--binlog-format=row --experimental=collections/default.experimental

mysql-test/collections/default.experimental

+funcs_1.charset_collation_1              # depends on compile-time decisions

mysql-test/collections/default.push

+perl mysql-test-run.pl --timer --force --comment=n_mix --mysqld=--binlog-format=mixed --experimental=collections/default.experimental
+perl mysql-test-run.pl --timer --force --comment=ps_row --ps-protocol --mysqld=--binlog-format=row --experimental=collections/default.experimental
+perl mysql-test-run.pl --timer --force --comment=embedded --embedded --experimental=collections/default.experimental
+perl mysql-test-run.pl --timer --force --comment=rpl_binlog_row --suite=rpl,binlog --mysqld=--binlog-format=row --experimental=collections/default.experimental
+perl mysql-test-run.pl --timer --force --comment=funcs_1 --suite=funcs_1 --experimental=collections/default.experimental

mysql-test/lib/mtr_report.pm

@@ -69,6 +69,8 @@ sub _mtr_report_test_name ($) {
 
   print _name(), _timestamp();
   printf "%-40s ", $tname;
+
+  return $tname;
 }
 
 
@@ -105,7 +107,7 @@ sub mtr_report_test_passed ($) {
 
 sub mtr_report_test ($) {
   my ($tinfo)= @_;
-  _mtr_report_test_name($tinfo);
+  my $test_name = _mtr_report_test_name($tinfo);
 
   my $comment=  $tinfo->{'comment'};
   my $logfile=  $tinfo->{'logfile'};
@@ -115,10 +117,37 @@ sub mtr_report_test ($) {
   if ($result eq 'MTR_RES_FAILED'){
 
     my $timest = format_time();
+    my $fail = "fail";
+
+    if ( $::opt_experimental )
+    {
+      # Find out if this test case is an experimental one, so we can treat
+      # the failure as an expected failure instead of a regression.
+      for my $exp ( @$::experimental_test_cases ) {
+        if ( $exp ne $test_name ) {
+          # if the expression is not the name of this test case, but has
+          # an asterisk at the end, determine if the characters up to
+          # but excluding the asterisk are the same
+          if ( $exp ne "" && substr($exp, -1, 1) eq "*" ) {
+            $exp = substr($exp, 0, length($exp) - 1);
+            if ( substr($test_name, 0, length($exp)) ne $exp ) {
+              # no match, try next entry
+              next;
+            }
+            # if yes, fall through to set the exp-fail status
+          } else {
+            # no match, try next entry
+            next;
+          }
+        }
+        $fail = "exp-fail";
+        last;
+      }
+    }
 
     if ( $warnings )
     {
-      mtr_report("[ fail ]  Found warnings/errors in server log file!");
+      mtr_report("[ $fail ]  Found warnings/errors in server log file!");
       mtr_report("        Test ended at $timest");
       mtr_report($warnings);
       return;
@@ -126,14 +155,14 @@ sub mtr_report_test ($) {
     my $timeout= $tinfo->{'timeout'};
     if ( $timeout )
     {
-      mtr_report("[ fail ]  timeout after $timeout seconds");
+      mtr_report("[ $fail ]  timeout after $timeout seconds");
       mtr_report("        Test ended at $timest");
       mtr_report("\n$tinfo->{'comment'}");
       return;
     }
     else
     {
-      mtr_report("[ fail ]\n        Test ended at $timest");
+      mtr_report("[ $fail ]\n        Test ended at $timest");
     }
 
     if ( $logfile )

mysql-test/mysql-test-run.pl

@@ -180,6 +180,9 @@ our $opt_client_debugger;
 my $config; # The currently running config
 my $current_config_name; # The currently running config file template
 
+our $opt_experimental;
+our $experimental_test_cases;
+
 my $baseport;
 my $opt_build_thread= $ENV{'MTR_BUILD_THREAD'} || "auto";
 my $build_thread= 0;
@@ -799,7 +802,7 @@ sub command_line_setup {
              'big-test'                 => \$opt_big_test,
 	     'combination=s'            => \@opt_combinations,
              'skip-combinations'        => \&collect_option,
-
+             'experimental=s'           => \$opt_experimental,
 	     'skip-im'                  => \&ignore_option,
 
              # Specify ports
@@ -966,6 +969,33 @@ sub command_line_setup {
     mtr_print_thick_line('#');
   }
 
+  if ( $opt_experimental )
+  {
+    # read the list of experimental test cases from the file specified on
+    # the command line
+    open(FILE, "<", $opt_experimental) or mtr_error("Can't read experimental file: $opt_experimental");
+    mtr_report("Using experimental file: $opt_experimental");
+    $experimental_test_cases = [];
+    while(<FILE>) {
+      chomp;
+      # remove comments (# foo) at the beginning of the line, or after a 
+      # blank at the end of the line
+      s/( +|^)#.*$//;
+      # remove whitespace
+      s/^ +//;              
+      s/ +$//;
+      # if nothing left, don't need to remember this line
+      if ( $_ eq "" ) {
+        next;
+      }
+      # remember what is left as the name of another test case that should be
+      # treated as experimental
+      print " - $_\n";
+      push @$experimental_test_cases, $_;
+    }
+    close FILE;
+  }
+
   foreach my $arg ( @ARGV )
   {
     if ( $arg =~ /^--skip-/ )