WL#3935 Improve mysqltest report and exitcode for diff

 - Move the code to generate test report to the test tool(in this
   case mysqltest) where the best control of what failed is
- Simplify the code in mysql-test-run.pl
- mysqltest will now find what diff to use in a best effort attempt
  using "diff -u", "diff -c" and finally dumping the two files verbatim
  in the report.


client/mysqltest.c:
  - Add new option --tail-lines indicating how many lines of the result to output
    when generating report for a failure
  - Remove eval_result, noone knows what it should do and it's not used.
  - Add support for best effort execution of systems builtin "diff",
    try with "diff -u" and "diff -c" and if that fails dump the whole content
     of teh two files to report
  - Use one common function when comapring two files, used when test 
    has completed and the result should be compared to the .result file
    as well as using it from myqltest's builtin diff_files command.
  - Output the last lines og the result so far  in the report that mysqltest
    generates in case of a test failure.
mysql-test/lib/mtr_report.pl:
  - Remove code for generating diff between .reject and .result file,
    that is handled by mysqltest(or the test tool) from now on.
  - Add better comments
mysql-test/mysql-test-run.pl:
  - Remove the --udiff flag to mysql-test-run.pl, if the systems diff supports
    unified diff it will be used automatically
  - Remove the code for "mtr_show_failed_diff", the report from mysqltest
    will know include the diff so if mysqltest(or another test tool)
    fails, just display it's report what went wrong
  - Pass --tail-lines=20 to mysqltest to it will shos the last 20 lines 
    from the result in the report in case of failure.	
mysql-test/r/mysqltest.result:
  Update result file now when the output from mysqltest has been sent to /dev/null
mysql-test/t/mysqltest.test:
  - Improve tests for --diff_files
  - Remove temporary files in var/tmp generated in the fly
  - Send output from test for --diff_files to /deb/null since only the error
    code it returns are of interest and tyhe outpu may vary depending
    on what builtin diff is being used.

mysql-test/lib/mtr_report.pl

@@ -26,7 +26,6 @@ sub mtr_report_test_failed($);
 sub mtr_report_test_skipped($);
 sub mtr_report_test_not_skipped_though_disabled($);
 
-sub mtr_show_failed_diff ($);
 sub mtr_report_stats ($);
 sub mtr_print_line ();
 sub mtr_print_thick_line ();
@@ -45,55 +44,6 @@ sub mtr_verbose (@);
 #
 ##############################################################################
 
-# We can't use diff -u or diff -a as these are not portable
-
-sub mtr_show_failed_diff ($) {
-  my $result_file_name=  shift;
-
-  # The reject and log files have been dumped to
-  # to filenames based on the result_file's name
-  my $tname= basename($result_file_name);
-  $tname=~ s/\..*$//;
-
-  my $reject_file=  "r/$tname.reject";
-  my $result_file=  "r/$tname.result";
-  my $log_file=  "r/$tname.log";
-  my $eval_file=    "r/$tname.eval";
-
-  if ( $::opt_suite ne "main" )
-  {
-    $reject_file= "$::glob_mysql_test_dir/suite/$::opt_suite/$reject_file";
-    $result_file= "$::glob_mysql_test_dir/suite/$::opt_suite/$result_file";
-    $eval_file=   "$::glob_mysql_test_dir/suite/$::opt_suite/$eval_file";
-    $log_file=   "$::glob_mysql_test_dir/suite/$::opt_suite/$log_file";
-  }
-
-  if ( -f $eval_file )
-  {
-    $result_file=  $eval_file;
-  }
-
-  my $diffopts= $::opt_udiff ? "-u" : "-c";
-
-  if ( -f $reject_file )
-  {
-    print "Below are the diffs between actual and expected results:\n";
-    print "-------------------------------------------------------\n";
-    # FIXME check result code?!
-    mtr_run("diff",[$diffopts,$result_file,$reject_file], "", "", "", "");
-    print "-------------------------------------------------------\n";
-    print "Please follow the instructions outlined at\n";
-    print "http://www.mysql.com/doc/en/Reporting_mysqltest_bugs.html\n";
-    print "to find the reason to this problem and how to report this.\n\n";
-  }
-
-  if ( -f $log_file )
-  {
-    print "Result from queries before failure can be found in $log_file\n";
-    # FIXME Maybe a tail -f -n 10 $log_file here
-  }
-}
-
 sub mtr_report_test_name ($) {
   my $tinfo= shift;
 
@@ -165,16 +115,23 @@ sub mtr_report_test_failed ($) {
 
   if ( $tinfo->{'comment'} )
   {
+    # The test failure has been detected by mysql-test-run.pl
+    # when starting the servers or due to other error, the reason for
+    # failing the test is saved in "comment"
     print "\nERROR: $tinfo->{'comment'}\n";
   }
   elsif ( -f $::path_timefile )
   {
-    print "\nErrors are (from $::path_timefile) :\n";
+    # Test failure was detected by test tool and it's report
+    # about what failed has been saved to file. Display the report.
+    print "\n";
     print mtr_fromfile($::path_timefile); # FIXME print_file() instead
-    print "\n(the last lines may be the most important ones)\n";
+    print "\n";
   }
   else
   {
+    # Neither this script or the test tool has recorded info
+    # about why the test has failed. Should be debugged.
     print "\nUnexpected termination, probably when starting mysqld\n";
   }
 }

mysql-test/mysql-test-run.pl

@@ -83,7 +83,6 @@ require "lib/mtr_io.pl";
 require "lib/mtr_gcov.pl";
 require "lib/mtr_gprof.pl";
 require "lib/mtr_report.pl";
-require "lib/mtr_diff.pl";
 require "lib/mtr_match.pl";
 require "lib/mtr_misc.pl";
 require "lib/mtr_stress.pl";
@@ -281,8 +280,6 @@ our $opt_wait_for_slave;
 
 our $opt_warnings;
 
-our $opt_udiff;
-
 our $opt_skip_ndbcluster= 0;
 our $opt_skip_ndbcluster_slave= 0;
 our $opt_with_ndbcluster= 0;
@@ -619,7 +616,6 @@ sub command_line_setup () {
              'start-dirty'              => \$opt_start_dirty,
              'start-and-exit'           => \$opt_start_and_exit,
              'timer!'                   => \$opt_timer,
-             'unified-diff|udiff'       => \$opt_udiff,
              'user=s'                   => \$opt_user,
              'testcase-timeout=i'       => \$opt_testcase_timeout,
              'suite-timeout=i'          => \$opt_suite_timeout,
@@ -3603,7 +3599,6 @@ sub report_failure_and_restart ($) {
   my $tinfo= shift;
 
   mtr_report_test_failed($tinfo);
-  mtr_show_failed_diff($tinfo->{'result_file'});
   print "\n";
   if ( $opt_force )
   {
@@ -4788,6 +4783,9 @@ sub run_mysqltest ($) {
 
   mtr_add_arg($args, "--test-file=%s", $tinfo->{'path'});
 
+  # Number of lines of resut to include in failure report
+  mtr_add_arg($args, "--tail-lines=20");
+
   if ( defined $tinfo->{'result_file'} ) {
     mtr_add_arg($args, "--result-file=%s", $tinfo->{'result_file'});
   }
@@ -5199,7 +5197,6 @@ Misc options
   fast                  Don't try to clean up from earlier runs
   reorder               Reorder tests to get fewer server restarts
   help                  Get this help text
-  unified-diff | udiff  When presenting differences, use unified diff
 
   testcase-timeout=MINUTES Max test case run time (default $default_testcase_timeout)
   suite-timeout=MINUTES Max test suite run time (default $default_suite_timeout)

mysql-test/r/mysqltest.result

@@ -461,7 +461,6 @@ root@localhost
 --------------------------------------------------------------------------------
 this will be executed
 this will be executed
-mysqltest: Result length mismatch
 mysqltest: The test didn't produce any output
 Failing multi statement query
 mysqltest: At line 3: query 'create table t1 (a int primary key);
@@ -473,6 +472,8 @@ mysqltest: At line 3: query 'create table t1 (a int primary key);
 insert into t1 values (1);
 select 'select-me';
 insertz 'error query'' failed: 1064: You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'insertz 'error query'' at line 1
+
+More results from queries before failure can be found in MYSQLTEST_VARDIR/tmp/bug11731.log
 drop table t1;
 Multi statement using expected error
 create table t1 (a int primary key);

mysql-test/t/mysqltest.test

@@ -436,6 +436,8 @@ EOF
 --error 1
 --exec $MYSQL_TEST < $MYSQLTEST_VARDIR/tmp/mysqltest.sql 2>&1
 
+remove_file $MYSQLTEST_VARDIR/tmp/mysqltest.sql;
+
 #
 # Extra delimiter
 #
@@ -786,6 +788,7 @@ echo $var3_var3;
 --replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR
 --error 1
 --exec echo "source $MYSQLTEST_VARDIR/tmp/recursive.sql;" | $MYSQL_TEST 2>&1
+remove_file $MYSQLTEST_VARDIR/tmp/recursive.sql;
 
 # Source a file with error
 --exec echo "garbage ;" > $MYSQLTEST_VARDIR/tmp/error.sql
@@ -793,6 +796,7 @@ echo $var3_var3;
 --error 1
 --exec echo "source $MYSQLTEST_VARDIR/tmp/error.sql;" | $MYSQL_TEST 2>&1
 
+remove_file $MYSQLTEST_VARDIR/tmp/error.sql;
 
 # Test execution of source in a while loop
 --write_file $MYSQLTEST_VARDIR/tmp/sourced.inc
@@ -1171,6 +1175,8 @@ EOF
 --error 1
 --exec $MYSQL_TEST < $MYSQLTEST_VARDIR/tmp/mysqltest.sql 2>&1
 
+remove_file $MYSQLTEST_VARDIR/tmp/mysqltest.sql;
+
 # ----------------------------------------------------------------------------
 # Test error messages returned from comments starting with a command
 # ----------------------------------------------------------------------------
@@ -1296,6 +1302,8 @@ EOF
 --error 1
 --exec echo "source $MYSQLTEST_VARDIR/tmp/mysqltest.sql;" | $MYSQL_TEST 2>&1
 
+remove_file $MYSQLTEST_VARDIR/tmp/mysqltest.sql;
+
 # connect when "disable_abort_on_error" caused "connection not found"
 --replace_result $MASTER_MYSOCK MASTER_SOCKET $MASTER_MYPORT MASTER_PORT
 --disable_abort_on_error
@@ -1399,7 +1407,11 @@ select "this will be executed";
 --exec touch $MYSQLTEST_VARDIR/tmp/zero_length_file.result
 --exec echo "echo ok;" > $MYSQLTEST_VARDIR/tmp/query.sql
 --error 1
---exec $MYSQL_TEST -x $MYSQLTEST_VARDIR/tmp/query.sql -R $MYSQLTEST_VARDIR/tmp/zero_length_file.result  2>&1
+--exec $MYSQL_TEST -x $MYSQLTEST_VARDIR/tmp/query.sql -R $MYSQLTEST_VARDIR/tmp/zero_length_file.result  > /dev/null 2>&1
+
+remove_file $MYSQLTEST_VARDIR/tmp/zero_length_file.result;
+remove_file $MYSQLTEST_VARDIR/tmp/zero_length_file.reject;
+
 #
 # Test that a test file that does not generate any output fails.
 #
@@ -1407,6 +1419,8 @@ select "this will be executed";
 --error 1
 --exec $MYSQL_TEST -x $MYSQLTEST_VARDIR/tmp/query.sql  2>&1
 
+remove_file $MYSQLTEST_VARDIR/tmp/query.sql;
+
 #
 # Test that mysqltest fails when there are no queries executed
 # but a result file exists
@@ -1436,6 +1450,7 @@ echo Failing multi statement query;
 --exec $MYSQL_TEST -x $MYSQLTEST_VARDIR/tmp/bug11731.sql 2>&1
 drop table t1;
 
+--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR
 --error 1
 --exec $MYSQL_TEST --record -x $MYSQLTEST_VARDIR/tmp/bug11731.sql -R $MYSQLTEST_VARDIR/tmp/bug11731.out 2>&1
 # The .out file should be non existent
@@ -1462,6 +1477,9 @@ drop table t1;
 # The .out file should exist
 --exec test -s $MYSQLTEST_VARDIR/tmp/bug11731.out
 drop table t1;
+remove_file $MYSQLTEST_VARDIR/tmp/bug11731.out;
+remove_file $MYSQLTEST_VARDIR/tmp/bug11731.log;
+remove_file $MYSQLTEST_VARDIR/tmp/bug11731.sql;
 
 #
 # Bug#19890  	mysqltest: "query" command is broken
@@ -1607,19 +1625,48 @@ for diff_file command
 of mysqltest
 EOF
 
+--write_file $MYSQLTEST_VARDIR/tmp/diff4.tmp
+Some data
+for diff_file command
+of musqltest
+EOF
+
 # Compare equal files
 --diff_files $MYSQLTEST_VARDIR/tmp/diff1.tmp $MYSQLTEST_VARDIR/tmp/diff2.tmp
 --diff_files $MYSQLTEST_VARDIR/tmp/diff2.tmp $MYSQLTEST_VARDIR/tmp/diff1.tmp
 
-# Compare files that differ
+# Write the below commands to a intermediary file and execute them with
+# mysqltest in --exec, since the output will vary depending on what "diff"
+# is available it is sent to /dev/null
+--write_file $MYSQLTEST_VARDIR/tmp/diff.test
+# Compare files that differ in size
+--error 2
+--diff_files $MYSQLTEST_VARDIR/tmp/diff1.tmp $MYSQLTEST_VARDIR/tmp/diff3.tmp
+--error 2
+--diff_files $MYSQLTEST_VARDIR/tmp/diff3.tmp $MYSQLTEST_VARDIR/tmp/diff1.tmp
+
+# Compare files that differ only in content
 --error 1
---diff_files $MYSQLTEST_VARDIR/tmp/diff3.tmp $MYSQLTEST_VARDIR/tmp/diff2.tmp
+--diff_files $MYSQLTEST_VARDIR/tmp/diff1.tmp $MYSQLTEST_VARDIR/tmp/diff4.tmp
 --error 1
---diff_files $MYSQLTEST_VARDIR/tmp/diff1.tmp $MYSQLTEST_VARDIR/tmp/diff3.tmp
+--diff_files $MYSQLTEST_VARDIR/tmp/diff4.tmp $MYSQLTEST_VARDIR/tmp/diff1.tmp
+EOF
+
+# Execute the above diffs, and send their output to /dev/null - only
+# interesting to see that it returns correct error codes
+--exec $MYSQL_TEST < $MYSQLTEST_VARDIR/tmp/diff.test > /dev/null 2>&1
+
 
 # Compare equal files, again...
 --diff_files $MYSQLTEST_VARDIR/tmp/diff1.tmp $MYSQLTEST_VARDIR/tmp/diff2.tmp
 
+--remove_file $MYSQLTEST_VARDIR/tmp/diff1.tmp
+--remove_file $MYSQLTEST_VARDIR/tmp/diff2.tmp
+--remove_file $MYSQLTEST_VARDIR/tmp/diff3.tmp
+--remove_file $MYSQLTEST_VARDIR/tmp/diff4.tmp
+--remove_file $MYSQLTEST_VARDIR/tmp/diff.test
+
+
 # ----------------------------------------------------------------------------
 # test for file_exist
 # ----------------------------------------------------------------------------