MDEV-22179 rr(record and replay) support for mtr

This feature adds the support for rr in mtr. These 2 options are added
--rr         run   the mysqld in rr record mode
--rr_option= run the rr with custom record option, for multiple
             options use --rr_option= for each option.
             For example
              ./mtr main.view --rr_option=-h --rr_option=-u --rr_option=-c=23
--boot-rr    run the mysqld performing bootstrap in rr record mode

Recording are stored in mysql-test/var/rr folder.
To run recording please run
rr replay var/rr/mysql-X

Limitations
Restart will create a new recording.
Repeat will work on same recording , So might be harder to debug.
If test create the multiple instance of mariadb all will be stored in var/rr

mysql-test/mysql-test-run.pl

@@ -267,8 +267,10 @@ our %gprof_dirs;
 
 our $glob_debugger= 0;
 our $opt_gdb;
+our $opt_rr;
 our $opt_client_gdb;
 my $opt_boot_gdb;
+my $opt_boot_rr;
 our $opt_dbx;
 our $opt_client_dbx;
 my $opt_boot_dbx;
@@ -338,6 +340,7 @@ my $opt_callgrind;
 my %mysqld_logs;
 my $opt_debug_sync_timeout= 300; # Default timeout for WAIT_FOR actions.
 my $warn_seconds = 60;
+my @rr_record_args;
 
 my $rebootstrap_re= '--innodb[-_](?:page[-_]size|checksum[-_]algorithm|undo[-_]tablespaces|log[-_]group[-_]home[-_]dir|data[-_]home[-_]dir)|data[-_]file[-_]path|force_rebootstrap';
 
@@ -1320,10 +1323,13 @@ sub command_line_setup {
              'debug-common'             => \$opt_debug_common,
              'debug-server'             => \$opt_debug_server,
              'gdb=s'                    => \$opt_gdb,
+             'rr'                       => \$opt_rr,
+             'rr_option=s'              => \@rr_record_args,
              'client-gdb'               => \$opt_client_gdb,
              'manual-gdb'               => \$opt_manual_gdb,
              'manual-lldb'              => \$opt_manual_lldb,
 	     'boot-gdb'                 => \$opt_boot_gdb,
+	     'boot-rr'                  => \$opt_boot_rr,
              'manual-debug'             => \$opt_manual_debug,
              'ddd'                      => \$opt_ddd,
              'client-ddd'               => \$opt_client_ddd,
@@ -2448,6 +2454,7 @@ sub environment_setup {
   $ENV{'MYSQL_TEST_DIR'}=     $glob_mysql_test_dir;
   $ENV{'DEFAULT_MASTER_PORT'}= $mysqld_variables{'port'};
   $ENV{'MYSQL_TMP_DIR'}=      $opt_tmpdir;
+  $ENV{'_RR_TRACE_DIR'}=      "$opt_vardir/rr";
   $ENV{'MYSQLTEST_VARDIR'}=   $opt_vardir;
   $ENV{'MYSQL_BINDIR'}=       $bindir;
   $ENV{'MYSQL_SHAREDIR'}=     $path_language;
@@ -2762,6 +2769,7 @@ sub setup_vardir() {
   # Create var/tmp and tmp - they might be different
   mkpath("$opt_vardir/tmp");
   mkpath($opt_tmpdir) if ($opt_tmpdir ne "$opt_vardir/tmp");
+  mkpath("$opt_vardir/rr");
 
   # On some operating systems, there is a limit to the length of a
   # UNIX domain socket's path far below PATH_MAX.
@@ -3417,6 +3425,10 @@ sub mysql_install_db {
       ddd_arguments(\$args, \$exe_mysqld_bootstrap, $mysqld->name(),
         $bootstrap_sql_file);
     }
+    if ($opt_boot_rr) {
+      $args= ["record", "$exe_mysqld_bootstrap", @$args];
+      $exe_mysqld_bootstrap= "rr";
+    }
 
     my $path_sql= my_find_file($install_basedir,
              ["mysql", "sql/share", "share/mariadb",
@@ -5408,6 +5420,18 @@ sub mysqld_start ($$) {
      # Indicate the exe should not be started
     $exe= undef;
   }
+  elsif ( $opt_rr || @rr_record_args)
+  {
+    if (@rr_record_args)
+    {
+      $args= ["record", @rr_record_args, "$exe", @$args];
+    }
+    else
+    {
+      $args= ["record", "$exe", @$args];
+    }
+    $exe= "rr";
+  }
   else
   {
     # Default to not wait until pid file has been created
@@ -6588,6 +6612,12 @@ Options for strace
                         specifying name and path to the trace program to use.
                         Example: $0 --strace-client=ktrace
 
+Options for rr(Record and Replay)
+  rr                    Run the "mysqld" executables using rr. Default run
+                        option is "rr record mysqld mysqld_options"
+  boot-rr               Start bootstrap server in rr
+  rr_option=ARG         Option to give rr record, can be specified more then once
+
 Misc options
   user=USER             User for connecting to mysqld(default: $opt_user)
   comment=STR           Write STR to the output