BUG#11745230: 12133: MASTER.INDEX FILE KEEPS MYSQLD FROM STARTING IF

              BIN LOG HAS BEEN MOVED 

When moving the binary/relay log files from one location to
another and restarting the server with a different log-bin or
relay-log paths, would cause the startup process to abort. The
root cause was that the server would not be able to find the log
files because it would consider old paths for entries in the
index file instead of the new location.  What's even worse, the
relative paths would not be considered relative to the path
provided in log-bin and relay-log, but to mysql_data_dir.
      
We fix the cases where the server contains relative paths. When
the server is reading from the index file, it checks whether the
entry contains relative paths. If it does, we replace it with the
absolute path set in log-bin/relay-log option. Absolute paths
remain unchanged and the index must be manually edited to
consider the new log-bin and/or relay-log path (this should be
documented). This is a fix for a GA version, that does not break
behavior (that much).
      
For development versions, we should go with Zhenxing's approach 
that removes paths altogether from index files.

mysql-test/include/begin_include_file.inc:
  Added parameter to keep the begin_include_file.inc silent. Useful when 
  including scripts that contain platform dependent parameters, for example:
  
  --let $rpl_server_parameters=--log-bin=$tmpdir/slave-bin --relay-log=$tmpdir/slave-relay-bin
  --let $keep_include_silent=1
  source include/rpl_start_server.inc;
  --let $keep_include_silent=0
  
  We want the paths ($tmpdir/slave-bin and $tmpdir/slave-relay-bin) not to be in the 
  result file.
mysql-test/suite/rpl/t/rpl_binlog_index.test:
  Test case.
sql/log.cc:
  When finding the corresponding log entry in the index file, we first 
  normalize the paths before doing the comparison. This will make relative
  paths to be turned into absolute paths (based on the opt_bin_logname or
  opt_relay_logname) and then compared against also, expanded paths entered, 
  through CHANGE MASTER for instance.
sql/log.h:
  Added normalize_binlog_name, which turns relative paths, into absolute paths
  given the parameter: is_relay_log ? opt_relay_logname : opt_bin_logname .
sql/mysqld.cc:
  Exposing opt_bin_logname.
sql/mysqld.h:
  Exposing opt_bin_logname.

mysql-test/include/begin_include_file.inc

@@ -48,6 +48,11 @@
 #     must be provided both for begin_include_file.inc and
 #     end_include_file.inc.
 #
+#   $keep_include_silent
+#     This specifies whether it should be echoed to the result file
+#     the following string: include/$include_filename
+#     If not set, the string will be echoed.
+#
 #   $rpl_debug
 #     If set, this script will print the following text:
 #      ==== BEGIN include/$include_filename.inc ====
@@ -59,7 +64,10 @@
 # recursively.
 if (!$_include_file_depth)
 {
-  --echo include/$include_filename
+  if (!$keep_include_silent)
+  {
+    --echo include/$include_filename
+  }
   --let $_include_file_depth= 0
 }
 --inc $_include_file_depth

mysql-test/suite/rpl/r/rpl_binlog_index.result

+include/master-slave.inc
+[connection master]
+CREATE TABLE t1 (a INT);
+FLUSH BINARY LOGS;
+INSERT INTO t1 VALUES (1);
+# Shutdown master
+include/rpl_stop_server.inc [server_number=1]
+# Move the master binlog files and the index file to a new place
+# Restart master with log-bin option set to the new path
+# Master has restarted successfully
+# Create the master-bin.index file with the old format
+# Shutdown master
+include/rpl_stop_server.inc [server_number=1]
+# Move back the master binlog files
+# Remove the unneeded master-bin.index file
+# Restart master with log-bin option set to default
+# Master has restarted successfully
+include/stop_slave.inc
+# Move the slave binlog and relay log files and index to the new place
+# Shutdown slave
+include/rpl_stop_server.inc [server_number=2]
+# Restart slave with options log-bin, relay-log set to the new paths
+# Slave has restarted successfully
+include/start_slave.inc
+include/stop_slave.inc
+FLUSH LOGS;
+INSERT INTO t1 VALUES (2);
+INSERT INTO t1 VALUES (2);
+INSERT INTO t1 VALUES (2);
+INSERT INTO t1 VALUES (2);
+FLUSH LOGS;
+FLUSH LOGS;
+include/start_slave.inc
+include/diff_tables.inc [master:t1,slave:t1]
+DROP TABLE t1;

mysql-test/suite/rpl/t/rpl_binlog_index.test

+# ==== Purpose ====
+#
+# Test that server can work fine after moving binlog or relay log
+# files to another directory and setting binlog or relay log paths to
+# the new path.
+# 
+# ==== Method ====
+#
+# Start replication, and then shutdown the master, move the binary
+# logs and the log index file to a another directory and then restart
+# the server with option to set the new binlog directory. After master
+# restarted successfully, do the similar on slave to check the relay
+# log of slave.
+#
+# ==== Reference ====
+#
+# BUG#12133 master.index file keeps mysqld from starting if bin log has been moved
+# BUG#42576 Relay logs in relay-log.info&localhost-relay-bin.index not processed after move
+
+source include/master-slave.inc;
+# There is no need to run this test case on all binlog format
+source include/have_binlog_format_row.inc;
+
+connection master;
+--let $master_datadir= `select @@datadir`
+connection slave;
+--let $slave_datadir= `select @@datadir`
+connection master;
+--let $tmpdir= $MYSQLTEST_VARDIR/tmp/rpl_binlog_index
+--mkdir $tmpdir
+
+CREATE TABLE t1 (a INT);
+# flush to generate one more binlog file.
+FLUSH BINARY LOGS;
+INSERT INTO t1 VALUES (1);
+
+sync_slave_with_master;
+
+
+#
+# Test on master
+#
+connection master;
+--echo # Shutdown master
+--let $rpl_server_number=1
+source include/rpl_stop_server.inc;
+
+--echo # Move the master binlog files and the index file to a new place
+--move_file $master_datadir/master-bin.000001 $tmpdir/master-bin.000001
+--move_file $master_datadir/master-bin.000002 $tmpdir/master-bin.000002
+--move_file $master_datadir/master-bin.index  $tmpdir/master-bin.index
+
+--echo # Restart master with log-bin option set to the new path
+--let $rpl_server_parameters=--log-bin=$tmpdir/master-bin
+--let $keep_include_silent=1
+source include/rpl_start_server.inc;
+--let $keep_include_silent=0
+
+--echo # Master has restarted successfully
+
+#
+# Test master can handle old format with directory path in index file
+#
+--let $is_windows= `select convert(@@version_compile_os using latin1) in ('Win32', 'Win64', 'Windows')`
+
+# write_var_to_file.inc will call SELECT INTO DUMPFILE, which has to be
+# done before shutdown the server
+--echo # Create the master-bin.index file with the old format
+--let $write_to_file= $master_datadir/master-bin.index
+if ($is_windows)
+{
+  --let $write_var= .\\master-bin.000001\n.\\master-bin.000002\n
+}
+if (!$is_windows)
+{
+  --let $write_var= ./master-bin.000001\n./master-bin.000002\n
+}
+--disable_query_log
+source include/write_var_to_file.inc;
+--enable_query_log
+
+--echo # Shutdown master
+--let $rpl_server_number=1
+source include/rpl_stop_server.inc;
+
+--echo # Move back the master binlog files
+--move_file $tmpdir/master-bin.000001 $master_datadir/master-bin.000001
+--move_file $tmpdir/master-bin.000002 $master_datadir/master-bin.000002
+
+--echo # Remove the unneeded master-bin.index file
+--remove_file $tmpdir/master-bin.index
+
+--echo # Restart master with log-bin option set to default
+--let $rpl_server_parameters=--log-bin=$master_datadir/master-bin
+--let $keep_include_silent=1
+source include/rpl_start_server.inc;
+--let $keep_include_silent=0
+
+--echo # Master has restarted successfully
+
+connection slave;
+source include/stop_slave.inc;
+
+--disable_query_log
+# slave-relay-bin.* files can vary, so read the slave-relay-bin.index
+# to figure out the slave-relay-bin.* files
+CREATE TEMPORARY TABLE tmp (id INT AUTO_INCREMENT PRIMARY KEY, filename VARCHAR(1024));
+# chmod to allow the following LOAD DATA
+--chmod 0666 $slave_datadir/slave-relay-bin.index
+--eval LOAD DATA INFILE '$slave_datadir/slave-relay-bin.index' INTO TABLE tmp (filename)
+--let $count= `SELECT count(*) FROM tmp`
+--echo # Move the slave binlog and relay log files and index to the new place
+--move_file $slave_datadir/slave-bin.index        $tmpdir/slave-bin.index
+--move_file $slave_datadir/slave-bin.000001       $tmpdir/slave-bin.000001
+--move_file $slave_datadir/slave-relay-bin.index  $tmpdir/slave-relay-bin.index
+while ($count)
+{
+  --let $filename= `select filename from tmp where id=$count`
+  --move_file $slave_datadir/$filename $tmpdir/$filename
+  --dec $count
+}
+DROP TEMPORARY TABLE tmp;
+--enable_query_log
+
+--echo # Shutdown slave
+--let $rpl_server_number=2
+source include/rpl_stop_server.inc;
+
+--echo # Restart slave with options log-bin, relay-log set to the new paths
+--let $rpl_server_parameters=--log-bin=$tmpdir/slave-bin --relay-log=$tmpdir/slave-relay-bin
+--let $keep_include_silent=1
+source include/rpl_start_server.inc;
+--let $keep_include_silent=0
+
+--echo # Slave has restarted successfully
+source include/start_slave.inc;
+--source include/stop_slave.inc
+
+connection master;
+FLUSH LOGS;
+INSERT INTO t1 VALUES (2);
+INSERT INTO t1 VALUES (2);
+INSERT INTO t1 VALUES (2);
+INSERT INTO t1 VALUES (2);
+
+FLUSH LOGS;
+
+connection slave;
+FLUSH LOGS;
+--source include/start_slave.inc
+connection master;
+sync_slave_with_master;
+--let $diff_tables= master:t1,slave:t1
+source include/diff_tables.inc;
+
+connection master;
+DROP TABLE t1;
+sync_slave_with_master;
+--remove_files_wildcard $tmpdir *
+--rmdir $tmpdir

sql/log.cc

@@ -3242,10 +3242,11 @@ int MYSQL_BIN_LOG::find_log_pos(LOG_INFO *linfo, const char *log_name,
 			    bool need_lock)
 {
   int error= 0;
-  char *fname= linfo->log_file_name;
-  uint log_name_len= log_name ? (uint) strlen(log_name) : 0;
+  char *full_fname= linfo->log_file_name;
+  char full_log_name[FN_REFLEN], fname[FN_REFLEN];
+  uint log_name_len= 0, fname_len= 0;
   DBUG_ENTER("find_log_pos");
-  DBUG_PRINT("enter",("log_name: %s", log_name ? log_name : "NULL"));
+  full_log_name[0]= full_fname[0]= 0;
 
   /*
     Mutex needed because we need to make sure the file pointer does not
@@ -3255,6 +3256,20 @@ int MYSQL_BIN_LOG::find_log_pos(LOG_INFO *linfo, const char *log_name,
     mysql_mutex_lock(&LOCK_index);
   mysql_mutex_assert_owner(&LOCK_index);
 
+  // extend relative paths for log_name to be searched
+  if (log_name)
+  {
+    if(normalize_binlog_name(full_log_name, log_name, is_relay_log))
+    {
+      error= LOG_INFO_EOF;
+      goto end;
+    }
+  }
+
+  log_name_len= log_name ? (uint) strlen(full_log_name) : 0;
+  DBUG_PRINT("enter", ("log_name: %s, full_log_name: %s", 
+                       log_name ? log_name : "NULL", full_log_name));
+
   /* As the file is flushed, we can't get an error here */
   (void) reinit_io_cache(&index_file, READ_CACHE, (my_off_t) 0, 0, 0);
 
@@ -3273,19 +3288,28 @@ int MYSQL_BIN_LOG::find_log_pos(LOG_INFO *linfo, const char *log_name,
       break;
     }
 
+    // extend relative paths and match against full path
+    if (normalize_binlog_name(full_fname, fname, is_relay_log))
+    {
+      error= LOG_INFO_EOF;
+      break;
+    }
+    fname_len= (uint) strlen(full_fname);
+
     // if the log entry matches, null string matching anything
     if (!log_name ||
-	(log_name_len == length-1 && fname[log_name_len] == '\n' &&
-	 !memcmp(fname, log_name, log_name_len)))
+	(log_name_len == fname_len-1 && full_fname[log_name_len] == '\n' &&
+	 !memcmp(full_fname, full_log_name, log_name_len)))
     {
-      DBUG_PRINT("info",("Found log file entry"));
-      fname[length-1]=0;			// remove last \n
+      DBUG_PRINT("info", ("Found log file entry"));
+      full_fname[fname_len-1]= 0;			// remove last \n
       linfo->index_file_start_offset= offset;
       linfo->index_file_offset = my_b_tell(&index_file);
       break;
     }
   }
 
+end:
   if (need_lock)
     mysql_mutex_unlock(&LOCK_index);
   DBUG_RETURN(error);
@@ -3320,7 +3344,8 @@ int MYSQL_BIN_LOG::find_next_log(LOG_INFO* linfo, bool need_lock)
 {
   int error= 0;
   uint length;
-  char *fname= linfo->log_file_name;
+  char fname[FN_REFLEN];
+  char *full_fname= linfo->log_file_name;
 
   if (need_lock)
     mysql_mutex_lock(&LOCK_index);
@@ -3336,8 +3361,19 @@ int MYSQL_BIN_LOG::find_next_log(LOG_INFO* linfo, bool need_lock)
     error = !index_file.error ? LOG_INFO_EOF : LOG_INFO_IO;
     goto err;
   }
-  fname[length-1]=0;				// kill \n
-  linfo->index_file_offset = my_b_tell(&index_file);
+
+  if (fname[0] != 0)
+  {
+    if(normalize_binlog_name(full_fname, fname, is_relay_log))
+    {
+      error= LOG_INFO_EOF;
+      goto err;
+    }
+    length= strlen(full_fname);
+  }
+
+  full_fname[length-1]= 0;			// kill \n
+  linfo->index_file_offset= my_b_tell(&index_file);
 
 err:
   if (need_lock)

sql/log.h

@@ -715,4 +715,66 @@ char *make_log_name(char *buff, const char *name, const char* log_ext);
 extern MYSQL_PLUGIN_IMPORT MYSQL_BIN_LOG mysql_bin_log;
 extern LOGGER logger;
 
+
+/**
+  Turns a relative log binary log path into a full path, based on the
+  opt_bin_logname or opt_relay_logname.
+
+  @param from         The log name we want to make into an absolute path.
+  @param to           The buffer where to put the results of the 
+                      normalization.
+  @param is_relay_log Switch that makes is used inside to choose which
+                      option (opt_bin_logname or opt_relay_logname) to
+                      use when calculating the base path.
+
+  @returns true if a problem occurs, false otherwise.
+ */
+
+inline bool normalize_binlog_name(char *to, const char *from, bool is_relay_log)
+{
+  DBUG_ENTER("normalize_binlog_name");
+  bool error= false;
+  char buff[FN_REFLEN];
+  char *ptr= (char*) from;
+  char *opt_name= is_relay_log ? opt_relay_logname : opt_bin_logname;
+
+  DBUG_ASSERT(from);
+
+  /* opt_name is not null and not empty and from is a relative path */
+  if (opt_name && opt_name[0] && from && !test_if_hard_path(from))
+  {
+    // take the path from opt_name
+    // take the filename from from 
+    char log_dirpart[FN_REFLEN], log_dirname[FN_REFLEN];
+    size_t log_dirpart_len, log_dirname_len;
+    dirname_part(log_dirpart, opt_name, &log_dirpart_len);
+    dirname_part(log_dirname, from, &log_dirname_len);
+
+    /* log may be empty => relay-log or log-bin did not 
+        hold paths, just filename pattern */
+    if (log_dirpart_len > 0)
+    {
+      /* create the new path name */
+      if(fn_format(buff, from+log_dirname_len, log_dirpart, "",
+                   MYF(MY_UNPACK_FILENAME | MY_SAFE_PATH)) == NULL)
+      {
+        error= true;
+        goto end;
+      }
+
+      ptr= buff;
+    }
+  }
+
+  DBUG_ASSERT(ptr);
+
+  if (ptr)
+    strmake(to, ptr, strlen(ptr));
+
+end:
+  DBUG_RETURN(error);
+}
+
+
+
 #endif /* LOG_H */

sql/mysqld.cc

@@ -652,7 +652,7 @@ ulong master_retry_count=0;
 char *master_info_file;
 char *relay_log_info_file, *report_user, *report_password, *report_host;
 char *opt_relay_logname = 0, *opt_relaylog_index_name=0;
-char *opt_logname, *opt_slow_logname;
+char *opt_logname, *opt_slow_logname, *opt_bin_logname;
 
 /* Static variables */
 
@@ -677,7 +677,6 @@ static char **defaults_argv;
 static int remaining_argc;
 /** Remaining command line arguments (arguments), filtered by handle_options().*/
 static char **remaining_argv;
-static char *opt_bin_logname;
 
 int orig_argc;
 char **orig_argv;

sql/mysqld.h

@@ -139,7 +139,8 @@ extern my_bool relay_log_recovery;
 extern uint test_flags,select_errors,ha_open_options;
 extern uint protocol_version, mysqld_port, dropping_tables;
 extern ulong delay_key_write_options;
-extern char *opt_logname, *opt_slow_logname;
+extern char *opt_logname, *opt_slow_logname, *opt_bin_logname, 
+            *opt_relay_logname;
 extern char *opt_backup_history_logname, *opt_backup_progress_logname,
             *opt_backup_settings_name;
 extern const char *log_output_str;