Remove not used files

Added option: --warning-for-wrong-transaction-id to maria_check


mysql-test/r/innodb-use-sys-malloc.result:
  Remove not used files
mysql-test/t/innodb-use-sys-malloc-master.opt:
  Remove not used files
storage/maria/ma_recovery.c:
  Added support for --lsn-end
storage/maria/ma_recovery.h:
  New prototype
storage/maria/ma_write.c:
  Added logging of KEY_OP_CHECK for all possible keyblock splits.
  (Helps us find error in log file handling)
storage/maria/maria_chk.c:
  Added option: --warning-for-wrong-transaction-id
  --update-status now resets open count if check succeds
storage/maria/maria_read_log.c:
  Added option --end-lsn for only appling part of log (good for debugging)
  Changed options to use '-' instead of '_'

mysql-test/r/innodb-use-sys-malloc.result

-SELECT @@GLOBAL.innodb_use_sys_malloc;
-@@GLOBAL.innodb_use_sys_malloc
-1
-1 Expected
-SET @@GLOBAL.innodb_use_sys_malloc=0;
-ERROR HY000: Variable 'innodb_use_sys_malloc' is a read only variable
-Expected error 'Read only variable'
-SELECT @@GLOBAL.innodb_use_sys_malloc;
-@@GLOBAL.innodb_use_sys_malloc
-1
-1 Expected
-drop table if exists t1;
-create table t1(a int not null) engine=innodb DEFAULT CHARSET=latin1;
-insert into t1 values (1),(2),(3),(4),(5),(6),(7);
-select * from t1;
-a
-1
-2
-3
-4
-5
-6
-7
-drop table t1;
-SELECT @@GLOBAL.innodb_use_sys_malloc;
-@@GLOBAL.innodb_use_sys_malloc
-1
-1 Expected
-SET @@GLOBAL.innodb_use_sys_malloc=0;
-ERROR HY000: Variable 'innodb_use_sys_malloc' is a read only variable
-Expected error 'Read only variable'
-SELECT @@GLOBAL.innodb_use_sys_malloc;
-@@GLOBAL.innodb_use_sys_malloc
-1
-1 Expected
-drop table if exists t1;
-create table t1(a int not null) engine=innodb DEFAULT CHARSET=latin1;
-insert into t1 values (1),(2),(3),(4),(5),(6),(7);
-select * from t1;
-a
-1
-2
-3
-4
-5
-6
-7
-drop table t1;

mysql-test/t/innodb-use-sys-malloc-master.opt

---loose-innodb-use-sys-malloc=true

storage/maria/ma_recovery.c

@@ -109,7 +109,8 @@ prototype_undo_exec_hook(UNDO_KEY_DELETE);
 prototype_undo_exec_hook(UNDO_KEY_DELETE_WITH_ROOT);
 prototype_undo_exec_hook(UNDO_BULK_INSERT);
 
-static int run_redo_phase(LSN lsn, enum maria_apply_log_way apply);
+static int run_redo_phase(LSN lsn, LSN end_lsn,
+                          enum maria_apply_log_way apply);
 static uint end_of_redo_phase(my_bool prepare_for_undo_phase);
 static int run_undo_phase(uint uncommitted);
 static void display_record_position(const LOG_DESC *log_desc,
@@ -217,8 +218,8 @@ int maria_recovery_from_log(void)
 #endif
   tprint(trace_file, "TRACE of the last MARIA recovery from mysqld\n");
   DBUG_ASSERT(maria_pagecache->inited);
-  res= maria_apply_log(LSN_IMPOSSIBLE, MARIA_LOG_APPLY, trace_file,
-                       TRUE, TRUE, TRUE, &warnings_count);
+  res= maria_apply_log(LSN_IMPOSSIBLE, LSN_IMPOSSIBLE, MARIA_LOG_APPLY,
+                       trace_file, TRUE, TRUE, TRUE, &warnings_count);
   if (!res)
   {
     if (warnings_count == 0 && recovery_found_crashed_tables == 0)
@@ -239,6 +240,7 @@ int maria_recovery_from_log(void)
 
    @param  from_lsn        LSN from which log reading/applying should start;
                            LSN_IMPOSSIBLE means "use last checkpoint"
+   @param  end_lsn         Apply until this. LSN_IMPOSSIBLE means until end.
    @param  apply           how log records should be applied or not
    @param  trace_file      trace file where progress/debug messages will go
    @param  skip_DDLs_arg   Should DDL records (CREATE/RENAME/DROP/REPAIR)
@@ -255,7 +257,8 @@ int maria_recovery_from_log(void)
      @retval !=0    Error
 */
 
-int maria_apply_log(LSN from_lsn, enum maria_apply_log_way apply,
+int maria_apply_log(LSN from_lsn, LSN end_lsn,
+                    enum maria_apply_log_way apply,
                     FILE *trace_file,
                     my_bool should_run_undo_phase, my_bool skip_DDLs_arg,
                     my_bool take_checkpoints, uint *warnings_count)
@@ -263,6 +266,7 @@ int maria_apply_log(LSN from_lsn, enum maria_apply_log_way apply,
   int error= 0;
   uint uncommitted_trans;
   ulonglong old_now;
+  my_bool abort_message_printed= 0;
   DBUG_ENTER("maria_apply_log");
 
   DBUG_ASSERT(apply == MARIA_LOG_APPLY || !should_run_undo_phase);
@@ -271,6 +275,7 @@ int maria_apply_log(LSN from_lsn, enum maria_apply_log_way apply,
   maria_recovery_changed_data= 0;
   /* checkpoints can happen only if TRNs have been built */
   DBUG_ASSERT(should_run_undo_phase || !take_checkpoints);
+  DBUG_ASSERT(end_lsn == LSN_IMPOSSIBLE || should_run_undo_phase == 0);
   all_active_trans= (struct st_trn_for_recovery *)
     my_malloc((SHORT_TRID_MAX + 1) * sizeof(struct st_trn_for_recovery),
               MYF(MY_ZEROFILL));
@@ -316,7 +321,7 @@ int maria_apply_log(LSN from_lsn, enum maria_apply_log_way apply,
   now= my_getsystime();
   in_redo_phase= TRUE;
   trnman_init(max_trid_in_control_file);
-  if (run_redo_phase(from_lsn, apply))
+  if (run_redo_phase(from_lsn, end_lsn, apply))
   {
     ma_message_no_user(0, "Redo phase failed");
     trnman_destroy();
@@ -324,6 +329,15 @@ int maria_apply_log(LSN from_lsn, enum maria_apply_log_way apply,
   }
   trnman_destroy();
 
+  if (end_lsn != LSN_IMPOSSIBLE)
+  {
+    abort_message_printed= 1;
+    my_message(HA_ERR_INITIALIZATION,
+               "Maria recovery aborted as end_lsn/end of file was reached",
+               MYF(0));
+    goto err2;
+  }
+
   if ((uncommitted_trans=
        end_of_redo_phase(should_run_undo_phase)) == (uint)-1)
   {
@@ -440,10 +454,15 @@ int maria_apply_log(LSN from_lsn, enum maria_apply_log_way apply,
 
   goto end;
 err:
-  error= 1;
   tprint(tracef, "\nRecovery of tables with transaction logs FAILED\n");
+err2:
   if (trns_created)
     delete_all_transactions();
+  error= 1;
+  if (close_all_tables())
+  {
+    ma_message_no_user(0, "closing of tables failed");
+  }
 end:
   error_handler_hook= save_error_handler_hook;
   hash_free(&all_dirty_pages);
@@ -480,7 +499,7 @@ int maria_apply_log(LSN from_lsn, enum maria_apply_log_way apply,
     maria_recovery_changed_data= 1;
   }
 
-  if (error)
+  if (error && !abort_message_printed)
     my_message(HA_ERR_INITIALIZATION,
                "Maria recovery failed. Please run maria_chk -r on all maria "
                "tables and delete all maria_log.######## files", MYF(0));
@@ -2370,7 +2389,7 @@ prototype_undo_exec_hook(UNDO_BULK_INSERT)
 }
 
 
-static int run_redo_phase(LSN lsn, enum maria_apply_log_way apply)
+static int run_redo_phase(LSN lsn, LSN lsn_end, enum maria_apply_log_way apply)
 {
   TRANSLOG_HEADER_BUFFER rec;
   struct st_translog_scanner_data scanner;
@@ -2498,6 +2517,17 @@ static int run_redo_phase(LSN lsn, enum maria_apply_log_way apply)
             tprint(tracef, "Cannot find record where it should be\n");
             goto err;
           }
+          if (lsn_end != LSN_IMPOSSIBLE && rec2.lsn >= lsn_end)
+          {
+            tprint(tracef,
+                   "lsn_end reached at (%lu,0x%lx). "
+                   "Skipping rest of redo entries",
+                   LSN_IN_PARTS(rec2.lsn));
+            translog_destroy_scanner(&scanner);
+            translog_free_record_header(&rec);
+            return(0);
+          }
+
           if (translog_scanner_init(rec2.lsn, 1, &scanner2, 1))
           {
             tprint(tracef, "Scanner2 init failed\n");

storage/maria/ma_recovery.h

@@ -26,7 +26,7 @@ C_MODE_START
 enum maria_apply_log_way
 { MARIA_LOG_APPLY, MARIA_LOG_DISPLAY_HEADER, MARIA_LOG_CHECK };
 int maria_recovery_from_log(void);
-int maria_apply_log(LSN lsn, enum maria_apply_log_way apply,
+int maria_apply_log(LSN lsn, LSN lsn_end, enum maria_apply_log_way apply,
                     FILE *trace_file,
                     my_bool execute_undo_phase, my_bool skip_DDLs,
                     my_bool take_checkpoints, uint *warnings_count);

storage/maria/ma_write.c

@@ -1987,9 +1987,9 @@ static my_bool _ma_log_split(MARIA_PAGE *ma_page,
                              uint changed_length)
 {
   LSN lsn;
-  uchar log_data[FILEID_STORE_SIZE + PAGE_STORE_SIZE + 3+3+3+3+3+2];
+  uchar log_data[FILEID_STORE_SIZE + PAGE_STORE_SIZE + 3+3+3+3+3+2 +7];
   uchar *log_pos;
-  LEX_CUSTRING log_array[TRANSLOG_INTERNAL_PARTS + 3];
+  LEX_CUSTRING log_array[TRANSLOG_INTERNAL_PARTS + 4];
   uint offset= (uint) (key_pos - ma_page->buff);
   uint translog_parts, extra_length;
   MARIA_HA *info= ma_page->info; 
@@ -2085,6 +2085,22 @@ static my_bool _ma_log_split(MARIA_PAGE *ma_page,
   log_array[TRANSLOG_INTERNAL_PARTS + 0].str=    log_data;
   log_array[TRANSLOG_INTERNAL_PARTS + 0].length= (uint) (log_pos -
                                                          log_data);
+#ifdef EXTRA_DEBUG_KEY_CHANGES
+  {
+    int page_length= ma_page->size;
+    ha_checksum crc;
+    crc= my_checksum(0, ma_page->buff + LSN_STORE_SIZE,
+                     page_length - LSN_STORE_SIZE);
+    log_pos[0]= KEY_OP_CHECK;
+    int2store(log_pos+1, page_length);
+    int4store(log_pos+3, crc);
+    log_array[TRANSLOG_INTERNAL_PARTS + translog_parts].str= log_pos;
+    log_array[TRANSLOG_INTERNAL_PARTS + translog_parts].length= 7;
+    extra_length+= 7;
+    translog_parts++;
+  }
+#endif
+
   DBUG_RETURN(translog_write_record(&lsn, LOGREC_REDO_INDEX,
                                     info->trn, info,
                                     (translog_size_t)
@@ -2122,8 +2138,8 @@ static my_bool _ma_log_del_prefix(MARIA_PAGE *ma_page,
                                   int move_length)
 {
   LSN lsn;
-  uchar log_data[FILEID_STORE_SIZE + PAGE_STORE_SIZE + 12], *log_pos;
-  LEX_CUSTRING log_array[TRANSLOG_INTERNAL_PARTS + 2];
+  uchar log_data[FILEID_STORE_SIZE + PAGE_STORE_SIZE + 12 + 7], *log_pos;
+  LEX_CUSTRING log_array[TRANSLOG_INTERNAL_PARTS + 3];
   uint offset= (uint) (key_pos - ma_page->buff);
   uint diff_length= org_length + move_length - new_length;
   uint translog_parts, extra_length;
@@ -2190,6 +2206,22 @@ static my_bool _ma_log_del_prefix(MARIA_PAGE *ma_page,
   log_array[TRANSLOG_INTERNAL_PARTS + 0].str=    log_data;
   log_array[TRANSLOG_INTERNAL_PARTS + 0].length= (uint) (log_pos -
                                                          log_data);
+#ifdef EXTRA_DEBUG_KEY_CHANGES
+  {
+    int page_length= ma_page->size;
+    ha_checksum crc;
+    crc= my_checksum(0, ma_page->buff + LSN_STORE_SIZE,
+                     page_length - LSN_STORE_SIZE);
+    log_pos[0]= KEY_OP_CHECK;
+    int2store(log_pos+1, page_length);
+    int4store(log_pos+3, crc);
+    log_array[TRANSLOG_INTERNAL_PARTS + translog_parts].str= log_pos;
+    log_array[TRANSLOG_INTERNAL_PARTS + translog_parts].length= 7;
+    extra_length+= 7;
+    translog_parts++;
+  }
+#endif
+
   DBUG_RETURN(translog_write_record(&lsn, LOGREC_REDO_INDEX,
                                     info->trn, info,
                                     (translog_size_t)
@@ -2215,9 +2247,9 @@ static my_bool _ma_log_key_middle(MARIA_PAGE *ma_page,
                                   uint key_length, int move_length)
 {
   LSN lsn;
-  uchar log_data[FILEID_STORE_SIZE + PAGE_STORE_SIZE + 3+5+3+3+3];
+  uchar log_data[FILEID_STORE_SIZE + PAGE_STORE_SIZE + 3+5+3+3+3 + 7];
   uchar *log_pos;
-  LEX_CUSTRING log_array[TRANSLOG_INTERNAL_PARTS + 4];
+  LEX_CUSTRING log_array[TRANSLOG_INTERNAL_PARTS + 5];
   uint key_offset;
   uint translog_parts, extra_length;
   my_off_t page;
@@ -2300,6 +2332,22 @@ static my_bool _ma_log_key_middle(MARIA_PAGE *ma_page,
                            key_length);
   }
 
+#ifdef EXTRA_DEBUG_KEY_CHANGES
+  {
+    int page_length= ma_page->size;
+    ha_checksum crc;
+    crc= my_checksum(0, ma_page->buff + LSN_STORE_SIZE,
+                     page_length - LSN_STORE_SIZE);
+    log_pos[0]= KEY_OP_CHECK;
+    int2store(log_pos+1, page_length);
+    int4store(log_pos+3, crc);
+    log_array[TRANSLOG_INTERNAL_PARTS + translog_parts].str= log_pos;
+    log_array[TRANSLOG_INTERNAL_PARTS + translog_parts].length= 7;
+    extra_length+= 7;
+    translog_parts++;
+  }
+#endif
+
   DBUG_RETURN(translog_write_record(&lsn, LOGREC_REDO_INDEX,
                                     info->trn, info,
                                     (translog_size_t)
@@ -2327,6 +2375,7 @@ static my_bool _ma_log_middle(MARIA_PAGE *ma_page,
   uchar log_data[FILEID_STORE_SIZE + PAGE_STORE_SIZE + 3 + 5], *log_pos;
   MARIA_HA *info= ma_page->info;
   my_off_t page;
+  uint translog_parts, extra_length;
   DBUG_ENTER("_ma_log_middle");
   DBUG_PRINT("enter", ("page: %lu", (ulong) page));
 
@@ -2352,12 +2401,31 @@ static my_bool _ma_log_middle(MARIA_PAGE *ma_page,
   log_array[TRANSLOG_INTERNAL_PARTS + 1].str=    ((char*) buff +
                                                   info->s->keypage_header);
   log_array[TRANSLOG_INTERNAL_PARTS + 1].length= data_changed_first;
+  translog_parts= 2;
+  extra_length= data_changed_first;
+
+#ifdef EXTRA_DEBUG_KEY_CHANGES
+  {
+    int page_length= ma_page->size;
+    ha_checksum crc;
+    crc= my_checksum(0, ma_page->buff + LSN_STORE_SIZE,
+                     page_length - LSN_STORE_SIZE);
+    log_pos[0]= KEY_OP_CHECK;
+    int2store(log_pos+1, page_length);
+    int4store(log_pos+3, crc);
+    log_array[TRANSLOG_INTERNAL_PARTS + translog_parts].str= log_pos;
+    log_array[TRANSLOG_INTERNAL_PARTS + translog_parts].length= 7;
+    extra_length+= 7;
+    translog_parts++;
+  }
+#endif
+
   DBUG_RETURN(translog_write_record(&lsn, LOGREC_REDO_INDEX,
                                     info->trn, info,
                                     (translog_size_t)
                                     log_array[TRANSLOG_INTERNAL_PARTS +
-                                              0].length + data_changed_first,
-                                    TRANSLOG_INTERNAL_PARTS + 2,
+                                              0].length + extra_length,
+                                    TRANSLOG_INTERNAL_PARTS + translog_parts,
                                     log_array, log_data, NULL));
 }
 #endif

storage/maria/maria_chk.c

@@ -42,6 +42,7 @@ static CHARSET_INFO *set_collation;
 static int stopwords_inited= 0;
 static MY_TMPDIR maria_chk_tmpdir;
 static my_bool opt_transaction_logging, opt_debug, opt_require_control_file;
+static my_bool opt_warning_for_wrong_transid;
 
 static const char *type_names[]=
 {
@@ -193,7 +194,7 @@ enum options_mc {
   OPT_FT_MAX_WORD_LEN, OPT_FT_STOPWORD_FILE,
   OPT_MAX_RECORD_LENGTH, OPT_AUTO_CLOSE, OPT_STATS_METHOD, OPT_TRANSACTION_LOG,
   OPT_SKIP_SAFEMALLOC, OPT_ZEROFILL_KEEP_LSN, OPT_REQUIRE_CONTROL_FILE,
-  OPT_LOG_DIR, OPT_DATADIR
+  OPT_LOG_DIR, OPT_DATADIR, OPT_WARNING_FOR_WRONG_TRANSID
 };
 
 static struct my_option my_long_options[] =
@@ -338,7 +339,9 @@ static struct my_option my_long_options[] =
    &opt_transaction_logging, &opt_transaction_logging,
    0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
   {"update-state", 'U',
-   "Mark tables as crashed if any errors were found.",
+   "Mark tables as crashed if any errors were found and clean if check didn't "
+   "find any errors. This allows one to get rid of warnings like 'table not "
+   "properly closed'",
    0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
   {"unpack", 'u',
    "Unpack file packed with mariapack.",
@@ -350,6 +353,11 @@ static struct my_option my_long_options[] =
    0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
   {"wait", 'w', "Wait if table is locked.",
    0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
+  {"warning-for-wrong-transaction-id", OPT_WARNING_FOR_WRONG_TRANSID,
+   "Give a warning if we find a transaction id in the table that is bigger"
+   "than what exists in the control file. Use --skip-... to disable warning",
+   &opt_warning_for_wrong_transid, &opt_warning_for_wrong_transid,
+   0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0},
   { "page_buffer_size", OPT_PAGE_BUFFER_SIZE,
     "Size of page buffer. Used by --safe-repair",
     &check_param.use_buffers, &check_param.use_buffers, 0,
@@ -1242,6 +1250,8 @@ static int maria_chk(HA_CHECK *param, char *filename)
              llstr(info->state->records,llbuff),
              llstr(info->state->del,llbuff2));
     maria_chk_init_for_check(param, info);
+    if (opt_warning_for_wrong_transid == 0)
+      param->max_trid= ~ (ulonglong) 0;
     error= maria_chk_status(param,info);
     maria_intersect_keys_active(share->state.key_map, param->keys_in_use);
     error|= maria_chk_size(param,info);
@@ -1278,8 +1288,11 @@ static int maria_chk(HA_CHECK *param, char *filename)
     }
     if (!error)
     {
-      if ((share->state.changed & STATE_CHANGED) &&
-          (param->testflag & T_UPDATE_STATE))
+      if (((share->state.changed &
+            (STATE_CHANGED | STATE_CRASHED | STATE_CRASHED_ON_REPAIR |
+             STATE_IN_REPAIR)) ||
+           share->state.open_count != 0)
+          && (param->testflag & T_UPDATE_STATE))
         info->update|=HA_STATE_CHANGED | HA_STATE_ROW_CHANGED;
       DBUG_PRINT("info", ("Reseting crashed state"));
       share->state.changed&= ~(STATE_CHANGED | STATE_CRASHED |

storage/maria/maria_read_log.c

@@ -33,7 +33,7 @@ static my_bool opt_display_only, opt_apply, opt_apply_undo, opt_silent;
 static my_bool opt_check;
 static const char *opt_tmpdir;
 static ulong opt_page_buffer_size;
-static ulonglong opt_start_from_lsn;
+static ulonglong opt_start_from_lsn, opt_end_lsn;
 static MY_TMPDIR maria_chk_tmpdir;
 
 
@@ -120,8 +120,14 @@ int main(int argc, char **argv)
             LSN_IN_PARTS(lsn));
   }
 
+  if (opt_end_lsn != LSN_IMPOSSIBLE)
+  {
+    /* We can't apply undo if we use end_lsn */
+    opt_apply_undo= 0;
+  }
+
   fprintf(stdout, "TRACE of the last maria_read_log\n");
-  if (maria_apply_log(lsn, opt_apply ?  MARIA_LOG_APPLY :
+  if (maria_apply_log(lsn, opt_end_lsn, opt_apply ?  MARIA_LOG_APPLY :
                       (opt_check ? MARIA_LOG_CHECK :
                        MARIA_LOG_DISPLAY_HEADER), opt_silent ? NULL : stdout,
                       opt_apply_undo, FALSE, FALSE, &warnings_count))
@@ -178,18 +184,21 @@ static struct my_option my_long_options[] =
   {"display-only", 'd', "display brief info read from records' header",
    (uchar **) &opt_display_only, (uchar **) &opt_display_only, 0, GET_BOOL,
    NO_ARG,0, 0, 0, 0, 0, 0},
-  {"maria_log_dir_path", 'l',
+  {"maria-log-dir-path", 'l',
     "Path to the directory where to store transactional log",
     (uchar **) &maria_data_root, (uchar **) &maria_data_root, 0,
     GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
-  { "page_buffer_size", 'P', "",
+  { "page-buffer-size", 'P', "",
     &opt_page_buffer_size, &opt_page_buffer_size, 0,
     GET_ULONG, REQUIRED_ARG, (long) USE_BUFFER_INIT,
     (long) USE_BUFFER_INIT, (long) ~(ulong) 0, (long) MALLOC_OVERHEAD,
     (long) IO_SIZE, 0},
-  { "start_from_lsn", 'o', "Start reading log from this lsn",
+  { "start-from-lsn", 'o', "Start reading log from this lsn",
     &opt_start_from_lsn, &opt_start_from_lsn,
     0, GET_ULL, REQUIRED_ARG, 0, 0, ~(longlong) 0, 0, 0, 0 },
+  { "end-lsn", 'e', "Stop applying at this lsn. If end-lsn is used, UNDO:s "
+    "will not be applied", &opt_end_lsn, &opt_end_lsn,
+    0, GET_ULL, REQUIRED_ARG, 0, 0, ~(longlong) 0, 0, 0, 0 },
   {"silent", 's', "Print less information during apply/undo phase",
    (uchar **) &opt_silent, (uchar **) &opt_silent, 0,
    GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},