Store maximum transaction id into control file at clean shutdown.

This can serve to maria_chk to check that trids found in rows and keys
are not too big. Also used by Recovery when logs are lost.
Options --require-control-file, --datadir, --log-dir (yes, the dashes are
inconsistent but I imitated mysqld --datadir and --maria-log-dir) for
maria_chk.
Lock control file _before_ reading its content.


storage/maria/ha_maria.cc:
  new prototype
storage/maria/ma_check.c:
  A function to find the max trid in the system (consults transaction
  manager and control file), to check tables.
storage/maria/ma_checkpoint.c:
  new prototype
storage/maria/ma_control_file.c:
  Store max trid into control file, in a backward-compatible way
  (can still read old control files).
  Parameter to ma_control_file_open(), to not create the log if it's
  missing (maria_chk needs that).
  Lock control file _before_ reading its content.
  Fix for a segfault when reading an old control file (bzero() with a
  negative second argument)
storage/maria/ma_control_file.h:
  changes to the control file module's API
storage/maria/ma_init.c:
  When Maria shuts down cleanly, store max trid into control file.
storage/maria/ma_loghandler.c:
  new prototype
storage/maria/ma_recovery.c:
  During recovery, consult max trid stored in control file, in case it is
  bigger than what we found in log (case of logs manually removed by user).
storage/maria/ma_test1.c:
  new prototype
storage/maria/ma_test2.c:
  new prototype
storage/maria/maria_chk.c:
  New option --require-control-file (abort if control file not found),
  --datadir (path for control file (and for logs if --log-dir not specified)),
  --log-dir (path for logs).
  Try to open control file when maria_chk starts.
storage/maria/maria_read_log.c:
  new prototype
storage/maria/trnman.c:
  A new function to know max trid in transaction manager
storage/maria/trnman_public.h:
  New function
storage/maria/unittest/ma_control_file-t.c:
  new prototypes. Testing storing and retrieving the max trid to/from
  control file
storage/maria/unittest/ma_test_loghandler-t.c:
  new prototype
storage/maria/unittest/ma_test_loghandler_first_lsn-t.c:
  new prototype
storage/maria/unittest/ma_test_loghandler_max_lsn-t.c:
  new prototype
storage/maria/unittest/ma_test_loghandler_multigroup-t.c:
  new prototype
storage/maria/unittest/ma_test_loghandler_multithread-t.c:
  new prototype
storage/maria/unittest/ma_test_loghandler_noflush-t.c:
  new prototype
storage/maria/unittest/ma_test_loghandler_nologs-t.c:
  new prototype
storage/maria/unittest/ma_test_loghandler_pagecache-t.c:
  new prototype
storage/maria/unittest/ma_test_loghandler_purge-t.c:
  new prototype

storage/maria/ha_maria.cc

@@ -2768,7 +2768,7 @@ static int ha_maria_init(void *p)
   maria_hton->flags= HTON_CAN_RECREATE | HTON_SUPPORT_LOG_TABLES;
   bzero(maria_log_pagecache, sizeof(*maria_log_pagecache));
   maria_tmpdir= &mysql_tmpdir_list;             /* For REDO */
-  res= maria_init() || ma_control_file_create_or_open() ||
+  res= maria_init() || ma_control_file_open(TRUE) ||
     !init_pagecache(maria_pagecache,
                     (size_t) pagecache_buffer_size, pagecache_division_limit,
                     pagecache_age_threshold, maria_block_size, 0) ||

storage/maria/ma_check.c

@@ -98,6 +98,7 @@ static void report_keypage_fault(HA_CHECK *param, MARIA_HA *info,
 static my_bool create_new_data_handle(MARIA_SORT_PARAM *param, File new_file);
 static my_bool _ma_flush_table_files_before_swap(HA_CHECK *param,
                                                  MARIA_HA *info);
+static TrID max_trid_in_system(void);
 
 
 void maria_chk_init(HA_CHECK *param)
@@ -6444,3 +6445,21 @@ static void report_keypage_fault(HA_CHECK *param, MARIA_HA *info,
                           "error: %d",
                           llstr(position / block_size, buff), my_errno);
 }
+
+
+/**
+  When we want to check a table, we verify that the transaction ids of rows
+  and keys are not bigger than the biggest id generated by Maria so far, which
+  is returned by the function below.
+
+  @note If control file is not open, 0 may be returned; to not confuse
+  this with a valid max trid of 0, the caller should notice that it failed to
+  open the control file (ma_control_file_inited() can serve for that).
+*/
+
+static TrID max_trid_in_system(void)
+{
+  TrID id= trnman_get_max_trid(); /* 0 if transac manager not initialized */
+  /* 'id' may be far bigger, if last shutdown is old */
+  return max(id, max_trid_in_control_file);
+}

storage/maria/ma_checkpoint.c

@@ -244,8 +244,8 @@ static int really_execute_checkpoint(void)
       such hook would be called before translog_flush (and we must be sure
       that log was flushed before we write to the control file).
     */
-    if (unlikely(ma_control_file_write_and_force(lsn, FILENO_IMPOSSIBLE,
-                                                 CONTROL_FILE_UPDATE_ONLY_LSN)))
+    if (unlikely(ma_control_file_write_and_force(lsn, last_logno,
+                                                 max_trid_in_control_file)))
     {
       translog_unlock();
       goto err;

storage/maria/ma_control_file.h

@@ -42,6 +42,8 @@ extern LSN last_checkpoint_lsn;
 */
 extern uint32 last_logno;
 
+extern TrID max_trid_in_control_file;
+
 extern my_bool maria_multi_threaded, maria_in_recovery;
 
 typedef enum enum_control_file_error {
@@ -58,33 +60,10 @@ typedef enum enum_control_file_error {
   CONTROL_FILE_UNKNOWN_ERROR /* any other error */
 } CONTROL_FILE_ERROR;
 
-#define CONTROL_FILE_UPDATE_ALL 0
-#define CONTROL_FILE_UPDATE_ONLY_LSN 1
-#define CONTROL_FILE_UPDATE_ONLY_LOGNO 2
-
-#ifdef	__cplusplus
-extern "C" {
-#endif
-
-/*
-  Looks for the control file. If none and creation was requested, creates file.
-  If present, reads it to find out last checkpoint's LSN and last log.
-  Called at engine's start.
-*/
-CONTROL_FILE_ERROR ma_control_file_create_or_open();
-/*
-  Write information durably to the control file.
-  Called when we have created a new log (after syncing this log's creation)
-  and when we have written a checkpoint (after syncing this log record).
-*/
-int ma_control_file_write_and_force(const LSN checkpoint_lsn, uint32 logno,
-                                    uint objs_to_write);
-
-
-/* Free resources taken by control file subsystem */
-int ma_control_file_end();
-
-#ifdef	__cplusplus
-}
-#endif
+C_MODE_START
+CONTROL_FILE_ERROR ma_control_file_open(my_bool create_if_missing);
+int ma_control_file_write_and_force(LSN checkpoint_lsn, uint32 logno, TrID trid);
+int ma_control_file_end(void);
+my_bool ma_control_file_inited(void);
+C_MODE_END
 #endif

storage/maria/ma_init.c

@@ -52,9 +52,20 @@ void maria_end(void)
 {
   if (maria_inited)
   {
+    TrID trid;
     maria_inited= maria_multi_threaded= FALSE;
     ft_free_stopwords();
     ma_checkpoint_end();
+    if (ma_control_file_inited() &&
+        ((trid= trnman_get_max_trid()) > max_trid_in_control_file))
+    {
+      /*
+        Store max transaction id into control file, in case logs are removed
+        by user, or maria_chk wants to check tables (it cannot access max trid
+        from the log, as it cannot process REDOs).
+      */
+      ma_control_file_write_and_force(last_checkpoint_lsn, last_logno, trid);
+    }
     trnman_destroy();
     if (translog_status == TRANSLOG_OK)
       translog_destroy();

storage/maria/ma_loghandler.c

@@ -1519,8 +1519,8 @@ static my_bool translog_create_new_file()
   if (translog_write_file_header())
     DBUG_RETURN(1);
 
-  if (ma_control_file_write_and_force(LSN_IMPOSSIBLE, file_no,
-                                      CONTROL_FILE_UPDATE_ONLY_LOGNO))
+  if (ma_control_file_write_and_force(last_checkpoint_lsn, file_no,
+                                      max_trid_in_control_file))
   {
     translog_stop_writing();
     DBUG_RETURN(1);
@@ -3697,7 +3697,7 @@ my_bool translog_init_with_table(const char *directory,
                 log_descriptor.open_files.elements);
 
     if (ma_control_file_write_and_force(checkpoint_lsn, start_file_num,
-                                        CONTROL_FILE_UPDATE_ALL))
+                                        max_trid_in_control_file))
       DBUG_RETURN(1);
     /* assign buffer 0 */
     translog_start_buffer(log_descriptor.buffers, &log_descriptor.bc, 0);

storage/maria/ma_recovery.c

@@ -2509,6 +2509,14 @@ static uint end_of_redo_phase(my_bool prepare_for_undo_phase)
 
   llstr(max_long_trid, llbuf);
   tprint(tracef, "Maximum transaction long id seen: %s\n", llbuf);
+  llstr(max_trid_in_control_file, llbuf);
+  tprint(tracef, "Maximum transaction long id seen in control file: %s\n",
+         llbuf);
+  /*
+    If logs were deleted, or lost, trid in control file is needed to set
+    trnman's generator:
+  */
+  set_if_bigger(max_long_trid, max_trid_in_control_file);
   if (prepare_for_undo_phase && trnman_init(max_long_trid))
     return -1;
 

storage/maria/ma_test1.c

@@ -77,7 +77,7 @@ int main(int argc,char *argv[])
   if (maria_init() ||
       (init_pagecache(maria_pagecache, maria_block_size * 16, 0, 0,
                       maria_block_size, MY_WME) == 0) ||
-      ma_control_file_create_or_open() ||
+      ma_control_file_open(TRUE) ||
       (init_pagecache(maria_log_pagecache,
                       TRANSLOG_PAGECACHE_SIZE, 0, 0,
                       TRANSLOG_PAGE_SIZE, MY_WME) == 0) ||

storage/maria/ma_test2.c

@@ -83,7 +83,7 @@ int main(int argc, char *argv[])
   if (maria_init() ||
       (init_pagecache(maria_pagecache, pagecache_size, 0, 0,
 		      maria_block_size, MY_WME) == 0) ||
-      ma_control_file_create_or_open() ||
+      ma_control_file_open(TRUE) ||
       (init_pagecache(maria_log_pagecache,
 		      TRANSLOG_PAGECACHE_SIZE, 0, 0,
 		      TRANSLOG_PAGE_SIZE, MY_WME) == 0) ||

storage/maria/maria_chk.c

@@ -37,11 +37,11 @@ SET_STACK_SIZE(9000)			/* Minimum stack size for program */
 static uint decode_bits;
 static char **default_argv;
 static const char *load_default_groups[]= { "maria_chk", 0 };
-static const char *set_collation_name, *opt_tmpdir;
+static const char *set_collation_name, *opt_tmpdir, *opt_log_dir;
 static CHARSET_INFO *set_collation;
 static int stopwords_inited= 0;
 static MY_TMPDIR maria_chk_tmpdir;
-static my_bool opt_transaction_logging, opt_debug;
+static my_bool opt_transaction_logging, opt_debug, opt_require_control_file;
 
 static const char *type_names[]=
 {
@@ -97,7 +97,7 @@ int main(int argc, char **argv)
   int error;
   MY_INIT(argv[0]);
 
-  maria_data_root= (char *)".";
+  opt_log_dir= maria_data_root= (char *)".";
   maria_chk_init(&check_param);
   check_param.opt_lock_memory= 1;		/* Lock memory if possible */
   check_param.using_global_keycache = 0;
@@ -110,20 +110,30 @@ int main(int argc, char **argv)
     If we are doing a repair, user may want to store this repair into the log
     so that the log has a complete history and can be used to replay.
   */
-  if (opt_transaction_logging && (check_param.testflag & T_REP_ANY) &&
-      (ma_control_file_create_or_open() ||
-       init_pagecache(maria_log_pagecache,
-                      TRANSLOG_PAGECACHE_SIZE, 0, 0,
-                      TRANSLOG_PAGE_SIZE, MY_WME) == 0 ||
-       translog_init(maria_data_root, TRANSLOG_FILE_SIZE,
-                     0, 0, maria_log_pagecache,
-                     TRANSLOG_DEFAULT_FLAGS, 0)))
+  if (opt_transaction_logging && (check_param.testflag & T_REP_ANY))
   {
-    _ma_check_print_error(&check_param,
-                          "Can't initialize transaction logging. Run "
-                          "recovery with switch --skip-transaction-log");
-    error= 1;
-    argc= 1;                                    /* Force loop out */
+    if (ma_control_file_open(FALSE) ||
+        init_pagecache(maria_log_pagecache,
+                       TRANSLOG_PAGECACHE_SIZE, 0, 0,
+                       TRANSLOG_PAGE_SIZE, MY_WME) == 0 ||
+        translog_init(opt_log_dir, TRANSLOG_FILE_SIZE,
+                      0, 0, maria_log_pagecache,
+                      TRANSLOG_DEFAULT_FLAGS, 0))
+    {
+      _ma_check_print_error(&check_param,
+                            "Can't initialize transaction logging. Run "
+                            "recovery with switch --skip-transaction-log");
+      error= 1;
+      goto end;
+    }
+  }
+  else
+  {
+    if (ma_control_file_open(FALSE) && opt_require_control_file)
+    {
+      error= 1;
+      goto end;
+    }
   }
 
   while (--argc >= 0)
@@ -156,6 +166,7 @@ int main(int argc, char **argv)
       VOID(fflush(stdout));
     }
   }
+end:
   if (check_param.total_files > 1)
   {					/* Only if descript */
     char buff[22],buff2[22];
@@ -183,7 +194,8 @@ enum options_mc {
   OPT_SORT_KEY_BLOCKS, OPT_DECODE_BITS, OPT_FT_MIN_WORD_LEN,
   OPT_FT_MAX_WORD_LEN, OPT_FT_STOPWORD_FILE,
   OPT_MAX_RECORD_LENGTH, OPT_AUTO_CLOSE, OPT_STATS_METHOD, OPT_TRANSACTION_LOG,
-  OPT_SKIP_SAFEMALLOC, OPT_ZEROFILL_KEEP_LSN
+  OPT_SKIP_SAFEMALLOC, OPT_ZEROFILL_KEEP_LSN, OPT_REQUIRE_CONTROL_FILE,
+  OPT_LOG_DIR, OPT_DATADIR
 };
 
 static struct my_option my_long_options[] =
@@ -249,6 +261,13 @@ static struct my_option my_long_options[] =
    (uchar**) &check_param.keys_in_use,
    (uchar**) &check_param.keys_in_use,
    0, GET_ULL, REQUIRED_ARG, -1, 0, 0, 0, 0, 0},
+  {"datadir", OPT_DATADIR,
+   "Path for control file (and logs if --log-dir not used).",
+   (uchar**) &maria_data_root, 0, 0, GET_STR, REQUIRED_ARG,
+   0, 0, 0, 0, 0, 0},
+  {"log-dir", OPT_LOG_DIR,
+   "Path for log files.",
+   (uchar**) &opt_log_dir, 0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
   {"max-record-length", OPT_MAX_RECORD_LENGTH,
    "Skip rows bigger than this if maria_chk can't allocate memory to hold it",
    (uchar**) &check_param.max_record_length,
@@ -274,6 +293,10 @@ static struct my_option my_long_options[] =
   {"sort-recover", 'n',
    "Force recovering with sorting even if the temporary file was very big.",
    0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
+  { "require-control-file", OPT_REQUIRE_CONTROL_FILE,
+    "Abort if cannot find control file",
+    (uchar**)&opt_require_control_file, 0, 0, GET_BOOL, NO_ARG,
+    0, 0, 0, 0, 0, 0},
 #ifdef DEBUG
   {"start-check-pos", OPT_START_CHECK_POS,
    "No help available.",

storage/maria/maria_read_log.c

@@ -56,7 +56,7 @@ int main(int argc, char **argv)
     goto err;
   }
   /* we don't want to create a control file, it MUST exist */
-  if (ma_control_file_create_or_open())
+  if (ma_control_file_open(FALSE))
   {
     fprintf(stderr, "Can't open control file (%d)\n", errno);
     goto err;

storage/maria/trnman.c

@@ -745,3 +745,18 @@ TRN *trnman_get_any_trn()
   TRN *trn= active_list_min.next;
   return (trn != &active_list_max) ? trn : NULL;
 }
+
+
+/**
+  Returns maximum transaction id given to a transaction so far.
+*/
+TrID trnman_get_max_trid()
+{
+  TrID id;
+  if (short_trid_to_active_trn == NULL)
+    return 0;
+  pthread_mutex_lock(&LOCK_trn_list);
+  id= global_trid_generator;
+  pthread_mutex_unlock(&LOCK_trn_list);
+  return id;
+}

storage/maria/trnman_public.h

@@ -54,7 +54,8 @@ uint trnman_decrement_locked_tables(TRN *trn);
 uint trnman_has_locked_tables(TRN *trn);
 void trnman_reset_locked_tables(TRN *trn, uint locked_tables);
 TRN *trnman_recreate_trn_from_recovery(uint16 shortid, TrID longid);
-TRN *trnman_get_any_trn();
+TRN *trnman_get_any_trn(void);
+TrID trnman_get_max_trid(void);
 #define TRANSID_SIZE		6
 #define transid_store(dst, id) int6store(dst,id)
 #define transid_korr(P) uint6korr(P)

storage/maria/unittest/ma_test_loghandler-t.c

@@ -196,7 +196,7 @@ int main(int argc __attribute__((unused)), char *argv[])
   }
 #endif
 
-  if (ma_control_file_create_or_open(TRUE))
+  if (ma_control_file_open(TRUE))
   {
     fprintf(stderr, "Can't init control file (%d)\n", errno);
     exit(1);

storage/maria/unittest/ma_test_loghandler_first_lsn-t.c

@@ -67,7 +67,7 @@ int main(int argc __attribute__((unused)), char *argv[])
   }
 #endif
 
-  if (ma_control_file_create_or_open(TRUE))
+  if (ma_control_file_open(TRUE))
   {
     fprintf(stderr, "Can't init control file (%d)\n", errno);
     exit(1);

storage/maria/unittest/ma_test_loghandler_max_lsn-t.c

@@ -64,7 +64,7 @@ int main(int argc __attribute__((unused)), char *argv[])
   }
 #endif
 
-  if (ma_control_file_create_or_open(TRUE))
+  if (ma_control_file_open(TRUE))
   {
     fprintf(stderr, "Can't init control file (%d)\n", errno);
     exit(1);

storage/maria/unittest/ma_test_loghandler_multigroup-t.c

@@ -184,7 +184,7 @@ int main(int argc __attribute__((unused)), char *argv[])
   }
 #endif
 
-  if (ma_control_file_create_or_open(TRUE))
+  if (ma_control_file_open(TRUE))
   {
     fprintf(stderr, "Can't init control file (%d)\n", errno);
     exit(1);
@@ -348,7 +348,7 @@ int main(int argc __attribute__((unused)), char *argv[])
   end_pagecache(&pagecache, 1);
   ma_control_file_end();
 
-  if (ma_control_file_create_or_open(TRUE))
+  if (ma_control_file_open(TRUE))
   {
     fprintf(stderr, "pass2: Can't init control file (%d)\n", errno);
     exit(1);

storage/maria/unittest/ma_test_loghandler_multithread-t.c

@@ -283,7 +283,7 @@ int main(int argc __attribute__((unused)),
 
   my_thread_global_init();
 
-  if (ma_control_file_create_or_open(TRUE))
+  if (ma_control_file_open(TRUE))
   {
     fprintf(stderr, "Can't init control file (%d)\n", errno);
     exit(1);

storage/maria/unittest/ma_test_loghandler_noflush-t.c

@@ -70,7 +70,7 @@ int main(int argc __attribute__((unused)), char *argv[])
   }
 #endif
 
-  if (ma_control_file_create_or_open(TRUE))
+  if (ma_control_file_open(TRUE))
   {
     fprintf(stderr, "Can't init control file (%d)\n", errno);
     exit(1);

storage/maria/unittest/ma_test_loghandler_nologs-t.c

@@ -67,7 +67,7 @@ int main(int argc __attribute__((unused)), char *argv[])
   }
 #endif
 
-  if (ma_control_file_create_or_open(TRUE))
+  if (ma_control_file_open(TRUE))
   {
     fprintf(stderr, "Can't init control file (%d)\n", errno);
     exit(1);
@@ -140,7 +140,7 @@ int main(int argc __attribute__((unused)), char *argv[])
     }
   }
 
-  if (ma_control_file_create_or_open(TRUE))
+  if (ma_control_file_open(TRUE))
   {
     fprintf(stderr, "Can't init control file (%d)\n", errno);
     exit(1);

storage/maria/unittest/ma_test_loghandler_pagecache-t.c

@@ -95,7 +95,7 @@ int main(int argc __attribute__((unused)), char *argv[])
   }
 #endif
 
-  if (ma_control_file_create_or_open(TRUE))
+  if (ma_control_file_open(TRUE))
   {
     fprintf(stderr, "Can't init control file (%d)\n", errno);
     exit(1);

storage/maria/unittest/ma_test_loghandler_purge-t.c

@@ -67,7 +67,7 @@ int main(int argc __attribute__((unused)), char *argv[])
   }
 #endif
 
-  if (ma_control_file_create_or_open(TRUE))
+  if (ma_control_file_open(TRUE))
   {
     fprintf(stderr, "Can't init control file (%d)\n", errno);
     exit(1);