Fixes BUG#60976 "Crash, valgrind warning and memory leak with partitioned archive tables"

Noted that there was no memory leak, just a lot of used partitioned tables.
Fixed old bug: 'show status' now shows memory usage when compiled with safemalloc.
Added option --flush to mysqlcheck.c to run a 'flush tables' between each check to keep down memory usage.
Changed '--safemalloc' options to mysqld so that one can use --safemalloc and --skip-safemalloc.
Now skip-safemalloc is default (ie, we only do checking of memory overrun during free()) to speed up tests.


client/client_priv.h:
  Added OPT_FLUSH_TABLES
client/mysqlcheck.c:
  Added option --flush to mysqlcheck.c to run a 'flush tables' between each check to keep down memory usage.
mysql-test/mysql-test-run.pl:
  Always run tests with --loose-skip-safemysqld for higher speed
sql/mysqld.cc:
  Changed '--safemalloc' options so that one can use --safemalloc and --skip-safemalloc.
  Now skip-safemalloc is default (ie, we only do checking of memory overrun during free()) to speed up tests
sql/sql_parse.cc:
  Fixed old bug: 'show status' now shows memory usage when compiled with safemalloc.
storage/archive/archive_reader.c:
  Changed all malloc() calls to use my_malloc()/my_free()
  Added checks of malloc() calls.
storage/archive/ha_archive.cc:
  Detect failure if init_archive_reader() and return errno. This fixed assert crash in my_seek().
  Changed all malloc() calls to use my_malloc()/my_free()

client/client_priv.h

@@ -60,6 +60,7 @@ enum options_client
   OPT_OPEN_FILES_LIMIT, OPT_SET_CHARSET, OPT_SERVER_ARG,
   OPT_POSITION, OPT_STOP_POSITION, OPT_START_DATETIME, OPT_STOP_DATETIME,
   OPT_SIGINT_IGNORE, OPT_HEXBLOB, OPT_ORDER_BY_PRIMARY, OPT_COUNT,
+  OPT_FLUSH_TABLES,
 #ifdef HAVE_NDBCLUSTER_DB
   OPT_NDBCLUSTER, OPT_NDB_CONNECTSTRING,
 #endif

client/mysqlcheck.c

@@ -16,7 +16,7 @@
 
 /* By Jani Tolonen, 2001-04-20, MySQL Development Team */
 
-#define CHECK_VERSION "2.6.0"
+#define CHECK_VERSION "2.7.0"
 
 #include "client_priv.h"
 #include <m_ctype.h>
@@ -35,8 +35,8 @@ static my_bool opt_alldbs = 0, opt_check_only_changed = 0, opt_extended = 0,
                opt_medium_check = 0, opt_quick = 0, opt_all_in_1 = 0,
                opt_silent = 0, opt_auto_repair = 0, ignore_errors = 0,
                tty_password= 0, opt_frm= 0, debug_info_flag= 0, debug_check_flag= 0,
-               opt_fix_table_names= 0, opt_fix_db_names= 0, opt_upgrade= 0,
-               opt_write_binlog= 1;
+               opt_fix_table_names= 0, opt_fix_db_names= 0, opt_upgrade= 0;
+static my_bool opt_write_binlog= 1, opt_flush_tables= 0;
 static uint verbose = 0, opt_mysql_port=0;
 static int my_end_arg;
 static char * opt_mysql_unix_port = 0;
@@ -121,6 +121,9 @@ static struct my_option my_long_options[] =
    "If you are using this option with CHECK TABLE, it will ensure that the table is 100 percent consistent, but will take a long time. If you are using this option with REPAIR TABLE, it will force using old slow repair with keycache method, instead of much faster repair by sorting.",
    &opt_extended, &opt_extended, 0, GET_BOOL, NO_ARG, 0, 0, 0,
    0, 0, 0},
+  {"flush", OPT_FLUSH_TABLES, "Flush each table after check. This is useful if you don't want to have the checked tables take up space in the caches after the check",
+   &opt_flush_tables, &opt_flush_tables, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0,
+   0, 0 },
   {"help", '?', "Display this help message and exit.", 0, 0, 0, GET_NO_ARG,
    NO_ARG, 0, 0, 0, 0, 0, 0},
   {"host",'h', "Connect to host.", &current_host,
@@ -685,6 +688,7 @@ static int disable_binlog()
 static int handle_request_for_tables(char *tables, uint length)
 {
   char *query, *end, options[100], message[100];
+  char table_name_buff[NAME_CHAR_LEN*2*2+1], *table_name;
   uint query_length= 0;
   const char *op = 0;
 
@@ -723,13 +727,17 @@ static int handle_request_for_tables(char *tables, uint length)
     /* No backticks here as we added them before */
     query_length= my_sprintf(query,
 			     (query, "%s TABLE %s %s", op, tables, options));
+    table_name= tables;
   }
   else
   {
-    char *ptr;
+    char *ptr, *org;
 
-    ptr= strmov(strmov(query, op), " TABLE ");
+    org= ptr= strmov(strmov(query, op), " TABLE ");
     ptr= fix_table_name(ptr, tables);
+    strmake(table_name_buff, org, min((int) sizeof(table_name_buff)-1,
+                                      (int) (ptr - org)));
+    table_name= table_name_buff;
     ptr= strxmov(ptr, " ", options, NullS);
     query_length= (uint) (ptr - query);
   }
@@ -737,9 +745,21 @@ static int handle_request_for_tables(char *tables, uint length)
   {
     sprintf(message, "when executing '%s TABLE ... %s'", op, options);
     DBerror(sock, message);
+    my_free(query, MYF(0));
     return 1;
   }
   print_result();
+  if (opt_flush_tables)
+  {
+    query_length= my_sprintf(query,
+			     (query, "FLUSH TABLES %s", table_name));
+    if (mysql_real_query(sock, query, query_length))
+    {
+      DBerror(sock, query);
+      my_free(query, MYF(0));
+      return 1;
+    }
+  }
   my_free(query, MYF(0));
   return 0;
 }

mysql-test/mysql-test-run.pl

@@ -4943,14 +4943,13 @@ sub mysqld_arguments ($$$) {
 
   if ( $opt_valgrind_mysqld )
   {
-    mtr_add_arg($args, "--skip-safemalloc");
-
     if ( $mysql_version_id < 50100 )
     {
       mtr_add_arg($args, "--skip-bdb");
     }
   }
 
+  mtr_add_arg($args, "--loose-skip-safemalloc");
   mtr_add_arg($args, "%s--disable-sync-frm");
 
   if (!using_extern() and $mysql_version_id >= 50106 && !$opt_user_args)

sql/ha_partition.cc

@@ -2456,7 +2456,7 @@ bool ha_partition::read_par_file(const char *name)
   fn_format(buff, name, "", ha_par_ext, MY_APPEND_EXT);
 
   /* Following could be done with my_stat to read in whole file */
-  if ((file= my_open(buff, O_RDONLY | O_SHARE, MYF(0))) < 0)
+  if ((file= my_open(buff, O_RDONLY | O_SHARE, MYF(MY_WME))) < 0)
     DBUG_RETURN(true);
   if (my_read(file, (uchar *) & buff[0], PAR_WORD_SIZE, MYF(MY_NABP)))
     goto err1;

sql/mysqld.cc

@@ -427,7 +427,7 @@ static bool volatile ready_to_exit;
 static my_bool opt_debugging= 0, opt_external_locking= 0, opt_console= 0;
 static my_bool opt_short_log_format= 0;
 static my_bool opt_ignore_wrong_options= 0, opt_expect_abort= 0;
-static my_bool opt_sync= 0;
+static my_bool opt_sync= 0, sf_malloc_trough_check= 0;
 static uint kill_cached_threads, wake_thread;
 ulong thread_created;
 uint thread_handling;
@@ -5944,7 +5944,7 @@ enum options_mysqld
   OPT_NDB_REPORT_THRESH_BINLOG_EPOCH_SLIP,
   OPT_NDB_REPORT_THRESH_BINLOG_MEM_USAGE,
   OPT_NDB_USE_COPYING_ALTER_TABLE,
-  OPT_SKIP_SAFEMALLOC, OPT_MUTEX_DEADLOCK_DETECTOR,
+  OPT_SAFEMALLOC, OPT_MUTEX_DEADLOCK_DETECTOR,
   OPT_TEMP_POOL, OPT_TX_ISOLATION, OPT_COMPLETION_TYPE,
   OPT_SKIP_SYMLINKS,
   OPT_MAX_BINLOG_DUMP_EVENTS, OPT_SPORADIC_BINLOG_DUMP_FAIL,
@@ -6898,12 +6898,11 @@ each time the SQL thread starts.",
   {"skip-networking", OPT_SKIP_NETWORKING,
    "Don't allow connection with TCP/IP.", 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0,
    0, 0, 0},
-#ifndef DBUG_OFF
-#ifdef SAFEMALLOC
-  {"skip-safemalloc", OPT_SKIP_SAFEMALLOC,
-   "Don't use the memory allocation checking.", 0, 0, 0, GET_NO_ARG, NO_ARG,
-   0, 0, 0, 0, 0, 0},
-#endif
+#if !defined(DBUG_OFF) && defined(SAFEMALLOC)
+  {"safemalloc", OPT_SAFEMALLOC,
+   "Check all memory allocation for every malloc/free call.",
+   &sf_malloc_trough_check, &sf_malloc_trough_check, 0,
+   GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0 },
 #endif
   {"skip-show-database", OPT_SKIP_SHOW_DB,
    "Don't allow 'SHOW DATABASE' commands.", 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0,
@@ -9238,11 +9237,6 @@ mysqld_get_one_option(int optid,
     }
     strmake(ft_boolean_syntax, argument, sizeof(ft_boolean_syntax)-1);
     break;
-  case OPT_SKIP_SAFEMALLOC:
-#ifdef SAFEMALLOC
-    sf_malloc_quick=1;
-#endif
-    break;
   case OPT_LOWER_CASE_TABLE_NAMES:
     lower_case_table_names= argument ? atoi(argument) : 1;
     lower_case_table_names_used= 1;
@@ -9436,6 +9430,9 @@ static int get_options(int *argc,char **argv)
 				  &global_system_variables.datetime_format))
     return 1;
 
+#ifdef SAFEMALLOC
+  sf_malloc_quick= !sf_malloc_trough_check;
+#endif
 #ifdef EMBEDDED_LIBRARY
   one_thread_scheduler(&thread_scheduler);
   one_thread_scheduler(&extra_thread_scheduler);

sql/sql_parse.cc

@@ -1540,7 +1540,7 @@ bool dispatch_command(enum enum_server_command command, THD *thd,
     {
       char *end= buff + length;
       length+= my_snprintf(end, buff_len - length - 1,
-                           end,"  Memory in use: %ldK  Max memory used: %ldK",
+                           "  Memory in use: %ldK  Max memory used: %ldK",
                            (sf_malloc_cur_memory+1023L)/1024L,
                            (sf_malloc_max_memory+1023L)/1024L);
     }

storage/archive/archive_reader.c

@@ -93,12 +93,16 @@ int main(int argc, char *argv[])
       printf("\tFRM length %u\n", reader_handle.frm_length);
     if (reader_handle.comment_start_pos)
     {
-      char *comment =
-        (char *) malloc(sizeof(char) * reader_handle.comment_length);
-      azread_comment(&reader_handle, comment);
-      printf("\tComment length %u\n\t\t%.*s\n", reader_handle.comment_length, 
-             reader_handle.comment_length, comment);
-      free(comment);
+      char *comment = (char *) my_malloc(reader_handle.comment_length,
+                                         MYF(MY_WME));
+      if (comment)
+      {
+        azread_comment(&reader_handle, comment);
+        printf("\tComment length %u\n\t\t%.*s\n",
+               reader_handle.comment_length, 
+               reader_handle.comment_length, comment);
+        my_free(comment,MYF(0));
+      }
     }
   }
   else
@@ -180,7 +184,7 @@ int main(int argc, char *argv[])
 
     azio_stream writer_handle;
 
-    buffer= (char *)malloc(reader_handle.longest_row);
+    buffer= (char *) my_malloc(reader_handle.longest_row, MYF(0));
     if (buffer == NULL)
     {
       printf("Could not allocate memory for row %llu\n", row_count);
@@ -251,7 +255,7 @@ int main(int argc, char *argv[])
         break;
     }
 
-    free(buffer);
+    my_free(buffer, MYF(0));
 
     azclose(&writer_handle);
   }

storage/archive/ha_archive.cc

@@ -835,7 +835,7 @@ int ha_archive::write_row(uchar *buf)
 
   if (!share->archive_write_open)
     if (init_archive_writer())
-      DBUG_RETURN(HA_ERR_CRASHED_ON_USAGE);
+      DBUG_RETURN(errno);
 
 
   if (table->next_number_field && record == table->record[0])
@@ -1020,7 +1020,8 @@ int ha_archive::rnd_init(bool scan)
   if (share->crashed)
       DBUG_RETURN(HA_ERR_CRASHED_ON_USAGE);
 
-  init_archive_reader();
+  if (init_archive_reader())
+      DBUG_RETURN(errno);
 
   /* We rewind the file so that we can read from the beginning if scan */
   if (scan)
@@ -1317,7 +1318,8 @@ int ha_archive::optimize(THD* thd, HA_CHECK_OPT* check_opt)
   char* frm_string;
   DBUG_ENTER("ha_archive::optimize");
 
-  init_archive_reader();
+  if (init_archive_reader())
+    DBUG_RETURN(errno);
 
   // now we close both our writer and our reader for the rename
   if (share->archive_write_open)
@@ -1326,7 +1328,7 @@ int ha_archive::optimize(THD* thd, HA_CHECK_OPT* check_opt)
     share->archive_write_open= FALSE;
   }
 
-  if (!(frm_string= (char*) malloc(archive.frm_length)))
+  if (!(frm_string= (char*) my_malloc(archive.frm_length, MYF(0))))
     return ENOMEM;
 
   azread_frm(&archive, frm_string);
@@ -1337,12 +1339,12 @@ int ha_archive::optimize(THD* thd, HA_CHECK_OPT* check_opt)
 
   if (!(azopen(&writer, writer_filename, O_CREAT|O_RDWR|O_BINARY)))
   {
-    free(frm_string);
+    my_free(frm_string, MYF(0));
     DBUG_RETURN(HA_ERR_CRASHED_ON_USAGE);
   }
 
   rc= azwrite_frm(&writer, frm_string, archive.frm_length);
-  free(frm_string);
+  my_free(frm_string, MYF(0));
   if (rc)
   {
     rc= HA_ERR_CRASHED_ON_USAGE;
@@ -1547,7 +1549,9 @@ int ha_archive::info(uint flag)
 
   if (flag & HA_STATUS_AUTO)
   {
-    init_archive_reader();
+    if (init_archive_reader())
+      DBUG_RETURN(errno);
+
     pthread_mutex_lock(&share->mutex);
     azflush(&archive, Z_SYNC_FLUSH);
     pthread_mutex_unlock(&share->mutex);
@@ -1626,7 +1630,9 @@ int ha_archive::check(THD* thd, HA_CHECK_OPT* check_opt)
     Now we will rewind the archive file so that we are positioned at the 
     start of the file.
   */
-  init_archive_reader();
+  if (init_archive_reader())
+    DBUG_RETURN(errno);
+
   read_data_header(&archive);
   while (!(rc= get_row(&archive, table->record[0])))
     count--;