merge mysql-5.0-bugteam to mysql-5.1-bugteam

include/my_global.h

@@ -868,7 +868,7 @@ typedef SOCKET_SIZE_TYPE size_socket;
 #define FLT_MAX		((float)3.40282346638528860e+38)
 #endif
 #ifndef SIZE_T_MAX
-#define SIZE_T_MAX ~((size_t) 0)
+#define SIZE_T_MAX      (~((size_t) 0))
 #endif
 
 #ifndef isfinite

include/myisam.h

@@ -259,6 +259,8 @@ extern ulong myisam_bulk_insert_tree_size, myisam_data_pointer_size;
 /* usually used to check if a symlink points into the mysql data home */
 /* which is normally forbidden                                        */
 extern int (*myisam_test_invalid_symlink)(const char *filename);
+extern ulonglong myisam_mmap_size, myisam_mmap_used;
+extern pthread_mutex_t THR_LOCK_myisam_mmap;
 
 	/* Prototypes for myisam-functions */
 
@@ -304,6 +306,7 @@ extern int mi_delete_all_rows(struct st_myisam_info *info);
 extern ulong _mi_calc_blob_length(uint length , const uchar *pos);
 extern uint mi_get_pointer_length(ulonglong file_length, uint def);
 
+#define MEMMAP_EXTRA_MARGIN     7       /* Write this as a suffix for mmap file */
 /* this is used to pass to mysql_myisamchk_table */
 
 #define   MYISAMCHK_REPAIR 1  /* equivalent to myisamchk -r */

mysql-test/r/variables.result

@@ -1018,6 +1018,12 @@ ERROR HY000: Variable 'hostname' is a read only variable
 show variables like 'hostname';
 Variable_name	Value
 hostname	#
+#
+# BUG#37408 - Compressed MyISAM files should not require/use mmap()
+#
+# Test 'myisam_mmap_size' option is not dynamic
+SET @@myisam_mmap_size= 500M;
+ERROR HY000: Variable 'myisam_mmap_size' is a read only variable
 End of 5.0 tests
 set join_buffer_size=1;
 Warnings:

mysql-test/t/variables.test

@@ -779,6 +779,12 @@ set @@hostname= "anothername";
 --replace_column 2 #
 show variables like 'hostname';
 
+--echo #
+--echo # BUG#37408 - Compressed MyISAM files should not require/use mmap()
+--echo #
+--echo # Test 'myisam_mmap_size' option is not dynamic
+--error ER_INCORRECT_GLOBAL_LOCAL_VAR
+SET @@myisam_mmap_size= 500M;
 --echo End of 5.0 tests
 
 #

mysys/my_thr_init.c

@@ -30,7 +30,9 @@ pthread_key(struct st_my_thread_var, THR_KEY_mysys);
 #endif /* USE_TLS */
 pthread_mutex_t THR_LOCK_malloc,THR_LOCK_open,
 	        THR_LOCK_lock,THR_LOCK_isam,THR_LOCK_myisam,THR_LOCK_heap,
-                THR_LOCK_net, THR_LOCK_charset, THR_LOCK_threads, THR_LOCK_time;
+                THR_LOCK_net, THR_LOCK_charset, THR_LOCK_threads, THR_LOCK_time,
+                THR_LOCK_myisam_mmap;
+
 pthread_cond_t  THR_COND_threads;
 uint            THR_thread_count= 0;
 uint 		my_thread_end_wait_time= 5;
@@ -144,6 +146,7 @@ my_bool my_thread_global_init(void)
   pthread_mutex_init(&THR_LOCK_lock,MY_MUTEX_INIT_FAST);
   pthread_mutex_init(&THR_LOCK_isam,MY_MUTEX_INIT_SLOW);
   pthread_mutex_init(&THR_LOCK_myisam,MY_MUTEX_INIT_SLOW);
+  pthread_mutex_init(&THR_LOCK_myisam_mmap,MY_MUTEX_INIT_FAST);
   pthread_mutex_init(&THR_LOCK_heap,MY_MUTEX_INIT_FAST);
   pthread_mutex_init(&THR_LOCK_net,MY_MUTEX_INIT_FAST);
   pthread_mutex_init(&THR_LOCK_charset,MY_MUTEX_INIT_FAST);
@@ -210,6 +213,7 @@ void my_thread_global_end(void)
   pthread_mutex_destroy(&THR_LOCK_lock);
   pthread_mutex_destroy(&THR_LOCK_isam);
   pthread_mutex_destroy(&THR_LOCK_myisam);
+  pthread_mutex_destroy(&THR_LOCK_myisam_mmap);
   pthread_mutex_destroy(&THR_LOCK_heap);
   pthread_mutex_destroy(&THR_LOCK_net);
   pthread_mutex_destroy(&THR_LOCK_time);

sql/mysqld.cc

@@ -5651,6 +5651,7 @@ enum options_mysqld
   OPT_MYISAM_BLOCK_SIZE, OPT_MYISAM_MAX_EXTRA_SORT_FILE_SIZE,
   OPT_MYISAM_MAX_SORT_FILE_SIZE, OPT_MYISAM_SORT_BUFFER_SIZE,
   OPT_MYISAM_USE_MMAP, OPT_MYISAM_REPAIR_THREADS,
+  OPT_MYISAM_MMAP_SIZE,
   OPT_MYISAM_STATS_METHOD,
   OPT_NET_BUFFER_LENGTH, OPT_NET_RETRY_COUNT,
   OPT_NET_READ_TIMEOUT, OPT_NET_WRITE_TIMEOUT,
@@ -6802,6 +6803,10 @@ The minimum value for this variable is 4096.",
    (uchar**) &max_system_variables.myisam_max_sort_file_size, 0,
    GET_ULL, REQUIRED_ARG, (longlong) LONG_MAX, 0, (ulonglong) MAX_FILE_SIZE,
    0, 1024*1024, 0},
+  {"myisam_mmap_size", OPT_MYISAM_MMAP_SIZE,
+   "Can be used to restrict the total memory used for memory mmaping of myisam files",
+   (uchar**) &myisam_mmap_size, (uchar**) &myisam_mmap_size, 0,
+   GET_ULL, REQUIRED_ARG, SIZE_T_MAX, MEMMAP_EXTRA_MARGIN, SIZE_T_MAX, 0, 1, 0},
   {"myisam_repair_threads", OPT_MYISAM_REPAIR_THREADS,
    "Number of threads to use when repairing MyISAM tables. The value of 1 disables parallel repair.",
    (uchar**) &global_system_variables.myisam_repair_threads,

sql/set_var.cc

@@ -147,6 +147,7 @@ static bool sys_update_general_log_path(THD *thd, set_var * var);
 static void sys_default_general_log_path(THD *thd, enum_var_type type);
 static bool sys_update_slow_log_path(THD *thd, set_var * var);
 static void sys_default_slow_log_path(THD *thd, enum_var_type type);
+static uchar *get_myisam_mmap_size(THD *thd);
 
 /*
   Variable definition list
@@ -898,6 +899,10 @@ sys_var_str sys_var_slow_log_path(&vars, "slow_query_log_file", sys_check_log_pa
 				  opt_slow_logname);
 static sys_var_log_output sys_var_log_output_state(&vars, "log_output", &log_output_options,
 					    &log_output_typelib, 0);
+static sys_var_readonly         sys_myisam_mmap_size(&vars, "myisam_mmap_size",
+                                                     OPT_GLOBAL,
+                                                     SHOW_LONGLONG,
+                                                     get_myisam_mmap_size);
 
 
 bool sys_var::check(THD *thd, set_var *var)
@@ -3175,6 +3180,12 @@ static uchar *get_tmpdir(THD *thd)
   return (uchar*)mysql_tmpdir;
 }
 
+static uchar *get_myisam_mmap_size(THD *thd)
+{
+  return (uchar *)&myisam_mmap_size;
+}
+
+
 /****************************************************************************
   Main handling of variables:
   - Initialisation

storage/myisam/mi_packrec.c

@@ -1492,21 +1492,55 @@ static int _mi_read_rnd_mempack_record(MI_INFO*, uchar *,my_off_t, my_bool);
 my_bool _mi_memmap_file(MI_INFO *info)
 {
   MYISAM_SHARE *share=info->s;
+  my_bool eom;
+
   DBUG_ENTER("mi_memmap_file");
 
   if (!info->s->file_map)
   {
+    my_off_t data_file_length= share->state.state.data_file_length;
+
+    if (myisam_mmap_size != SIZE_T_MAX)
+    {
+      pthread_mutex_lock(&THR_LOCK_myisam_mmap);
+      eom= data_file_length > myisam_mmap_size - myisam_mmap_used - MEMMAP_EXTRA_MARGIN;
+      if (!eom)
+        myisam_mmap_used+= data_file_length + MEMMAP_EXTRA_MARGIN;
+      pthread_mutex_unlock(&THR_LOCK_myisam_mmap);
+    }
+    else
+      eom= data_file_length > myisam_mmap_size - MEMMAP_EXTRA_MARGIN;
+
+    if (eom)
+    {
+      DBUG_PRINT("warning", ("File is too large for mmap"));
+      DBUG_RETURN(0);
+    }
     if (my_seek(info->dfile,0L,MY_SEEK_END,MYF(0)) <
         share->state.state.data_file_length+MEMMAP_EXTRA_MARGIN)
     {
       DBUG_PRINT("warning",("File isn't extended for memmap"));
+      if (myisam_mmap_size != SIZE_T_MAX)
+      {
+        pthread_mutex_lock(&THR_LOCK_myisam_mmap);
+        myisam_mmap_used-= data_file_length + MEMMAP_EXTRA_MARGIN;
+        pthread_mutex_unlock(&THR_LOCK_myisam_mmap);
+      }
       DBUG_RETURN(0);
     }
     if (mi_dynmap_file(info,
                        share->state.state.data_file_length + 
                          MEMMAP_EXTRA_MARGIN))
+    {
+      if (myisam_mmap_size != SIZE_T_MAX)
+      {
+        pthread_mutex_lock(&THR_LOCK_myisam_mmap);
+        myisam_mmap_used-= data_file_length + MEMMAP_EXTRA_MARGIN;
+        pthread_mutex_unlock(&THR_LOCK_myisam_mmap);
+      }
       DBUG_RETURN(0);
     }
+  }
   info->opt_flag|= MEMMAP_USED;
   info->read_record= share->read_record= _mi_read_mempack_record;
   share->read_rnd= _mi_read_rnd_mempack_record;
@@ -1518,6 +1552,13 @@ void _mi_unmap_file(MI_INFO *info)
 {
   VOID(my_munmap((char*) info->s->file_map, 
                  (size_t) info->s->mmaped_length + MEMMAP_EXTRA_MARGIN));
+
+  if (myisam_mmap_size != SIZE_T_MAX)
+  {
+    pthread_mutex_lock(&THR_LOCK_myisam_mmap);
+    myisam_mmap_used-= info->s->mmaped_length + MEMMAP_EXTRA_MARGIN;
+    pthread_mutex_unlock(&THR_LOCK_myisam_mmap);
+  }
 }
 
 

storage/myisam/mi_static.c

@@ -40,7 +40,8 @@ ulong myisam_concurrent_insert= 0;
 my_off_t myisam_max_temp_length= MAX_FILE_SIZE;
 ulong    myisam_bulk_insert_tree_size=8192*1024;
 ulong    myisam_data_pointer_size=4;
-
+ulonglong    myisam_mmap_size= SIZE_T_MAX, myisam_mmap_used= 0;
+pthread_mutex_t  THR_LOCK_myisam_mmap;
 
 static int always_valid(const char *filename __attribute__((unused)))
 {

storage/myisam/myisamdef.h

@@ -437,7 +437,6 @@ typedef struct st_mi_sort_param
 #define MI_MAX_BLOCK_LENGTH	((((ulong) 1 << 24)-1) & (~ (ulong) (MI_DYN_ALIGN_SIZE-1)))
 #define MI_REC_BUFF_OFFSET      ALIGN_SIZE(MI_DYN_DELETE_BLOCK_HEADER+sizeof(uint32))
 
-#define MEMMAP_EXTRA_MARGIN	7	/* Write this as a suffix for file */
 
 #define PACK_TYPE_SELECTED	1	/* Bits in field->pack_type */
 #define PACK_TYPE_SPACE_FIELDS	2