Bug#21074 Large query_cache freezes mysql server sporadically under heavy load

Invaldating a subset of a sufficiently large query cache can take a long time.
During this time the server is efficiently frozen and no other operation can
be executed. This patch addresses this problem by moving the locks which cause
the freezing and also by temporarily disable the query cache while the 
invalidation takes place.

sql/ha_ndbcluster.cc

@@ -6996,7 +6996,6 @@ int ndbcluster_find_files(handlerton *hton, THD *thd,
 
   // Lock mutex before deleting and creating frm files
   pthread_mutex_lock(&LOCK_open);
-
   if (!global_read_lock)
   {
     // Delete old files
@@ -7010,10 +7009,12 @@ int ndbcluster_find_files(handlerton *hton, THD *thd,
       table_list.db= (char*) db;
       table_list.alias= table_list.table_name= (char*)file_name;
       (void)mysql_rm_table_part2(thd, &table_list,
-                                                                 /* if_exists */ FALSE,
-                                                                 /* drop_temporary */ FALSE,
-                                                                 /* drop_view */ FALSE,
-                                                                 /* dont_log_query*/ TRUE);
+                                 FALSE,   /* if_exists */
+                                 FALSE,   /* drop_temporary */ 
+                                 FALSE,   /* drop_view */
+                                 TRUE,    /* dont_log_query*/ 
+                                 FALSE);  /* need lock open */
+
       /* Clear error message that is returned when table is deleted */
       thd->clear_error();
     }
@@ -7029,7 +7030,7 @@ int ndbcluster_find_files(handlerton *hton, THD *thd,
   }
 
   pthread_mutex_unlock(&LOCK_open);
-  
+
   hash_free(&ok_tables);
   hash_free(&ndb_tables);
 

sql/lock.cc

@@ -1027,6 +1027,102 @@ bool lock_table_names(THD *thd, TABLE_LIST *table_list)
 }
 
 
+/**
+  @brief Lock all tables in list with an exclusive table name lock.
+
+  @param thd Thread handle.
+  @param table_list Names of tables to lock.
+
+  @note This function needs to be protected by LOCK_open. If we're 
+    under LOCK TABLES, this function does not work as advertised. Namely,
+    it does not exclude other threads from using this table and does not
+    put an exclusive name lock on this table into the table cache.
+
+  @see lock_table_names
+  @see unlock_table_names
+
+  @retval TRUE An error occured.
+  @retval FALSE Name lock successfully acquired.
+*/
+
+bool lock_table_names_exclusively(THD *thd, TABLE_LIST *table_list)
+{
+  if (lock_table_names(thd, table_list))
+    return TRUE;
+
+  /*
+    Upgrade the table name locks from semi-exclusive to exclusive locks.
+  */
+  for (TABLE_LIST *table= table_list; table; table= table->next_global)
+  {
+    if (table->table)
+      table->table->open_placeholder= 1;
+  }
+  return FALSE;
+}
+
+
+/**
+  @brief Test is 'table' is protected by an exclusive name lock.
+
+  @param[in] thd The current thread handler
+  @param[in] table Table container containing the single table to be tested
+
+  @note Needs to be protected by LOCK_open mutex.
+
+  @return Error status code
+    @retval TRUE Table is protected
+    @retval FALSE Table is not protected
+*/
+
+bool
+is_table_name_exclusively_locked_by_this_thread(THD *thd,
+                                                TABLE_LIST *table_list)
+{
+  char  key[MAX_DBKEY_LENGTH];
+  uint  key_length;
+
+  key_length= create_table_def_key(thd, key, table_list, 0);
+
+  return is_table_name_exclusively_locked_by_this_thread(thd, (uchar *)key,
+                                                         key_length);
+}
+
+
+/**
+  @brief Test is 'table key' is protected by an exclusive name lock.
+
+  @param[in] thd The current thread handler.
+  @param[in] table Table container containing the single table to be tested.
+
+  @note Needs to be protected by LOCK_open mutex
+
+  @retval TRUE Table is protected
+  @retval FALSE Table is not protected
+ */
+
+bool
+is_table_name_exclusively_locked_by_this_thread(THD *thd, uchar *key,
+                                                int key_length)
+{
+  HASH_SEARCH_STATE state;
+  TABLE *table;
+
+  for (table= (TABLE*) hash_first(&open_cache, key,
+                                  key_length, &state);
+       table ;
+       table= (TABLE*) hash_next(&open_cache, key,
+                                 key_length, &state))
+  {
+    if (table->in_use == thd &&
+        table->open_placeholder == 1 &&
+        table->s->version == 0)
+      return TRUE;
+  }
+
+  return FALSE;
+}
+
 /*
   Unlock all tables in list with a name lock
 

sql/mysql_priv.h

@@ -820,10 +820,8 @@ void mysql_client_binlog_statement(THD *thd);
 bool mysql_rm_table(THD *thd,TABLE_LIST *tables, my_bool if_exists,
                     my_bool drop_temporary);
 int mysql_rm_table_part2(THD *thd, TABLE_LIST *tables, bool if_exists,
-			 bool drop_temporary, bool drop_view, bool log_query);
-int mysql_rm_table_part2_with_lock(THD *thd, TABLE_LIST *tables,
-				   bool if_exists, bool drop_temporary,
-				   bool log_query);
+                         bool drop_temporary, bool drop_view, bool log_query,
+                         bool need_lock_open);
 bool quick_rm_table(handlerton *base,const char *db,
                     const char *table_name, uint flags);
 void close_cached_table(THD *thd, TABLE *table);
@@ -1799,6 +1797,11 @@ bool wait_for_locked_table_names(THD *thd, TABLE_LIST *table_list);
 bool lock_table_names(THD *thd, TABLE_LIST *table_list);
 void unlock_table_names(THD *thd, TABLE_LIST *table_list,
 			TABLE_LIST *last_table);
+bool lock_table_names_exclusively(THD *thd, TABLE_LIST *table_list);
+bool is_table_name_exclusively_locked_by_this_thread(THD *thd, 
+                                                     TABLE_LIST *table_list);
+bool is_table_name_exclusively_locked_by_this_thread(THD *thd, uchar *key,
+                                                     int key_length);
 
 
 /* old unireg functions */

sql/sql_cache.h

@@ -65,17 +65,44 @@ struct Query_cache_query;
 struct Query_cache_result;
 class Query_cache;
 
+/**
+  @brief This class represents a node in the linked chain of queries
+         belonging to one table.
 
+  @note The root of this linked list is not a query-type block, but the table-
+        type block which all queries has in common.
+*/
 struct Query_cache_block_table
 {
   Query_cache_block_table() {}                /* Remove gcc warning */
-  TABLE_COUNTER_TYPE n;		// numbr in table (from 0)
+
+  /**
+    This node holds a position in a static table list belonging
+    to the associated query (base 0).
+  */
+  TABLE_COUNTER_TYPE n;
+
+  /**
+    Pointers to the next and previous node, linking all queries with 
+    a common table.
+  */
   Query_cache_block_table *next, *prev;
+
+  /**
+    A pointer to the table-type block which all
+    linked queries has in common.
+  */
   Query_cache_table *parent;
+
+  /**
+    A method to calculate the address of the query cache block
+    owning this node. The purpose of this calculation is to 
+    make it easier to move the query cache block without having
+    to modify all the pointer addresses.
+  */
   inline Query_cache_block *block();
 };
 
-
 struct Query_cache_block
 {
   Query_cache_block() {}                      /* Remove gcc warning */
@@ -151,6 +178,11 @@ struct Query_cache_table
   /* data need by some engines */
   ulonglong engine_data_buff;
 
+  /**
+    The number of queries depending of this table.
+  */
+  int32 m_cached_query_count;
+
   inline char *db()			     { return (char *) data(); }
   inline char *table()			     { return tbl; }
   inline void table(char *table_arg)	     { tbl= table_arg; }
@@ -237,9 +269,14 @@ class Query_cache
   ulong free_memory, queries_in_cache, hits, inserts, refused,
     free_memory_blocks, total_blocks, lowmem_prunes;
 
+
 private:
-  pthread_cond_t COND_flush_finished;
-  bool flush_in_progress;
+  pthread_cond_t COND_cache_status_changed;
+
+  enum Cache_status { NO_FLUSH_IN_PROGRESS, FLUSH_IN_PROGRESS,
+                      TABLE_FLUSH_IN_PROGRESS };
+
+  Cache_status m_cache_status;
 
   void free_query_internal(Query_cache_block *point);
 
@@ -253,7 +290,7 @@ class Query_cache
       2. query block (for operation inside query (query block/results))
 
     Thread doing cache flush releases the mutex once it sets
-    flush_in_progress flag, so other threads may bypass the cache as
+    m_cache_status flag, so other threads may bypass the cache as
     if it is disabled, not waiting for reset to finish.  The exception
     is other threads that were going to do cache flush---they'll wait
     till the end of a flush operation.
@@ -270,6 +307,7 @@ class Query_cache
   /* options */
   ulong min_allocation_unit, min_result_data_size;
   uint def_query_hash_size, def_table_hash_size;
+  
   uint mem_bin_num, mem_bin_steps;		// See at init_cache & find_bin
 
   my_bool initialized;
@@ -295,10 +333,13 @@ class Query_cache
 			      ulong data_len,
 			      Query_cache_block *query_block,
 			      my_bool first_block);
-  void invalidate_table(TABLE_LIST *table);
-  void invalidate_table(TABLE *table);
-  void invalidate_table(uchar *key, uint32  key_length);
-  void invalidate_table(Query_cache_block *table_block);
+  void invalidate_table(THD *thd, TABLE_LIST *table);
+  void invalidate_table(THD *thd, TABLE *table);
+  void invalidate_table(THD *thd, uchar *key, uint32  key_length);
+  void invalidate_table(THD *thd, Query_cache_block *table_block);
+  void invalidate_query_block_list(THD *thd, 
+                                   Query_cache_block_table *list_root);
+
   TABLE_COUNTER_TYPE
     register_tables_from_list(TABLE_LIST *tables_used,
                               TABLE_COUNTER_TYPE counter,
@@ -337,6 +378,8 @@ class Query_cache
 	      Query_cache_block *pprev);
   my_bool join_results(ulong join_limit);
 
+  void wait_while_table_flush_is_in_progress(bool *interrupt);
+
   /*
     Following function control structure_guard_mutex
     by themself or don't need structure_guard_mutex
@@ -347,8 +390,7 @@ class Query_cache
   Query_cache_block *write_block_data(ulong data_len, uchar* data,
 				       ulong header_len,
 				       Query_cache_block::block_type type,
-				       TABLE_COUNTER_TYPE ntab = 0,
-				       my_bool under_guard=0);
+				       TABLE_COUNTER_TYPE ntab = 0);
   my_bool append_result_data(Query_cache_block **result,
 			     ulong data_len, uchar* data,
 			     Query_cache_block *parent);
@@ -360,8 +402,7 @@ class Query_cache
   inline ulong get_min_first_result_data_size();
   inline ulong get_min_append_result_data_size();
   Query_cache_block *allocate_block(ulong len, my_bool not_less,
-				     ulong min,
-				     my_bool under_guard=0);
+				     ulong min);
   /*
     If query is cacheable return number tables in query
     (query without tables not cached)
@@ -424,6 +465,11 @@ class Query_cache
   friend void query_cache_end_of_result(THD *thd);
   friend void query_cache_abort(NET *net);
 
+  bool is_flushing(void) 
+  { 
+    return (m_cache_status != Query_cache::NO_FLUSH_IN_PROGRESS);
+  }
+
   /*
     The following functions are only used when debugging
     We don't protect these with ifndef DBUG_OFF to not have to recompile

sql/sql_db.cc

@@ -1084,7 +1084,7 @@ static long mysql_rm_known_files(THD *thd, MY_DIR *dirp, const char *db,
     }
   }
   if (thd->killed ||
-      (tot_list && mysql_rm_table_part2_with_lock(thd, tot_list, 1, 0, 1)))
+      (tot_list && mysql_rm_table_part2(thd, tot_list, 1, 0, 1, 1, 1)))
     goto err;
 
   /* Remove RAID directories */

sql/sql_parse.cc

@@ -2445,7 +2445,7 @@ mysql_execute_command(THD *thd)
           check_grant(thd, INSERT_ACL | CREATE_ACL, &new_list, 0, 1, 0)))
         goto error;
     }
-    query_cache_invalidate3(thd, first_table, 0);
+
     if (end_active_trans(thd) || mysql_rename_tables(thd, first_table, 0))
       goto error;
     break;

sql/sql_rename.cc

@@ -144,10 +144,14 @@ bool mysql_rename_tables(THD *thd, TABLE_LIST *table_list, bool silent)
     }
   }
 
-  VOID(pthread_mutex_lock(&LOCK_open));
-  if (lock_table_names(thd, table_list))
+  pthread_mutex_lock(&LOCK_open);
+  if (lock_table_names_exclusively(thd, table_list))
+  {
+    pthread_mutex_unlock(&LOCK_open);
     goto err;
-  
+  }
+  pthread_mutex_unlock(&LOCK_open);
+
   error=0;
   if ((ren_table=rename_tables(thd,table_list,0)))
   {
@@ -183,10 +187,14 @@ bool mysql_rename_tables(THD *thd, TABLE_LIST *table_list, bool silent)
     send_ok(thd);
   }
 
+  if (!error)
+    query_cache_invalidate3(thd, table_list, 0);
+
+  pthread_mutex_lock(&LOCK_open);
   unlock_table_names(thd, table_list, (TABLE_LIST*) 0);
+  pthread_mutex_unlock(&LOCK_open);
 
 err:
-  pthread_mutex_unlock(&LOCK_open);
   /* enable logging back if needed */
   if (disable_logs)
   {

sql/sql_table.cc

@@ -1411,14 +1411,7 @@ bool mysql_rm_table(THD *thd,TABLE_LIST *tables, my_bool if_exists,
     LOCK_open during wait_if_global_read_lock(), other threads could not
     close their tables. This would make a pretty deadlock.
   */
-  thd->mysys_var->current_mutex= &LOCK_open;
-  thd->mysys_var->current_cond= &COND_refresh;
-  VOID(pthread_mutex_lock(&LOCK_open));
-
-  error= mysql_rm_table_part2(thd, tables, if_exists, drop_temporary, 0, 0);
-
-  pthread_mutex_unlock(&LOCK_open);
-
+  error= mysql_rm_table_part2(thd, tables, if_exists, drop_temporary, 0, 0, 1);
   pthread_mutex_lock(&thd->mysys_var->mutex);
   thd->mysys_var->current_mutex= 0;
   thd->mysys_var->current_cond= 0;
@@ -1433,49 +1426,6 @@ bool mysql_rm_table(THD *thd,TABLE_LIST *tables, my_bool if_exists,
   DBUG_RETURN(FALSE);
 }
 
-
-/*
- delete (drop) tables.
-
-  SYNOPSIS
-    mysql_rm_table_part2_with_lock()
-    thd			Thread handle
-    tables		List of tables to delete
-    if_exists		If 1, don't give error if one table doesn't exists
-    dont_log_query	Don't write query to log files. This will also not
-                        generate warnings if the handler files doesn't exists
-
- NOTES
-   Works like documented in mysql_rm_table(), but don't check
-   global_read_lock and don't send_ok packet to server.
-
- RETURN
-  0	ok
-  1	error
-*/
-
-int mysql_rm_table_part2_with_lock(THD *thd,
-				   TABLE_LIST *tables, bool if_exists,
-				   bool drop_temporary, bool dont_log_query)
-{
-  int error;
-  thd->mysys_var->current_mutex= &LOCK_open;
-  thd->mysys_var->current_cond= &COND_refresh;
-  VOID(pthread_mutex_lock(&LOCK_open));
-
-  error= mysql_rm_table_part2(thd, tables, if_exists, drop_temporary, 1,
-			      dont_log_query);
-
-  pthread_mutex_unlock(&LOCK_open);
-
-  pthread_mutex_lock(&thd->mysys_var->mutex);
-  thd->mysys_var->current_mutex= 0;
-  thd->mysys_var->current_cond= 0;
-  pthread_mutex_unlock(&thd->mysys_var->mutex);
-  return error;
-}
-
-
 /*
   Execute the drop of a normal or temporary table
 
@@ -1508,7 +1458,7 @@ int mysql_rm_table_part2_with_lock(THD *thd,
 
 int mysql_rm_table_part2(THD *thd, TABLE_LIST *tables, bool if_exists,
 			 bool drop_temporary, bool drop_view,
-			 bool dont_log_query)
+			 bool dont_log_query, bool need_lock_open)
 {
   TABLE_LIST *table;
   char path[FN_REFLEN], *alias;
@@ -1520,9 +1470,11 @@ int mysql_rm_table_part2(THD *thd, TABLE_LIST *tables, bool if_exists,
   String built_query;
   DBUG_ENTER("mysql_rm_table_part2");
 
+  if (need_lock_open)
+    pthread_mutex_lock(&LOCK_open);
+
   LINT_INIT(alias);
   LINT_INIT(path_length);
-  safe_mutex_assert_owner(&LOCK_open);
 
   if (thd->current_stmt_binlog_row_based && !dont_log_query)
   {
@@ -1555,8 +1507,15 @@ int mysql_rm_table_part2(THD *thd, TABLE_LIST *tables, bool if_exists,
     }
   }
 
-  if (!drop_temporary && lock_table_names(thd, tables))
+  if (!drop_temporary && lock_table_names_exclusively(thd, tables))
+  {
+    if (need_lock_open)
+      pthread_mutex_unlock(&LOCK_open);
     DBUG_RETURN(1);
+  }
+
+  if (need_lock_open)
+    pthread_mutex_unlock(&LOCK_open);
 
   /* Don't give warnings for not found errors, as we already generate notes */
   thd->no_warnings_for_error= 1;
@@ -1567,7 +1526,7 @@ int mysql_rm_table_part2(THD *thd, TABLE_LIST *tables, bool if_exists,
     handlerton *table_type;
     enum legacy_db_type frm_db_type;
 
-    mysql_ha_flush(thd, table, MYSQL_HA_CLOSE_FINAL, TRUE);
+    mysql_ha_flush(thd, table, MYSQL_HA_CLOSE_FINAL, !need_lock_open);
     if (!close_temporary_table(thd, table))
     {
       tmp_table_deleted=1;
@@ -1604,6 +1563,8 @@ int mysql_rm_table_part2(THD *thd, TABLE_LIST *tables, bool if_exists,
     {
       TABLE *locked_table;
       abort_locked_tables(thd, db, table->table_name);
+      if (need_lock_open)
+        pthread_mutex_lock(&LOCK_open);
       remove_table_from_cache(thd, db, table->table_name,
 	                      RTFC_WAIT_OTHER_THREAD_FLAG |
 			      RTFC_CHECK_KILLED_FLAG);
@@ -1614,6 +1575,9 @@ int mysql_rm_table_part2(THD *thd, TABLE_LIST *tables, bool if_exists,
       if ((locked_table= drop_locked_tables(thd, db, table->table_name)))
         table->table= locked_table;
 
+      if (need_lock_open)
+        pthread_mutex_unlock(&LOCK_open);
+
       if (thd->killed)
       {
         thd->no_warnings_for_error= 0;
@@ -1739,9 +1703,11 @@ int mysql_rm_table_part2(THD *thd, TABLE_LIST *tables, bool if_exists,
       */
     }
   }
-
-  if (!drop_temporary)
-    unlock_table_names(thd, tables, (TABLE_LIST*) 0);
+  if (need_lock_open)
+    pthread_mutex_lock(&LOCK_open);
+  unlock_table_names(thd, tables, (TABLE_LIST*) 0);
+  if (need_lock_open)
+    pthread_mutex_unlock(&LOCK_open);
   thd->no_warnings_for_error= 0;
   DBUG_RETURN(error);
 }

sql/sql_trigger.cc

@@ -1268,8 +1268,6 @@ bool Table_triggers_list::drop_all_triggers(THD *thd, char *db, char *name)
   bzero(&table, sizeof(table));
   init_alloc_root(&table.mem_root, 8192, 0);
 
-  safe_mutex_assert_owner(&LOCK_open);
-
   if (Table_triggers_list::check_n_load(thd, db, name, &table, 1))
   {
     result= 1;
@@ -1431,26 +1429,24 @@ Table_triggers_list::change_table_name_in_trignames(const char *db_name,
 }
 
 
-/*
-  Update .TRG and .TRN files after renaming triggers' subject table.
+/**
+  @brief Update .TRG and .TRN files after renaming triggers' subject table.
 
-  SYNOPSIS
-    change_table_name()
-      thd        Thread context
-      db         Old database of subject table
-      old_table  Old name of subject table
-      new_db     New database for subject table
-      new_table  New name of subject table
+  @param[in,out] thd Thread context
+  @param[in] db Old database of subject table
+  @param[in] old_table Old name of subject table
+  @param[in] new_db New database for subject table
+  @param[in] new_table New name of subject table
 
-  NOTE
+  @note
     This method tries to leave trigger related files in consistent state,
     i.e. it either will complete successfully, or will fail leaving files
     in their initial state.
     Also this method assumes that subject table is not renamed to itself.
+    This method needs to be called under an exclusive table name lock.
 
-  RETURN VALUE
-    FALSE  Success
-    TRUE   Error
+  @retval FALSE Success
+  @retval TRUE  Error
 */
 
 bool Table_triggers_list::change_table_name(THD *thd, const char *db,
@@ -1466,7 +1462,19 @@ bool Table_triggers_list::change_table_name(THD *thd, const char *db,
   bzero(&table, sizeof(table));
   init_alloc_root(&table.mem_root, 8192, 0);
 
-  safe_mutex_assert_owner(&LOCK_open);
+  uchar key[MAX_DBKEY_LENGTH];
+  uint key_length= (uint) (strmov(strmov((char*)&key[0], db)+1,
+                    old_table)-(char*)&key[0])+1;
+
+  /*
+    This method interfaces the mysql server code protected by
+    either LOCK_open mutex or with an exclusive table name lock.
+    In the future, only an exclusive table name lock will be enough.
+  */
+#ifndef DBUG_OFF
+  if (!is_table_name_exclusively_locked_by_this_thread(thd, key, key_length))
+    safe_mutex_assert_owner(&LOCK_open);
+#endif
 
   DBUG_ASSERT(my_strcasecmp(table_alias_charset, db, new_db) ||
               my_strcasecmp(table_alias_charset, old_table, new_table));