Implementation of Monty's idea: Now we can open mysql.proc table for lookup

of stored routines definitions even if we already have some tables open and
locked. To avoid deadlocks in this case we have to put certain restrictions
on locking of mysql.proc table.

This allows to use stored routines safely under LOCK TABLES without explicitly
mentioning mysql.proc in the list of locked tables. It also fixes bug #11554
"Server crashes on statement indirectly using non-cached function".

mysql-test/r/sp-error.result

@@ -286,6 +286,19 @@ ERROR 42000: OUT or INOUT argument 2 for routine test.p is not a variable
 call p(42, @tmp_y, 43)|
 ERROR 42000: OUT or INOUT argument 3 for routine test.p is not a variable
 drop procedure p|
+create procedure p() begin end|
+lock table t1 read|
+call p()|
+unlock tables|
+drop procedure p|
+lock tables t1 read, mysql.proc write|
+ERROR HY000: You can't combine write-locking of system 'mysql.proc' table with other tables
+lock tables mysql.proc write, mysql.user write|
+ERROR HY000: You can't combine write-locking of system 'mysql.proc' table with other tables
+lock tables t1 read, mysql.proc read|
+unlock tables|
+lock tables mysql.proc write|
+unlock tables|
 create procedure bug1965()
 begin
 declare c cursor for select val from t1 order by valname;
@@ -477,7 +490,7 @@ begin
 select * from t1;
 end|
 lock table t1 read|
-call bug9566()|
+alter procedure bug9566 comment 'Some comment'|
 ERROR HY000: Table 'proc' was not locked with LOCK TABLES
 unlock tables|
 drop procedure bug9566|

mysql-test/r/sp-threads.result

@@ -55,3 +55,12 @@ call bug11158();
 unlock tables;
 drop procedure bug11158;
 drop table t1, t2;
+drop function if exists bug11554;
+drop view if exists v1;
+create table t1 (i int);
+create function bug11554 () returns int return 1;
+create view v1 as select bug11554() as f;
+insert into t1 (select f from v1);
+drop function bug11554;
+drop table t1;
+drop view v1;

mysql-test/t/sp-error.test

@@ -386,6 +386,29 @@ call p(42, @tmp_y, 43)|
 drop procedure p|
 
 
+#
+# Let us test that we can access mysql.proc table for routines
+# definitions lookup without locking it explicitly.
+#
+create procedure p() begin end|
+lock table t1 read|
+# This should succeed
+call p()|
+unlock tables|
+drop procedure p|
+# Let us check restrictions which this ability puts on mysql.proc locking.
+--error ER_WRONG_LOCK_OF_SYSTEM_TABLE
+lock tables t1 read, mysql.proc write|
+--error ER_WRONG_LOCK_OF_SYSTEM_TABLE
+lock tables mysql.proc write, mysql.user write|
+# Locking for read should be OK
+lock tables t1 read, mysql.proc read|
+unlock tables|
+# You also should be able lock only mysql.proc for write
+lock tables mysql.proc write|
+unlock tables|
+
+
 #
 # BUG#1965
 #
@@ -676,9 +699,7 @@ create procedure bug6600()
 # BUG#9566: explicit LOCK TABLE and store procedures result in illegal state
 #
 # We should not think that mysql.proc table does not exist if we are unable
-# to open it under LOCK TABLE or in prelocked mode. Probably this test
-# should be removed when Monty will allow access to mysql.proc without
-# locking it.
+# to open it under LOCK TABLE or in prelocked mode.
 #
 --disable_warnings
 drop procedure if exists bug9566|
@@ -688,9 +709,11 @@ begin
   select * from t1;
 end|
 lock table t1 read|
-# This should fail because we forgot to lock mysql.proc table explicitly
+# This should fail since we forgot to lock mysql.proc for writing
+# explicitly, and we can't open mysql.proc for _writing_ if there
+# are locked tables.
 --error 1100
-call bug9566()|
+alter procedure bug9566 comment 'Some comment'|
 unlock tables|
 # This should succeed
 drop procedure bug9566|

mysql-test/t/sp-threads.test

@@ -111,6 +111,25 @@ connection con1root;
 drop procedure bug11158;
 drop table t1, t2;
 
+#
+# BUG#11554: Server crashes on statement indirectly using non-cached function
+#
+--disable_warnings
+drop function if exists bug11554;
+drop view if exists v1;
+--enable_warnings
+create table t1 (i int);
+create function bug11554 () returns int return 1;
+create view v1 as select bug11554() as f;
+connection con2root;
+# This should not crash server
+insert into t1 (select f from v1);
+# Clean-up
+connection con1root;
+drop function bug11554;
+drop table t1;
+drop view v1;
+
 #
 # BUG#NNNN: New bug synopsis
 #

sql/lock.cc

@@ -425,6 +425,19 @@ static MYSQL_LOCK *get_lock_data(THD *thd, TABLE **table_ptr, uint count,
       tables+=table_ptr[i]->file->lock_count();
       lock_count++;
     }
+    /*
+      To be able to open and lock for reading system tables like 'mysql.proc',
+      when we already have some tables opened and locked, and avoid deadlocks
+      we have to disallow write-locking of these tables with any other tables.
+    */
+    if (table_ptr[i]->s->system_table &&
+        table_ptr[i]->reginfo.lock_type >= TL_WRITE_ALLOW_WRITE &&
+        count != 1)
+    {
+      my_error(ER_WRONG_LOCK_OF_SYSTEM_TABLE, MYF(0), table_ptr[i]->s->db,
+               table_ptr[i]->s->table_name);
+      return 0;
+    }
   }
 
   if (!(sql_lock= (MYSQL_LOCK*)

sql/mysql_priv.h

@@ -735,7 +735,7 @@ bool mysql_truncate(THD *thd, TABLE_LIST *table_list, bool dont_send_ok);
 bool mysql_create_or_drop_trigger(THD *thd, TABLE_LIST *tables, bool create);
 TABLE *open_ltable(THD *thd, TABLE_LIST *table_list, thr_lock_type update);
 TABLE *open_table(THD *thd, TABLE_LIST *table_list, MEM_ROOT* mem,
-		  bool *refresh);
+		  bool *refresh, uint flags);
 TABLE *reopen_name_locked_table(THD* thd, TABLE_LIST* table);
 TABLE *find_locked_table(THD *thd, const char *db,const char *table_name);
 bool reopen_table(TABLE *table,bool locked);

sql/share/errmsg.txt

@@ -5362,3 +5362,5 @@ ER_NO_DEFAULT_FOR_VIEW_FIELD
         eng "Field of view '%-.64s.%-.64s' underlying table doesn't have a default value"
 ER_SP_NO_RECURSION
         eng "Recursive stored routines are not allowed."
+ER_WRONG_LOCK_OF_SYSTEM_TABLE
+        eng "You can't combine write-locking of system '%-.64s.%-.64s' table with other tables"

sql/sp.h

@@ -74,9 +74,6 @@ sp_show_create_function(THD *thd, sp_name *name);
 int
 sp_show_status_function(THD *thd, const char *wild);
 
-bool
-sp_function_exists(THD *thd, sp_name *name);
-
 
 /*
  *  For precaching of functions and procedures

sql/sql_base.cc

@@ -542,10 +542,9 @@ void close_thread_tables(THD *thd, bool lock_in_use, bool skip_derived,
 
 bool close_thread_table(THD *thd, TABLE **table_ptr)
 {
-  DBUG_ENTER("close_thread_table");
-
   bool found_old_table= 0;
   TABLE *table= *table_ptr;
+  DBUG_ENTER("close_thread_table");
   DBUG_ASSERT(table->key_read == 0);
   DBUG_ASSERT(table->file->inited == handler::NONE);
 
@@ -972,18 +971,34 @@ TABLE *reopen_name_locked_table(THD* thd, TABLE_LIST* table_list)
 }
 
 
-/******************************************************************************
-** open a table
-** Uses a cache of open tables to find a table not in use.
-** If refresh is a NULL pointer, then the is no version number checking and
-** the table is not put in the thread-open-list
-** If the return value is NULL and refresh is set then one must close
-** all tables and retry the open
-******************************************************************************/
+/*
+  Open a table.
+
+  SYNOPSIS
+    open_table()
+      thd         Thread context
+      table_list  Open first table in list
+      refresh     Pointer to memory that will be set to 1 if
+                  we need to close all tables and reopen them
+                  If this is a NULL pointer, then the is no version
+                  number checking and the table is not put in the
+                  thread-open-list
+      flags       Bitmap of flags to modify how open works:
+                    MYSQL_LOCK_IGNORE_FLUSH - Open table even if someone
+                    has done a flush or namelock on it.
+
+  IMPLEMENTATION
+    Uses a cache of open tables to find a table not in use.
+
+  RETURN
+    NULL  Open failed.  If refresh is set then one should close
+          all other tables and retry the open
+    #     Success. Pointer to TABLE object for open table.
+*/
 
 
 TABLE *open_table(THD *thd, TABLE_LIST *table_list, MEM_ROOT *mem_root,
-		  bool *refresh)
+		  bool *refresh, uint flags)
 {
   reg1	TABLE *table;
   char	key[MAX_DBKEY_LENGTH];
@@ -1096,9 +1111,16 @@ TABLE *open_table(THD *thd, TABLE_LIST *table_list, MEM_ROOT *mem_root,
   {
     if (table->s->version != refresh_version)
     {
+      if (flags & MYSQL_LOCK_IGNORE_FLUSH)
+      {
+        /* Force close at once after usage */
+        thd->version= table->s->version;
+        continue;
+      }
+
       /*
-      ** There is a refresh in progress for this table
-      ** Wait until the table is freed or the thread is killed.
+        There is a refresh in progress for this table
+        Wait until the table is freed or the thread is killed.
       */
       close_old_data_files(thd,thd->open_tables,0,0);
       if (table->in_use != thd)
@@ -1681,6 +1703,15 @@ static int open_unireg_entry(THD *thd, TABLE *entry, const char *db,
   if (error == 5)
     DBUG_RETURN(0);	// we have just opened VIEW
 
+  /*
+    We can't mark all tables in 'mysql' database as system since we don't
+    allow to lock such tables for writing with any other tables (even with
+    other system tables) and some privilege tables need this.
+  */
+  if (!my_strcasecmp(system_charset_info, db, "mysql") &&
+      !my_strcasecmp(system_charset_info, name, "proc"))
+    entry->s->system_table= 1;
+
   if (Table_triggers_list::check_n_load(thd, db, name, entry))
     goto err;
 
@@ -1835,7 +1866,7 @@ int open_tables(THD *thd, TABLE_LIST **start, uint *counter)
     }
     (*counter)++;
     if (!tables->table &&
-	!(tables->table= open_table(thd, tables, &new_frm_mem, &refresh)))
+	!(tables->table= open_table(thd, tables, &new_frm_mem, &refresh, 0)))
     {
       free_root(&new_frm_mem, MYF(MY_KEEP_PREALLOC));
       if (tables->view)
@@ -2010,7 +2041,7 @@ TABLE *open_ltable(THD *thd, TABLE_LIST *table_list, thr_lock_type lock_type)
   thd->current_tablenr= 0;
   /* open_ltable can be used only for BASIC TABLEs */
   table_list->required_type= FRMTYPE_TABLE;
-  while (!(table= open_table(thd, table_list, thd->mem_root, &refresh)) &&
+  while (!(table= open_table(thd, table_list, thd->mem_root, &refresh, 0)) &&
          refresh)
     ;
 

sql/sql_class.cc

@@ -156,6 +156,14 @@ bool foreign_key_prefix(Key *a, Key *b)
 /****************************************************************************
 ** Thread specific functions
 ****************************************************************************/
+
+Open_tables_state::Open_tables_state()
+  :version(refresh_version)
+{
+  reset_open_tables_state();
+}
+
+
 /*
   Pass nominal parameters to Statement constructor only to ensure that
   the destructor works OK in case of error. The main_mem_root will be
@@ -164,6 +172,7 @@ bool foreign_key_prefix(Key *a, Key *b)
 
 THD::THD()
   :Statement(CONVENTIONAL_EXECUTION, 0, ALLOC_ROOT_MIN_BLOCK_SIZE, 0),
+   Open_tables_state(),
    user_time(0), global_read_lock(0), is_fatal_error(0),
    rand_used(0), time_zone_used(0),
    last_insert_id_used(0), insert_id_used(0), clear_next_insert_id(0),
@@ -181,10 +190,8 @@ THD::THD()
   db_length= col_access=0;
   query_error= tmp_table_used= 0;
   next_insert_id=last_insert_id=0;
-  open_tables= temporary_tables= handler_tables= derived_tables= 0;
   hash_clear(&handler_tables_hash);
   tmp_table=0;
-  lock=locked_tables=0;
   used_tables=0;
   cuted_fields= sent_row_count= 0L;
   limit_found_rows= 0;
@@ -230,7 +237,6 @@ THD::THD()
 #ifndef NO_EMBEDDED_ACCESS_CHECKS
   db_access=NO_ACCESS;
 #endif
-  version=refresh_version;			// For boot
   *scramble= '\0';
 
   init();
@@ -259,7 +265,6 @@ THD::THD()
   tablespace_op=FALSE;
   ulong tmp=sql_rnd_with_mutex();
   randominit(&rand, tmp + (ulong) &rand, tmp + (ulong) ::query_id);
-  prelocked_mode= NON_PRELOCKED;
 }
 
 
@@ -1775,3 +1780,40 @@ void THD::set_status_var_init()
 {
   bzero((char*) &status_var, sizeof(status_var));
 }
+
+/****************************************************************************
+  Handling of open and locked tables states.
+
+  This is used when we want to open/lock (and then close) some tables when
+  we already have a set of tables open and locked. We use these methods for
+  access to mysql.proc table to find definitions of stored routines.
+****************************************************************************/
+
+bool THD::push_open_tables_state()
+{
+  Open_tables_state *state;
+  DBUG_ENTER("push_open_table_state");
+  /* Currently we only push things one level */
+  DBUG_ASSERT(open_state_list.elements == 0);
+
+  if (!(state= (Open_tables_state*) alloc(sizeof(*state))))
+    DBUG_RETURN(1);                             // Fatal error is set
+  /* Store state for currently open tables */
+  state->set_open_tables_state(this);
+  if (open_state_list.push_back(state, mem_root))
+    DBUG_RETURN(1);                             // Fatal error is set
+  reset_open_tables_state();
+  DBUG_RETURN(0);
+}
+
+void THD::pop_open_tables_state()
+{
+  Open_tables_state *state;
+  DBUG_ENTER("pop_open_table_state");
+  /* Currently we only push things one level */
+  DBUG_ASSERT(open_state_list.elements == 1);
+
+  state= open_state_list.pop();
+  set_open_tables_state(state);
+  DBUG_VOID_RETURN;
+}

sql/sql_class.h

@@ -932,13 +932,94 @@ enum prelocked_mode_type {NON_PRELOCKED= 0, PRELOCKED= 1,
                           PRELOCKED_UNDER_LOCK_TABLES= 2};
 
 
+/*
+  Class that holds information about tables which were open and locked
+  by the thread. It is also used to save/restore this information in
+  push_open_tables_state()/pop_open_tables_state().
+*/
+
+class Open_tables_state
+{
+public:
+  /*
+    open_tables - list of regular tables in use by this thread
+    temporary_tables - list of temp tables in use by this thread
+    handler_tables - list of tables that were opened with HANDLER OPEN
+     and are still in use by this thread
+  */
+  TABLE *open_tables, *temporary_tables, *handler_tables, *derived_tables;
+  /*
+    During a MySQL session, one can lock tables in two modes: automatic
+    or manual. In automatic mode all necessary tables are locked just before
+    statement execution, and all acquired locks are stored in 'lock'
+    member. Unlocking takes place automatically as well, when the
+    statement ends.
+    Manual mode comes into play when a user issues a 'LOCK TABLES'
+    statement. In this mode the user can only use the locked tables.
+    Trying to use any other tables will give an error. The locked tables are
+    stored in 'locked_tables' member.  Manual locking is described in
+    the 'LOCK_TABLES' chapter of the MySQL manual.
+    See also lock_tables() for details.
+  */
+  MYSQL_LOCK *lock;
+  /*
+    Tables that were locked with explicit or implicit LOCK TABLES.
+    (Implicit LOCK TABLES happens when we are prelocking tables for
+     execution of statement which uses stored routines. See description
+     THD::prelocked_mode for more info.)
+  */
+  MYSQL_LOCK *locked_tables;
+  /*
+    prelocked_mode_type enum and prelocked_mode member are used for
+    indicating whenever "prelocked mode" is on, and what type of
+    "prelocked mode" is it.
+
+    Prelocked mode is used for execution of queries which explicitly
+    or implicitly (via views or triggers) use functions, thus may need
+    some additional tables (mentioned in query table list) for their
+    execution.
+
+    First open_tables() call for such query will analyse all functions
+    used by it and add all additional tables to table its list. It will
+    also mark this query as requiring prelocking. After that lock_tables()
+    will issue implicit LOCK TABLES for the whole table list and change
+    thd::prelocked_mode to non-0. All queries called in functions invoked
+    by the main query will use prelocked tables. Non-0 prelocked_mode
+    will also surpress mentioned analysys in those queries thus saving
+    cycles. Prelocked mode will be turned off once close_thread_tables()
+    for the main query will be called.
+
+    Note: Since not all "tables" present in table list are really locked
+    thd::prelocked_mode does not imply thd::locked_tables.
+  */
+  prelocked_mode_type prelocked_mode;
+  ulong	version;
+  uint current_tablenr;
+
+  Open_tables_state();
+
+  void set_open_tables_state(Open_tables_state *state)
+  {
+    *this= *state;
+  }
+
+  void reset_open_tables_state()
+  {
+    open_tables= temporary_tables= handler_tables= derived_tables= 0;
+    lock= locked_tables= 0;
+    prelocked_mode= NON_PRELOCKED;
+  }
+};
+
+
 /*
   For each client connection we create a separate thread with THD serving as
   a thread/connection descriptor
 */
 
 class THD :public ilink,
-           public Statement
+           public Statement,
+           public Open_tables_state
 {
 public:
 #ifdef EMBEDDED_LIBRARY
@@ -1006,34 +1087,6 @@ class THD :public ilink,
   ulong master_access;			/* Global privileges from mysql.user */
   ulong db_access;			/* Privileges for current db */
 
-  /*
-    open_tables - list of regular tables in use by this thread
-    temporary_tables - list of temp tables in use by this thread
-    handler_tables - list of tables that were opened with HANDLER OPEN
-     and are still in use by this thread
-  */
-  TABLE   *open_tables,*temporary_tables, *handler_tables, *derived_tables;
-  /*
-    During a MySQL session, one can lock tables in two modes: automatic
-    or manual. In automatic mode all necessary tables are locked just before
-    statement execution, and all acquired locks are stored in 'lock'
-    member. Unlocking takes place automatically as well, when the
-    statement ends.
-    Manual mode comes into play when a user issues a 'LOCK TABLES'
-    statement. In this mode the user can only use the locked tables.
-    Trying to use any other tables will give an error. The locked tables are
-    stored in 'locked_tables' member.  Manual locking is described in
-    the 'LOCK_TABLES' chapter of the MySQL manual.
-    See also lock_tables() for details.
-  */
-  MYSQL_LOCK	*lock;				/* Current locks */
-  /*
-    Tables that were locked with explicit or implicit LOCK TABLES.
-    (Implicit LOCK TABLES happens when we are prelocking tables for
-     execution of statement which uses stored routines. See description
-     THD::prelocked_mode for more info.)
-  */
-  MYSQL_LOCK	*locked_tables;
   HASH		handler_tables_hash;
   /*
     One thread can hold up to one named user-level lock. This variable
@@ -1150,6 +1203,7 @@ class THD :public ilink,
   List	     <MYSQL_ERROR> warn_list;
   uint	     warn_count[(uint) MYSQL_ERROR::WARN_LEVEL_END];
   uint	     total_warn_count;
+  List	     <Open_tables_state> open_state_list;
   /*
     Id of current query. Statement can be reused to execute several queries
     query_id is global in context of the whole MySQL server.
@@ -1159,7 +1213,7 @@ class THD :public ilink,
     update auto-updatable fields (like auto_increment and timestamp).
   */
   query_id_t query_id, warn_id;
-  ulong	     version, options, thread_id, col_access;
+  ulong	     options, thread_id, col_access;
 
   /* Statement id is thread-wide. This counter is used to generate ids */
   ulong      statement_id_counter;
@@ -1167,7 +1221,7 @@ class THD :public ilink,
   ulong      row_count;  // Row counter, mainly for errors and warnings
   long	     dbug_thread_id;
   pthread_t  real_id;
-  uint	     current_tablenr,tmp_table,global_read_lock;
+  uint	     tmp_table, global_read_lock;
   uint	     server_status,open_options,system_thread;
   uint32     db_length;
   uint       select_number;             //number of select (used for EXPLAIN)
@@ -1218,31 +1272,6 @@ class THD :public ilink,
     long    long_value;
   } sys_var_tmp;
 
-  /*
-    prelocked_mode_type enum and prelocked_mode member are used for
-    indicating whenever "prelocked mode" is on, and what type of
-    "prelocked mode" is it.
-
-    Prelocked mode is used for execution of queries which explicitly
-    or implicitly (via views or triggers) use functions, thus may need
-    some additional tables (mentioned in query table list) for their
-    execution.
-
-    First open_tables() call for such query will analyse all functions
-    used by it and add all additional tables to table its list. It will
-    also mark this query as requiring prelocking. After that lock_tables()
-    will issue implicit LOCK TABLES for the whole table list and change
-    thd::prelocked_mode to non-0. All queries called in functions invoked
-    by the main query will use prelocked tables. Non-0 prelocked_mode
-    will also surpress mentioned analysys in those queries thus saving
-    cycles. Prelocked mode will be turned off once close_thread_tables()
-    for the main query will be called.
-
-    Note: Since not all "tables" present in table list are really locked
-    thd::relocked_mode does not imply thd::locked_tables.
-  */
-  prelocked_mode_type prelocked_mode;
-
   THD();
   ~THD();
 
@@ -1428,8 +1457,11 @@ class THD :public ilink,
              (variables.sql_mode & MODE_STRICT_ALL_TABLES)));
   }
   void set_status_var_init();
+  bool push_open_tables_state();
+  void pop_open_tables_state();
 };
 
+
 #define tmp_disable_binlog(A)       \
   {ulong tmp_disable_binlog__save_options= (A)->options; \
   (A)->options&= ~OPTION_BIN_LOG

sql/sql_lex.cc

@@ -145,7 +145,7 @@ void lex_start(THD *thd, uchar *buf,uint length)
   lex->found_semicolon= 0;
   lex->safe_to_cache_query= 1;
   lex->time_zone_tables_used= 0;
-  lex->leaf_tables_insert= lex->proc_table= lex->query_tables= 0;
+  lex->leaf_tables_insert= lex->query_tables= 0;
   lex->query_tables_last= &lex->query_tables;
   lex->variables_used= 0;
   lex->select_lex.parent_lex= lex;

sql/sql_lex.h

@@ -719,7 +719,6 @@ typedef struct st_lex
     function)
   */
   TABLE_LIST **query_tables_last;
-  TABLE_LIST *proc_table; /* refer to mysql.proc if it was opened by VIEW */
   /* store original leaf_tables for INSERT SELECT and PS/SP */
   TABLE_LIST *leaf_tables_insert;
 

sql/sql_table.cc

@@ -1760,7 +1760,7 @@ TABLE *create_table_from_items(THD *thd, HA_CREATE_INFO *create_info,
                             create_info, *extra_fields, *keys, 0,
                             select_field_count))
     {
-      if (!(table= open_table(thd, create_table, thd->mem_root, (bool*) 0)))
+      if (!(table= open_table(thd, create_table, thd->mem_root, (bool*)0, 0)))
         quick_rm_table(create_info->db_type, create_table->db,
                        table_case_name(create_info, create_table->table_name));
     }
@@ -3571,7 +3571,7 @@ bool mysql_alter_table(THD *thd,char *new_db, char *new_name,
       bzero((void*) &tbl, sizeof(tbl));
       tbl.db= new_db;
       tbl.table_name= tbl.alias= tmp_name;
-      new_table= open_table(thd, &tbl, thd->mem_root, 0);
+      new_table= open_table(thd, &tbl, thd->mem_root, 0, 0);
     }
     else
     {

sql/table.h

@@ -163,6 +163,13 @@ typedef struct st_table_share
   my_bool crashed;
   my_bool is_view;
   my_bool name_lock, replace_with_name_lock;
+  /*
+    TRUE if this is a system table like 'mysql.proc', which we want to be
+    able to open and lock even when we already have some tables open and
+    locked. To avoid deadlocks we have to put certain restrictions on
+    locking of this table for writing. FALSE - otherwise.
+  */
+  my_bool system_table;
 } TABLE_SHARE;