WL#4165 "Prepared statements: validation".

Add metadata validation to ~20 more SQL commands. Make sure that
these commands actually work in ps-protocol, since until now they
were enabled, but not carefully tested.
Fixes the ml003 bug found by Matthias during internal testing of the
patch.


mysql-test/r/ps_ddl.result:
  Update test results (WL#4165)
mysql-test/t/ps_ddl.test:
  Cover with tests metadata validation of 26 SQL statements.
sql/mysql_priv.h:
  Fix the name in the comment.
sql/sp_head.cc:
  Changed the way the observer is removed in case of stored procedures
  to support validation prepare stmt from "call p1(<expr>)": whereas
  tables used in the expression must be validated, substatements
  of p1 must not.
  The previous scheme used to silence the observer only in stored
  functions and triggers.
sql/sql_class.cc:
  Now the observer is silenced in sp_head::execute(). Remove it from
  Sub_statement_state.
sql/sql_class.h:
  Now the observer is silenced in sp_head::execute(). Remove it from
  Sub_statement_state.
sql/sql_parse.cc:
  Add CF_REEXECUTION_FRAGILE to 20 more SQLCOMs that need it.
sql/sql_prepare.cc:
  Add metadata validation to ~20 new SQLCOMs that need it.
  Fix memory leaks with expressions used in SHOW DATABASES and CALL
  (and prepared statements).
  We need to fix all expressions at prepare, since if these expressions
  use subqueries, there are one-time transformations of the parse
  tree that must be done at prepare. 
  List of fixed commands includes: SHOW TABLES, SHOW DATABASES,
  SHOW TRIGGERS, SHOW EVENTS, SHOW OPEN TABLES,SHOW KEYS, SHOW FIELDS, 
  SHOW COLLATIONS, SHOW CHARSETS, SHOW VARIABLES, SHOW TATUS, SHOW TABLE
  STATUS, SHOW PROCEDURE STATUS, SHOW FUNCTION STATUS, CALL.
  Add comment to set_parameters().
sql/table.h:
  Update comments.

sql/mysql_priv.h

@@ -695,7 +695,7 @@ const char *set_thd_proc_info(THD *thd, const char *info,
   prepare matches the type of the object we obtained from the
   table definition cache.
 
-  @sa check_and_update_metadata_version()
+  @sa check_and_update_table_version()
   @sa Execute_observer
   @sa Prepared_statement::reprepare()
 */

sql/sp_head.cc

@@ -1068,6 +1068,7 @@ sp_head::execute(THD *thd)
   LEX *old_lex;
   Item_change_list old_change_list;
   String old_packet;
+  Metadata_version_observer *save_metadata_observer= thd->m_metadata_observer;
 
   Object_creation_ctx *saved_creation_ctx;
 
@@ -1135,6 +1136,25 @@ sp_head::execute(THD *thd)
   thd->variables.sql_mode= m_sql_mode;
   save_abort_on_warning= thd->abort_on_warning;
   thd->abort_on_warning= 0;
+  /**
+    When inside a substatement (a stored function or trigger
+    statement), clear the metadata observer in THD, if any.
+    Remember the value of the observer here, to be able
+    to restore it when leaving the substatement.
+
+    We reset the observer to suppress errors when a substatement
+    uses temporary tables. If a temporary table does not exist
+    at start of the main statement, it's not prelocked
+    and thus is not validated with other prelocked tables.
+
+    Later on, when the temporary table is opened, metadata
+    versions mismatch, expectedly.
+
+    The proper solution for the problem is to re-validate tables
+    of substatements (Bug#12257, Bug#27011, Bug#32868, Bug#33000),
+    but it's not implemented yet.
+  */
+  thd->m_metadata_observer= 0;
 
   /*
     It is also more efficient to save/restore current thd->lex once when
@@ -1297,6 +1317,7 @@ sp_head::execute(THD *thd)
   thd->derived_tables= old_derived_tables;
   thd->variables.sql_mode= save_sql_mode;
   thd->abort_on_warning= save_abort_on_warning;
+  thd->m_metadata_observer= save_metadata_observer;
 
   thd->stmt_arena= old_arena;
   state= EXECUTED;

sql/sql_class.cc

@@ -2876,7 +2876,6 @@ void THD::reset_sub_statement_state(Sub_statement_state *backup,
     first_successful_insert_id_in_prev_stmt;
   backup->first_successful_insert_id_in_cur_stmt= 
     first_successful_insert_id_in_cur_stmt;
-  backup->m_metadata_observer= m_metadata_observer;
 
   if ((!lex->requires_prelocking() || is_update_query(lex->sql_command)) &&
       !current_stmt_binlog_row_based)
@@ -2896,7 +2895,6 @@ void THD::reset_sub_statement_state(Sub_statement_state *backup,
   cuted_fields= 0;
   transaction.savepoints= 0;
   first_successful_insert_id_in_cur_stmt= 0;
-  m_metadata_observer= 0;
 }
 
 
@@ -2945,7 +2943,6 @@ void THD::restore_sub_statement_state(Sub_statement_state *backup)
   */
   examined_row_count+= backup->examined_row_count;
   cuted_fields+=       backup->cuted_fields;
-  m_metadata_observer= backup->m_metadata_observer;
 }
 
 

sql/sql_class.h

@@ -45,7 +45,7 @@
   At most 1 instance of this class is active at a time, in which
   case THD::m_metadata_observer is not NULL.
 
-  @sa check_and_update_metadata_version() for details of the
+  @sa check_and_update_table_version() for details of the
   version tracking algorithm 
 
   @sa Execute_observer for details of how we detect that
@@ -847,7 +847,7 @@ class Open_tables_state
     to avoid spurious ER_NEED_REPREPARE errors -- system and
     INFORMATION_SCHEMA tables are not subject to metadata version
     tracking.
-    @sa check_and_update_metadata_version()
+    @sa check_and_update_table_version()
   */
   Metadata_version_observer *m_metadata_observer;
 
@@ -983,25 +983,6 @@ class Sub_statement_state
   bool enable_slow_log;
   bool last_insert_id_used;
   SAVEPOINT *savepoints;
-  /**
-    When inside a substatement (a stored function or trigger
-    statement), clear the metadata observer in THD, if any.
-    Remember the value of the observer here, to be able
-    to restore it when leaving the substatement.
-
-    We reset the observer to suppress errors when a substatement
-    uses temporary tables. If a temporary table does not exist
-    at start of the main statement, it's not prelocked
-    and thus is not validated with other prelocked tables.
-
-    Later on, when the temporary table is opened, metadata
-    versions mismatch, expectedly.
-
-    The proper solution for the problem is to re-validate tables
-    of substatements (Bug#12257, Bug#27011, Bug#32868, Bug#33000),
-    but it's not implemented yet.
-  */
-  Metadata_version_observer *m_metadata_observer;
 };
 
 

sql/sql_parse.cc

@@ -200,19 +200,19 @@ void init_update_queries(void)
 {
   bzero((uchar*) &sql_command_flags, sizeof(sql_command_flags));
 
-  sql_command_flags[SQLCOM_CREATE_TABLE]=   CF_CHANGES_DATA;
+  sql_command_flags[SQLCOM_CREATE_TABLE]=   CF_CHANGES_DATA | CF_REEXECUTION_FRAGILE;
   sql_command_flags[SQLCOM_CREATE_INDEX]=   CF_CHANGES_DATA;
   sql_command_flags[SQLCOM_ALTER_TABLE]=    CF_CHANGES_DATA | CF_WRITE_LOGS_COMMAND;
   sql_command_flags[SQLCOM_TRUNCATE]=       CF_CHANGES_DATA | CF_WRITE_LOGS_COMMAND;
   sql_command_flags[SQLCOM_DROP_TABLE]=     CF_CHANGES_DATA;
-  sql_command_flags[SQLCOM_LOAD]=           CF_CHANGES_DATA;
+  sql_command_flags[SQLCOM_LOAD]=           CF_CHANGES_DATA | CF_REEXECUTION_FRAGILE;
   sql_command_flags[SQLCOM_CREATE_DB]=      CF_CHANGES_DATA;
   sql_command_flags[SQLCOM_DROP_DB]=        CF_CHANGES_DATA;
   sql_command_flags[SQLCOM_RENAME_TABLE]=   CF_CHANGES_DATA;
   sql_command_flags[SQLCOM_BACKUP_TABLE]=   CF_CHANGES_DATA;
   sql_command_flags[SQLCOM_RESTORE_TABLE]=  CF_CHANGES_DATA;
   sql_command_flags[SQLCOM_DROP_INDEX]=     CF_CHANGES_DATA;
-  sql_command_flags[SQLCOM_CREATE_VIEW]=    CF_CHANGES_DATA;
+  sql_command_flags[SQLCOM_CREATE_VIEW]=    CF_CHANGES_DATA | CF_REEXECUTION_FRAGILE;
   sql_command_flags[SQLCOM_DROP_VIEW]=      CF_CHANGES_DATA;
   sql_command_flags[SQLCOM_CREATE_EVENT]=   CF_CHANGES_DATA;
   sql_command_flags[SQLCOM_ALTER_EVENT]=    CF_CHANGES_DATA;
@@ -235,19 +235,21 @@ void init_update_queries(void)
   sql_command_flags[SQLCOM_REPLACE_SELECT]= CF_CHANGES_DATA | CF_HAS_ROW_COUNT |
                                             CF_REEXECUTION_FRAGILE;
   sql_command_flags[SQLCOM_SELECT]=         CF_REEXECUTION_FRAGILE;
-
-  sql_command_flags[SQLCOM_SHOW_STATUS_PROC]= CF_STATUS_COMMAND;
-  sql_command_flags[SQLCOM_SHOW_STATUS]=      CF_STATUS_COMMAND;
-  sql_command_flags[SQLCOM_SHOW_DATABASES]=   CF_STATUS_COMMAND;
-  sql_command_flags[SQLCOM_SHOW_TRIGGERS]=    CF_STATUS_COMMAND;
-  sql_command_flags[SQLCOM_SHOW_EVENTS]=      CF_STATUS_COMMAND;
-  sql_command_flags[SQLCOM_SHOW_OPEN_TABLES]= CF_STATUS_COMMAND;
+  sql_command_flags[SQLCOM_SET_OPTION]=     CF_REEXECUTION_FRAGILE;
+  sql_command_flags[SQLCOM_DO]=             CF_REEXECUTION_FRAGILE;
+
+  sql_command_flags[SQLCOM_SHOW_STATUS_PROC]= CF_STATUS_COMMAND | CF_REEXECUTION_FRAGILE;
+  sql_command_flags[SQLCOM_SHOW_STATUS]=      CF_STATUS_COMMAND | CF_REEXECUTION_FRAGILE;
+  sql_command_flags[SQLCOM_SHOW_DATABASES]=   CF_STATUS_COMMAND | CF_REEXECUTION_FRAGILE;
+  sql_command_flags[SQLCOM_SHOW_TRIGGERS]=    CF_STATUS_COMMAND | CF_REEXECUTION_FRAGILE;
+  sql_command_flags[SQLCOM_SHOW_EVENTS]=      CF_STATUS_COMMAND | CF_REEXECUTION_FRAGILE;
+  sql_command_flags[SQLCOM_SHOW_OPEN_TABLES]= CF_STATUS_COMMAND | CF_REEXECUTION_FRAGILE;
   sql_command_flags[SQLCOM_SHOW_PLUGINS]=     CF_STATUS_COMMAND;
-  sql_command_flags[SQLCOM_SHOW_FIELDS]=      CF_STATUS_COMMAND;
-  sql_command_flags[SQLCOM_SHOW_KEYS]=        CF_STATUS_COMMAND;
-  sql_command_flags[SQLCOM_SHOW_VARIABLES]=   CF_STATUS_COMMAND;
-  sql_command_flags[SQLCOM_SHOW_CHARSETS]=    CF_STATUS_COMMAND;
-  sql_command_flags[SQLCOM_SHOW_COLLATIONS]=  CF_STATUS_COMMAND;
+  sql_command_flags[SQLCOM_SHOW_FIELDS]=      CF_STATUS_COMMAND | CF_REEXECUTION_FRAGILE;
+  sql_command_flags[SQLCOM_SHOW_KEYS]=        CF_STATUS_COMMAND | CF_REEXECUTION_FRAGILE;
+  sql_command_flags[SQLCOM_SHOW_VARIABLES]=   CF_STATUS_COMMAND | CF_REEXECUTION_FRAGILE;
+  sql_command_flags[SQLCOM_SHOW_CHARSETS]=    CF_STATUS_COMMAND | CF_REEXECUTION_FRAGILE;
+  sql_command_flags[SQLCOM_SHOW_COLLATIONS]=  CF_STATUS_COMMAND | CF_REEXECUTION_FRAGILE;
   sql_command_flags[SQLCOM_SHOW_NEW_MASTER]= CF_STATUS_COMMAND;
   sql_command_flags[SQLCOM_SHOW_BINLOGS]= CF_STATUS_COMMAND;
   sql_command_flags[SQLCOM_SHOW_SLAVE_HOSTS]= CF_STATUS_COMMAND;
@@ -271,7 +273,7 @@ void init_update_queries(void)
   sql_command_flags[SQLCOM_SHOW_CREATE_PROC]=  CF_STATUS_COMMAND;
   sql_command_flags[SQLCOM_SHOW_CREATE_FUNC]=  CF_STATUS_COMMAND;
   sql_command_flags[SQLCOM_SHOW_CREATE_TRIGGER]=  CF_STATUS_COMMAND;
-  sql_command_flags[SQLCOM_SHOW_STATUS_FUNC]=  CF_STATUS_COMMAND;
+  sql_command_flags[SQLCOM_SHOW_STATUS_FUNC]=  CF_STATUS_COMMAND | CF_REEXECUTION_FRAGILE;
   sql_command_flags[SQLCOM_SHOW_PROC_CODE]=  CF_STATUS_COMMAND;
   sql_command_flags[SQLCOM_SHOW_FUNC_CODE]=  CF_STATUS_COMMAND;
   sql_command_flags[SQLCOM_SHOW_CREATE_EVENT]=  CF_STATUS_COMMAND;
@@ -279,9 +281,11 @@ void init_update_queries(void)
   sql_command_flags[SQLCOM_SHOW_PROFILE]= CF_STATUS_COMMAND;
 
    sql_command_flags[SQLCOM_SHOW_TABLES]=       (CF_STATUS_COMMAND |
-                                               CF_SHOW_TABLE_COMMAND);
+                                                 CF_SHOW_TABLE_COMMAND |
+                                                 CF_REEXECUTION_FRAGILE);
   sql_command_flags[SQLCOM_SHOW_TABLE_STATUS]= (CF_STATUS_COMMAND |
-                                                CF_SHOW_TABLE_COMMAND);
+                                                CF_SHOW_TABLE_COMMAND |
+                                                CF_REEXECUTION_FRAGILE);
 
   /*
     The following is used to preserver CF_ROW_COUNT during the
@@ -289,7 +293,7 @@ void init_update_queries(void)
     last called (or executed) statement is preserved.
     See mysql_execute_command() for how CF_ROW_COUNT is used.
   */
-  sql_command_flags[SQLCOM_CALL]= 		CF_HAS_ROW_COUNT;
+  sql_command_flags[SQLCOM_CALL]= 		CF_HAS_ROW_COUNT | CF_REEXECUTION_FRAGILE;
   sql_command_flags[SQLCOM_EXECUTE]= 		CF_HAS_ROW_COUNT;
 
   /*

sql/sql_prepare.cc

@@ -1480,6 +1480,43 @@ static bool mysql_test_set_fields(Prepared_statement *stmt,
 }
 
 
+/**
+  Validate and prepare for execution CALL statement expressions.
+
+  @param stmt               prepared statement
+  @param tables             list of tables used in this query
+  @param value_list         list of expressions
+
+  @retval FALSE             success
+  @retval TRUE              error, error message is set in THD
+*/
+
+static bool mysql_test_call_fields(Prepared_statement *stmt,
+                                   TABLE_LIST *tables,
+                                   List<Item> *value_list)
+{
+  DBUG_ENTER("mysql_test_call_fields");
+
+  List_iterator<Item> it(*value_list);
+  THD *thd= stmt->thd;
+  Item *item;
+
+  if (tables && check_table_access(thd, SELECT_ACL, tables, UINT_MAX, FALSE) ||
+      open_normal_and_derived_tables(thd, tables, 0))
+    goto err;
+
+  while ((item= it++))
+  {
+    if (!item->fixed && item->fix_fields(thd, it.ref()) ||
+        item->check_cols(1))
+      goto err;
+  }
+  DBUG_RETURN(FALSE);
+err:
+  DBUG_RETURN(TRUE);
+}
+
+
 /**
   Check internal SELECT of the prepared command.
 
@@ -1601,6 +1638,17 @@ static bool mysql_test_create_table(Prepared_statement *stmt)
 
     res= select_like_stmt_test(stmt, 0, 0);
   }
+  else if (lex->create_info.options & HA_LEX_CREATE_TABLE_LIKE)
+  {
+    /*
+      Check that the source table exist, and also record
+      its metadata version. Even though not strictly necessary,
+      we validate metadata of all CREATE TABLE statements,
+      which keeps metadata validation code simple.
+    */
+    if (open_normal_and_derived_tables(stmt->thd, lex->query_tables, 0))
+      DBUG_RETURN(TRUE);
+  }
 
   /* put tables back for PS rexecuting */
   lex->link_first_table_back(create_table, link_to_local);
@@ -1838,7 +1886,21 @@ static bool check_prepared_statement(Prepared_statement *stmt)
   case SQLCOM_DELETE:
     res= mysql_test_delete(stmt, tables);
     break;
-
+  /* The following allow WHERE clause, so they must be tested like SELECT */
+  case SQLCOM_SHOW_DATABASES:
+  case SQLCOM_SHOW_TABLES:
+  case SQLCOM_SHOW_TRIGGERS:
+  case SQLCOM_SHOW_EVENTS:
+  case SQLCOM_SHOW_OPEN_TABLES:
+  case SQLCOM_SHOW_FIELDS:
+  case SQLCOM_SHOW_KEYS:
+  case SQLCOM_SHOW_COLLATIONS:
+  case SQLCOM_SHOW_CHARSETS:
+  case SQLCOM_SHOW_VARIABLES:
+  case SQLCOM_SHOW_STATUS:
+  case SQLCOM_SHOW_TABLE_STATUS:
+  case SQLCOM_SHOW_STATUS_PROC:
+  case SQLCOM_SHOW_STATUS_FUNC:
   case SQLCOM_SELECT:
     res= mysql_test_select(stmt, tables);
     if (res == 2)
@@ -1863,6 +1925,9 @@ static bool check_prepared_statement(Prepared_statement *stmt)
     res= mysql_test_do_fields(stmt, tables, lex->insert_list);
     break;
 
+  case SQLCOM_CALL:
+    res= mysql_test_call_fields(stmt, tables, &lex->value_list);
+    break;
   case SQLCOM_SET_OPTION:
     res= mysql_test_set_fields(stmt, tables, &lex->var_list);
     break;
@@ -1912,7 +1977,6 @@ static bool check_prepared_statement(Prepared_statement *stmt)
   case SQLCOM_DROP_INDEX:
   case SQLCOM_ROLLBACK:
   case SQLCOM_TRUNCATE:
-  case SQLCOM_CALL:
   case SQLCOM_DROP_VIEW:
   case SQLCOM_REPAIR:
   case SQLCOM_ANALYZE:
@@ -3064,6 +3128,26 @@ bool Prepared_statement::prepare(const char *packet, uint packet_len)
 }
 
 
+/**
+  Assign parameter values either from variables, in case of SQL PS
+  or from the execute packet.
+
+  @param expanded_query  a container with the original SQL statement.
+                         '?' placeholders will be replaced with
+                         their values in case of success.
+                         The result is used for logging and replication
+  @param packet          pointer to execute packet.
+                         NULL in case of SQL PS
+  @param packet_end      end of the packet. NULL in case of SQL PS
+
+  @todo Use a paremeter source class family instead of 'if's, and
+  support stored procedure variables.
+
+  @retval TRUE an error occurred when assigning a parameter (likely
+          a conversion error or out of memory, or malformed packet)
+  @retval FALSE success
+*/
+
 bool
 Prepared_statement::set_parameters(String *expanded_query,
                                    uchar *packet, uchar *packet_end)

sql/table.h

@@ -1336,7 +1336,7 @@ struct TABLE_LIST
     (if any) with values obtained from the current table
     definition cache element.
 
-    @sa check_and_update_metadata_version()
+    @sa check_and_update_table_version()
   */
   inline
   bool is_metadata_version_equal(TABLE_SHARE *s) const
@@ -1349,7 +1349,7 @@ struct TABLE_LIST
     Record the value of metadata version of the corresponding
     table definition cache element in this parse tree node.
 
-    @sa check_and_update_metadata_version()
+    @sa check_and_update_table_version()
   */
   inline
   void set_metadata_version(TABLE_SHARE *s)