Bug#25411 (trigger code truncated), PART I

The issue found with bug 25411 is due to the function skip_rear_comments()
which damages the source code while implementing a work around.
The root cause of the problem is in the lexical analyser, which does not
process special comments properly.
For special comments like :
[1] aaa /*!50000 bbb */ ccc
since 5.0 is a version older that the current code, the parser is in lining
the content of the special comment, so that the query to process is
[2] aaa bbb ccc
However, the text of the query captured when processing a stored procedure,
stored function or trigger (or event in 5.1), can be after rebuilding it:
[3] aaa bbb */ ccc
which is wrong.

To fix bug 25411 properly, the lexical analyser needs to return [2] when
in lining special comments.
In order to implement this, some preliminary cleanup is required in the code,
which is implemented by this patch.

Before this change, the structure named LEX (or st_lex) contains attributes
that belong to lexical analysis, as well as attributes that represents the
abstract syntax tree (AST) of a statement.
Creating a new LEX structure for each statements (which makes sense for the
AST part) also re-initialized the lexical analysis phase each time, which
is conceptually wrong.

With this patch, the previous st_lex structure has been split in two:
- st_lex represents the Abstract Syntax Tree for a statement. The name "lex"
has not been changed to avoid a bigger impact in the code base.
- class lex_input_stream represents the internal state of the lexical
  analyser, which by definition should *not* be reinitialized when parsing
  multiple statements from the same input stream.

This change is a pre-requisite for bug 25411, since the implementation of
lex_input_stream will later improve to deal properly with special comments,
and this processing can not be done with the current implementation of
sp_head::reset_lex and sp_head::restore_lex, which interfere with the lexer.

This change set alone does not fix bug 25411.


sql/item_func.cc:
  Refactoring, separate lex_input_stream from st_lex.
sql/log_event.cc:
  Refactoring, separate lex_input_stream from st_lex.
sql/mysql_priv.h:
  Refactoring, separate lex_input_stream from st_lex.
sql/slave.cc:
  Refactoring, separate lex_input_stream from st_lex.
sql/sp.cc:
  Refactoring, separate lex_input_stream from st_lex.
sql/sp_head.cc:
  Refactoring, separate lex_input_stream from st_lex.
sql/sp_head.h:
  Refactoring, separate lex_input_stream from st_lex.
sql/sql_class.cc:
  Refactoring, separate lex_input_stream from st_lex.
sql/sql_class.h:
  Refactoring, separate lex_input_stream from st_lex.
sql/sql_lex.cc:
  Refactoring, separate lex_input_stream from st_lex.
sql/sql_lex.h:
  Refactoring, separate lex_input_stream from st_lex.
sql/sql_parse.cc:
  Refactoring, separate lex_input_stream from st_lex.
sql/sql_prepare.cc:
  Refactoring, separate lex_input_stream from st_lex.
sql/sql_trigger.cc:
  Refactoring, separate lex_input_stream from st_lex.
sql/sql_view.cc:
  Refactoring, separate lex_input_stream from st_lex.
sql/sql_yacc.yy:
  Refactoring, separate lex_input_stream from st_lex.

mysql-test/r/bdb_notembedded.result

+set autocommit=1;
+reset master;
+create table bug16206 (a int);
+insert into bug16206 values(1);
+start transaction;
+insert into bug16206 values(2);
+commit;
+show binlog events;
+Log_name	Pos	Event_type	Server_id	End_log_pos	Info
+f	n	Format_desc	1	n	Server ver: VERSION, Binlog ver: 4
+f	n	Query	1	n	use `test`; create table bug16206 (a int)
+f	n	Query	1	n	use `test`; insert into bug16206 values(1)
+f	n	Query	1	n	use `test`; insert into bug16206 values(2)
+drop table bug16206;
+reset master;
+create table bug16206 (a int) engine=         bdb;
+insert into bug16206 values(0);
+insert into bug16206 values(1);
+start transaction;
+insert into bug16206 values(2);
+commit;
+insert into bug16206 values(3);
+show binlog events;
+Log_name	Pos	Event_type	Server_id	End_log_pos	Info
+f	n	Format_desc	1	n	Server ver: VERSION, Binlog ver: 4
+f	n	Query	1	n	use `test`; create table bug16206 (a int) engine=         bdb
+f	n	Query	1	n	use `test`; insert into bug16206 values(0)
+f	n	Query	1	n	use `test`; insert into bug16206 values(1)
+f	n	Query	1	n	use `test`; BEGIN
+f	n	Query	1	n	use `test`; insert into bug16206 values(2)
+f	n	Query	1	n	use `test`; COMMIT
+f	n	Query	1	n	use `test`; insert into bug16206 values(3)
+drop table bug16206;
+set autocommit=0;
+End of 5.0 tests

mysql-test/t/bdb_notembedded.test

+-- source include/not_embedded.inc
+-- source include/have_bdb.inc
+
+#
+# Bug #16206: Superfluous COMMIT event in binlog when updating BDB in autocommit mode
+#
+set autocommit=1;
+
+let $VERSION=`select version()`;
+
+reset master;
+create table bug16206 (a int);
+insert into bug16206 values(1);
+start transaction;
+insert into bug16206 values(2);
+commit;
+--replace_result $VERSION VERSION
+--replace_column 1 f 2 n 5 n
+show binlog events;
+drop table bug16206;
+
+reset master;
+create table bug16206 (a int) engine=         bdb;
+insert into bug16206 values(0);
+insert into bug16206 values(1);
+start transaction;
+insert into bug16206 values(2);
+commit;
+insert into bug16206 values(3);
+--replace_result $VERSION VERSION
+--replace_column 1 f 2 n 5 n
+show binlog events;
+drop table bug16206;
+
+set autocommit=0;
+
+
+--echo End of 5.0 tests

sql/item_func.cc

@@ -4272,7 +4272,7 @@ int get_var_with_binlog(THD *thd, enum_sql_command sql_command,
     List<set_var_base> tmp_var_list;
     LEX *sav_lex= thd->lex, lex_tmp;
     thd->lex= &lex_tmp;
-    lex_start(thd, NULL, 0);
+    lex_start(thd);
     tmp_var_list.push_back(new set_var_user(new Item_func_set_user_var(name,
                                                                        new Item_null())));
     /* Create the variable */

sql/log_event.cc

@@ -1879,7 +1879,8 @@ int Query_log_event::exec_event(struct st_relay_log_info* rli,
         thd->variables.collation_database= thd->db_charset;
       
       /* Execute the query (note that we bypass dispatch_command()) */
-      mysql_parse(thd, thd->query, thd->query_length);
+      const char* found_semicolon= NULL;
+      mysql_parse(thd, thd->query, thd->query_length, &found_semicolon);
 
     }
     else
@@ -2987,10 +2988,12 @@ int Load_log_event::exec_event(NET* net, struct st_relay_log_info* rli,
   thd->query_error= 0;
   clear_all_errors(thd, rli);
   /*
-    Usually mysql_init_query() is called by mysql_parse(), but we need it here
+    Usually lex_start() is called by mysql_parse(), but we need it here
     as the present method does not call mysql_parse().
   */
-  mysql_init_query(thd, 0, 0);
+  lex_start(thd);
+  mysql_reset_thd_for_next_command(thd);
+
   if (!use_rli_only_for_errors)
   {
     /* Saved for InnoDB, see comment in Query_log_event::exec_event() */

sql/mysql_priv.h

@@ -697,13 +697,15 @@ bool do_rename(THD *thd, TABLE_LIST *ren_table, char *new_db,
                       char *new_table_name, char *new_table_alias,
                       bool skip_error);
 bool mysql_change_db(THD *thd,const char *name,bool no_access_check);
-void mysql_parse(THD *thd,char *inBuf,uint length);
+
+void mysql_parse(THD *thd, const char *inBuf, uint length,
+                 const char ** semicolon);
+
 bool mysql_test_parse_for_slave(THD *thd,char *inBuf,uint length);
 bool is_update_query(enum enum_sql_command command);
 bool alloc_query(THD *thd, const char *packet, uint packet_length);
 void mysql_init_select(LEX *lex);
 void mysql_reset_thd_for_next_command(THD *thd);
-void mysql_init_query(THD *thd, uchar *buf, uint length);
 bool mysql_new_select(LEX *lex, bool move_down);
 void create_select_for_variable(const char *var_name);
 void mysql_init_multi_delete(LEX *lex);

sql/slave.cc

@@ -1517,6 +1517,7 @@ static int create_table_from_dump(THD* thd, MYSQL *mysql, const char* db,
   handler *file;
   ulonglong save_options;
   NET *net= &mysql->net;
+  const char *found_semicolon= NULL;
   DBUG_ENTER("create_table_from_dump");  
 
   packet_len= my_net_read(net); // read create table statement
@@ -1567,7 +1568,7 @@ static int create_table_from_dump(THD* thd, MYSQL *mysql, const char* db,
   save_db_length= thd->db_length;
   DBUG_ASSERT(db != 0);
   thd->reset_db((char*)db, strlen(db));
-  mysql_parse(thd, thd->query, packet_len); // run create table
+  mysql_parse(thd, thd->query, packet_len, &found_semicolon); // run create table
   thd->db = save_db;		// leave things the way the were before
   thd->db_length= save_db_length;
   thd->options = save_options;

sql/sp.cc

@@ -434,10 +434,15 @@ db_load_routine(THD *thd, int type, sp_name *name, sp_head **sphp,
   if ((ret= sp_use_new_db(thd, name->m_db, &old_db, 1, &dbchanged)))
     goto end;
 
-  lex_start(thd, (uchar*)defstr.c_ptr(), defstr.length());
+  {
+    Lex_input_stream lip(thd, defstr.c_ptr(), defstr.length());
+    thd->m_lip= &lip;
+    lex_start(thd);
+    ret= MYSQLparse(thd);
+  }
 
   thd->spcont= 0;
-  if (MYSQLparse(thd) || thd->is_fatal_error || newlex.sphead == NULL)
+  if (ret || thd->is_fatal_error || newlex.sphead == NULL)
   {
     sp_head *sp= newlex.sphead;
 

sql/sp_head.cc

@@ -519,9 +519,10 @@ void
 sp_head::init_strings(THD *thd, LEX *lex)
 {
   DBUG_ENTER("sp_head::init_strings");
-  uchar *endp;                  /* Used to trim the end */
+  const char *endp;                  /* Used to trim the end */
   /* During parsing, we must use thd->mem_root */
   MEM_ROOT *root= thd->mem_root;
+  Lex_input_stream *lip=thd->m_lip;
 
   if (m_param_begin && m_param_end)
   {
@@ -531,17 +532,17 @@ sp_head::init_strings(THD *thd, LEX *lex)
   }
 
   /* If ptr has overrun end_of_query then end_of_query is the end */
-  endp= (lex->ptr > lex->end_of_query ? lex->end_of_query : lex->ptr);
+  endp= (lip->ptr > lip->end_of_query ? lip->end_of_query : lip->ptr);
   /*
     Trim "garbage" at the end. This is sometimes needed with the
     "/ * ! VERSION... * /" wrapper in dump files.
   */
-  endp= skip_rear_comments(m_body_begin, endp);
+  endp= skip_rear_comments((char*) m_body_begin, (char*) endp);
 
   m_body.length= endp - m_body_begin;
-  m_body.str= strmake_root(root, (char *)m_body_begin, m_body.length);
-  m_defstr.length= endp - lex->buf;
-  m_defstr.str= strmake_root(root, (char *)lex->buf, m_defstr.length);
+  m_body.str= strmake_root(root, m_body_begin, m_body.length);
+  m_defstr.length= endp - lip->buf;
+  m_defstr.str= strmake_root(root, lip->buf, m_defstr.length);
   DBUG_VOID_RETURN;
 }
 
@@ -1756,24 +1757,13 @@ sp_head::reset_lex(THD *thd)
   DBUG_ENTER("sp_head::reset_lex");
   LEX *sublex;
   LEX *oldlex= thd->lex;
-  my_lex_states org_next_state= oldlex->next_state;
 
   (void)m_lex.push_front(oldlex);
   thd->lex= sublex= new st_lex;
 
-  /* Reset most stuff. The length arguments doesn't matter here. */
-  lex_start(thd, oldlex->buf, (ulong) (oldlex->end_of_query - oldlex->ptr));
+  /* Reset most stuff. */
+  lex_start(thd);
 
-  /*
-    next_state is normally the same (0), but it happens that we swap lex in
-    "mid-sentence", so we must restore it.
-   */
-  sublex->next_state= org_next_state;
-  /* We must reset ptr and end_of_query again */
-  sublex->ptr= oldlex->ptr;
-  sublex->end_of_query= oldlex->end_of_query;
-  sublex->tok_start= oldlex->tok_start;
-  sublex->yylineno= oldlex->yylineno;
   /* And keep the SP stuff too */
   sublex->sphead= oldlex->sphead;
   sublex->spcont= oldlex->spcont;
@@ -1806,9 +1796,6 @@ sp_head::restore_lex(THD *thd)
   if (! oldlex)
     return;			// Nothing to restore
 
-  // Update some state in the old one first
-  oldlex->ptr= sublex->ptr;
-  oldlex->next_state= sublex->next_state;
   oldlex->trg_table_fields.push_back(&sublex->trg_table_fields);
 
   /*

sql/sp_head.h

@@ -125,7 +125,7 @@ class sp_head :private Query_arena
 
   create_field m_return_field_def; /* This is used for FUNCTIONs only. */
 
-  uchar *m_tmp_query;		// Temporary pointer to sub query string
+  const char *m_tmp_query;	// Temporary pointer to sub query string
   uint m_old_cmq;		// Old CLIENT_MULTI_QUERIES value
   st_sp_chistics *m_chistics;
   ulong m_sql_mode;		// For SHOW CREATE and execution
@@ -174,7 +174,9 @@ class sp_head :private Query_arena
   */
   HASH m_sroutines;
   // Pointers set during parsing
-  uchar *m_param_begin, *m_param_end, *m_body_begin;
+  const char *m_param_begin;
+  const char *m_param_end;
+  const char *m_body_begin;
 
   /*
     Security context for stored routine which should be run under

sql/sql_class.cc

@@ -176,7 +176,7 @@ THD::THD()
    rand_used(0), time_zone_used(0),
    last_insert_id_used(0), last_insert_id_used_bin_log(0), insert_id_used(0),
    clear_next_insert_id(0), in_lock_tables(0), bootstrap(0),
-   derived_tables_processing(FALSE), spcont(NULL)
+   derived_tables_processing(FALSE), spcont(NULL), m_lip(NULL)
 {
   ulong tmp;
 

sql/sql_class.h

@@ -28,6 +28,7 @@ class Slave_log_event;
 class Format_description_log_event;
 class sp_rcontext;
 class sp_cache;
+class Lex_input_stream;
 
 enum enum_enable_or_disable { LEAVE_AS_IS, ENABLE, DISABLE };
 enum enum_ha_read_modes { RFIRST, RNEXT, RPREV, RLAST, RKEY, RNEXT_SAME };
@@ -1492,6 +1493,15 @@ class THD :public Statement,
     query_id_t first_query_id;
   } binlog_evt_union;
 
+  /**
+    Character input stream consumed by the lexical analyser,
+    used during parsing.
+    Note that since the parser is not re-entrant, we keep only one input
+    stream here. This member is valid only when executing code during parsing,
+    and may point to invalid memory after that.
+  */
+  Lex_input_stream *m_lip;
+
   THD();
   ~THD();
 

sql/sql_lex.h

@@ -469,7 +469,7 @@ class st_select_lex_unit: public st_select_lex_node {
   void set_limit(st_select_lex *values);
   void set_thd(THD *thd_arg) { thd= thd_arg; }
 
-  friend void lex_start(THD *thd, uchar *buf, uint length);
+  friend void lex_start(THD *thd);
   friend int subselect_union_engine::exec();
 
   List<Item> *get_unit_column_types();
@@ -675,7 +675,7 @@ class st_select_lex: public st_select_lex_node
   void cut_subtree() { slave= 0; }
   bool test_limit();
 
-  friend void lex_start(THD *thd, uchar *buf, uint length);
+  friend void lex_start(THD *thd);
   st_select_lex() : n_sum_items(0), n_child_sum_items(0) {}
   void make_empty_select()
   {
@@ -898,30 +898,70 @@ struct st_parsing_options
 };
 
 
+/**
+  This class represents the character input stream consumed during
+  lexical analysis.
+*/
+class Lex_input_stream
+{
+public:
+  Lex_input_stream(THD *thd, const char* buff, unsigned int length);
+  ~Lex_input_stream();
+
+  /** Current thread. */
+  THD *m_thd;
+
+  /** Current line number. */
+  uint yylineno;
+
+  /** Length of the last token parsed. */
+  uint yytoklen;
+
+  /** Interface with bison, value of the last token parsed. */
+  LEX_YYSTYPE yylval;
+
+  /** Pointer to the current position in the input stream. */
+  const char* ptr;
+
+  /** Starting position of the last token parsed. */
+  const char* tok_start;
+
+  /** Ending position of the last token parsed. */
+  const char* tok_end;
+
+  /** End of the query text in the input stream. */
+  const char* end_of_query;
+
+  /** Starting position of the previous token parsed. */
+  const char* tok_start_prev;
+
+  /** Begining of the query text in the input stream. */
+  const char* buf;
+
+  /** Current state of the lexical analyser. */
+  enum my_lex_states next_state;
+
+  /** Position of ';' in the stream, to delimit multiple queries. */
+  const char* found_semicolon;
+};
+
+
 /* The state of the lex parsing. This is saved in the THD struct */
 
 typedef struct st_lex : public Query_tables_list
 {
-  uint	 yylineno,yytoklen;			/* Simulate lex */
-  LEX_YYSTYPE yylval;
   SELECT_LEX_UNIT unit;                         /* most upper unit */
   SELECT_LEX select_lex;                        /* first SELECT_LEX */
   /* current SELECT_LEX in parsing */
   SELECT_LEX *current_select;
   /* list of all SELECT_LEX */
   SELECT_LEX *all_selects_list;
-  uchar *buf;			/* The beginning of string, used by SPs */
-  uchar *ptr,*tok_start,*tok_end,*end_of_query;
-  
-  /* The values of tok_start/tok_end as they were one call of MYSQLlex before */
-  uchar *tok_start_prev, *tok_end_prev;
 
   char *length,*dec,*change,*name;
   char *help_arg;
   char *backup_dir;				/* For RESTORE/BACKUP */
   char* to_log;                                 /* For PURGE MASTER LOGS TO */
   char* x509_subject,*x509_issuer,*ssl_cipher;
-  char* found_semicolon;                        /* For multi queries - next query */
   String *wild;
   sql_exchange *exchange;
   select_result *result;
@@ -990,7 +1030,6 @@ typedef struct st_lex : public Query_tables_list
   enum_sql_command sql_command, orig_sql_command;
   thr_lock_type lock_option;
   enum SSL_type ssl_type;			/* defined in violite.h */
-  enum my_lex_states next_state;
   enum enum_duplicates duplicates;
   enum enum_tx_isolation tx_isolation;
   enum enum_ha_read_modes ha_read_mode;
@@ -1101,8 +1140,9 @@ typedef struct st_lex : public Query_tables_list
     Pointers to part of LOAD DATA statement that should be rewritten
     during replication ("LOCAL 'filename' REPLACE INTO" part).
   */
-  uchar *fname_start, *fname_end;
-  
+  const char *fname_start;
+  const char *fname_end;
+
   bool escape_used;
 
   st_lex();
@@ -1211,7 +1251,7 @@ struct st_lex_local: public st_lex
 
 extern void lex_init(void);
 extern void lex_free(void);
-extern void lex_start(THD *thd, uchar *buf,uint length);
+extern void lex_start(THD *thd);
 extern void lex_end(LEX *lex);
 extern int MYSQLlex(void *arg, void *yythd);
-extern uchar *skip_rear_comments(uchar *begin, uchar *end);
+extern char *skip_rear_comments(char *begin, char *end);

sql/sql_parse.cc

@@ -1236,6 +1236,7 @@ pthread_handler_t handle_bootstrap(void *arg)
   THD *thd=(THD*) arg;
   FILE *file=bootstrap_file;
   char *buff;
+  const char* found_semicolon= NULL;
 
   /* The following must be called before DBUG_ENTER */
   thd->thread_stack= (char*) &thd;
@@ -1312,7 +1313,7 @@ pthread_handler_t handle_bootstrap(void *arg)
     */
     thd->query_id=next_query_id();
     thd->set_time();
-    mysql_parse(thd,thd->query,length);
+    mysql_parse(thd, thd->query, length, & found_semicolon);
     close_thread_tables(thd);			// Free tables
 
     if (thd->is_fatal_error)
@@ -1789,17 +1790,19 @@ bool dispatch_command(enum enum_server_command command, THD *thd,
     char *packet_end= thd->query + thd->query_length;
     /* 'b' stands for 'buffer' parameter', special for 'my_snprintf' */
     const char *format= "%.*b";
+    const char* found_semicolon= NULL;
+
     mysql_log.write(thd,command, format, thd->query_length, thd->query);
     DBUG_PRINT("query",("%-.4096s",thd->query));
 
     if (!(specialflag & SPECIAL_NO_PRIOR))
       my_pthread_setprio(pthread_self(),QUERY_PRIOR);
 
-    mysql_parse(thd,thd->query, thd->query_length);
+    mysql_parse(thd, thd->query, thd->query_length, & found_semicolon);
 
-    while (!thd->killed && thd->lex->found_semicolon && !thd->net.report_error)
+    while (!thd->killed && found_semicolon && !thd->net.report_error)
     {
-      char *next_packet= thd->lex->found_semicolon;
+      char *next_packet= (char*) found_semicolon;
       net->no_send_error= 0;
       /*
         Multiple queries exits, execute them individually
@@ -1824,7 +1827,7 @@ bool dispatch_command(enum enum_server_command command, THD *thd,
       thd->set_time(); /* Reset the query start time. */
       /* TODO: set thd->lex->sql_command to SQLCOM_END here */
       VOID(pthread_mutex_unlock(&LOCK_thread_count));
-      mysql_parse(thd, next_packet, length);
+      mysql_parse(thd, next_packet, length, & found_semicolon);
     }
 
     if (!(specialflag & SPECIAL_NO_PRIOR))
@@ -1845,7 +1848,7 @@ bool dispatch_command(enum enum_server_command command, THD *thd,
     LEX_STRING conv_name;
 
     /* used as fields initializator */
-    lex_start(thd, 0, 0);
+    lex_start(thd);
 
     statistic_increment(thd->status_var.com_stat[SQLCOM_SHOW_FIELDS],
 			&LOCK_status);
@@ -1882,7 +1885,10 @@ bool dispatch_command(enum enum_server_command command, THD *thd,
       break;
     /* init structures for VIEW processing */
     table_list.select_lex= &(thd->lex->select_lex);
-    mysql_init_query(thd, (uchar*)"", 0);
+
+    lex_start(thd);
+    mysql_reset_thd_for_next_command(thd);
+
     thd->lex->
       select_lex.table_list.link_in_list((byte*) &table_list,
                                          (byte**) &table_list.next_local);
@@ -5687,20 +5693,6 @@ bool my_yyoverflow(short **yyss, YYSTYPE **yyvs, ulong *yystacksize)
 }
 
 
-/****************************************************************************
-  Initialize global thd variables needed for query
-****************************************************************************/
-
-void
-mysql_init_query(THD *thd, uchar *buf, uint length)
-{
-  DBUG_ENTER("mysql_init_query");
-  lex_start(thd, buf, length);
-  mysql_reset_thd_for_next_command(thd);
-  DBUG_VOID_RETURN;
-}
-
-
 /*
  Reset THD part responsible for command processing state.
 
@@ -5887,21 +5879,54 @@ void mysql_init_multi_delete(LEX *lex)
   mysql_test_parse_for_slave() in this same file.
 */
 
-void mysql_parse(THD *thd, char *inBuf, uint length)
+/**
+  Parse a query.
+  @param thd Current thread
+  @param inBuf Begining of the query text
+  @param length Length of the query text
+  @param [out] semicolon For multi queries, position of the character of
+  the next query in the query text.
+*/
+
+void mysql_parse(THD *thd, const char *inBuf, uint length,
+                 const char ** found_semicolon)
 {
   DBUG_ENTER("mysql_parse");
 
   DBUG_EXECUTE_IF("parser_debug", turn_parser_debug_on(););
 
-  mysql_init_query(thd, (uchar*) inBuf, length);
-  if (query_cache_send_result_to_client(thd, inBuf, length) <= 0)
+  /*
+    Warning.
+    The purpose of query_cache_send_result_to_client() is to lookup the
+    query in the query cache first, to avoid parsing and executing it.
+    So, the natural implementation would be to:
+    - first, call query_cache_send_result_to_client,
+    - second, if caching failed, initialise the lexical and syntactic parser.
+    The problem is that the query cache depends on a clean initialization
+    of the thd and thd->lex structures, which happen to be implemented
+    by:
+    - lex_start()
+    - mysql_reset_thd_for_next_command()
+    So, initializing the lexical analyser *before* using the query cache
+    is required for the cache to work properly.
+    FIXME: cleanup the dependencies in the code to simplify this.
+  */
+  Lex_input_stream lip(thd, inBuf, length);
+  thd->m_lip= &lip;
+  lex_start(thd);
+  mysql_reset_thd_for_next_command(thd);
+
+  if (query_cache_send_result_to_client(thd, (char*) inBuf, length) <= 0)
   {
     LEX *lex= thd->lex;
-    
+
     sp_cache_flush_obsolete(&thd->sp_proc_cache);
     sp_cache_flush_obsolete(&thd->sp_func_cache);
-    
-    if (!MYSQLparse((void *)thd) && ! thd->is_fatal_error)
+
+    int err= MYSQLparse(thd);
+    *found_semicolon= lip.found_semicolon;
+
+    if (!err && ! thd->is_fatal_error)
     {
 #ifndef NO_EMBEDDED_ACCESS_CHECKS
       if (mqh_used && thd->user_connect &&
@@ -5924,8 +5949,8 @@ void mysql_parse(THD *thd, char *inBuf, uint length)
             PROCESSLIST.
             Note that we don't need LOCK_thread_count to modify query_length.
           */
-          if (lex->found_semicolon &&
-              (thd->query_length= (ulong)(lex->found_semicolon - thd->query)))
+          if (lip.found_semicolon &&
+              (thd->query_length= (ulong)(lip.found_semicolon - thd->query)))
             thd->query_length--;
           /* Actually execute the query */
 	  mysql_execute_command(thd);
@@ -5952,6 +5977,12 @@ void mysql_parse(THD *thd, char *inBuf, uint length)
     thd->cleanup_after_query();
     DBUG_ASSERT(thd->change_list.is_empty());
   }
+  else
+  {
+    /* There are no multi queries in the cache. */
+    *found_semicolon= NULL;
+  }
+
   DBUG_VOID_RETURN;
 }
 
@@ -5972,8 +6003,13 @@ bool mysql_test_parse_for_slave(THD *thd, char *inBuf, uint length)
   bool error= 0;
   DBUG_ENTER("mysql_test_parse_for_slave");
 
-  mysql_init_query(thd, (uchar*) inBuf, length);
-  if (!MYSQLparse((void*) thd) && ! thd->is_fatal_error &&
+  Lex_input_stream lip(thd, inBuf, length);
+  thd->m_lip= &lip;
+  lex_start(thd);
+  mysql_reset_thd_for_next_command(thd);
+  int err= MYSQLparse((void*) thd);
+
+  if (!err && ! thd->is_fatal_error &&
       all_tables_not_ok(thd,(TABLE_LIST*) lex->select_lex.table_list.first))
     error= 1;                  /* Ignore question */
   thd->end_statement();
@@ -7032,8 +7068,9 @@ bool check_simple_select()
   if (lex->current_select != &lex->select_lex)
   {
     char command[80];
-    strmake(command, lex->yylval->symbol.str,
-	    min(lex->yylval->symbol.length, sizeof(command)-1));
+    Lex_input_stream *lip= thd->m_lip;
+    strmake(command, lip->yylval->symbol.str,
+	    min(lip->yylval->symbol.length, sizeof(command)-1));
     my_error(ER_CANT_USE_OPTION_HERE, MYF(0), command);
     return 1;
   }

sql/sql_prepare.cc

@@ -2799,11 +2799,15 @@ bool Prepared_statement::prepare(const char *packet, uint packet_len)
 
   old_stmt_arena= thd->stmt_arena;
   thd->stmt_arena= this;
-  lex_start(thd, (uchar*) thd->query, thd->query_length);
+
+  Lex_input_stream lip(thd, thd->query, thd->query_length);
+  thd->m_lip= &lip;
+  lex_start(thd);
   lex->safe_to_cache_query= FALSE;
   lex->stmt_prepare_mode= TRUE;
+  int err= MYSQLparse((void *)thd);
 
-  error= MYSQLparse((void *)thd) || thd->is_fatal_error ||
+  error= err || thd->is_fatal_error ||
       thd->net.report_error || init_param_array(this);
 
   /*

sql/sql_trigger.cc

@@ -978,10 +978,14 @@ bool Table_triggers_list::check_n_load(THD *thd, const char *db,
         LEX_STRING *trg_definer= it_definer++;
 
         thd->variables.sql_mode= (ulong)*trg_sql_mode;
-        lex_start(thd, (uchar*)trg_create_str->str, trg_create_str->length);
 
+        Lex_input_stream lip(thd, trg_create_str->str, trg_create_str->length);
+        thd->m_lip= &lip;
+        lex_start(thd);
 	thd->spcont= 0;
-        if (MYSQLparse((void *)thd) || thd->is_fatal_error)
+        int err= MYSQLparse((void *)thd);
+
+        if (err || thd->is_fatal_error)
         {
           /* Currently sphead is always deleted in case of a parse error */
           DBUG_ASSERT(lex.sphead == 0);

sql/sql_view.cc

@@ -761,8 +761,8 @@ static int mysql_register_view(THD *thd, TABLE_LIST *view,
   view->query.str= (char*)str.ptr();
   view->query.length= str.length()-1; // we do not need last \0
   view->source.str= thd->query + thd->lex->create_view_select_start;
-  view->source.length= (char *)skip_rear_comments((uchar *)view->source.str,
-                                                  (uchar *)thd->query +
+  view->source.length= (char *)skip_rear_comments((char *)view->source.str,
+                                                  (char *)thd->query +
                                                   thd->query_length) -
                         view->source.str;
   view->file_version= 1;
@@ -973,10 +973,14 @@ bool mysql_make_view(THD *thd, File_parser *parser, TABLE_LIST *table,
     now Lex placed in statement memory
   */
   table->view= lex= thd->lex= (LEX*) new(thd->mem_root) st_lex_local;
-  lex_start(thd, (uchar*)table->query.str, table->query.length);
-  view_select= &lex->select_lex;
-  view_select->select_number= ++thd->select_number;
+
   {
+    Lex_input_stream lip(thd, table->query.str, table->query.length);
+    thd->m_lip= &lip;
+    lex_start(thd);
+    view_select= &lex->select_lex;
+    view_select->select_number= ++thd->select_number;
+
     ulong save_mode= thd->variables.sql_mode;
     /* switch off modes which can prevent normal parsing of VIEW
       - MODE_REAL_AS_FLOAT            affect only CREATE TABLE parsing