Patch for Bug#27863 (excessive memory usage for many small queries in a

multiquery packet).

Background:

  - a query can contain multiple SQL statements;

  - the server frees resources allocated to process a query when the
    whole query is handled. In other words, resources allocated to process
    one SQL statement from a multi-statement query are freed when all SQL
    statements are handled.

The problem was that the parser allocated a buffer of size of the whole
query for each SQL statement in a multi-statement query. Thus, if a query
had many SQL-statements (so, the query was long), but each SQL statement
was short, ther parser tried to allocate huge amount of memory (number of
small SQL statements * length of the whole query).

The memory was allocated for a so-called "cpp buffer", which is intended to
store pre-processed SQL statement -- SQL text without version specific
comments.

The fix is to allocate memory for the "cpp buffer" once for all SQL
statements (once for a query).

mysql-test/t/parser_stack.test

@@ -399,4 +399,12 @@ delimiter ;$$
 
 drop procedure p_37228;
 
+#
+# Bug#27863 (excessive memory usage for many small queries in a multiquery
+# packet).
+#
 
+let $i=`select repeat("set @a=1;", 65535)`;
+--disable_query_log
+eval $i;
+--enable_query_log

sql/ha_ndbcluster_binlog.cc

@@ -263,7 +263,7 @@ static void run_query(THD *thd, char *buf, char *end,
   ulonglong save_thd_options= thd->variables.option_bits;
   DBUG_ASSERT(sizeof(save_thd_options) == sizeof(thd->variables.option_bits));
   NET save_thd_net= thd->net;
-  const char* found_semicolon= NULL;
+  Parser_state parser_state(thd, thd->query(), thd->query_length());
 
   bzero((char*) &thd->net, sizeof(NET));
   thd->set_query(buf, (uint) (end - buf));
@@ -277,7 +277,7 @@ static void run_query(THD *thd, char *buf, char *end,
   DBUG_ASSERT(!thd->in_sub_stmt);
   DBUG_ASSERT(!thd->locked_tables_mode);
 
-  mysql_parse(thd, thd->query(), thd->query_length(), &found_semicolon);
+  mysql_parse(thd, thd->query(), thd->query_length(), &parser_state);
 
   if (no_print_error && thd->is_slave_error)
   {

sql/log_event.cc

@@ -3290,8 +3290,8 @@ int Query_log_event::do_apply_event(Relay_log_info const *rli,
       thd->table_map_for_update= (table_map)table_map_for_update;
       
       /* Execute the query (note that we bypass dispatch_command()) */
-      const char* found_semicolon= NULL;
-      mysql_parse(thd, thd->query(), thd->query_length(), &found_semicolon);
+      Parser_state parser_state(thd, thd->query(), thd->query_length());
+      mysql_parse(thd, thd->query(), thd->query_length(), &parser_state);
       log_slow_statement(thd);
 
       /*

sql/sql_lex.cc

@@ -142,37 +142,64 @@ st_parsing_options::reset()
   allows_derived= TRUE;
 }
 
+
+/**
+  Perform initialization of Lex_input_stream instance.
+
+  Basically, a buffer for pre-processed query. This buffer should be large
+  enough to keep multi-statement query. The allocation is done once in the
+  Lex_input_stream constructor in order to prevent memory pollution when
+  the server is processing large multi-statement queries.
+
+  @todo Check return value of THD::alloc().
+*/
+
 Lex_input_stream::Lex_input_stream(THD *thd,
                                    const char* buffer,
                                    unsigned int length)
-: m_thd(thd),
-  yylineno(1),
-  yytoklen(0),
-  yylval(NULL),
-  lookahead_token(-1),
-  lookahead_yylval(NULL),
-  m_ptr(buffer),
-  m_tok_start(NULL),
-  m_tok_end(NULL),
-  m_end_of_query(buffer + length),
-  m_tok_start_prev(NULL),
-  m_buf(buffer),
-  m_buf_length(length),
-  m_echo(TRUE),
-  m_cpp_tok_start(NULL),
-  m_cpp_tok_start_prev(NULL),
-  m_cpp_tok_end(NULL),
-  m_body_utf8(NULL),
-  m_cpp_utf8_processed_ptr(NULL),
-  next_state(MY_LEX_START),
-  found_semicolon(NULL),
-  ignore_space(test(thd->variables.sql_mode & MODE_IGNORE_SPACE)),
-  stmt_prepare_mode(FALSE),
-  multi_statements(TRUE),
-  in_comment(NO_COMMENT),
-  m_underscore_cs(NULL)
+  :m_thd(thd)
 {
   m_cpp_buf= (char*) thd->alloc(length + 1);
+  reset(buffer, length);
+}
+
+
+/**
+  Prepare Lex_input_stream instance state for use for handling next SQL statement.
+
+  It should be called between two statements in a multi-statement query.
+  The operation resets the input stream to the beginning-of-parse state,
+  but does not reallocate m_cpp_buf.
+*/
+
+void
+Lex_input_stream::reset(const char *buffer, unsigned int length)
+{
+  yylineno= 1;
+  yytoklen= 0;
+  yylval= NULL;
+  lookahead_token= -1;
+  lookahead_yylval= NULL;
+  m_ptr= buffer;
+  m_tok_start= NULL;
+  m_tok_end= NULL;
+  m_end_of_query= buffer + length;
+  m_tok_start_prev= NULL;
+  m_buf= buffer;
+  m_buf_length= length;
+  m_echo= TRUE;
+  m_cpp_tok_start= NULL;
+  m_cpp_tok_start_prev= NULL;
+  m_cpp_tok_end= NULL;
+  m_body_utf8= NULL;
+  m_cpp_utf8_processed_ptr= NULL;
+  next_state= MY_LEX_START;
+  found_semicolon= NULL;
+  ignore_space= test(m_thd->variables.sql_mode & MODE_IGNORE_SPACE);
+  stmt_prepare_mode= FALSE;
+  multi_statements= TRUE;
+  in_comment=NO_COMMENT;
+  m_underscore_cs= NULL;
   m_cpp_ptr= m_cpp_buf;
 }
 

sql/sql_lex.h

@@ -1377,6 +1377,8 @@ public:
   Lex_input_stream(THD *thd, const char* buff, unsigned int length);
   ~Lex_input_stream();
 
+  void reset(const char *buff, unsigned int length);
+
   /**
     Set the echo mode.
 
@@ -2206,8 +2208,8 @@ struct LEX: public Query_tables_list
 class Set_signal_information
 {
 public:
-  /** Constructor. */
-  Set_signal_information();
+  /** Empty default constructor, use clear() */
+ Set_signal_information() {} 
 
   /** Copy constructor. */
   Set_signal_information(const Set_signal_information& set);
@@ -2220,7 +2222,7 @@ public:
   void clear();
 
   /**
-    For each contition item assignment, m_item[] contains the parsed tree
+    For each condition item assignment, m_item[] contains the parsed tree
     that represents the expression assigned, if any.
     m_item[] is an array indexed by Diag_condition_item_name.
   */
@@ -2237,8 +2239,16 @@ class Yacc_state
 {
 public:
   Yacc_state()
-    : yacc_yyss(NULL), yacc_yyvs(NULL)
-  {}
+  {
+    reset();
+  }
+
+  void reset()
+  {
+    yacc_yyss= NULL;
+    yacc_yyvs= NULL;
+    m_set_signal_info.clear();
+  }
 
   ~Yacc_state();
 
@@ -2283,6 +2293,12 @@ public:
 
   Lex_input_stream m_lip;
   Yacc_state m_yacc;
+
+  void reset(const char *found_semicolon, unsigned int length)
+  {
+    m_lip.reset(found_semicolon, length);
+    m_yacc.reset();
+  }
 };
 
 

sql/sql_parse.cc

@@ -458,7 +458,6 @@ static void handle_bootstrap_impl(THD *thd)
 {
   MYSQL_FILE *file= bootstrap_file;
   char *buff;
-  const char* found_semicolon= NULL;
 
   DBUG_ENTER("handle_bootstrap");
 
@@ -531,7 +530,8 @@ static void handle_bootstrap_impl(THD *thd)
       mode we have only one thread.
     */
     thd->set_time();
-    mysql_parse(thd, thd->query(), length, & found_semicolon);
+    Parser_state parser_state(thd, thd->query(), length);
+    mysql_parse(thd, thd->query(), length, &parser_state);
     close_thread_tables(thd);			// Free tables
 
     bootstrap_error= thd->is_error();
@@ -1074,19 +1074,21 @@ bool dispatch_command(enum enum_server_command command, THD *thd,
                       (char *) thd->security_ctx->host_or_ip);
     char *packet_end= thd->query() + thd->query_length();
     /* 'b' stands for 'buffer' parameter', special for 'my_snprintf' */
-    const char* end_of_stmt= NULL;
 
     general_log_write(thd, command, thd->query(), thd->query_length());
     DBUG_PRINT("query",("%-.4096s",thd->query()));
 #if defined(ENABLED_PROFILING)
     thd->profiling.set_query_source(thd->query(), thd->query_length());
 #endif
+    Parser_state parser_state(thd, thd->query(), thd->query_length());
 
-    mysql_parse(thd, thd->query(), thd->query_length(), &end_of_stmt);
+    mysql_parse(thd, thd->query(), thd->query_length(), &parser_state);
 
-    while (!thd->killed && (end_of_stmt != NULL) && ! thd->is_error())
+    while (!thd->killed && (parser_state.m_lip.found_semicolon != NULL) &&
+           ! thd->is_error())
     {
-      char *beginning_of_next_stmt= (char*) end_of_stmt;
+      char *beginning_of_next_stmt= (char*)
+        parser_state.m_lip.found_semicolon;
 
       thd->protocol->end_statement();
       query_cache_end_of_result(thd);
@@ -1127,8 +1129,9 @@ bool dispatch_command(enum enum_server_command command, THD *thd,
       */
       statistic_increment(thd->status_var.questions, &LOCK_status);
       thd->set_time(); /* Reset the query start time. */
+      parser_state.reset(beginning_of_next_stmt, length);
       /* TODO: set thd->lex->sql_command to SQLCOM_END here */
-      mysql_parse(thd, beginning_of_next_stmt, length, &end_of_stmt);
+      mysql_parse(thd, beginning_of_next_stmt, length, &parser_state);
     }
 
     DBUG_PRINT("info",("query ready"));
@@ -5721,7 +5724,7 @@ void mysql_init_multi_delete(LEX *lex)
 */
 
 void mysql_parse(THD *thd, const char *inBuf, uint length,
-                 const char ** found_semicolon)
+                 Parser_state *parser_state)
 {
   int error;
   DBUG_ENTER("mysql_parse");
@@ -5751,10 +5754,7 @@ void mysql_parse(THD *thd, const char *inBuf, uint length,
   {
     LEX *lex= thd->lex;
 
-    Parser_state parser_state(thd, inBuf, length);
-
-    bool err= parse_sql(thd, & parser_state, NULL);
-    *found_semicolon= parser_state.m_lip.found_semicolon;
+    bool err= parse_sql(thd, parser_state, NULL);
 
     if (!err)
     {
@@ -5769,6 +5769,7 @@ void mysql_parse(THD *thd, const char *inBuf, uint length,
       {
 	if (! thd->is_error())
 	{
+          const char *found_semicolon= parser_state->m_lip.found_semicolon;
           /*
             Binlog logs a string starting from thd->query and having length
             thd->query_length; so we set thd->query_length correctly (to not
@@ -5779,12 +5780,12 @@ void mysql_parse(THD *thd, const char *inBuf, uint length,
             PROCESSLIST.
             Note that we don't need LOCK_thread_count to modify query_length.
           */
-          if (*found_semicolon && (ulong) (*found_semicolon - thd->query()))
+          if (found_semicolon && (ulong) (found_semicolon - thd->query()))
             thd->set_query_inner(thd->query(),
-                                 (uint32) (*found_semicolon -
+                                 (uint32) (found_semicolon -
                                            thd->query() - 1));
           /* Actually execute the query */
-          if (*found_semicolon)
+          if (found_semicolon)
           {
             lex->safe_to_cache_query= 0;
             thd->server_status|= SERVER_MORE_RESULTS_EXISTS;
@@ -5821,11 +5822,6 @@ void mysql_parse(THD *thd, const char *inBuf, uint length,
     thd->cleanup_after_query();
     DBUG_ASSERT(thd->change_list.is_empty());
   }
-  else
-  {
-    /* There are no multi queries in the cache. */
-    *found_semicolon= NULL;
-  }
 
   DBUG_VOID_RETURN;
 }

sql/sql_parse.h

@@ -84,7 +84,7 @@ bool is_log_table_write_query(enum enum_sql_command command);
 bool alloc_query(THD *thd, const char *packet, uint packet_length);
 void mysql_init_select(LEX *lex);
 void mysql_parse(THD *thd, const char *inBuf, uint length,
-                 const char ** semicolon);
+                 Parser_state *parser_state);
 void mysql_reset_thd_for_next_command(THD *thd);
 bool mysql_new_select(LEX *lex, bool move_down);
 void create_select_for_variable(const char *var_name);

sql/sql_signal.cc

@@ -75,10 +75,6 @@ const LEX_STRING Diag_statement_item_names[]=
   { C_STRING_WITH_LEN("TRANSACTION_ACTIVE") }
 };
 
-Set_signal_information::Set_signal_information()
-{
-  clear();
-}
 
 Set_signal_information::Set_signal_information(
   const Set_signal_information& set)