Bug#27430 "Crash in subquery code when in PS and table DDL changed after

 PREPARE", review fixes:
- make the patch follow the specification of WL#4166 and remove  
the new error that was originally introduced.
Now the client never gets an error from reprepare, unless it failed.
I.e. even if the statement at hand returns a completely different
result set, this is not considered a server error.
The C API library, that can not handle this situation, was modified to
return a client error.
Added additional test coverage.


include/errmsg.h:
  Add a new client side error: now when we automatically
  reprepare a statement, the new result set may contain a different
  number of columns.
include/mysql_com.h:
  Add a new server status to be sent to the client if the 
  number of columns in the result set is different.
libmysql/errmsg.c:
  Add a new error message.
libmysql/libmysql.c:
  Make the client library robust against a result set that
  contains a different number of columns from prepare time.
  Previously that could never happen, and we simply had an assert.
  That means in particular that all clients are advised to upgrade
  with transition to 5.1, if they are using prepared statements C API.
  Make mysql_stmt_store_result() and mysql_stmt_execute() robust against 
  "broken" statement handles (those that have an error).
sql/sql_parse.cc:
  Clear transient server status flags at start of statement more 
  systematically.
sql/share/errmsg.txt:
  Remove an error that is unused and is not part of any public release.
sql/sql_prepare.cc:
  Instead of returning an error in case the number of result set columns
  has changed, simply update the client in server status.
  That will allow modern clients automatically recover from an error.
tests/mysql_client_test.c:
  Add additional coverage to the cases when the number of result
  set columns changed as a result of reprepare.
  Cover conversion and truncation of result set columns.

include/errmsg.h

@@ -96,6 +96,7 @@ extern const char *client_errors[];	/* Error messages */
 #define CR_NOT_IMPLEMENTED                      2054
 #define CR_SERVER_LOST_EXTENDED			2055
 #define CR_STMT_CLOSED				2056
-#define CR_ERROR_LAST  /*Copy last error nr:*/  2056
+#define CR_NEW_STMT_METADATA                    2057
+#define CR_ERROR_LAST  /*Copy last error nr:*/  2057
 /* Add error numbers before CR_ERROR_LAST and change it accordingly. */
 

include/mysql_com.h

@@ -184,19 +184,38 @@ enum enum_server_command
 #define SERVER_MORE_RESULTS_EXISTS 8    /* Multi query - next query exists */
 #define SERVER_QUERY_NO_GOOD_INDEX_USED 16
 #define SERVER_QUERY_NO_INDEX_USED      32
-/*
+/**
   The server was able to fulfill the clients request and opened a
   read-only non-scrollable cursor for a query. This flag comes
   in reply to COM_STMT_EXECUTE and COM_STMT_FETCH commands.
 */
 #define SERVER_STATUS_CURSOR_EXISTS 64
-/*
+/**
   This flag is sent when a read-only cursor is exhausted, in reply to
   COM_STMT_FETCH command.
 */
 #define SERVER_STATUS_LAST_ROW_SENT 128
 #define SERVER_STATUS_DB_DROPPED        256 /* A database was dropped */
 #define SERVER_STATUS_NO_BACKSLASH_ESCAPES 512
+/**
+  Sent to the client if after a prepared statement reprepare
+  we discovered that the new statement returns a different 
+  number of result set columns.
+*/
+#define SERVER_STATUS_METADATA_CHANGED 1024
+
+/**
+  Server status flags that must be cleared when starting
+  execution of a new SQL statement.
+  Flags from this set are only added to the
+  current server status by the execution engine, but 
+  never removed -- the execution engine expects them 
+  to disappear automagically by the next command.
+*/
+#define SERVER_STATUS_CLEAR_SET (SERVER_QUERY_NO_GOOD_INDEX_USED| \
+                                 SERVER_QUERY_NO_INDEX_USED|\
+                                 SERVER_MORE_RESULTS_EXISTS|\
+                                 SERVER_STATUS_METADATA_CHANGED)
 
 #define MYSQL_ERRMSG_SIZE	512
 #define NET_READ_TIMEOUT	30		/* Timeout on read */
@@ -205,6 +224,7 @@ enum enum_server_command
 
 #define ONLY_KILL_QUERY         1
 
+
 struct st_vio;					/* Only C */
 typedef struct st_vio Vio;
 

libmysql/errmsg.c

@@ -84,6 +84,7 @@ const char *client_errors[]=
   "This feature is not implemented yet",
   "Lost connection to MySQL server at '%s', system error: %d",
   "Statement closed indirectly because of a preceeding %s() call",
+  "The number of columns in the result set differs from the number of bound buffers. You must reset the statement, rebind the result set columns, and execute the statement again",
   ""
 };
 
@@ -149,6 +150,7 @@ const char *client_errors[]=
   "This feature is not implemented yet",
   "Lost connection to MySQL server at '%s', system error: %d",
   "Statement closed indirectly because of a preceeding %s() call",
+  "The number of columns in the result set differs from the number of bound buffers. You must reset the statement, rebind the result set columns, and execute the statement again",
   ""
 };
 
@@ -212,6 +214,7 @@ const char *client_errors[]=
   "This feature is not implemented yet",
   "Lost connection to MySQL server at '%s', system error: %d",
   "Statement closed indirectly because of a preceeding %s() call",
+  "The number of columns in the result set differs from the number of bound buffers. You must reset the statement, rebind the result set columns, and execute the statement again",
   ""
 };
 #endif

libmysql/libmysql.c

@@ -1706,6 +1706,7 @@ static my_bool setup_one_fetch_function(MYSQL_BIND *, MYSQL_FIELD *field);
 #define RESET_SERVER_SIDE 1
 #define RESET_LONG_DATA 2
 #define RESET_STORE_RESULT 4
+#define RESET_CLEAR_ERROR 8
 
 static my_bool reset_stmt_handle(MYSQL_STMT *stmt, uint flags);
 
@@ -2090,7 +2091,7 @@ mysql_stmt_prepare(MYSQL_STMT *stmt, const char *query, ulong length)
   To be removed when all commands will fully support prepared mode.
 */
 
-static unsigned int alloc_stmt_fields(MYSQL_STMT *stmt)
+static void alloc_stmt_fields(MYSQL_STMT *stmt)
 {
   MYSQL_FIELD *fields, *field, *end;
   MEM_ROOT *alloc= &stmt->mem_root;
@@ -2108,7 +2109,10 @@ static unsigned int alloc_stmt_fields(MYSQL_STMT *stmt)
       !(stmt->bind= (MYSQL_BIND *) alloc_root(alloc,
 					      sizeof(MYSQL_BIND) *
 					      stmt->field_count)))
-    return 0;
+  {
+    set_stmt_error(stmt, CR_OUT_OF_MEMORY, unknown_sqlstate, NULL);
+    return;
+  }
 
   for (fields= mysql->fields, end= fields+stmt->field_count,
 	 field= stmt->fields;
@@ -2127,13 +2131,15 @@ static unsigned int alloc_stmt_fields(MYSQL_STMT *stmt)
     field->def      = fields->def ? strdup_root(alloc,fields->def): 0;
     field->max_length= 0;
   }
-  return stmt->field_count;
 }
 
 
-/*
+/**
   Update result set columns metadata if it was sent again in
   reply to COM_STMT_EXECUTE.
+
+  @note If the new field count is different from the original one,
+        an error is set and no update is performed.
 */
 
 static void update_stmt_fields(MYSQL_STMT *stmt)
@@ -2143,7 +2149,22 @@ static void update_stmt_fields(MYSQL_STMT *stmt)
   MYSQL_FIELD *stmt_field= stmt->fields;
   MYSQL_BIND *my_bind= stmt->bind_result_done ? stmt->bind : 0;
 
-  DBUG_ASSERT(stmt->field_count == stmt->mysql->field_count);
+  if (stmt->field_count != stmt->mysql->field_count)
+  {
+    /*
+      The tables used in the statement were altered,
+      and the query now returns a different number of columns.
+      There is no way to continue without reallocating the bind
+      array:
+      - if the number of columns increased, mysql_stmt_fetch()
+      will write beyond allocated memory
+      - if the number of columns decreased, some user-bound
+      buffers will be left unassigned without user knowing
+      that.
+    */
+    set_stmt_error(stmt, CR_NEW_STMT_METADATA, unknown_sqlstate, NULL);
+    return;
+  }
 
   for (; field < field_end; ++field, ++stmt_field)
   {
@@ -2792,6 +2813,50 @@ my_bool STDCALL mysql_stmt_attr_get(MYSQL_STMT *stmt,
 }
 
 
+/**
+  Update statement result set metadata from with the new field
+  information sent during statement execute.
+
+  @pre mysql->field_count is not zero
+
+  @retval TRUE   if error: out of memory or the new
+                 result set has a different number of columns
+  @retval FALSE  success
+*/
+
+static void reinit_result_set_metadata(MYSQL_STMT *stmt)
+{
+  /* Server has sent result set metadata */
+  if (stmt->field_count == 0)
+  {
+    /*
+      This is 'SHOW'/'EXPLAIN'-like query. Current implementation of
+      prepared statements can't send result set metadata for these queries
+      on prepare stage. Read it now.
+    */
+    alloc_stmt_fields(stmt);
+  }
+  else
+  {
+    /*
+      Update result set metadata if it for some reason changed between
+      prepare and execute, i.e.:
+      - in case of 'SELECT ?' we don't know column type unless data was
+      supplied to mysql_stmt_execute, so updated column type is sent
+      now.
+      - if data dictionary changed between prepare and execute, for
+      example a table used in the query was altered.
+      Note, that now (4.1.3) we always send metadata in reply to
+      COM_STMT_EXECUTE (even if it is not necessary), so either this or
+      previous branch always works.
+      TODO: send metadata only when it's really necessary and add a warning
+      'Metadata changed' when it's sent twice.
+      */
+    update_stmt_fields(stmt);
+  }
+}
+
+
 /*
   Send placeholders data to server (if there are placeholders)
   and execute prepared statement.
@@ -2847,7 +2912,7 @@ int STDCALL mysql_stmt_execute(MYSQL_STMT *stmt)
     DBUG_RETURN(1);
   }
 
-  if (reset_stmt_handle(stmt, RESET_STORE_RESULT))
+  if (reset_stmt_handle(stmt, RESET_STORE_RESULT | RESET_CLEAR_ERROR))
     DBUG_RETURN(1);
   /*
     No need to check for stmt->state: if the statement wasn't
@@ -2855,40 +2920,10 @@ int STDCALL mysql_stmt_execute(MYSQL_STMT *stmt)
   */
   if (mysql->methods->stmt_execute(stmt))
     DBUG_RETURN(1);
-  if (mysql->field_count)
-  {
-    /* Server has sent result set metadata */
-    if (stmt->field_count == 0)
-    {
-      /*
-        This is 'SHOW'/'EXPLAIN'-like query. Current implementation of
-        prepared statements can't send result set metadata for these queries
-        on prepare stage. Read it now.
-      */
-      alloc_stmt_fields(stmt);
-    }
-    else
-    {
-      /*
-        Update result set metadata if it for some reason changed between
-        prepare and execute, i.e.:
-        - in case of 'SELECT ?' we don't know column type unless data was
-          supplied to mysql_stmt_execute, so updated column type is sent
-          now.
-        - if data dictionary changed between prepare and execute, for
-          example a table used in the query was altered.
-        Note, that now (4.1.3) we always send metadata in reply to
-        COM_STMT_EXECUTE (even if it is not necessary), so either this or
-        previous branch always works.
-        TODO: send metadata only when it's really necessary and add a warning
-        'Metadata changed' when it's sent twice.
-      */
-      update_stmt_fields(stmt);
-    }
-  }
   stmt->state= MYSQL_STMT_EXECUTE_DONE;
-  if (stmt->field_count)
+  if (mysql->field_count)
   {
+    reinit_result_set_metadata(stmt);
     if (stmt->server_status & SERVER_STATUS_CURSOR_EXISTS)
     {
       mysql->status= MYSQL_STATUS_READY;
@@ -2903,7 +2938,7 @@ int STDCALL mysql_stmt_execute(MYSQL_STMT *stmt)
         network or b) is more efficient if all (few) result set rows are
         precached on client and server's resources are freed.
       */
-      DBUG_RETURN(mysql_stmt_store_result(stmt));
+      mysql_stmt_store_result(stmt);
     }
     else
     {
@@ -2912,7 +2947,7 @@ int STDCALL mysql_stmt_execute(MYSQL_STMT *stmt)
       stmt->read_row_func= stmt_read_row_unbuffered;
     }
   }
-  DBUG_RETURN(0);
+  DBUG_RETURN(test(stmt->last_errno));
 }
 
 
@@ -4766,6 +4801,12 @@ int STDCALL mysql_stmt_store_result(MYSQL_STMT *stmt)
     DBUG_RETURN(1);
   }
 
+  if (stmt->last_errno)
+  {
+    /* An attempt to use an invalid statement handle. */
+    DBUG_RETURN(1);
+  }
+
   if (mysql->status == MYSQL_STATUS_READY &&
       stmt->server_status & SERVER_STATUS_CURSOR_EXISTS)
   {
@@ -4973,9 +5014,10 @@ static my_bool reset_stmt_handle(MYSQL_STMT *stmt, uint flags)
           stmt->state= MYSQL_STMT_INIT_DONE;
           return 1;
         }
-        stmt_clear_error(stmt);
       }
     }
+    if (flags & RESET_CLEAR_ERROR)
+      stmt_clear_error(stmt);
     stmt->state= MYSQL_STMT_PREPARE_DONE;
   }
   return 0;
@@ -4986,7 +5028,8 @@ my_bool STDCALL mysql_stmt_free_result(MYSQL_STMT *stmt)
   DBUG_ENTER("mysql_stmt_free_result");
 
   /* Free the client side and close the server side cursor if there is one */
-  DBUG_RETURN(reset_stmt_handle(stmt, RESET_LONG_DATA | RESET_STORE_RESULT));
+  DBUG_RETURN(reset_stmt_handle(stmt, RESET_LONG_DATA | RESET_STORE_RESULT |
+                                RESET_CLEAR_ERROR));
 }
 
 /********************************************************************
@@ -5067,7 +5110,9 @@ my_bool STDCALL mysql_stmt_reset(MYSQL_STMT *stmt)
     DBUG_RETURN(1);
   }
   /* Reset the client and server sides of the prepared statement */
-  DBUG_RETURN(reset_stmt_handle(stmt, RESET_SERVER_SIDE | RESET_LONG_DATA));
+  DBUG_RETURN(reset_stmt_handle(stmt,
+                                RESET_SERVER_SIDE | RESET_LONG_DATA |
+                                RESET_CLEAR_ERROR));
 }
 
 /*

sql/share/errmsg.txt

@@ -6130,6 +6130,3 @@ ER_LOG_PURGE_NO_FILE
 
 ER_NEED_REPREPARE
   eng "Prepared statement needs to be re-prepared"
-
-ER_PS_REBIND
-  eng "Prepared statement result set has changed, a rebind needed"

sql/sql_parse.cc

@@ -914,8 +914,11 @@ bool dispatch_command(enum enum_server_command command, THD *thd,
   /* TODO: set thd->lex->sql_command to SQLCOM_END here */
   VOID(pthread_mutex_unlock(&LOCK_thread_count));
 
-  thd->server_status&=
-           ~(SERVER_QUERY_NO_INDEX_USED | SERVER_QUERY_NO_GOOD_INDEX_USED);
+  /**
+    Clear the set of flags that are expected to be cleared at the
+    beginning of each command.
+  */
+  thd->server_status&= ~SERVER_STATUS_CLEAR_SET;
   switch (command) {
   case COM_INIT_DB:
   {
@@ -5377,9 +5380,11 @@ void mysql_reset_thd_for_next_command(THD *thd)
 
   thd->query_start_used= 0;
   thd->is_fatal_error= thd->time_zone_used= 0;
-  thd->server_status&= ~ (SERVER_MORE_RESULTS_EXISTS | 
-                          SERVER_QUERY_NO_INDEX_USED |
-                          SERVER_QUERY_NO_GOOD_INDEX_USED);
+  /*
+    Clear the status flag that are expected to be cleared at the
+    beginning of each SQL statement.
+  */
+  thd->server_status&= ~SERVER_STATUS_CLEAR_SET;
   /*
     If in autocommit mode and not in a transaction, reset
     OPTION_STATUS_NO_TRANS_UPDATE | OPTION_KEEP_LOG to not get warnings

sql/sql_prepare.cc

@@ -3315,7 +3315,7 @@ Prepared_statement::reprepare()
   @param[in]  copy  the re-prepared prepared statement to verify
                     the metadata of
 
-  @retval TRUE  error, ER_PS_NEED_REBIND is reported
+  @retval TRUE  error, ER_PS_REBIND is reported
   @retval FALSE statement return no or compatible metadata
 */
 
@@ -3333,9 +3333,8 @@ bool Prepared_statement::validate_metadata(Prepared_statement *copy)
   if (lex->select_lex.item_list.elements !=
       copy->lex->select_lex.item_list.elements)
   {
-    /** Column counts mismatch. */
-    my_error(ER_PS_REBIND, MYF(0));
-    return TRUE;
+    /** Column counts mismatch, update the client */
+    thd->server_status|= SERVER_STATUS_METADATA_CHANGED;
   }
 
   return FALSE;

tests/mysql_client_test.c

@@ -6668,10 +6668,10 @@ static void test_pure_coverage()
   check_execute_r(stmt, rc); /* unsupported buffer type */
 
   rc= mysql_stmt_store_result(stmt);
-  check_execute(stmt, rc);
+  DIE_UNLESS(rc);
 
   rc= mysql_stmt_store_result(stmt);
-  check_execute_r(stmt, rc); /* commands out of sync */
+  DIE_UNLESS(rc); /* Old error must be reset first */
 
   mysql_stmt_close(stmt);
 
@@ -8423,6 +8423,9 @@ static void test_fetch_offset()
   rc= mysql_stmt_fetch_column(stmt, my_bind, 0, 0);
   check_execute_r(stmt, rc);
 
+  rc= mysql_stmt_execute(stmt);
+  check_execute(stmt, rc);
+
   rc= mysql_stmt_bind_result(stmt, my_bind);
   check_execute(stmt, rc);
 
@@ -9901,7 +9904,7 @@ static void test_rename()
   MYSQL_STMT *stmt;
   const char *query= "rename table t1 to t2, t3 to t4";
   int rc;
-  myheader("test_table_manipulation");
+  myheader("test_table_rename");
 
   rc= mysql_query(mysql, "DROP TABLE IF EXISTS t1, t2, t3, t4");
   myquery(rc);
@@ -17383,7 +17386,7 @@ static void test_bug28386()
   DBUG_VOID_RETURN;
 }
 
-static void test_wl4166()
+static void test_wl4166_1()
 {
   MYSQL_STMT *stmt;
   int        int_data;
@@ -17399,7 +17402,7 @@ static void test_wl4166()
   int rc;
   int i;
 
-  myheader("test_wl4166");
+  myheader("test_wl4166_1");
 
   rc= mysql_query(mysql, "DROP TABLE IF EXISTS table_4166");
   myquery(rc);
@@ -17498,6 +17501,97 @@ static void test_wl4166()
 
   rc= mysql_query(mysql, "DROP TABLE table_4166");
   myquery(rc);
+}
+
+
+static void test_wl4166_2()
+{
+  MYSQL_STMT *stmt;
+  int        c_int;
+  MYSQL_TIME d_date;
+  MYSQL_BIND bind_out[2];
+  int rc;
+  int i;
+
+  myheader("test_wl4166_2");
+
+  rc= mysql_query(mysql, "drop table if exists t1");
+  myquery(rc);
+  rc= mysql_query(mysql, "create table t1 (c_int int, d_date date)");
+  myquery(rc);
+  rc= mysql_query(mysql,
+                  "insert into t1 (c_int, d_date) values (42, '1948-05-15')");
+  myquery(rc);
+
+  stmt= mysql_simple_prepare(mysql, "select * from t1");
+  check_stmt(stmt);
+
+  bzero(bind_out, sizeof(bind_out));
+  bind_out[0].buffer_type= MYSQL_TYPE_LONG;
+  bind_out[0].buffer= (void*) &c_int;
+
+  bind_out[1].buffer_type= MYSQL_TYPE_DATE;
+  bind_out[1].buffer= (void*) &d_date;
+
+  rc= mysql_stmt_bind_result(stmt, bind_out);
+  check_execute(stmt, rc);
+
+  /* int -> varchar transition */
+
+  rc= mysql_query(mysql,
+                  "alter table t1 change column c_int c_int varchar(11)");
+  myquery(rc);
+
+  rc= mysql_stmt_execute(stmt);
+  check_execute(stmt, rc);
+
+  rc= mysql_stmt_fetch(stmt);
+  check_execute(stmt, rc);
+
+  DIE_UNLESS(c_int == 42);
+  DIE_UNLESS(d_date.year == 1948);
+  DIE_UNLESS(d_date.month == 5);
+  DIE_UNLESS(d_date.day == 15);
+
+  rc= mysql_stmt_fetch(stmt);
+  DIE_UNLESS(rc == MYSQL_NO_DATA);
+
+  /* varchar to int retrieval with truncation */
+
+  rc= mysql_query(mysql, "update t1 set c_int='abcde'");
+  myquery(rc);
+
+  rc= mysql_stmt_execute(stmt);
+  check_execute(stmt, rc);
+
+  rc= mysql_stmt_fetch(stmt);
+  check_execute_r(stmt, rc);
+
+  DIE_UNLESS(c_int == 0);
+
+  rc= mysql_stmt_fetch(stmt);
+  DIE_UNLESS(rc == MYSQL_NO_DATA);
+
+  /* alter table and increase the number of columns */
+  rc= mysql_query(mysql, "alter table t1 add column d_int int");
+  myquery(rc);
+
+  rc= mysql_stmt_execute(stmt);
+  check_execute_r(stmt, rc);
+
+  rc= mysql_stmt_reset(stmt);
+  check_execute(stmt, rc);
+
+  /* decrease the number of columns */
+  rc= mysql_query(mysql, "alter table t1 drop d_date, drop d_int");
+  myquery(rc);
+
+  rc= mysql_stmt_execute(stmt);
+  check_execute_r(stmt, rc);
+
+  mysql_stmt_close(stmt);
+  rc= mysql_query(mysql, "drop table t1");
+  myquery(rc);
 
 }
 
@@ -17807,7 +17901,8 @@ static struct my_tests_st my_tests[]= {
   { "test_bug31418", test_bug31418 },
   { "test_bug31669", test_bug31669 },
   { "test_bug28386", test_bug28386 },
-  { "test_wl4166", test_wl4166 },
+  { "test_wl4166_1", test_wl4166_1 },
+  { "test_wl4166_2", test_wl4166_2 },
   { 0, 0 }
 };