MDEV-19237 Skip sending metadata when possible for binary protocol.

Do not resend metadata, if metadata does not change between prepare and
execute of prepared statement, or between executes.

Currently, metadata of *every* prepared statement will be checksummed,
and change is detected once checksum changes.

This is not from ideal, performance-wise. The code for
better/faster detection of unchanged metadata, is already in place, but
currently disabled due to PS bugs, such as MDEV-23913.

include/mysql_com.h

@@ -305,6 +305,9 @@ enum enum_indicator_type
 /* support of extended metadata (e.g. type/format information) */
 #define MARIADB_CLIENT_EXTENDED_METADATA (1ULL << 35)
 
+/* Do not resend metadata for prepared statements, since 10.6*/
+#define MARIADB_CLIENT_CACHE_METADATA (1ULL << 36)
+
 #ifdef HAVE_COMPRESS
 #define CAN_CLIENT_COMPRESS CLIENT_COMPRESS
 #else
@@ -345,8 +348,8 @@ enum enum_indicator_type
                            CLIENT_CONNECT_ATTRS |\
                            MARIADB_CLIENT_STMT_BULK_OPERATIONS |\
                            MARIADB_CLIENT_EXTENDED_METADATA|\
+                           MARIADB_CLIENT_CACHE_METADATA |\
                            CLIENT_CAN_HANDLE_EXPIRED_PASSWORDS)
-
 /*
   Switch off the flags that are optional and depending on build flags
   If any of the optional flags is supported by the build it will be switched

libmariadb @ 3cf45a2c

-Subproject commit e38244220646a7e95c9be22576460aa7a4eb715f
+Subproject commit 3cf45a2c5711936840bdbdb3a0d74869d12e69a7

sql/mysqld.cc

@@ -7169,6 +7169,7 @@ SHOW_VAR status_vars[]= {
   {"Max_used_connections",     (char*) &max_used_connections,  SHOW_LONG},
   {"Memory_used",              (char*) &show_memory_used, SHOW_SIMPLE_FUNC},
   {"Memory_used_initial",      (char*) &start_memory_used, SHOW_LONGLONG},
+  {"Resultset_metadata_skipped", (char *) offsetof(STATUS_VAR, skip_metadata_count),SHOW_LONG_STATUS},
   {"Not_flushed_delayed_rows", (char*) &delayed_rows_in_use,    SHOW_LONG_NOFLUSH},
   {"Open_files",               (char*) &my_file_opened,         SHOW_SINT},
   {"Open_streams",             (char*) &my_stream_opened,       SHOW_LONG_NOFLUSH},

sql/protocol.cc

@@ -913,6 +913,241 @@ bool Protocol_text::store_field_metadata(const THD * thd,
 }
 
 
+/*
+  MARIADB_CLIENT_CACHE_METADATA  support.
+
+  Bulk of the code below is dedicated to detecting whether column metadata has
+  changed after prepare, or between executions of a prepared statement.
+
+  For some prepared statements, metadata can't change without going through
+  Prepared_Statement::reprepare(), which makes detecting changes easy.
+
+  Others, "SELECT ?" & Co, are more fragile, and sensitive to input parameters,
+  or user variables. Detecting metadata change for this class of PS is harder,
+  we calculate signature (hash value), and check whether this changes between
+  executions. This is a more expensive method.
+*/
+
+
+/**
+  Detect whether column info can be changed without
+  PS repreparing.
+
+  Such colum info is called fragile. The opposite of
+  fragile is.
+
+
+  @param  it - Item representing column info
+  @return true, if columninfo is "fragile", false if it is stable
+
+
+  @todo does not work due to MDEV-23913. Currently,
+  everything about prepared statements is fragile.
+*/
+
+static bool is_fragile_columnifo(Item *it)
+{
+#define MDEV_23913_FIXED 0
+#if MDEV_23913_FIXED
+  if (dynamic_cast<Item_param *>(it))
+    return true;
+
+  if (dynamic_cast<Item_func_user_var *>(it))
+    return true;
+
+  if (dynamic_cast <Item_sp_variable*>(it))
+    return true;
+
+  /* Check arguments of functions.*/
+  auto item_args= dynamic_cast<Item_args *>(it);
+  if (!item_args)
+    return false;
+  auto args= item_args->arguments();
+  auto arg_count= item_args->argument_count();
+  for (uint i= 0; i < arg_count; i++)
+  {
+    if (is_fragile_columnifo(args[i]))
+      return true;
+  }
+  return false;
+#else /* MDEV-23913 fixed*/
+  return true;
+#endif
+}
+
+
+#define INVALID_METADATA_CHECKSUM 0
+
+
+/**
+  Calculate signature for column info sent to the client as CRC32 over data, 
+  that goes into the column info packet.
+  We assume that if checksum does not change, then column info was not
+  modified.
+
+  @param thd THD
+  @param list column info
+
+  @return CRC32 of the metadata
+*/
+
+static uint32 calc_metadata_hash(THD *thd, List<Item> *list)
+{
+  List_iterator_fast<Item> it(*list);
+  Item *item;
+  uint32 crc32_c= 0;
+  while ((item= it++))
+  {
+    Send_field field(thd, item);
+    auto field_type= item->type_handler()->field_type();
+    auto charset= item->charset_for_protocol();
+    /*
+       The data below should contain everything that influences
+       content of the column info packet.
+    */
+    LEX_CSTRING data[]=
+    {
+      field.table_name,
+      field.org_table_name,
+      field.col_name,
+      field.org_col_name,
+      field.db_name,
+      field.attr(MARIADB_FIELD_ATTR_DATA_TYPE_NAME),
+      field.attr(MARIADB_FIELD_ATTR_FORMAT_NAME),
+      {(const char *) &field.length, sizeof(field.length)},
+      {(const char *) &field.flags, sizeof(field.flags)},
+      {(const char *) &field.decimals, sizeof(field.decimals)},
+      {(const char *) &charset, sizeof(charset)},
+      {(const char *) &field_type, sizeof(field_type)},
+    };
+    for (const auto &chunk : data)
+      crc32_c= my_crc32c(crc32_c, chunk.str, chunk.length);
+  }
+
+  if (crc32_c == INVALID_METADATA_CHECKSUM)
+     return 1;
+  return crc32_c;
+}
+
+
+
+/**
+  Check if metadata columns have changed since last call to this
+  function.
+
+  @param send_column_info_state  saved state, changed if the function
+         return true. 
+  @param thd THD
+  @param list columninfo Items
+  @return true,if metadata columns have changed since last call,
+          false otherwise 
+*/
+
+static bool metadata_columns_changed(send_column_info_state &state, THD *thd,
+                                     List<Item> &list)
+{
+  if (!state.initialized)
+  {
+    state.initialized= true;
+    state.immutable= true;
+    Item *item;
+    List_iterator_fast<Item> it(list);
+    while ((item= it++))
+    {
+      if (is_fragile_columnifo(item))
+      {
+        state.immutable= false;
+        state.checksum= calc_metadata_hash(thd, &list);
+        break;
+      }
+    }
+    state.last_charset= thd->variables.character_set_client;
+    return true;
+  }
+  
+  /*
+    Since column info can change under our feet, we use more expensive
+    checksumming to check if column metadata has not changed since last time.
+  */
+  if (!state.immutable)
+  {
+    uint32 checksum= calc_metadata_hash(thd, &list);
+    if (checksum != state.checksum)
+    {
+      state.checksum= checksum;
+      state.last_charset= thd->variables.character_set_client;
+      return true;
+    }
+  }
+
+  /*
+    Character_set_client influences result set metadata, thus resend metadata
+    whenever it changes.
+  */
+  if (state.last_charset != thd->variables.character_set_client)
+  {
+    state.last_charset= thd->variables.character_set_client;
+    return true;
+  }
+
+  return false;
+}
+
+
+/**
+  Determine whether column info must be sent to the client.
+  Skip column info, if client supports caching, and (prepared) statement
+  output fields have not changed.
+
+  @param thd THD
+  @param list column info
+  @param flags send flags. If Protocol::SEND_FORCE_COLUMN_INFO is set,
+         this function will return true
+  @return true, if column info must be sent to the client.
+          false otherwise
+*/
+
+static bool should_send_column_info(THD* thd, List<Item>* list, uint flags) 
+{
+  if (!(thd->client_capabilities & MARIADB_CLIENT_CACHE_METADATA))
+  {
+    /* Client does not support abbreviated metadata.*/
+    return true;
+  }
+
+  if (!thd->cur_stmt)
+  {
+    /* Neither COM_PREPARE nor COM_EXECUTE run.*/
+    return true;
+  }
+
+  if (thd->spcont)
+  {
+    /* Always sent full metadata from inside the stored procedure.*/
+    return true;
+  }
+
+  if (flags & Protocol::SEND_FORCE_COLUMN_INFO)
+    return true;
+
+  auto &column_info_state= thd->cur_stmt->column_info_state;
+#ifndef DBUG_OFF
+  auto cmd= thd->get_command();
+#endif
+
+  DBUG_ASSERT(cmd == COM_STMT_EXECUTE || cmd == COM_STMT_PREPARE);
+  DBUG_ASSERT(cmd != COM_STMT_PREPARE || !column_info_state.initialized);
+
+  bool ret= metadata_columns_changed(column_info_state, thd, *list);
+
+  DBUG_ASSERT(cmd != COM_STMT_PREPARE || ret);
+  if (!ret)
+    thd->status_var.skip_metadata_count++;
+
+  return ret;
+}
+
+
 /**
   Send name and type of result to client.
 
@@ -938,30 +1173,44 @@ bool Protocol::send_result_set_metadata(List<Item> *list, uint flags)
   Protocol_text prot(thd, thd->variables.net_buffer_length);
   DBUG_ENTER("Protocol::send_result_set_metadata");
 
+  bool send_column_info= should_send_column_info(thd, list, flags);
+
   if (flags & SEND_NUM_ROWS)
-  {				// Packet with number of elements
-    uchar buff[MAX_INT_WIDTH];
+  {
+    /*
+      Packet with number of columns.
+
+      Will also have a 1 byte column info indicator, in case
+      MARIADB_CLIENT_CACHE_METADATA client capability is set.
+    */
+    uchar buff[MAX_INT_WIDTH+1];
     uchar *pos= net_store_length(buff, list->elements);
+    if (thd->client_capabilities & MARIADB_CLIENT_CACHE_METADATA)
+      *pos++= (uchar)send_column_info;
+
     DBUG_ASSERT(pos <= buff + sizeof(buff));
     if (my_net_write(&thd->net, buff, (size_t) (pos-buff)))
       DBUG_RETURN(1);
   }
 
+  if (send_column_info)
+  {
 #ifndef DBUG_OFF
-  field_handlers= (const Type_handler**) thd->alloc(sizeof(field_handlers[0]) *
-                                                    list->elements);
+    field_handlers= (const Type_handler **) thd->alloc(
+        sizeof(field_handlers[0]) * list->elements);
 #endif
 
-  for (uint pos= 0; (item=it++); pos++)
-  {
-    prot.prepare_for_resend();
-    if (prot.store_item_metadata(thd, item, pos))
-      goto err;
-    if (prot.write())
-      DBUG_RETURN(1);
+    for (uint pos= 0; (item= it++); pos++)
+    {
+      prot.prepare_for_resend();
+      if (prot.store_item_metadata(thd, item, pos))
+        goto err;
+      if (prot.write())
+        DBUG_RETURN(1);
 #ifndef DBUG_OFF
-    field_handlers[pos]= item->type_handler();
+      field_handlers[pos]= item->type_handler();
 #endif
+    }
   }
 
   if (flags & SEND_EOF)
@@ -1685,7 +1934,8 @@ bool Protocol_binary::send_out_parameters(List<Item_param> *sp_params)
   thd->server_status|= SERVER_PS_OUT_PARAMS | SERVER_MORE_RESULTS_EXISTS;
 
   /* Send meta-data. */
-  if (send_result_set_metadata(&out_param_lst, SEND_NUM_ROWS | SEND_EOF))
+  if (send_result_set_metadata(&out_param_lst,
+        SEND_NUM_ROWS | SEND_EOF | SEND_FORCE_COLUMN_INFO))
     return TRUE;
 
   /* Send data. */

sql/protocol.h

@@ -93,7 +93,7 @@ class Protocol
   virtual ~Protocol() {}
   void init(THD* thd_arg);
 
-  enum { SEND_NUM_ROWS= 1, SEND_EOF= 2 };
+  enum { SEND_NUM_ROWS= 1, SEND_EOF= 2, SEND_FORCE_COLUMN_INFO= 4 };
   virtual bool send_result_set_metadata(List<Item> *list, uint flags);
   bool send_list_fields(List<Field> *list, const TABLE_LIST *table_list);
   bool send_result_set_row(List<Item> *row_items);

sql/sql_class.h

@@ -929,6 +929,12 @@ typedef struct system_status_var
   ulong access_denied_errors;
   ulong lost_connections;
   ulong max_statement_time_exceeded;
+  /*
+   Number of times where column info was not
+   sent with prepared statement metadata.
+  */
+  ulong skip_metadata_count;
+
   /*
     Number of statements sent from the client
   */
@@ -948,6 +954,7 @@ typedef struct system_status_var
   ulonglong table_open_cache_hits;
   ulonglong table_open_cache_misses;
   ulonglong table_open_cache_overflows;
+  ulonglong send_metadata_skips;
   double last_query_cost;
   double cpu_time, busy_time;
   uint32 threads_running;
@@ -1193,6 +1200,38 @@ class Query_arena_stmt
 
 class Server_side_cursor;
 
+/*
+  Struct to catch changes in column metadata that is sent to client. 
+  in the "result set metadata". Used to support 
+  MARIADB_CLIENT_CACHE_METADATA.
+*/
+struct send_column_info_state
+{
+  /* Last client charset (affects metadata) */
+  CHARSET_INFO *last_charset= nullptr;
+
+  /* Checksum, only used to check changes if 'immutable' is false*/
+  uint32 checksum= 0;
+
+  /*
+    Column info can only be changed by PreparedStatement::reprepare()
+ 
+    There is a class of "weird" prepared statements like SELECT ? or SELECT @a
+    that are not immutable, and depend on input parameters or user variables
+  */
+  bool immutable= false;
+
+  bool initialized= false;
+
+  /*  Used by PreparedStatement::reprepare()*/
+  void reset()
+  {
+    initialized= false;
+    checksum= 0;
+  }
+};
+
+
 /**
   @class Statement
   @brief State of a single command executed against this connection.
@@ -1282,6 +1321,8 @@ class Statement: public ilink, public Query_arena
 
   LEX_CSTRING db;
 
+  send_column_info_state column_info_state;
+ 
   /* This is set to 1 of last call to send_result_to_client() was ok */
   my_bool query_cache_is_applicable;
 
@@ -2385,6 +2426,8 @@ class THD: public THD_count, /* this must be first */
 
   /* Last created prepared statement */
   Statement *last_stmt;
+  Statement *cur_stmt= 0;
+
   inline void set_last_stmt(Statement *stmt)
   { last_stmt= (is_error() ? NULL : stmt); }
   inline void clear_last_stmt() { last_stmt= NULL; }

sql/sql_prepare.cc

@@ -334,9 +334,13 @@ static bool send_prep_stmt(Prepared_statement *stmt, uint columns)
   error= my_net_write(net, buff, sizeof(buff));
   if (stmt->param_count && likely(!error))
   {
-    error= thd->protocol_text.send_result_set_metadata((List<Item> *)
-                                          &stmt->lex->param_list,
-                                          Protocol::SEND_EOF);
+    /*
+      Force the column info to be written
+      (in this case PS parameter type info).
+    */
+    error= thd->protocol_text.send_result_set_metadata(
+                (List<Item> *)&stmt->lex->param_list,
+                Protocol::SEND_EOF | Protocol::SEND_FORCE_COLUMN_INFO);
   }
 
   if (likely(!error))
@@ -3444,10 +3448,15 @@ static void mysql_stmt_execute_common(THD *thd,
   thd->protocol= &thd->protocol_binary;
   MYSQL_EXECUTE_PS(thd->m_statement_psi, stmt->m_prepared_stmt);
 
+  auto save_cur_stmt= thd->cur_stmt;
+  thd->cur_stmt= stmt;
+
   if (!bulk_op)
     stmt->execute_loop(&expanded_query, open_cursor, packet, packet_end);
   else
     stmt->execute_bulk_loop(&expanded_query, open_cursor, packet, packet_end);
+
+  thd->cur_stmt= save_cur_stmt;
   thd->protocol= save_protocol;
 
   sp_cache_enforce_limit(thd->sp_proc_cache, stored_program_cache_size);
@@ -4206,6 +4215,8 @@ bool Prepared_statement::prepare(const char *packet, uint packet_len)
 
   old_stmt_arena= thd->stmt_arena;
   thd->stmt_arena= this;
+  auto save_cur_stmt= thd->cur_stmt;
+  thd->cur_stmt= this;
 
   Parser_state parser_state;
   if (parser_state.init(thd, thd->query(), thd->query_length()))
@@ -4213,6 +4224,7 @@ bool Prepared_statement::prepare(const char *packet, uint packet_len)
     thd->restore_backup_statement(this, &stmt_backup);
     thd->restore_active_arena(this, &stmt_backup);
     thd->stmt_arena= old_stmt_arena;
+    thd->cur_stmt = save_cur_stmt;
     DBUG_RETURN(TRUE);
   }
 
@@ -4222,6 +4234,7 @@ bool Prepared_statement::prepare(const char *packet, uint packet_len)
   lex_start(thd);
   lex->context_analysis_only|= CONTEXT_ANALYSIS_ONLY_PREPARE;
 
+
   error= (parse_sql(thd, & parser_state, NULL) ||
           thd->is_error() ||
           init_param_array(this));
@@ -4297,6 +4310,7 @@ bool Prepared_statement::prepare(const char *packet, uint packet_len)
   cleanup_stmt();
   thd->restore_backup_statement(this, &stmt_backup);
   thd->stmt_arena= old_stmt_arena;
+  thd->cur_stmt= save_cur_stmt;
 
   if (likely(error == 0))
   {
@@ -4755,6 +4769,7 @@ Prepared_statement::reprepare()
       it's failed, we need to return all the warnings to the user.
     */
     thd->get_stmt_da()->clear_warning_info(thd->query_id);
+    column_info_state.reset();
   }
   else
   {

tests/mysql_client_test.c

@@ -21147,6 +21147,124 @@ static void test_execute_direct()
 #endif
 }
 
+
+static void assert_metadata_skipped_count_equals(MYSQL *mysql, int val)
+{
+  MYSQL_ROW row;
+  MYSQL_RES *result;
+  int rc= mysql_query(mysql, "SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.SESSION_STATUS WHERE "
+                         "VARIABLE_NAME='Resultset_metadata_skipped'");
+  myquery(rc);
+  result= mysql_use_result(mysql);
+  mytest(result);
+  row= mysql_fetch_row(result);
+  DIE_UNLESS(atoi(row[0]) == val);
+  mysql_free_result(result);
+}
+
+
+static void flush_session_status(MYSQL* mysql)
+{
+  int rc= mysql_query(mysql, "flush status");
+  myquery(rc);
+}
+
+
+static void exec_stmt(MYSQL_STMT* stmt)
+{
+  int rc= mysql_stmt_execute(stmt);
+  check_execute(stmt, rc);
+  rc= mysql_stmt_store_result(stmt);
+  check_execute(stmt, rc);
+}
+
+
+
+static void test_cache_metadata()
+{
+  char char_val[]= "blah";
+  int int_val = 1;
+  MYSQL_BIND param= {0};
+  my_bool is_null= FALSE;
+
+
+  MYSQL_STMT* stmt= mysql_stmt_init(mysql);
+  check_stmt(stmt);
+  int rc= mysql_stmt_prepare(stmt, "SELECT ?", -1);
+  myquery(rc);
+
+  param.buffer= char_val;
+  param.buffer_type= MYSQL_TYPE_STRING;
+  param.is_null= &is_null;
+  param.buffer_length = 4;
+
+  rc= mysql_stmt_bind_param(stmt,&param);
+  exec_stmt(stmt);
+
+  flush_session_status(mysql);
+  /* Execute the statement again, check that metadata is skipped*/
+  exec_stmt(stmt);
+  assert_metadata_skipped_count_equals(mysql, 1);
+
+  flush_session_status(mysql);
+  /*
+    Execute the statement again, such that metadata changes,
+    (using LONG parameter in bind for "SELECT ?", instead of string.
+    Check that metadata is NOT skipped.
+  */
+  param.buffer= &int_val;
+  param.buffer_type= MYSQL_TYPE_LONG;
+  param.is_null= &is_null;
+  rc= mysql_stmt_bind_param(stmt, &param);
+  exec_stmt(stmt);
+  assert_metadata_skipped_count_equals(mysql, 0);
+  mysql_stmt_close(stmt);
+
+
+  /*
+    Test with real table, and DDL which causes column info to be
+    changed.
+  */
+  stmt= mysql_stmt_init(mysql);
+
+  rc= mysql_query(
+      mysql, "CREATE OR REPLACE TABLE t1 (a int, b bigint) engine=memory");
+  myquery(rc);
+
+  flush_session_status(mysql);
+  check_stmt(stmt);
+  rc= mysql_stmt_prepare(stmt, "SELECT * from t1", -1);
+  myquery(rc);
+
+  exec_stmt(stmt);
+  /* Metadata skipped, since already sent with COM_STMT_PREPARE result.*/
+  assert_metadata_skipped_count_equals(mysql, 1);
+
+  flush_session_status(mysql);
+  exec_stmt(stmt);
+  /* Metadata skipped again*/
+  assert_metadata_skipped_count_equals(mysql, 1);
+
+  rc= mysql_query(mysql, "ALTER TABLE t1 MODIFY b CHAR(10)");
+  myquery(rc);
+
+  /* Column metadata WILL change for the next execution due to DDL*/
+  flush_session_status(mysql);
+  exec_stmt(stmt);
+  assert_metadata_skipped_count_equals(mysql, 0);
+
+  /* On reexecution, PS column metadata will NOT change. */
+  flush_session_status(mysql);
+  exec_stmt(stmt);
+  assert_metadata_skipped_count_equals(mysql, 1);
+
+  rc= mysql_query(mysql, "DROP TABLE t1");
+  myquery(rc);
+
+  mysql_stmt_close(stmt);
+}
+
+
 static struct my_tests_st my_tests[]= {
   { "disable_query_logs", disable_query_logs },
   { "test_view_sp_list_fields", test_view_sp_list_fields },
@@ -21446,6 +21564,7 @@ static struct my_tests_st my_tests[]= {
   { "test_mdev18408", test_mdev18408 },
   { "test_mdev20261", test_mdev20261 },
   { "test_execute_direct", test_execute_direct },
+  { "test_cache_metadata", test_cache_metadata},
   { 0, 0 }
 };