Commit 295f3e4c authored by Vladislav Vaintroub's avatar Vladislav Vaintroub

MDEV-19237 Skip sending metadata when possible for binary protocol.

Do not resend metadata, if metadata does not change between prepare and
execute of prepared statement, or between executes.

Currently, metadata of *every* prepared statement will be checksummed,
and change is detected once checksum changes.

This is not from ideal, performance-wise. The code for
better/faster detection of unchanged metadata, is already in place, but
currently disabled due to PS bugs, such as MDEV-23913.
parent a62a675f
......@@ -305,6 +305,9 @@ enum enum_indicator_type
/* support of extended metadata (e.g. type/format information) */
#define MARIADB_CLIENT_EXTENDED_METADATA (1ULL << 35)
/* Do not resend metadata for prepared statements, since 10.6*/
#define MARIADB_CLIENT_CACHE_METADATA (1ULL << 36)
#ifdef HAVE_COMPRESS
#define CAN_CLIENT_COMPRESS CLIENT_COMPRESS
#else
......@@ -345,8 +348,8 @@ enum enum_indicator_type
CLIENT_CONNECT_ATTRS |\
MARIADB_CLIENT_STMT_BULK_OPERATIONS |\
MARIADB_CLIENT_EXTENDED_METADATA|\
MARIADB_CLIENT_CACHE_METADATA |\
CLIENT_CAN_HANDLE_EXPIRED_PASSWORDS)
/*
Switch off the flags that are optional and depending on build flags
If any of the optional flags is supported by the build it will be switched
......
Subproject commit e38244220646a7e95c9be22576460aa7a4eb715f
Subproject commit 3cf45a2c5711936840bdbdb3a0d74869d12e69a7
......@@ -7169,6 +7169,7 @@ SHOW_VAR status_vars[]= {
{"Max_used_connections", (char*) &max_used_connections, SHOW_LONG},
{"Memory_used", (char*) &show_memory_used, SHOW_SIMPLE_FUNC},
{"Memory_used_initial", (char*) &start_memory_used, SHOW_LONGLONG},
{"Resultset_metadata_skipped", (char *) offsetof(STATUS_VAR, skip_metadata_count),SHOW_LONG_STATUS},
{"Not_flushed_delayed_rows", (char*) &delayed_rows_in_use, SHOW_LONG_NOFLUSH},
{"Open_files", (char*) &my_file_opened, SHOW_SINT},
{"Open_streams", (char*) &my_stream_opened, SHOW_LONG_NOFLUSH},
......
......@@ -913,6 +913,241 @@ bool Protocol_text::store_field_metadata(const THD * thd,
}
/*
MARIADB_CLIENT_CACHE_METADATA support.
Bulk of the code below is dedicated to detecting whether column metadata has
changed after prepare, or between executions of a prepared statement.
For some prepared statements, metadata can't change without going through
Prepared_Statement::reprepare(), which makes detecting changes easy.
Others, "SELECT ?" & Co, are more fragile, and sensitive to input parameters,
or user variables. Detecting metadata change for this class of PS is harder,
we calculate signature (hash value), and check whether this changes between
executions. This is a more expensive method.
*/
/**
Detect whether column info can be changed without
PS repreparing.
Such colum info is called fragile. The opposite of
fragile is.
@param it - Item representing column info
@return true, if columninfo is "fragile", false if it is stable
@todo does not work due to MDEV-23913. Currently,
everything about prepared statements is fragile.
*/
static bool is_fragile_columnifo(Item *it)
{
#define MDEV_23913_FIXED 0
#if MDEV_23913_FIXED
if (dynamic_cast<Item_param *>(it))
return true;
if (dynamic_cast<Item_func_user_var *>(it))
return true;
if (dynamic_cast <Item_sp_variable*>(it))
return true;
/* Check arguments of functions.*/
auto item_args= dynamic_cast<Item_args *>(it);
if (!item_args)
return false;
auto args= item_args->arguments();
auto arg_count= item_args->argument_count();
for (uint i= 0; i < arg_count; i++)
{
if (is_fragile_columnifo(args[i]))
return true;
}
return false;
#else /* MDEV-23913 fixed*/
return true;
#endif
}
#define INVALID_METADATA_CHECKSUM 0
/**
Calculate signature for column info sent to the client as CRC32 over data,
that goes into the column info packet.
We assume that if checksum does not change, then column info was not
modified.
@param thd THD
@param list column info
@return CRC32 of the metadata
*/
static uint32 calc_metadata_hash(THD *thd, List<Item> *list)
{
List_iterator_fast<Item> it(*list);
Item *item;
uint32 crc32_c= 0;
while ((item= it++))
{
Send_field field(thd, item);
auto field_type= item->type_handler()->field_type();
auto charset= item->charset_for_protocol();
/*
The data below should contain everything that influences
content of the column info packet.
*/
LEX_CSTRING data[]=
{
field.table_name,
field.org_table_name,
field.col_name,
field.org_col_name,
field.db_name,
field.attr(MARIADB_FIELD_ATTR_DATA_TYPE_NAME),
field.attr(MARIADB_FIELD_ATTR_FORMAT_NAME),
{(const char *) &field.length, sizeof(field.length)},
{(const char *) &field.flags, sizeof(field.flags)},
{(const char *) &field.decimals, sizeof(field.decimals)},
{(const char *) &charset, sizeof(charset)},
{(const char *) &field_type, sizeof(field_type)},
};
for (const auto &chunk : data)
crc32_c= my_crc32c(crc32_c, chunk.str, chunk.length);
}
if (crc32_c == INVALID_METADATA_CHECKSUM)
return 1;
return crc32_c;
}
/**
Check if metadata columns have changed since last call to this
function.
@param send_column_info_state saved state, changed if the function
return true.
@param thd THD
@param list columninfo Items
@return true,if metadata columns have changed since last call,
false otherwise
*/
static bool metadata_columns_changed(send_column_info_state &state, THD *thd,
List<Item> &list)
{
if (!state.initialized)
{
state.initialized= true;
state.immutable= true;
Item *item;
List_iterator_fast<Item> it(list);
while ((item= it++))
{
if (is_fragile_columnifo(item))
{
state.immutable= false;
state.checksum= calc_metadata_hash(thd, &list);
break;
}
}
state.last_charset= thd->variables.character_set_client;
return true;
}
/*
Since column info can change under our feet, we use more expensive
checksumming to check if column metadata has not changed since last time.
*/
if (!state.immutable)
{
uint32 checksum= calc_metadata_hash(thd, &list);
if (checksum != state.checksum)
{
state.checksum= checksum;
state.last_charset= thd->variables.character_set_client;
return true;
}
}
/*
Character_set_client influences result set metadata, thus resend metadata
whenever it changes.
*/
if (state.last_charset != thd->variables.character_set_client)
{
state.last_charset= thd->variables.character_set_client;
return true;
}
return false;
}
/**
Determine whether column info must be sent to the client.
Skip column info, if client supports caching, and (prepared) statement
output fields have not changed.
@param thd THD
@param list column info
@param flags send flags. If Protocol::SEND_FORCE_COLUMN_INFO is set,
this function will return true
@return true, if column info must be sent to the client.
false otherwise
*/
static bool should_send_column_info(THD* thd, List<Item>* list, uint flags)
{
if (!(thd->client_capabilities & MARIADB_CLIENT_CACHE_METADATA))
{
/* Client does not support abbreviated metadata.*/
return true;
}
if (!thd->cur_stmt)
{
/* Neither COM_PREPARE nor COM_EXECUTE run.*/
return true;
}
if (thd->spcont)
{
/* Always sent full metadata from inside the stored procedure.*/
return true;
}
if (flags & Protocol::SEND_FORCE_COLUMN_INFO)
return true;
auto &column_info_state= thd->cur_stmt->column_info_state;
#ifndef DBUG_OFF
auto cmd= thd->get_command();
#endif
DBUG_ASSERT(cmd == COM_STMT_EXECUTE || cmd == COM_STMT_PREPARE);
DBUG_ASSERT(cmd != COM_STMT_PREPARE || !column_info_state.initialized);
bool ret= metadata_columns_changed(column_info_state, thd, *list);
DBUG_ASSERT(cmd != COM_STMT_PREPARE || ret);
if (!ret)
thd->status_var.skip_metadata_count++;
return ret;
}
/**
Send name and type of result to client.
......@@ -938,21 +1173,34 @@ bool Protocol::send_result_set_metadata(List<Item> *list, uint flags)
Protocol_text prot(thd, thd->variables.net_buffer_length);
DBUG_ENTER("Protocol::send_result_set_metadata");
bool send_column_info= should_send_column_info(thd, list, flags);
if (flags & SEND_NUM_ROWS)
{ // Packet with number of elements
uchar buff[MAX_INT_WIDTH];
{
/*
Packet with number of columns.
Will also have a 1 byte column info indicator, in case
MARIADB_CLIENT_CACHE_METADATA client capability is set.
*/
uchar buff[MAX_INT_WIDTH+1];
uchar *pos= net_store_length(buff, list->elements);
if (thd->client_capabilities & MARIADB_CLIENT_CACHE_METADATA)
*pos++= (uchar)send_column_info;
DBUG_ASSERT(pos <= buff + sizeof(buff));
if (my_net_write(&thd->net, buff, (size_t) (pos-buff)))
DBUG_RETURN(1);
}
if (send_column_info)
{
#ifndef DBUG_OFF
field_handlers= (const Type_handler**) thd->alloc(sizeof(field_handlers[0]) *
list->elements);
field_handlers= (const Type_handler **) thd->alloc(
sizeof(field_handlers[0]) * list->elements);
#endif
for (uint pos= 0; (item=it++); pos++)
for (uint pos= 0; (item= it++); pos++)
{
prot.prepare_for_resend();
if (prot.store_item_metadata(thd, item, pos))
......@@ -963,6 +1211,7 @@ bool Protocol::send_result_set_metadata(List<Item> *list, uint flags)
field_handlers[pos]= item->type_handler();
#endif
}
}
if (flags & SEND_EOF)
{
......@@ -1685,7 +1934,8 @@ bool Protocol_binary::send_out_parameters(List<Item_param> *sp_params)
thd->server_status|= SERVER_PS_OUT_PARAMS | SERVER_MORE_RESULTS_EXISTS;
/* Send meta-data. */
if (send_result_set_metadata(&out_param_lst, SEND_NUM_ROWS | SEND_EOF))
if (send_result_set_metadata(&out_param_lst,
SEND_NUM_ROWS | SEND_EOF | SEND_FORCE_COLUMN_INFO))
return TRUE;
/* Send data. */
......
......@@ -93,7 +93,7 @@ class Protocol
virtual ~Protocol() {}
void init(THD* thd_arg);
enum { SEND_NUM_ROWS= 1, SEND_EOF= 2 };
enum { SEND_NUM_ROWS= 1, SEND_EOF= 2, SEND_FORCE_COLUMN_INFO= 4 };
virtual bool send_result_set_metadata(List<Item> *list, uint flags);
bool send_list_fields(List<Field> *list, const TABLE_LIST *table_list);
bool send_result_set_row(List<Item> *row_items);
......
......@@ -929,6 +929,12 @@ typedef struct system_status_var
ulong access_denied_errors;
ulong lost_connections;
ulong max_statement_time_exceeded;
/*
Number of times where column info was not
sent with prepared statement metadata.
*/
ulong skip_metadata_count;
/*
Number of statements sent from the client
*/
......@@ -948,6 +954,7 @@ typedef struct system_status_var
ulonglong table_open_cache_hits;
ulonglong table_open_cache_misses;
ulonglong table_open_cache_overflows;
ulonglong send_metadata_skips;
double last_query_cost;
double cpu_time, busy_time;
uint32 threads_running;
......@@ -1193,6 +1200,38 @@ class Query_arena_stmt
class Server_side_cursor;
/*
Struct to catch changes in column metadata that is sent to client.
in the "result set metadata". Used to support
MARIADB_CLIENT_CACHE_METADATA.
*/
struct send_column_info_state
{
/* Last client charset (affects metadata) */
CHARSET_INFO *last_charset= nullptr;
/* Checksum, only used to check changes if 'immutable' is false*/
uint32 checksum= 0;
/*
Column info can only be changed by PreparedStatement::reprepare()
There is a class of "weird" prepared statements like SELECT ? or SELECT @a
that are not immutable, and depend on input parameters or user variables
*/
bool immutable= false;
bool initialized= false;
/* Used by PreparedStatement::reprepare()*/
void reset()
{
initialized= false;
checksum= 0;
}
};
/**
@class Statement
@brief State of a single command executed against this connection.
......@@ -1282,6 +1321,8 @@ class Statement: public ilink, public Query_arena
LEX_CSTRING db;
send_column_info_state column_info_state;
/* This is set to 1 of last call to send_result_to_client() was ok */
my_bool query_cache_is_applicable;
......@@ -2385,6 +2426,8 @@ class THD: public THD_count, /* this must be first */
/* Last created prepared statement */
Statement *last_stmt;
Statement *cur_stmt= 0;
inline void set_last_stmt(Statement *stmt)
{ last_stmt= (is_error() ? NULL : stmt); }
inline void clear_last_stmt() { last_stmt= NULL; }
......
......@@ -334,9 +334,13 @@ static bool send_prep_stmt(Prepared_statement *stmt, uint columns)
error= my_net_write(net, buff, sizeof(buff));
if (stmt->param_count && likely(!error))
{
error= thd->protocol_text.send_result_set_metadata((List<Item> *)
&stmt->lex->param_list,
Protocol::SEND_EOF);
/*
Force the column info to be written
(in this case PS parameter type info).
*/
error= thd->protocol_text.send_result_set_metadata(
(List<Item> *)&stmt->lex->param_list,
Protocol::SEND_EOF | Protocol::SEND_FORCE_COLUMN_INFO);
}
if (likely(!error))
......@@ -3444,10 +3448,15 @@ static void mysql_stmt_execute_common(THD *thd,
thd->protocol= &thd->protocol_binary;
MYSQL_EXECUTE_PS(thd->m_statement_psi, stmt->m_prepared_stmt);
auto save_cur_stmt= thd->cur_stmt;
thd->cur_stmt= stmt;
if (!bulk_op)
stmt->execute_loop(&expanded_query, open_cursor, packet, packet_end);
else
stmt->execute_bulk_loop(&expanded_query, open_cursor, packet, packet_end);
thd->cur_stmt= save_cur_stmt;
thd->protocol= save_protocol;
sp_cache_enforce_limit(thd->sp_proc_cache, stored_program_cache_size);
......@@ -4206,6 +4215,8 @@ bool Prepared_statement::prepare(const char *packet, uint packet_len)
old_stmt_arena= thd->stmt_arena;
thd->stmt_arena= this;
auto save_cur_stmt= thd->cur_stmt;
thd->cur_stmt= this;
Parser_state parser_state;
if (parser_state.init(thd, thd->query(), thd->query_length()))
......@@ -4213,6 +4224,7 @@ bool Prepared_statement::prepare(const char *packet, uint packet_len)
thd->restore_backup_statement(this, &stmt_backup);
thd->restore_active_arena(this, &stmt_backup);
thd->stmt_arena= old_stmt_arena;
thd->cur_stmt = save_cur_stmt;
DBUG_RETURN(TRUE);
}
......@@ -4222,6 +4234,7 @@ bool Prepared_statement::prepare(const char *packet, uint packet_len)
lex_start(thd);
lex->context_analysis_only|= CONTEXT_ANALYSIS_ONLY_PREPARE;
error= (parse_sql(thd, & parser_state, NULL) ||
thd->is_error() ||
init_param_array(this));
......@@ -4297,6 +4310,7 @@ bool Prepared_statement::prepare(const char *packet, uint packet_len)
cleanup_stmt();
thd->restore_backup_statement(this, &stmt_backup);
thd->stmt_arena= old_stmt_arena;
thd->cur_stmt= save_cur_stmt;
if (likely(error == 0))
{
......@@ -4755,6 +4769,7 @@ Prepared_statement::reprepare()
it's failed, we need to return all the warnings to the user.
*/
thd->get_stmt_da()->clear_warning_info(thd->query_id);
column_info_state.reset();
}
else
{
......
......@@ -21147,6 +21147,124 @@ static void test_execute_direct()
#endif
}
static void assert_metadata_skipped_count_equals(MYSQL *mysql, int val)
{
MYSQL_ROW row;
MYSQL_RES *result;
int rc= mysql_query(mysql, "SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.SESSION_STATUS WHERE "
"VARIABLE_NAME='Resultset_metadata_skipped'");
myquery(rc);
result= mysql_use_result(mysql);
mytest(result);
row= mysql_fetch_row(result);
DIE_UNLESS(atoi(row[0]) == val);
mysql_free_result(result);
}
static void flush_session_status(MYSQL* mysql)
{
int rc= mysql_query(mysql, "flush status");
myquery(rc);
}
static void exec_stmt(MYSQL_STMT* stmt)
{
int rc= mysql_stmt_execute(stmt);
check_execute(stmt, rc);
rc= mysql_stmt_store_result(stmt);
check_execute(stmt, rc);
}
static void test_cache_metadata()
{
char char_val[]= "blah";
int int_val = 1;
MYSQL_BIND param= {0};
my_bool is_null= FALSE;
MYSQL_STMT* stmt= mysql_stmt_init(mysql);
check_stmt(stmt);
int rc= mysql_stmt_prepare(stmt, "SELECT ?", -1);
myquery(rc);
param.buffer= char_val;
param.buffer_type= MYSQL_TYPE_STRING;
param.is_null= &is_null;
param.buffer_length = 4;
rc= mysql_stmt_bind_param(stmt,&param);
exec_stmt(stmt);
flush_session_status(mysql);
/* Execute the statement again, check that metadata is skipped*/
exec_stmt(stmt);
assert_metadata_skipped_count_equals(mysql, 1);
flush_session_status(mysql);
/*
Execute the statement again, such that metadata changes,
(using LONG parameter in bind for "SELECT ?", instead of string.
Check that metadata is NOT skipped.
*/
param.buffer= &int_val;
param.buffer_type= MYSQL_TYPE_LONG;
param.is_null= &is_null;
rc= mysql_stmt_bind_param(stmt, &param);
exec_stmt(stmt);
assert_metadata_skipped_count_equals(mysql, 0);
mysql_stmt_close(stmt);
/*
Test with real table, and DDL which causes column info to be
changed.
*/
stmt= mysql_stmt_init(mysql);
rc= mysql_query(
mysql, "CREATE OR REPLACE TABLE t1 (a int, b bigint) engine=memory");
myquery(rc);
flush_session_status(mysql);
check_stmt(stmt);
rc= mysql_stmt_prepare(stmt, "SELECT * from t1", -1);
myquery(rc);
exec_stmt(stmt);
/* Metadata skipped, since already sent with COM_STMT_PREPARE result.*/
assert_metadata_skipped_count_equals(mysql, 1);
flush_session_status(mysql);
exec_stmt(stmt);
/* Metadata skipped again*/
assert_metadata_skipped_count_equals(mysql, 1);
rc= mysql_query(mysql, "ALTER TABLE t1 MODIFY b CHAR(10)");
myquery(rc);
/* Column metadata WILL change for the next execution due to DDL*/
flush_session_status(mysql);
exec_stmt(stmt);
assert_metadata_skipped_count_equals(mysql, 0);
/* On reexecution, PS column metadata will NOT change. */
flush_session_status(mysql);
exec_stmt(stmt);
assert_metadata_skipped_count_equals(mysql, 1);
rc= mysql_query(mysql, "DROP TABLE t1");
myquery(rc);
mysql_stmt_close(stmt);
}
static struct my_tests_st my_tests[]= {
{ "disable_query_logs", disable_query_logs },
{ "test_view_sp_list_fields", test_view_sp_list_fields },
......@@ -21446,6 +21564,7 @@ static struct my_tests_st my_tests[]= {
{ "test_mdev18408", test_mdev18408 },
{ "test_mdev20261", test_mdev20261 },
{ "test_execute_direct", test_execute_direct },
{ "test_cache_metadata", test_cache_metadata},
{ 0, 0 }
};
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment