MDEV-33152 Add QUERIES to INDEX_STATISTICS

MDEV-33901 INDEX_STATISTICS.QUERIES is incremented additionally for subqueries

Other changes:
- Do not collect index statistics for system tables like index_stats
  table_stats, performance_schema, information_schema etc as the user
  has no control of these and the generate noise in the statistics.

mysql-test/main/information_schema_stats.result

@@ -16,22 +16,22 @@ select count(*) from just_a_test where state = 'California';
 count(*)
 2
 select * from information_schema.index_statistics where table_schema='test' and table_name='just_a_test';
-TABLE_SCHEMA	TABLE_NAME	INDEX_NAME	ROWS_READ
-test	just_a_test	IND_just_a_test_first_name_last_name	1
-test	just_a_test	IND_just_a_test_state	2
+TABLE_SCHEMA	TABLE_NAME	INDEX_NAME	ROWS_READ	QUERIES
+test	just_a_test	IND_just_a_test_first_name_last_name	1	1
+test	just_a_test	IND_just_a_test_state	2	1
 select * from information_schema.table_statistics where table_schema='test' and table_name='just_a_test';
 TABLE_SCHEMA	TABLE_NAME	ROWS_READ	ROWS_CHANGED	ROWS_CHANGED_X_INDEXES	ROWS_INSERTED	ROWS_UPDATED	ROWS_DELETED	KEY_READ_HITS	KEY_READ_MISSES
 test	just_a_test	18	5	5	5	0	0	2	0
 alter table just_a_test drop key IND_just_a_test_first_name_last_name;
 select * from information_schema.index_statistics where table_schema='test' and table_name='just_a_test';
-TABLE_SCHEMA	TABLE_NAME	INDEX_NAME	ROWS_READ
-test	just_a_test	IND_just_a_test_state	2
+TABLE_SCHEMA	TABLE_NAME	INDEX_NAME	ROWS_READ	QUERIES
+test	just_a_test	IND_just_a_test_state	2	1
 select * from information_schema.table_statistics where table_schema='test' and table_name='just_a_test';
 TABLE_SCHEMA	TABLE_NAME	ROWS_READ	ROWS_CHANGED	ROWS_CHANGED_X_INDEXES	ROWS_INSERTED	ROWS_UPDATED	ROWS_DELETED	KEY_READ_HITS	KEY_READ_MISSES
 test	just_a_test	23	5	5	5	0	0	2	0
 alter table just_a_test drop column state;
 select * from information_schema.index_statistics where table_schema='test' and table_name='just_a_test';
-TABLE_SCHEMA	TABLE_NAME	INDEX_NAME	ROWS_READ
+TABLE_SCHEMA	TABLE_NAME	INDEX_NAME	ROWS_READ	QUERIES
 select * from just_a_test force index(primary) where id between 1 and 2;
 id	first_name	last_name	address	phone	email
 1	fa	la	china_a	11111111	fa_la@163.com
@@ -46,11 +46,14 @@ delete from just_a_test where id=5;
 select * from information_schema.table_statistics where table_schema='test' and table_name='just_a_test';
 TABLE_SCHEMA	TABLE_NAME	ROWS_READ	ROWS_CHANGED	ROWS_CHANGED_X_INDEXES	ROWS_INSERTED	ROWS_UPDATED	ROWS_DELETED	KEY_READ_HITS	KEY_READ_MISSES
 test	just_a_test	32	7	7	5	1	1	5	1
-drop table just_a_test;
 select * from information_schema.index_statistics where table_schema='test' and table_name='just_a_test';
-TABLE_SCHEMA	TABLE_NAME	INDEX_NAME	ROWS_READ
+TABLE_SCHEMA	TABLE_NAME	INDEX_NAME	ROWS_READ	QUERIES
+test	just_a_test	PRIMARY	4	3
+drop table just_a_test;
 select * from information_schema.table_statistics where table_schema='test' and table_name='just_a_test';
 TABLE_SCHEMA	TABLE_NAME	ROWS_READ	ROWS_CHANGED	ROWS_CHANGED_X_INDEXES	ROWS_INSERTED	ROWS_UPDATED	ROWS_DELETED	KEY_READ_HITS	KEY_READ_MISSES
+select * from information_schema.index_statistics where table_schema='test' and table_name='just_a_test';
+TABLE_SCHEMA	TABLE_NAME	INDEX_NAME	ROWS_READ	QUERIES
 create table just_a_test(id int not null primary key,first_name varchar(10),last_name varchar(10),address varchar(100),phone bigint,email varchar(30), state varchar(30),key(first_name,last_name),key(state));
 insert into just_a_test values(1,'fa','la','china_a',11111111,'fa_la@163.com','California'),
 (2,'fb','lb','china_b',22222222,'fb_lb@163.com','Arizona'),
@@ -67,16 +70,16 @@ select count(*) from just_a_test where id between 2 and 4;
 count(*)
 3
 select * from information_schema.index_statistics where table_schema='test' and table_name='just_a_test';
-TABLE_SCHEMA	TABLE_NAME	INDEX_NAME	ROWS_READ
-test	just_a_test	PRIMARY	4
-test	just_a_test	first_name	1
-test	just_a_test	state	2
+TABLE_SCHEMA	TABLE_NAME	INDEX_NAME	ROWS_READ	QUERIES
+test	just_a_test	PRIMARY	4	1
+test	just_a_test	first_name	1	1
+test	just_a_test	state	2	1
 select * from information_schema.table_statistics where table_schema='test' and table_name='just_a_test';
 TABLE_SCHEMA	TABLE_NAME	ROWS_READ	ROWS_CHANGED	ROWS_CHANGED_X_INDEXES	ROWS_INSERTED	ROWS_UPDATED	ROWS_DELETED	KEY_READ_HITS	KEY_READ_MISSES
 test	just_a_test	7	5	15	5	0	0	3	0
 drop table just_a_test;
 select * from information_schema.index_statistics where table_schema='test' and table_name='just_a_test';
-TABLE_SCHEMA	TABLE_NAME	INDEX_NAME	ROWS_READ
+TABLE_SCHEMA	TABLE_NAME	INDEX_NAME	ROWS_READ	QUERIES
 select * from information_schema.table_statistics where table_schema='test' and table_name='just_a_test';
 TABLE_SCHEMA	TABLE_NAME	ROWS_READ	ROWS_CHANGED	ROWS_CHANGED_X_INDEXES	ROWS_INSERTED	ROWS_UPDATED	ROWS_DELETED	KEY_READ_HITS	KEY_READ_MISSES
 set global userstat=@save_userstat;

mysql-test/main/information_schema_stats.test

@@ -30,9 +30,10 @@ update just_a_test set first_name="unlucky" where id=5;
 select * from information_schema.table_statistics where table_schema='test' and table_name='just_a_test';
 delete from just_a_test where id=5;
 select * from information_schema.table_statistics where table_schema='test' and table_name='just_a_test';
-drop table just_a_test;
 select * from information_schema.index_statistics where table_schema='test' and table_name='just_a_test';
+drop table just_a_test;
 select * from information_schema.table_statistics where table_schema='test' and table_name='just_a_test';
+select * from information_schema.index_statistics where table_schema='test' and table_name='just_a_test';
 #
 # Test direct drop table
 #

mysql-test/main/subselect.result

@@ -7131,8 +7131,8 @@ test	t1	2	0	0	0	0	0	0	0
 test	t2	3	0	0	0	0	0	1	0
 test	t3	0	0	0	0	0	0	0	1
 show index_statistics;
-Table_schema	Table_name	Index_name	Rows_read
-test	t2	b	1
+Table_schema	Table_name	Index_name	Rows_read	Queries
+test	t2	b	1	1
 set global userstat=@tmp_mdev410;
 DROP TABLE t1,t2,t3,t4;
 #

mysql-test/main/subselect_no_exists_to_in.result

@@ -7133,8 +7133,8 @@ test	t1	2	0	0	0	0	0	0	0
 test	t2	3	0	0	0	0	0	1	0
 test	t3	0	0	0	0	0	0	0	1
 show index_statistics;
-Table_schema	Table_name	Index_name	Rows_read
-test	t2	b	1
+Table_schema	Table_name	Index_name	Rows_read	Queries
+test	t2	b	1	1
 set global userstat=@tmp_mdev410;
 DROP TABLE t1,t2,t3,t4;
 #

mysql-test/main/subselect_no_mat.result

@@ -7129,8 +7129,8 @@ test	t1	2	0	0	0	0	0	0	0
 test	t2	3	0	0	0	0	0	1	0
 test	t3	0	0	0	0	0	0	0	1
 show index_statistics;
-Table_schema	Table_name	Index_name	Rows_read
-test	t2	b	1
+Table_schema	Table_name	Index_name	Rows_read	Queries
+test	t2	b	1	1
 set global userstat=@tmp_mdev410;
 DROP TABLE t1,t2,t3,t4;
 #

mysql-test/main/subselect_no_opts.result

@@ -7126,8 +7126,8 @@ test	t1	2	0	0	0	0	0	0	0
 test	t2	3	0	0	0	0	0	1	0
 test	t3	0	0	0	0	0	0	0	1
 show index_statistics;
-Table_schema	Table_name	Index_name	Rows_read
-test	t2	b	1
+Table_schema	Table_name	Index_name	Rows_read	Queries
+test	t2	b	1	1
 set global userstat=@tmp_mdev410;
 DROP TABLE t1,t2,t3,t4;
 #

mysql-test/main/subselect_no_scache.result

@@ -7137,8 +7137,8 @@ test	t1	2	0	0	0	0	0	0	0
 test	t2	3	0	0	0	0	0	1	0
 test	t3	0	0	0	0	0	0	0	1
 show index_statistics;
-Table_schema	Table_name	Index_name	Rows_read
-test	t2	b	1
+Table_schema	Table_name	Index_name	Rows_read	Queries
+test	t2	b	1	1
 set global userstat=@tmp_mdev410;
 DROP TABLE t1,t2,t3,t4;
 #

mysql-test/main/subselect_no_semijoin.result

@@ -7126,8 +7126,8 @@ test	t1	2	0	0	0	0	0	0	0
 test	t2	3	0	0	0	0	0	1	0
 test	t3	0	0	0	0	0	0	0	1
 show index_statistics;
-Table_schema	Table_name	Index_name	Rows_read
-test	t2	b	1
+Table_schema	Table_name	Index_name	Rows_read	Queries
+test	t2	b	1	1
 set global userstat=@tmp_mdev410;
 DROP TABLE t1,t2,t3,t4;
 #

mysql-test/main/userstat.result

@@ -63,6 +63,7 @@ TABLE_SCHEMA	varchar(192)	NO		NULL
 TABLE_NAME	varchar(192)	NO		NULL	
 INDEX_NAME	varchar(192)	NO		NULL	
 ROWS_READ	bigint(21)	NO		NULL	
+QUERIES	bigint(21)	NO		NULL	
 show columns from information_schema.table_statistics;
 Field	Type	Null	Key	Default	Extra
 TABLE_SCHEMA	varchar(192)	NO		NULL	
@@ -147,13 +148,13 @@ handler_read_key
 disconnect ssl_con;
 set @@global.userstat=0;
 select * from information_schema.index_statistics;
-TABLE_SCHEMA	TABLE_NAME	INDEX_NAME	ROWS_READ
+TABLE_SCHEMA	TABLE_NAME	INDEX_NAME	ROWS_READ	QUERIES
 select * from information_schema.table_statistics;
 TABLE_SCHEMA	TABLE_NAME	ROWS_READ	ROWS_CHANGED	ROWS_CHANGED_X_INDEXES	ROWS_INSERTED	ROWS_UPDATED	ROWS_DELETED	KEY_READ_HITS	KEY_READ_MISSES
 show table_statistics;
 Table_schema	Table_name	Rows_read	Rows_changed	Rows_changed_x_#indexes	Rows_inserted	Rows_updated	Rows_deleted	Key_read_hits	Key_read_misses
 show index_statistics;
-Table_schema	Table_name	Index_name	Rows_read
+Table_schema	Table_name	Index_name	Rows_read	Queries
 select TOTAL_CONNECTIONS, TOTAL_SSL_CONNECTIONS, CONCURRENT_CONNECTIONS, ROWS_READ, ROWS_SENT, ROWS_DELETED, ROWS_INSERTED, ROWS_UPDATED, SELECT_COMMANDS, UPDATE_COMMANDS, COMMIT_TRANSACTIONS, ROLLBACK_TRANSACTIONS, DENIED_CONNECTIONS, LOST_CONNECTIONS, ACCESS_DENIED, EMPTY_QUERIES from information_schema.client_statistics;;
 TOTAL_CONNECTIONS	2
 TOTAL_SSL_CONNECTIONS	1
@@ -197,7 +198,7 @@ OTHER_COMMANDS IN (7,8)
 flush table_statistics;
 flush index_statistics;
 select * from information_schema.index_statistics;
-TABLE_SCHEMA	TABLE_NAME	INDEX_NAME	ROWS_READ
+TABLE_SCHEMA	TABLE_NAME	INDEX_NAME	ROWS_READ	QUERIES
 select * from information_schema.table_statistics;
 TABLE_SCHEMA	TABLE_NAME	ROWS_READ	ROWS_CHANGED	ROWS_CHANGED_X_INDEXES	ROWS_INSERTED	ROWS_UPDATED	ROWS_DELETED	KEY_READ_HITS	KEY_READ_MISSES
 show status like "%generic%";
@@ -263,3 +264,26 @@ drop table t1;
 #
 # End of 10.2 tests
 #
+#
+# MDEV-33901 INDEX_STATISTICS.QUERIES is incremented additionally for
+# subqueries
+#
+SET @save_userstat= @@userstat;
+set global userstat= 1;
+create or replace table t1 (a int, key(a)) engine=MyISAM;
+insert into t1 values (1),(2),(3),(4);
+flush index_statistics;
+select a from t1 where a in ( select a from t1 );
+a
+1
+2
+3
+4
+show index_statistics;
+Table_schema	Table_name	Index_name	Rows_read	Queries
+test	t1	a	8	1
+drop table t1;
+set global userstat=@save_userstat;
+#
+# End of 11.5 tests
+#

mysql-test/main/userstat.test

@@ -137,3 +137,25 @@ drop table t1;
 --echo #
 --echo # End of 10.2 tests
 --echo #
+
+--echo #
+--echo # MDEV-33901 INDEX_STATISTICS.QUERIES is incremented additionally for
+--echo # subqueries
+--echo #
+
+SET @save_userstat= @@userstat;
+set global userstat= 1;
+
+--disable_ps_protocol
+create or replace table t1 (a int, key(a)) engine=MyISAM;
+insert into t1 values (1),(2),(3),(4);
+flush index_statistics;
+select a from t1 where a in ( select a from t1 );
+show index_statistics;
+drop table t1;
+set global userstat=@save_userstat;
+--enable_ps_protocol
+
+--echo #
+--echo # End of 11.5 tests
+--echo #

mysql-test/suite/funcs_1/r/is_columns_is.result

@@ -186,6 +186,7 @@ def	information_schema	GLOBAL_STATUS	VARIABLE_VALUE	2	NULL	NO	varchar	4096	12288
 def	information_schema	GLOBAL_VARIABLES	VARIABLE_NAME	1	NULL	NO	varchar	64	192	NULL	NULL	NULL	utf8mb3	utf8mb3_general_ci	varchar(64)			select		NEVER	NULL	NO	NO
 def	information_schema	GLOBAL_VARIABLES	VARIABLE_VALUE	2	NULL	NO	varchar	4096	12288	NULL	NULL	NULL	utf8mb3	utf8mb3_general_ci	varchar(4096)			select		NEVER	NULL	NO	NO
 def	information_schema	INDEX_STATISTICS	INDEX_NAME	3	NULL	NO	varchar	192	576	NULL	NULL	NULL	utf8mb3	utf8mb3_general_ci	varchar(192)			select		NEVER	NULL	NO	NO
+def	information_schema	INDEX_STATISTICS	QUERIES	5	NULL	NO	bigint	NULL	NULL	19	0	NULL	NULL	NULL	bigint(21)			select		NEVER	NULL	NO	NO
 def	information_schema	INDEX_STATISTICS	ROWS_READ	4	NULL	NO	bigint	NULL	NULL	19	0	NULL	NULL	NULL	bigint(21)			select		NEVER	NULL	NO	NO
 def	information_schema	INDEX_STATISTICS	TABLE_NAME	2	NULL	NO	varchar	192	576	NULL	NULL	NULL	utf8mb3	utf8mb3_general_ci	varchar(192)			select		NEVER	NULL	NO	NO
 def	information_schema	INDEX_STATISTICS	TABLE_SCHEMA	1	NULL	NO	varchar	192	576	NULL	NULL	NULL	utf8mb3	utf8mb3_general_ci	varchar(192)			select		NEVER	NULL	NO	NO
@@ -791,6 +792,7 @@ NULL	information_schema	GEOMETRY_COLUMNS	SRID	smallint	NULL	NULL	NULL	NULL	small
 3.0000	information_schema	INDEX_STATISTICS	TABLE_NAME	varchar	192	576	utf8mb3	utf8mb3_general_ci	varchar(192)
 3.0000	information_schema	INDEX_STATISTICS	INDEX_NAME	varchar	192	576	utf8mb3	utf8mb3_general_ci	varchar(192)
 NULL	information_schema	INDEX_STATISTICS	ROWS_READ	bigint	NULL	NULL	NULL	NULL	bigint(21)
+NULL	information_schema	INDEX_STATISTICS	QUERIES	bigint	NULL	NULL	NULL	NULL	bigint(21)
 3.0000	information_schema	KEYWORDS	WORD	varchar	64	192	utf8mb3	utf8mb3_general_ci	varchar(64)
 3.0000	information_schema	KEY_CACHES	KEY_CACHE_NAME	varchar	192	576	utf8mb3	utf8mb3_general_ci	varchar(192)
 NULL	information_schema	KEY_CACHES	SEGMENTS	int	NULL	NULL	NULL	NULL	int(3) unsigned

mysql-test/suite/funcs_1/r/is_columns_is_embedded.result

@@ -186,6 +186,7 @@ def	information_schema	GLOBAL_STATUS	VARIABLE_VALUE	2	NULL	NO	varchar	4096	12288
 def	information_schema	GLOBAL_VARIABLES	VARIABLE_NAME	1	NULL	NO	varchar	64	192	NULL	NULL	NULL	utf8mb3	utf8mb3_general_ci	varchar(64)					NEVER	NULL	NO	NO
 def	information_schema	GLOBAL_VARIABLES	VARIABLE_VALUE	2	NULL	NO	varchar	4096	12288	NULL	NULL	NULL	utf8mb3	utf8mb3_general_ci	varchar(4096)					NEVER	NULL	NO	NO
 def	information_schema	INDEX_STATISTICS	INDEX_NAME	3	NULL	NO	varchar	192	576	NULL	NULL	NULL	utf8mb3	utf8mb3_general_ci	varchar(192)					NEVER	NULL	NO	NO
+def	information_schema	INDEX_STATISTICS	QUERIES	5	NULL	NO	bigint	NULL	NULL	19	0	NULL	NULL	NULL	bigint(21)					NEVER	NULL	NO	NO
 def	information_schema	INDEX_STATISTICS	ROWS_READ	4	NULL	NO	bigint	NULL	NULL	19	0	NULL	NULL	NULL	bigint(21)					NEVER	NULL	NO	NO
 def	information_schema	INDEX_STATISTICS	TABLE_NAME	2	NULL	NO	varchar	192	576	NULL	NULL	NULL	utf8mb3	utf8mb3_general_ci	varchar(192)					NEVER	NULL	NO	NO
 def	information_schema	INDEX_STATISTICS	TABLE_SCHEMA	1	NULL	NO	varchar	192	576	NULL	NULL	NULL	utf8mb3	utf8mb3_general_ci	varchar(192)					NEVER	NULL	NO	NO
@@ -791,6 +792,7 @@ NULL	information_schema	GEOMETRY_COLUMNS	SRID	smallint	NULL	NULL	NULL	NULL	small
 3.0000	information_schema	INDEX_STATISTICS	TABLE_NAME	varchar	192	576	utf8mb3	utf8mb3_general_ci	varchar(192)
 3.0000	information_schema	INDEX_STATISTICS	INDEX_NAME	varchar	192	576	utf8mb3	utf8mb3_general_ci	varchar(192)
 NULL	information_schema	INDEX_STATISTICS	ROWS_READ	bigint	NULL	NULL	NULL	NULL	bigint(21)
+NULL	information_schema	INDEX_STATISTICS	QUERIES	bigint	NULL	NULL	NULL	NULL	bigint(21)
 3.0000	information_schema	KEYWORDS	WORD	varchar	64	192	utf8mb3	utf8mb3_general_ci	varchar(64)
 3.0000	information_schema	KEY_CACHES	KEY_CACHE_NAME	varchar	192	576	utf8mb3	utf8mb3_general_ci	varchar(192)
 NULL	information_schema	KEY_CACHES	SEGMENTS	int	NULL	NULL	NULL	NULL	int(3) unsigned

plugin/userstat/index_stats.cc

@@ -6,6 +6,7 @@ static ST_FIELD_INFO index_stats_fields[]=
   Column("TABLE_NAME",   Varchar(NAME_LEN),   NOT_NULL, "Table_name"),
   Column("INDEX_NAME",   Varchar(NAME_LEN),   NOT_NULL, "Index_name"),
   Column("ROWS_READ",    SLonglong(),         NOT_NULL, "Rows_read"),
+  Column("QUERIES",      SLonglong(),         NOT_NULL, "Queries"),
   CEnd()
 };
 
@@ -44,6 +45,7 @@ static int index_stats_fill(THD *thd, TABLE_LIST *tables, COND *cond)
                            system_charset_info);
     table->field[2]->store(index_name, (uint) index_name_length, system_charset_info);
     table->field[3]->store((longlong)index_stats->rows_read, TRUE);
+    table->field[4]->store((longlong)index_stats->queries, TRUE);
 
     if (schema_table_store_record(thd, table))
     {

sql/handler.cc

@@ -5984,6 +5984,9 @@ void handler::update_global_index_stats()
 {
   DBUG_ASSERT(table->s);
 
+  if (table->s->table_category != TABLE_CATEGORY_USER)
+    return; // Ignore stat tables, performance_schema, information_schema etc.
+
   if (!table->in_use->userstat_running)
   {
     /* Reset all index read values */
@@ -6024,6 +6027,15 @@ void handler::update_global_index_stats()
       }
       /* Updates the global index stats. */
       index_stats->rows_read+= index_rows_read[index];
+      /*
+        Ensure we do not update queries if the table is used
+        twice in the same statement.
+      */
+      if (index_stats->query_id != table->in_use->query_id)
+      {
+        index_stats->query_id= table->in_use->query_id;
+        index_stats->queries++;
+      }
       index_rows_read[index]= 0;
 end:
       mysql_mutex_unlock(&LOCK_global_index_stats);

sql/structs.h

@@ -29,6 +29,7 @@
 #include <mysql_com.h>                  /* USERNAME_LENGTH */
 #include "sql_bitmap.h"
 #include "lex_charset.h"
+#include "sql_basic_types.h"           /* query_id_t */
 
 struct TABLE;
 class Type_handler;
@@ -382,6 +383,8 @@ typedef struct st_index_stats
   char index[NAME_LEN * 3 + 3];
   size_t index_name_length;                       /* Length of 'index' */
   ulonglong rows_read;
+  ulonglong queries;
+  query_id_t query_id;
 } INDEX_STATS;