MDEV-30659 Server crash on EXPLAIN SELECT/SELECT on table with engine Aria for LooseScan Strategy

The issue was that Loose_scan_opt::save_to_position() did not take
into account records_out from best_access_path()

mysql-test/main/selectivity.result

@@ -2011,3 +2011,57 @@ INSERT INTO t2 VALUES (1),(2);
 SELECT STRAIGHT_JOIN pk FROM t1 JOIN t2 ON a = pk WHERE b >= 'A' ORDER BY t2.pk LIMIT 8 OFFSET 1;
 pk
 DROP TABLE t1, t2;
+#
+# MDEV-30659 Server crash on EXPLAIN SELECT/SELECT on table with
+# engine Aria for LooseScan Strategy
+#
+create table t1 (old_c1 integer, old_c2 integer, c1 integer,
+c2 integer, c3 integer) engine=aria;
+insert into t1(c1,c2,c3)
+values (1,1,1), (1,2,2), (1,3,3),
+(2,1,4), (2,2,5), (2,3,6),
+(2,4,7), (2,5,8);
+create index t1_c2 on t1 (c2,c1);
+explain select * from t1 where t1.c2 in (select a.c2 from t1 a) and
+c2 >= 3 order by c2;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	a	range	t1_c2	t1_c2	5	NULL	5	Using where; Using index; LooseScan
+1	PRIMARY	t1	ref	t1_c2	t1_c2	5	test.a.c2	1	
+drop table t1;
+create table t1 (old_c1 integer, old_c2 integer, c1 integer,
+c2 integer, c3 integer) engine=aria;
+create trigger trg_t1 before update on t1 for each row
+begin
+set new.old_c1=old.c1;
+set new.old_c2=old.c2;
+end;
+/
+insert into t1 (c1,c2,c3) values
+(1,1,1), (1,2,2), (1,3,3), (2,1,4), (2,2,5), (2,3,6), (2,4,7), (2,5,8);
+create index t1_c2 on t1 (c2,c1);
+analyze table t1 persistent for all;
+Table	Op	Msg_type	Msg_text
+test.t1	analyze	status	Engine-independent statistics collected
+test.t1	analyze	status	OK
+create table t2 as select * from t1;
+truncate table t1;
+insert into t1 select * from t2;
+explain select * from t1 where t1.c2 in (select a.c2 from t1 a) and c2 >= 3 order by c2;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	a	range	t1_c2	t1_c2	5	NULL	5	Using where; Using index; LooseScan
+1	PRIMARY	t1	ref	t1_c2	t1_c2	5	test.a.c2	1	
+drop trigger trg_t1;
+drop table t1,t2;
+create table t1 (old_c1 integer, old_c2 integer, c1 integer,
+c2 integer, c3 integer) engine=aria;
+insert into t1 (c1,c2,c3) values
+(1,1,1), (1,2,2), (1,3,3), (2,1,4), (2,2,5), (2,3,6), (2,4,7), (2,5,8);
+create index t1_c2 on t1 (c2,c1);
+create table t2 as select * from t1;
+truncate table t1;
+insert into t1 select * from t2;
+explain select * from t1 where t1.c2 in (select a.c2 from t1 a) and c2 >= 3 order by c2;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	a	range	t1_c2	t1_c2	5	NULL	5	Using where; Using index; LooseScan
+1	PRIMARY	t1	ref	t1_c2	t1_c2	5	test.a.c2	2	
+drop table t1,t2;

mysql-test/main/selectivity.test

@@ -1376,3 +1376,54 @@ INSERT INTO t2 VALUES (1),(2);
 SELECT STRAIGHT_JOIN pk FROM t1 JOIN t2 ON a = pk WHERE b >= 'A' ORDER BY t2.pk LIMIT 8 OFFSET 1;
 
 DROP TABLE t1, t2;
+
+--echo #
+--echo # MDEV-30659 Server crash on EXPLAIN SELECT/SELECT on table with
+--echo # engine Aria for LooseScan Strategy
+--echo #
+
+create table t1 (old_c1 integer, old_c2 integer, c1 integer,
+                 c2 integer, c3 integer) engine=aria;
+insert into t1(c1,c2,c3)
+        values (1,1,1), (1,2,2), (1,3,3),
+               (2,1,4), (2,2,5), (2,3,6),
+               (2,4,7), (2,5,8);
+create index t1_c2 on t1 (c2,c1);
+
+explain select * from t1 where t1.c2 in (select a.c2 from t1 a) and
+c2 >= 3 order by c2;
+drop table t1;
+
+create table t1 (old_c1 integer, old_c2 integer, c1 integer,
+                 c2 integer, c3 integer) engine=aria;
+
+delimiter /;
+create trigger trg_t1 before update on t1 for each row
+begin
+  set new.old_c1=old.c1;
+  set new.old_c2=old.c2;
+end;
+/
+delimiter ;/
+
+insert into t1 (c1,c2,c3) values
+       (1,1,1), (1,2,2), (1,3,3), (2,1,4), (2,2,5), (2,3,6), (2,4,7), (2,5,8);
+create index t1_c2 on t1 (c2,c1);
+analyze table t1 persistent for all;
+create table t2 as select * from t1;
+truncate table t1;
+insert into t1 select * from t2;
+explain select * from t1 where t1.c2 in (select a.c2 from t1 a) and c2 >= 3 order by c2;
+drop trigger trg_t1;
+drop table t1,t2;
+
+create table t1 (old_c1 integer, old_c2 integer, c1 integer,
+                 c2 integer, c3 integer) engine=aria;
+insert into t1 (c1,c2,c3) values
+       (1,1,1), (1,2,2), (1,3,3), (2,1,4), (2,2,5), (2,3,6), (2,4,7), (2,5,8);
+create index t1_c2 on t1 (c2,c1);
+create table t2 as select * from t1;
+truncate table t1;
+insert into t1 select * from t2;
+explain select * from t1 where t1.c2 in (select a.c2 from t1 a) and c2 >= 3 order by c2;
+drop table t1,t2;

mysql-test/main/selectivity_innodb.result

@@ -2018,6 +2018,60 @@ INSERT INTO t2 VALUES (1),(2);
 SELECT STRAIGHT_JOIN pk FROM t1 JOIN t2 ON a = pk WHERE b >= 'A' ORDER BY t2.pk LIMIT 8 OFFSET 1;
 pk
 DROP TABLE t1, t2;
+#
+# MDEV-30659 Server crash on EXPLAIN SELECT/SELECT on table with
+# engine Aria for LooseScan Strategy
+#
+create table t1 (old_c1 integer, old_c2 integer, c1 integer,
+c2 integer, c3 integer) engine=aria;
+insert into t1(c1,c2,c3)
+values (1,1,1), (1,2,2), (1,3,3),
+(2,1,4), (2,2,5), (2,3,6),
+(2,4,7), (2,5,8);
+create index t1_c2 on t1 (c2,c1);
+explain select * from t1 where t1.c2 in (select a.c2 from t1 a) and
+c2 >= 3 order by c2;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	a	range	t1_c2	t1_c2	5	NULL	5	Using where; Using index; LooseScan
+1	PRIMARY	t1	ref	t1_c2	t1_c2	5	test.a.c2	1	
+drop table t1;
+create table t1 (old_c1 integer, old_c2 integer, c1 integer,
+c2 integer, c3 integer) engine=aria;
+create trigger trg_t1 before update on t1 for each row
+begin
+set new.old_c1=old.c1;
+set new.old_c2=old.c2;
+end;
+/
+insert into t1 (c1,c2,c3) values
+(1,1,1), (1,2,2), (1,3,3), (2,1,4), (2,2,5), (2,3,6), (2,4,7), (2,5,8);
+create index t1_c2 on t1 (c2,c1);
+analyze table t1 persistent for all;
+Table	Op	Msg_type	Msg_text
+test.t1	analyze	status	Engine-independent statistics collected
+test.t1	analyze	status	OK
+create table t2 as select * from t1;
+truncate table t1;
+insert into t1 select * from t2;
+explain select * from t1 where t1.c2 in (select a.c2 from t1 a) and c2 >= 3 order by c2;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	a	range	t1_c2	t1_c2	5	NULL	5	Using where; Using index; LooseScan
+1	PRIMARY	t1	ref	t1_c2	t1_c2	5	test.a.c2	1	
+drop trigger trg_t1;
+drop table t1,t2;
+create table t1 (old_c1 integer, old_c2 integer, c1 integer,
+c2 integer, c3 integer) engine=aria;
+insert into t1 (c1,c2,c3) values
+(1,1,1), (1,2,2), (1,3,3), (2,1,4), (2,2,5), (2,3,6), (2,4,7), (2,5,8);
+create index t1_c2 on t1 (c2,c1);
+create table t2 as select * from t1;
+truncate table t1;
+insert into t1 select * from t2;
+explain select * from t1 where t1.c2 in (select a.c2 from t1 a) and c2 >= 3 order by c2;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	a	range	t1_c2	t1_c2	5	NULL	5	Using where; Using index; LooseScan
+1	PRIMARY	t1	ref	t1_c2	t1_c2	5	test.a.c2	2	
+drop table t1,t2;
 set optimizer_switch=@save_optimizer_switch_for_selectivity_test;
 set @tmp_ust= @@use_stat_tables;
 set @tmp_oucs= @@optimizer_use_condition_selectivity;

mysql-test/main/subselect_sj2.result

@@ -736,8 +736,8 @@ alter table t3 add primary key(id), add key(a);
 The following must use loose index scan over t3, key a:
 explain select count(a) from t2 where a in ( SELECT  a FROM t3);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
-1	PRIMARY	t2	index	a	a	5	NULL	1000	Using where; Using index
-1	PRIMARY	t3	ref	a	a	5	test.t2.a	30	Using index; FirstMatch(t2)
+1	PRIMARY	t3	index	a	a	5	NULL	30000	Using where; Using index; LooseScan
+1	PRIMARY	t2	ref	a	a	5	test.t3.a	1	Using index
 select count(a) from t2 where a in ( SELECT  a FROM t3);
 count(a)
 1000

mysql-test/main/subselect_sj2_jcl6.result

@@ -745,8 +745,8 @@ alter table t3 add primary key(id), add key(a);
 The following must use loose index scan over t3, key a:
 explain select count(a) from t2 where a in ( SELECT  a FROM t3);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
-1	PRIMARY	t2	index	a	a	5	NULL	1000	Using where; Using index
-1	PRIMARY	t3	ref	a	a	5	test.t2.a	30	Using index; FirstMatch(t2)
+1	PRIMARY	t3	index	a	a	5	NULL	30000	Using where; Using index; LooseScan
+1	PRIMARY	t2	ref	a	a	5	test.t3.a	1	Using index
 select count(a) from t2 where a in ( SELECT  a FROM t3);
 count(a)
 1000

mysql-test/main/subselect_sj2_mat.result

@@ -738,8 +738,8 @@ alter table t3 add primary key(id), add key(a);
 The following must use loose index scan over t3, key a:
 explain select count(a) from t2 where a in ( SELECT  a FROM t3);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
-1	PRIMARY	t2	index	a	a	5	NULL	1000	Using where; Using index
-1	PRIMARY	t3	ref	a	a	5	test.t2.a	30	Using index; FirstMatch(t2)
+1	PRIMARY	t3	index	a	a	5	NULL	30000	Using where; Using index; LooseScan
+1	PRIMARY	t2	ref	a	a	5	test.t3.a	1	Using index
 select count(a) from t2 where a in ( SELECT  a FROM t3);
 count(a)
 1000

mysql-test/main/subselect_sj_mat.result

@@ -108,7 +108,7 @@ explain extended
 select * from t1i where a1 in (select b1 from t2i where b1 > '0');
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
 1	PRIMARY	t2i	index	it2i1,it2i3	it2i1	#	NULL	5	50.00	Using where; Using index; LooseScan
-1	PRIMARY	t1i	ref	_it1_idx	_it1_idx	#	_ref_	1	20.00	
+1	PRIMARY	t1i	ref	_it1_idx	_it1_idx	#	_ref_	1	40.00	
 Warnings:
 Note	1003	select `test`.`t1i`.`a1` AS `a1`,`test`.`t1i`.`a2` AS `a2` from `test`.`t1i` semi join (`test`.`t2i`) where `test`.`t1i`.`a1` = `test`.`t2i`.`b1` and `test`.`t2i`.`b1` > '0'
 select * from t1i where a1 in (select b1 from t2i where b1 > '0');
@@ -131,7 +131,7 @@ explain extended
 select * from t1i where (a1, a2) in (select b1, b2 from t2i where b1 > '0');
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
 1	PRIMARY	t2i	index	it2i1,it2i2,it2i3	it2i3	#	NULL	5	50.00	Using where; Using index; LooseScan
-1	PRIMARY	t1i	ref	_it1_idx	_it1_idx	#	_ref_	1	20.00	
+1	PRIMARY	t1i	ref	_it1_idx	_it1_idx	#	_ref_	1	40.00	
 Warnings:
 Note	1003	select `test`.`t1i`.`a1` AS `a1`,`test`.`t1i`.`a2` AS `a2` from `test`.`t1i` semi join (`test`.`t2i`) where `test`.`t1i`.`a1` = `test`.`t2i`.`b1` and `test`.`t1i`.`a2` = `test`.`t2i`.`b2` and `test`.`t2i`.`b1` > '0'
 select * from t1i where (a1, a2) in (select b1, b2 from t2i where b1 > '0');
@@ -277,7 +277,7 @@ explain extended
 select * from t1i where (a1, a2) in (select b1, b2 from t2i order by b1, b2);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
 1	PRIMARY	t2i	index	it2i1,it2i2,it2i3	it2i3	18	NULL	5	50.00	Using where; Using index; LooseScan
-1	PRIMARY	t1i	ref	it1i1,it1i2,it1i3	it1i3	18	test.t2i.b1,test.t2i.b2	1	20.00	Using index
+1	PRIMARY	t1i	ref	it1i1,it1i2,it1i3	it1i3	18	test.t2i.b1,test.t2i.b2	1	40.00	Using index
 Warnings:
 Note	1003	select `test`.`t1i`.`a1` AS `a1`,`test`.`t1i`.`a2` AS `a2` from `test`.`t1i` semi join (`test`.`t2i`) where `test`.`t1i`.`a1` = `test`.`t2i`.`b1` and `test`.`t1i`.`a2` = `test`.`t2i`.`b2`
 select * from t1i where (a1, a2) in (select b1, b2 from t2i order by b1, b2);
@@ -349,7 +349,7 @@ where (a1, a2) in (select b1, b2 from t2i where b1 >  '0') and
 where (c1, c2) in (select b1, b2 from t2i where b2 > '0'));
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
 1	PRIMARY	t2i	index	it2i1,it2i2,it2i3	#	#	#	5	50.00	#
-1	PRIMARY	t1i	ref	it1i1,it1i2,it1i3	#	#	#	1	20.00	#
+1	PRIMARY	t1i	ref	it1i1,it1i2,it1i3	#	#	#	1	40.00	#
 1	PRIMARY	t3i	ref	it3i1,it3i2,it3i3	#	#	#	1	100.00	#
 1	PRIMARY	t2i	ref	it2i1,it2i2,it2i3	#	#	#	1	60.00	#
 Warnings:
@@ -433,7 +433,7 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
 4	MATERIALIZED	t3	ALL	NULL	#	#	#	4	100.00	#
 3	MATERIALIZED	t3	ALL	NULL	#	#	#	4	100.00	#
 7	UNION	t2i	index	it2i1,it2i2,it2i3	#	#	#	5	50.00	#
-7	UNION	t1i	ref	it1i1,it1i2,it1i3	#	#	#	1	20.00	#
+7	UNION	t1i	ref	it1i1,it1i2,it1i3	#	#	#	1	40.00	#
 7	UNION	t3i	ref	it3i1,it3i2,it3i3	#	#	#	1	100.00	#
 7	UNION	t2i	ref	it2i1,it2i2,it2i3	#	#	#	1	60.00	#
 NULL	UNION RESULT	<union1,7>	ALL	NULL	#	#	#	NULL	NULL	#

sql/opt_subselect.h

@@ -294,14 +294,30 @@ class Loose_scan_opt
     }
   }
 
-  void save_to_position(JOIN_TAB *tab, double record_count, POSITION *pos)
+  void save_to_position(JOIN_TAB *tab, double record_count,
+                        double *records_out,
+                        POSITION *pos)
   {
     pos->read_time=       best_loose_scan_cost;
     if (best_loose_scan_cost != DBL_MAX)
     {
+      /*
+        best_loose_scan_records can be smaller than records_out in
+        the case where we use index statistics on a result column
+        to estimate the number of unique row combinations.
+        Example from selectivity.test:
+          SELECT * from t1 where t1.c2 in (select a.c2 from t1 a)
+                 and c2 >= 3 order by c2;
+          In this case range optimizer tells us that a has records_out of 5
+          while index statistics tells us that a.c2 has only 4 unique
+          values.
+      */
+      set_if_smaller(*records_out, best_loose_scan_records);
+
       pos->loops= record_count;
       pos->records_read=    best_loose_scan_records;
-      pos->records_init= pos->records_out= pos->records_read;
+      pos->records_init=    pos->records_read;
+      pos->records_out=     *records_out;
       pos->key=             best_loose_scan_start_key;
       pos->cond_selectivity= 1.0;
       pos->loosescan_picker.loosescan_key=   best_loose_scan_key;

sql/sql_select.cc

@@ -8151,7 +8151,6 @@ best_access_path(JOIN      *join,
   best.use_join_buffer= FALSE;
   best.spl_plan= 0;
 
-  pos->loops=        record_count;
   disable_jbuf= disable_jbuf || idx == join->const_tables;
 
   trace_wrapper.add_table_name(s);
@@ -9303,8 +9302,8 @@ best_access_path(JOIN      *join,
   crash_if_first_double_is_bigger(best.records_out, best.records_read);
 
   /* Update the cost information for the current partial plan */
+  pos->loops=        record_count;
   pos->records_init= best.records_read;
-  pos->record_combinations= record_count;
   pos->records_after_filter= best.records_after_filter;
   pos->records_read= best.records;
   pos->records_out=  best.records_out;
@@ -9324,7 +9323,8 @@ best_access_path(JOIN      *join,
                        key_dependent & remaining_tables);
   pos->refills=  best.refills;
 
-  loose_scan_opt.save_to_position(s, record_count, loose_scan_pos);
+  loose_scan_opt.save_to_position(s, record_count, &pos->records_out,
+                                  loose_scan_pos);
 
   if (!best.key &&
       idx == join->const_tables &&              // First table
@@ -11793,7 +11793,7 @@ prev_record_reads(const POSITION *position, uint idx, table_map found_ref,
         if (pos->type == JT_EQ_REF)
           found= COST_MULT(found, pos->identical_keys);
         else if (pos->use_join_buffer)
-          found= COST_MULT(found, pos->record_combinations / pos->refills);
+          found= COST_MULT(found, pos->loops / pos->refills);
       }
       break;
     }

sql/sql_select.h

@@ -1012,9 +1012,6 @@ class POSITION
   */
   double records_out;
 
-  /* record combinations before this table */
-  double record_combinations;
-
   /* Values from prev_record_reads call for EQ_REF table*/
   double        prev_record_reads, identical_keys;
 
@@ -1028,6 +1025,7 @@ class POSITION
   */
   double read_time;
 
+  /* record combinations before this table */
   double loops;
 
   double    prefix_record_count;