MDEV-30659 Server crash on EXPLAIN SELECT/SELECT on table with engine Aria for LooseScan Strategy

Amended patch from Monty:

The issue was that Loose_scan_opt::save_to_position() did not take
into account records_out from best_access_path()

Make sure that POSITION object filled by Loose_scan_opt::save_to_position()
has records_out not higher than any other possible access method.

mysql-test/main/selectivity.result

@@ -2011,3 +2011,57 @@ INSERT INTO t2 VALUES (1),(2);
 SELECT STRAIGHT_JOIN pk FROM t1 JOIN t2 ON a = pk WHERE b >= 'A' ORDER BY t2.pk LIMIT 8 OFFSET 1;
 pk
 DROP TABLE t1, t2;
+#
+# MDEV-30659 Server crash on EXPLAIN SELECT/SELECT on table with
+# engine Aria for LooseScan Strategy
+#
+create table t1 (old_c1 integer, old_c2 integer, c1 integer,
+c2 integer, c3 integer) engine=aria;
+insert into t1(c1,c2,c3)
+values (1,1,1), (1,2,2), (1,3,3),
+(2,1,4), (2,2,5), (2,3,6),
+(2,4,7), (2,5,8);
+create index t1_c2 on t1 (c2,c1);
+explain select * from t1 where t1.c2 in (select a.c2 from t1 a) and
+c2 >= 3 order by c2;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	a	range	t1_c2	t1_c2	5	NULL	5	Using where; Using index; LooseScan
+1	PRIMARY	t1	ref	t1_c2	t1_c2	5	test.a.c2	1	
+drop table t1;
+create table t1 (old_c1 integer, old_c2 integer, c1 integer,
+c2 integer, c3 integer) engine=aria;
+create trigger trg_t1 before update on t1 for each row
+begin
+set new.old_c1=old.c1;
+set new.old_c2=old.c2;
+end;
+/
+insert into t1 (c1,c2,c3) values
+(1,1,1), (1,2,2), (1,3,3), (2,1,4), (2,2,5), (2,3,6), (2,4,7), (2,5,8);
+create index t1_c2 on t1 (c2,c1);
+analyze table t1 persistent for all;
+Table	Op	Msg_type	Msg_text
+test.t1	analyze	status	Engine-independent statistics collected
+test.t1	analyze	status	OK
+create table t2 as select * from t1;
+truncate table t1;
+insert into t1 select * from t2;
+explain select * from t1 where t1.c2 in (select a.c2 from t1 a) and c2 >= 3 order by c2;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	a	range	t1_c2	t1_c2	5	NULL	5	Using where; Using index; LooseScan
+1	PRIMARY	t1	ref	t1_c2	t1_c2	5	test.a.c2	1	
+drop trigger trg_t1;
+drop table t1,t2;
+create table t1 (old_c1 integer, old_c2 integer, c1 integer,
+c2 integer, c3 integer) engine=aria;
+insert into t1 (c1,c2,c3) values
+(1,1,1), (1,2,2), (1,3,3), (2,1,4), (2,2,5), (2,3,6), (2,4,7), (2,5,8);
+create index t1_c2 on t1 (c2,c1);
+create table t2 as select * from t1;
+truncate table t1;
+insert into t1 select * from t2;
+explain select * from t1 where t1.c2 in (select a.c2 from t1 a) and c2 >= 3 order by c2;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	a	range	t1_c2	t1_c2	5	NULL	5	Using where; Using index; LooseScan
+1	PRIMARY	t1	ref	t1_c2	t1_c2	5	test.a.c2	2	
+drop table t1,t2;

mysql-test/main/selectivity.test

@@ -1376,3 +1376,54 @@ INSERT INTO t2 VALUES (1),(2);
 SELECT STRAIGHT_JOIN pk FROM t1 JOIN t2 ON a = pk WHERE b >= 'A' ORDER BY t2.pk LIMIT 8 OFFSET 1;
 
 DROP TABLE t1, t2;
+
+--echo #
+--echo # MDEV-30659 Server crash on EXPLAIN SELECT/SELECT on table with
+--echo # engine Aria for LooseScan Strategy
+--echo #
+
+create table t1 (old_c1 integer, old_c2 integer, c1 integer,
+                 c2 integer, c3 integer) engine=aria;
+insert into t1(c1,c2,c3)
+        values (1,1,1), (1,2,2), (1,3,3),
+               (2,1,4), (2,2,5), (2,3,6),
+               (2,4,7), (2,5,8);
+create index t1_c2 on t1 (c2,c1);
+
+explain select * from t1 where t1.c2 in (select a.c2 from t1 a) and
+c2 >= 3 order by c2;
+drop table t1;
+
+create table t1 (old_c1 integer, old_c2 integer, c1 integer,
+                 c2 integer, c3 integer) engine=aria;
+
+delimiter /;
+create trigger trg_t1 before update on t1 for each row
+begin
+  set new.old_c1=old.c1;
+  set new.old_c2=old.c2;
+end;
+/
+delimiter ;/
+
+insert into t1 (c1,c2,c3) values
+       (1,1,1), (1,2,2), (1,3,3), (2,1,4), (2,2,5), (2,3,6), (2,4,7), (2,5,8);
+create index t1_c2 on t1 (c2,c1);
+analyze table t1 persistent for all;
+create table t2 as select * from t1;
+truncate table t1;
+insert into t1 select * from t2;
+explain select * from t1 where t1.c2 in (select a.c2 from t1 a) and c2 >= 3 order by c2;
+drop trigger trg_t1;
+drop table t1,t2;
+
+create table t1 (old_c1 integer, old_c2 integer, c1 integer,
+                 c2 integer, c3 integer) engine=aria;
+insert into t1 (c1,c2,c3) values
+       (1,1,1), (1,2,2), (1,3,3), (2,1,4), (2,2,5), (2,3,6), (2,4,7), (2,5,8);
+create index t1_c2 on t1 (c2,c1);
+create table t2 as select * from t1;
+truncate table t1;
+insert into t1 select * from t2;
+explain select * from t1 where t1.c2 in (select a.c2 from t1 a) and c2 >= 3 order by c2;
+drop table t1,t2;

mysql-test/main/selectivity_innodb.result

@@ -2018,6 +2018,60 @@ INSERT INTO t2 VALUES (1),(2);
 SELECT STRAIGHT_JOIN pk FROM t1 JOIN t2 ON a = pk WHERE b >= 'A' ORDER BY t2.pk LIMIT 8 OFFSET 1;
 pk
 DROP TABLE t1, t2;
+#
+# MDEV-30659 Server crash on EXPLAIN SELECT/SELECT on table with
+# engine Aria for LooseScan Strategy
+#
+create table t1 (old_c1 integer, old_c2 integer, c1 integer,
+c2 integer, c3 integer) engine=aria;
+insert into t1(c1,c2,c3)
+values (1,1,1), (1,2,2), (1,3,3),
+(2,1,4), (2,2,5), (2,3,6),
+(2,4,7), (2,5,8);
+create index t1_c2 on t1 (c2,c1);
+explain select * from t1 where t1.c2 in (select a.c2 from t1 a) and
+c2 >= 3 order by c2;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	a	range	t1_c2	t1_c2	5	NULL	5	Using where; Using index; LooseScan
+1	PRIMARY	t1	ref	t1_c2	t1_c2	5	test.a.c2	1	
+drop table t1;
+create table t1 (old_c1 integer, old_c2 integer, c1 integer,
+c2 integer, c3 integer) engine=aria;
+create trigger trg_t1 before update on t1 for each row
+begin
+set new.old_c1=old.c1;
+set new.old_c2=old.c2;
+end;
+/
+insert into t1 (c1,c2,c3) values
+(1,1,1), (1,2,2), (1,3,3), (2,1,4), (2,2,5), (2,3,6), (2,4,7), (2,5,8);
+create index t1_c2 on t1 (c2,c1);
+analyze table t1 persistent for all;
+Table	Op	Msg_type	Msg_text
+test.t1	analyze	status	Engine-independent statistics collected
+test.t1	analyze	status	OK
+create table t2 as select * from t1;
+truncate table t1;
+insert into t1 select * from t2;
+explain select * from t1 where t1.c2 in (select a.c2 from t1 a) and c2 >= 3 order by c2;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	a	range	t1_c2	t1_c2	5	NULL	5	Using where; Using index; LooseScan
+1	PRIMARY	t1	ref	t1_c2	t1_c2	5	test.a.c2	1	
+drop trigger trg_t1;
+drop table t1,t2;
+create table t1 (old_c1 integer, old_c2 integer, c1 integer,
+c2 integer, c3 integer) engine=aria;
+insert into t1 (c1,c2,c3) values
+(1,1,1), (1,2,2), (1,3,3), (2,1,4), (2,2,5), (2,3,6), (2,4,7), (2,5,8);
+create index t1_c2 on t1 (c2,c1);
+create table t2 as select * from t1;
+truncate table t1;
+insert into t1 select * from t2;
+explain select * from t1 where t1.c2 in (select a.c2 from t1 a) and c2 >= 3 order by c2;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	a	range	t1_c2	t1_c2	5	NULL	5	Using where; Using index; LooseScan
+1	PRIMARY	t1	ref	t1_c2	t1_c2	5	test.a.c2	2	
+drop table t1,t2;
 set optimizer_switch=@save_optimizer_switch_for_selectivity_test;
 set @tmp_ust= @@use_stat_tables;
 set @tmp_oucs= @@optimizer_use_condition_selectivity;

sql/opt_subselect.h

@@ -294,14 +294,23 @@ class Loose_scan_opt
     }
   }
 
-  void save_to_position(JOIN_TAB *tab, double record_count, POSITION *pos)
+  void save_to_position(JOIN_TAB *tab, double record_count,
+                        double records_out,
+                        POSITION *pos)
   {
     pos->read_time=       best_loose_scan_cost;
     if (best_loose_scan_cost != DBL_MAX)
     {
+      /*
+        Make sure LooseScan plan doesn't produce more rows than
+        the records_out of other table access method.
+      */
+      set_if_smaller(best_loose_scan_records, records_out);
+
       pos->loops= record_count;
       pos->records_read=    best_loose_scan_records;
-      pos->records_init= pos->records_out= pos->records_read;
+      pos->records_init=    pos->records_read;
+      pos->records_out=     best_loose_scan_records;
       pos->key=             best_loose_scan_start_key;
       pos->cond_selectivity= 1.0;
       pos->loosescan_picker.loosescan_key=   best_loose_scan_key;

sql/sql_select.cc

@@ -8155,7 +8155,6 @@ best_access_path(JOIN      *join,
   best.uses_jbuf= FALSE;
   best.spl_plan= 0;
 
-  pos->loops=        record_count;
   disable_jbuf= disable_jbuf || idx == join->const_tables;
 
   trace_wrapper.add_table_name(s);
@@ -9282,6 +9281,7 @@ best_access_path(JOIN      *join,
   crash_if_first_double_is_bigger(best.records_out, best.records_read);
 
   /* Update the cost information for the current partial plan */
+  pos->loops=        record_count;
   pos->records_init= best.records_read;
   pos->records_after_filter= best.records_after_filter;
   pos->records_read= best.records;
@@ -9299,7 +9299,8 @@ best_access_path(JOIN      *join,
   pos->key_dependent= (best.type == JT_EQ_REF ? (table_map) 0 :
                        key_dependent & remaining_tables);
 
-  loose_scan_opt.save_to_position(s, record_count, loose_scan_pos);
+  loose_scan_opt.save_to_position(s, record_count, pos->records_out,
+                                  loose_scan_pos);
 
   if (!best.key &&
       idx == join->const_tables &&              // First table
@@ -30525,9 +30526,8 @@ static bool get_range_limit_read_cost(const POSITION *pos,
         cond_selectivity= best_rows / range_rows;
       else
         cond_selectivity= 1.0;
-#if 0 // FIXME: cond_selectivity=8/4 = 2 in main.update_use_source
+
       DBUG_ASSERT(cond_selectivity <= 1.000000001);
-#endif
       set_if_smaller(cond_selectivity, 1.0);
 
       /*

sql/sql_select.h

@@ -1022,6 +1022,7 @@ class POSITION
   */
   double read_time;
 
+  /* record combinations before this table */
   double loops;
 
   double    prefix_record_count;