MDEV-31380: Assertion `s->table->opt_range_condition_rows <= s->found_records' failed

LooseScan code set opt_range_condition_rows to be the

  MIN(loose_scan_plan->records, table->records)

totally ignoring possible quick range selects.  If there was a quick
select $QUICK on another index with

  $QUICK->records < loose_scan_plan->records

this would create a situation where

   opt_range_condition_rows > $QUICK->records

which causes an assert in 10.6+ and potentially wrong query plan
choice in 10.5.

Fixed by making opt_range_condition_rows to be the minimum #rows
of any quick select.
Approved-by: Monty <monty@mariadb.org>

mysql-test/main/group_min_max.result

@@ -4204,6 +4204,30 @@ a	b	s1
 2	2	t2:t2a-null;min_t3_b:t3b-null
 3	3	t2:1;min_t3_b:3
 drop table t1,t2,t3;
+# 
+# MDEV-31380: Assertion `s->table->opt_range_condition_rows <= s->found_records' failed
+#  (assertion in 10.6+, DBL_MAX costs in 10.5)
+# 
+CREATE TABLE t1 (a INT, b INT, PRIMARY KEY(a), KEY(b)) ENGINE=InnoDB;
+INSERT INTO t1 SELECT seq, seq FROM seq_1_to_100;
+SET
+@tmp=@@optimizer_use_condition_selectivity,
+optimizer_use_condition_selectivity = 1,
+@tmp2=@@optimizer_trace,
+optimizer_trace=1;
+SELECT DISTINCT * FROM t1 WHERE a IN (1, 2);
+a	b
+1	1
+2	2
+select 
+CAST(json_value(json_extract(trace, '$**.chosen_access_method.cost'), '$[0]')
+as DOUBLE) < 1.0e100
+from information_schema.optimizer_trace;
+CAST(json_value(json_extract(trace, '$**.chosen_access_method.cost'), '$[0]')
+as DOUBLE) < 1.0e100
+1
+set optimizer_use_condition_selectivity = @tmp, optimizer_trace=@tmp2;
+drop table t1;
 #
 # End of 10.5 tests
 #

mysql-test/main/group_min_max.test

@@ -6,6 +6,7 @@
 --source include/no_valgrind_without_big.inc
 --source include/default_optimizer_switch.inc
 --source include/have_innodb.inc
+--source include/have_sequence.inc
 #
 # TODO:
 # Add queries with:
@@ -1858,6 +1859,30 @@ from t1;
 
 drop table t1,t2,t3;
 
+--echo # 
+--echo # MDEV-31380: Assertion `s->table->opt_range_condition_rows <= s->found_records' failed
+--echo #  (assertion in 10.6+, DBL_MAX costs in 10.5)
+--echo # 
+
+CREATE TABLE t1 (a INT, b INT, PRIMARY KEY(a), KEY(b)) ENGINE=InnoDB;
+INSERT INTO t1 SELECT seq, seq FROM seq_1_to_100;
+
+SET
+  @tmp=@@optimizer_use_condition_selectivity,
+  optimizer_use_condition_selectivity = 1,
+  @tmp2=@@optimizer_trace,
+  optimizer_trace=1;
+
+SELECT DISTINCT * FROM t1 WHERE a IN (1, 2);
+
+select 
+  CAST(json_value(json_extract(trace, '$**.chosen_access_method.cost'), '$[0]')
+       as DOUBLE) < 1.0e100
+from information_schema.optimizer_trace;
+
+set optimizer_use_condition_selectivity = @tmp, optimizer_trace=@tmp2;
+drop table t1;
+
 --echo #
 --echo # End of 10.5 tests
 --echo #

sql/opt_range.cc

@@ -3030,8 +3030,8 @@ int SQL_SELECT::test_quick_select(THD *thd, key_map keys_to_use,
         restore_nonrange_trees(&param, tree, backup_keys);
       if ((group_trp= get_best_group_min_max(&param, tree, read_time)))
       {
-        param.table->opt_range_condition_rows= MY_MIN(group_trp->records,
-                                                  head->stat_records());
+        set_if_smaller(param.table->opt_range_condition_rows,
+                       group_trp->records);
         Json_writer_object grp_summary(thd, "best_group_range_summary");
 
         if (unlikely(thd->trace_started()))