MDEV-6209: Assertion `join->best_read < double(1.79769313486231570815e+308L ...

- Use floating-point division in selectivity calculations.

mysql-test/r/selectivity_no_engine.result

@@ -205,6 +205,16 @@ Warnings:
 Note	1003	select `test`.`t0`.`a` AS `a`,`test`.`t1`.`kp1` AS `kp1`,`test`.`t1`.`kp2` AS `kp2`,`test`.`t1`.`filler1` AS `filler1`,`test`.`t1`.`filler2` AS `filler2` from `test`.`t0` join `test`.`t1` where ((`test`.`t1`.`kp1` = `test`.`t0`.`a`) and (`test`.`t1`.`kp2` = 4))
 drop table t0, t1;
 # 
+# MDEV-6209: Assertion `join->best_read < double(1.79769313486231570815e+308L)' 
+#            failed in bool greedy_search with optimizer_use_condition_selectivity>1
+# 
+SET optimizer_use_condition_selectivity = 2;
+CREATE TABLE t1 (a CHAR(6), b INT, PRIMARY KEY (a,b)) ENGINE=MyISAM;
+INSERT INTO t1 VALUES ('foo',1),('bar',2);
+SELECT * FROM t1 AS t1_1, t1 AS t1_2 WHERE NOT ( t1_1.a <> 'baz');
+a	b	a	b
+DROP TABLE t1;
+# 
 # End of the test file
 # 
 set use_stat_tables= @save_use_stat_tables;

mysql-test/t/selectivity_no_engine.test

@@ -148,6 +148,19 @@ explain extended select * from t0, t1 where t1.kp1=t0.a and t1.kp2=t0.a+1;
 --echo # NOTE: t0: 10*100% is ok,  t1: 10*9.90% is bad. t1 should have 10*100%.
 explain extended select * from t0, t1 where t1.kp1=t0.a and t1.kp2=4;
 drop table t0, t1;
+
+--echo # 
+--echo # MDEV-6209: Assertion `join->best_read < double(1.79769313486231570815e+308L)' 
+--echo #            failed in bool greedy_search with optimizer_use_condition_selectivity>1
+--echo # 
+SET optimizer_use_condition_selectivity = 2;
+
+CREATE TABLE t1 (a CHAR(6), b INT, PRIMARY KEY (a,b)) ENGINE=MyISAM;
+INSERT INTO t1 VALUES ('foo',1),('bar',2);
+
+SELECT * FROM t1 AS t1_1, t1 AS t1_2 WHERE NOT ( t1_1.a <> 'baz');
+DROP TABLE t1;
+
 --echo # 
 --echo # End of the test file
 --echo # 

sql/sql_select.cc

@@ -7349,7 +7349,7 @@ double table_cond_selectivity(JOIN *join, uint idx, JOIN_TAB *s,
           }
           keyparts++;
         }
-        sel /= table->quick_rows[key] / table->stat_records();
+        sel /= (double)table->quick_rows[key] / (double) table->stat_records();
       }
     }