compatibility fix

test results corrected

mysql-test/r/range.result

@@ -560,10 +560,14 @@ count(*)
 0
 explain select count(*) from t1 where x > -16;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
-1	SIMPLE	t1	range	PRIMARY	PRIMARY	8	NULL	1	Using where; Using index
+1	SIMPLE	t1	index	PRIMARY	PRIMARY	8	NULL	2	Using where; Using index
 select count(*) from t1 where x > -16;
 count(*)
-1
+2
+select * from t1 where x > -16;
+x
+18446744073709551600
+18446744073709551601
 select count(*) from t1 where x = 18446744073709551601;
 count(*)
 1

mysql-test/t/range.test

@@ -429,6 +429,7 @@ explain select count(*) from t1 where x > -16;
 # The following result should be (2). To be fixed when we add 'unsigned flag' to
 # Field::store(longlong)
 select count(*) from t1 where x > -16;
+select * from t1 where x > -16;
 select count(*) from t1 where x = 18446744073709551601;
 
 drop table t1;

sql/opt_range.cc

@@ -7189,7 +7189,7 @@ void cost_group_min_max(TABLE* table, KEY *index_info, uint used_key_parts,
   {
     quick_prefix_selectivity= (double) quick_prefix_records /
                               (double) table_records;
-    num_groups= (uint) round(num_groups * quick_prefix_selectivity);
+    num_groups= (uint) rint(num_groups * quick_prefix_selectivity);
   }
 
   if (used_key_parts > group_key_parts)