Fix for bug #28748: "Select" returning one value too few

Problem: we may get unexpected results comparing [u]longlong values as doubles.
Fix: adjust the test to use integer comparators.
Note: it's not a real fix, we have to implement some new comparators 
to completely solve the original problem (see my comment in the bug report).

mysql-test/r/func_in.result

@@ -431,15 +431,17 @@ insert into t2 values(13491727406643098568),
 (0x8000000400000001),
 (0x8000004000000001),
 (0x8000040000000001);
-SELECT HEX(a) FROM t2 WHERE a IN (0xBB3C3E98175D33C8, 42);
+SELECT HEX(a) FROM t2 WHERE a IN 
+(CAST(0xBB3C3E98175D33C8 AS UNSIGNED),
+42);
 HEX(a)
 BB3C3E98175D33C8
 SELECT HEX(a) FROM t2 WHERE a IN
-(0xBB3C3E98175D33C8,
-0x7fffffffffffffff,
-0x8000000000000000,
-0x8000000000000400,
-0x8000000000000401,
+(CAST(0xBB3C3E98175D33C8 AS UNSIGNED),
+CAST(0x7fffffffffffffff AS UNSIGNED),
+CAST(0x8000000000000000 AS UNSIGNED),
+CAST(0x8000000000000400 AS UNSIGNED),
+CAST(0x8000000000000401 AS UNSIGNED),
 42);
 HEX(a)
 BB3C3E98175D33C8
@@ -447,15 +449,22 @@ BB3C3E98175D33C8
 8000000000000000
 8000000000000400
 8000000000000401
-SELECT HEX(a) FROM t2 WHERE a IN (0x7fffffffffffffff,0x8000000000000001);
+SELECT HEX(a) FROM t2 WHERE a IN 
+(CAST(0x7fffffffffffffff AS UNSIGNED), 
+CAST(0x8000000000000001 AS UNSIGNED));
 HEX(a)
 7FFFFFFFFFFFFFFF
 8000000000000001
-SELECT HEX(a) FROM t2 WHERE a IN (0x7ffffffffffffffe,0x7fffffffffffffff);
+SELECT HEX(a) FROM t2 WHERE a IN 
+(CAST(0x7ffffffffffffffe AS UNSIGNED), 
+CAST(0x7fffffffffffffff AS UNSIGNED));
 HEX(a)
 7FFFFFFFFFFFFFFE
 7FFFFFFFFFFFFFFF
-SELECT HEX(a) FROM t2 WHERE a IN (0x7ffffffffffffffe,0x7fffffffffffffff,'abc');
+SELECT HEX(a) FROM t2 WHERE a IN 
+(0x7ffffffffffffffe, 
+0x7fffffffffffffff,
+'abc');
 HEX(a)
 7FFFFFFFFFFFFFFE
 7FFFFFFFFFFFFFFF

mysql-test/t/func_in.test

@@ -335,19 +335,28 @@ insert into t2 values(13491727406643098568),
        (0x8000004000000001),
        (0x8000040000000001);
 
-SELECT HEX(a) FROM t2 WHERE a IN (0xBB3C3E98175D33C8, 42);
+SELECT HEX(a) FROM t2 WHERE a IN 
+  (CAST(0xBB3C3E98175D33C8 AS UNSIGNED),
+   42);
 
 SELECT HEX(a) FROM t2 WHERE a IN
-(0xBB3C3E98175D33C8,
- 0x7fffffffffffffff,
- 0x8000000000000000,
- 0x8000000000000400,
- 0x8000000000000401,
- 42);
-
-SELECT HEX(a) FROM t2 WHERE a IN (0x7fffffffffffffff,0x8000000000000001);
-SELECT HEX(a) FROM t2 WHERE a IN (0x7ffffffffffffffe,0x7fffffffffffffff);
-SELECT HEX(a) FROM t2 WHERE a IN (0x7ffffffffffffffe,0x7fffffffffffffff,'abc');
+  (CAST(0xBB3C3E98175D33C8 AS UNSIGNED),
+   CAST(0x7fffffffffffffff AS UNSIGNED),
+   CAST(0x8000000000000000 AS UNSIGNED),
+   CAST(0x8000000000000400 AS UNSIGNED),
+   CAST(0x8000000000000401 AS UNSIGNED),
+   42);
+
+SELECT HEX(a) FROM t2 WHERE a IN 
+  (CAST(0x7fffffffffffffff AS UNSIGNED), 
+   CAST(0x8000000000000001 AS UNSIGNED));
+SELECT HEX(a) FROM t2 WHERE a IN 
+  (CAST(0x7ffffffffffffffe AS UNSIGNED), 
+   CAST(0x7fffffffffffffff AS UNSIGNED));
+SELECT HEX(a) FROM t2 WHERE a IN 
+  (0x7ffffffffffffffe, 
+   0x7fffffffffffffff,
+   'abc');
 
 CREATE TABLE t3 (a BIGINT UNSIGNED);
 INSERT INTO t3 VALUES (9223372036854775551);