MDEV-31303 Key not used when IN clause has both signed and usigned values

Summary:

This patch enables possible index optimization when
the WHERE clause has an IN condition of the form:

signed_or_unsigned_column IN (signed_or_unsigned_constant,
                              signed_or_unsigned_constant
                              [,signed_or_unsigned_constant]*)

when the IN list constants are of different signess, e.g.:
  WHERE signed_column   IN (signed_constant, unsigned_constant ...)
  WHERE unsigned_column IN (signed_constant, unsigned_constant ...)

Details:

In a condition like:
   WHERE unsigned_predicant IN (1, LONGLONG_MAX + 1)

comparison handlers for individual (predicant,value) pairs are
calculated as follows:

* unsigned_predicant and 1 produce &type_handler_newdecimal
* unsigned_predicant and (LONGLONG_MAX + 1) produce &type_handler_slonglong

The old code decided that it could not use bisection because
the two pairs had different comparison handlers.
As a result, bisection was not allowed, and, in case of
an indexed integer column predicant the index on the column was not used.

The new code catches special cases like:
    signed_predicant   IN (signed_constant, unsigned_constant)
    unsigned_predicant IN (signed_constant, unsigned_constant)

It enables bisection using in_longlong, which supports a mixture
of predicant and values of different signess.
In case when the predicant is an indexed column this change
automatically enables index range optimization.

Thanks to Vicențiu Ciorbaru for proposing the idea and for preparing MTR tests.

mysql-test/main/func_debug.result

@@ -912,7 +912,8 @@ a IN (CAST(1 AS SIGNED), CAST(1 AS UNSIGNED))
 Warnings:
 Note	1105	DBUG: [0] arg=1 handler=0 (bigint)
 Note	1105	DBUG: [1] arg=2 handler=1 (decimal)
-Note	1105	DBUG: types_compatible=no bisect=no
+Note	1105	DBUG: found a mix of UINT and SINT
+Note	1105	DBUG: types_compatible=yes bisect=yes
 SELECT a IN (CAST(1 AS SIGNED), CAST(1 AS UNSIGNED),NULL) FROM t1;
 a IN (CAST(1 AS SIGNED), CAST(1 AS UNSIGNED),NULL)
 Warnings:
@@ -950,7 +951,8 @@ a NOT IN (CAST(1 AS SIGNED), CAST(1 AS UNSIGNED))
 Warnings:
 Note	1105	DBUG: [0] arg=1 handler=0 (bigint)
 Note	1105	DBUG: [1] arg=2 handler=1 (decimal)
-Note	1105	DBUG: types_compatible=no bisect=no
+Note	1105	DBUG: found a mix of UINT and SINT
+Note	1105	DBUG: types_compatible=yes bisect=yes
 SELECT a NOT IN (CAST(1 AS SIGNED), CAST(1 AS UNSIGNED),NULL) FROM t1;
 a NOT IN (CAST(1 AS SIGNED), CAST(1 AS UNSIGNED),NULL)
 Warnings:
@@ -1624,7 +1626,8 @@ a	b
 Warnings:
 Note	1105	DBUG: [0] arg=1 handler=0 (bigint)
 Note	1105	DBUG: [1] arg=2 handler=1 (decimal)
-Note	1105	DBUG: types_compatible=no bisect=no
+Note	1105	DBUG: found a mix of UINT and SINT
+Note	1105	DBUG: types_compatible=yes bisect=no
 DROP TABLE t1;
 #
 # MDEV-11554 Wrong result for CASE on a mixture of signed and unsigned expressions

mysql-test/main/func_in.result

@@ -944,3 +944,40 @@ Warning	1292	Truncated incorrect DECIMAL value: '0x'
 #
 # End of 10.4 tests
 #
+#
+# Start of 10.5 tests
+#
+#
+# MDEV-31303: Key not used
+#
+CREATE TABLE `a` (
+`id` bigint AUTO_INCREMENT PRIMARY KEY,
+`c1` bigint unsigned,
+KEY (`c1`)
+);
+INSERT INTO `a` VALUES (1,9223382399205928659),(2,9223384207280813348),
+(3,9223385953115437234),(4,9223387250780556749),(5,9223387354282558788),
+(6,9223387603870501596),(7,9223389270813433667),(8,9223389903231468827),
+(9,9223390280789586779),(10,9223391591398222899),(11,9223391875473564350),
+(12,9223393152250049433),(13,9223393939696790223),(14,9223394417225350415),
+(15,9223397646397141015),(16,9223398025879291243),(17,9223399038671098072),
+(18,9223399534968874556),(19,9223400449518009285),(20,9223400860292643549),
+(21,9223400940692256924),(22,9223401073791948119),(23,9223402820804649616),
+(24,9223403470951992681),(25,9223405581879567267),(26,9223405754978563829),
+(27,9223405972966828221), (28, 9223372036854775808), (29, 9223372036854775807) ;
+explain SELECT c1 FROM a WHERE c1 IN ( 1, 9223372036854775807 );
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	a	range	c1	c1	9	NULL	2	Using where; Using index
+explain SELECT c1 FROM a WHERE c1 IN ( 1, 9223372036854775808 );
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	a	range	c1	c1	9	NULL	2	Using where; Using index
+SELECT c1 FROM a WHERE c1 IN ( 1, 9223372036854775807 );
+c1
+9223372036854775807
+SELECT c1 FROM a WHERE c1 IN ( 1, 9223372036854775808 );
+c1
+9223372036854775808
+drop table `a`;
+#
+# End of 10.5 tests
+#

mysql-test/main/func_in.test

@@ -726,3 +726,37 @@ SELECT ('0x',1) IN ((0,1),(1,1));
 --echo # End of 10.4 tests
 --echo #
 
+--echo #
+--echo # Start of 10.5 tests
+--echo #
+
+--echo #
+--echo # MDEV-31303: Key not used
+--echo #
+CREATE TABLE `a` (
+  `id` bigint AUTO_INCREMENT PRIMARY KEY,
+  `c1` bigint unsigned,
+  KEY (`c1`)
+);
+
+INSERT INTO `a` VALUES (1,9223382399205928659),(2,9223384207280813348),
+(3,9223385953115437234),(4,9223387250780556749),(5,9223387354282558788),
+(6,9223387603870501596),(7,9223389270813433667),(8,9223389903231468827),
+(9,9223390280789586779),(10,9223391591398222899),(11,9223391875473564350),
+(12,9223393152250049433),(13,9223393939696790223),(14,9223394417225350415),
+(15,9223397646397141015),(16,9223398025879291243),(17,9223399038671098072),
+(18,9223399534968874556),(19,9223400449518009285),(20,9223400860292643549),
+(21,9223400940692256924),(22,9223401073791948119),(23,9223402820804649616),
+(24,9223403470951992681),(25,9223405581879567267),(26,9223405754978563829),
+(27,9223405972966828221), (28, 9223372036854775808), (29, 9223372036854775807) ;
+
+explain SELECT c1 FROM a WHERE c1 IN ( 1, 9223372036854775807 );
+explain SELECT c1 FROM a WHERE c1 IN ( 1, 9223372036854775808 );
+SELECT c1 FROM a WHERE c1 IN ( 1, 9223372036854775807 );
+SELECT c1 FROM a WHERE c1 IN ( 1, 9223372036854775808 );
+drop table `a`;
+
+--echo #
+--echo # End of 10.5 tests
+--echo #
+

sql/item_cmpfunc.cc

@@ -4442,6 +4442,42 @@ bool Item_func_in::fix_length_and_dec()
     return TRUE;
   }
 
+  if (!arg_types_compatible && comparator_count() == 2)
+  {
+    /*
+      Catch a special case: a mixture of signed and unsigned integer types.
+      in_longlong can handle such cases.
+
+      Note, prepare_predicant_and_values() aggregates this mixture as follows:
+      - signed+unsigned produce &type_handler_newdecimal.
+      - signed+signed or unsigned+unsigned produce &type_handler_slonglong
+      So we have extactly two distinct handlers.
+
+      The code below assumes that unsigned longlong is handled
+      by &type_handler_slonglong in comparison context,
+      which may change in the future to &type_handler_ulonglong.
+      The DBUG_ASSERT is needed to address this change here properly.
+    */
+    DBUG_ASSERT(type_handler_ulonglong.type_handler_for_comparison() ==
+                &type_handler_slonglong);
+    // Let's check if all arguments are of integer types
+    uint found_int_args= 0;
+    for (uint i= 0; i < arg_count; i++, found_int_args++)
+    {
+      if (args[i]->type_handler_for_comparison() != &type_handler_slonglong)
+        break;
+    }
+    if (found_int_args == arg_count)
+    {
+      // All arguments are integers. Switch to integer comparison.
+      arg_types_compatible= true;
+      DBUG_EXECUTE_IF("Item_func_in",
+                      push_warning_printf(thd, Sql_condition::WARN_LEVEL_NOTE,
+                      ER_UNKNOWN_ERROR, "DBUG: found a mix of UINT and SINT"););
+      m_comparator.set_handler(&type_handler_slonglong);
+    }
+  }
+
   if (arg_types_compatible) // Bisection condition #1
   {
     if (m_comparator.type_handler()->