Bug#22855:

Conversion errors when constructing the condition for an
IN predicates were treated as if the affected column contains
NULL. If such a IN predicate is inside NOT we get wrong 
results.
Corrected the handling of conversion errors in an IN predicate 
that is resolved by unique_subquery (through 
subselect_uniquesubquery_engine).


mysql-test/r/subselect3.result:
  Bug#22855: test case
mysql-test/t/subselect3.test:
  Bug#22855: test case
sql/item_subselect.cc:
  Bug#22855: corrected the handling of conversion errors and
  NULL key values in IN predicate that is resolved by index
  lookup.

mysql-test/r/bdb_notembedded.result

+set autocommit=1;
+reset master;
+create table bug16206 (a int);
+insert into bug16206 values(1);
+start transaction;
+insert into bug16206 values(2);
+commit;
+show binlog events;
+Log_name	Pos	Event_type	Server_id	End_log_pos	Info
+f	n	Format_desc	1	n	Server ver: VERSION, Binlog ver: 4
+f	n	Query	1	n	use `test`; create table bug16206 (a int)
+f	n	Query	1	n	use `test`; insert into bug16206 values(1)
+f	n	Query	1	n	use `test`; insert into bug16206 values(2)
+drop table bug16206;
+reset master;
+create table bug16206 (a int) engine=         bdb;
+insert into bug16206 values(0);
+insert into bug16206 values(1);
+start transaction;
+insert into bug16206 values(2);
+commit;
+insert into bug16206 values(3);
+show binlog events;
+Log_name	Pos	Event_type	Server_id	End_log_pos	Info
+f	n	Format_desc	1	n	Server ver: VERSION, Binlog ver: 4
+f	n	Query	1	n	use `test`; create table bug16206 (a int) engine=         bdb
+f	n	Query	1	n	use `test`; insert into bug16206 values(0)
+f	n	Query	1	n	use `test`; insert into bug16206 values(1)
+f	n	Query	1	n	use `test`; BEGIN
+f	n	Query	1	n	use `test`; insert into bug16206 values(2)
+f	n	Query	1	n	use `test`; COMMIT
+f	n	Query	1	n	use `test`; insert into bug16206 values(3)
+drop table bug16206;
+set autocommit=0;
+End of 5.0 tests

mysql-test/r/subselect3.result

@@ -692,3 +692,21 @@ a	MAX(b)	test
 2	3	h
 3	4	i
 DROP TABLE t1, t2;
+CREATE TABLE t1 (a INT NOT NULL);
+INSERT INTO t1 VALUES (1),(-1), (65),(66);
+CREATE TABLE t2 (a INT UNSIGNED NOT NULL PRIMARY KEY);
+INSERT INTO t2 VALUES (65),(66);
+SELECT a FROM t1 WHERE a NOT IN (65,66);
+a
+1
+-1
+SELECT a FROM t1 WHERE a NOT IN (SELECT a FROM t2);
+a
+1
+-1
+EXPLAIN SELECT a FROM t1 WHERE a NOT IN (SELECT a FROM t2);
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	t1	ALL	NULL	NULL	NULL	NULL	4	Using where
+2	DEPENDENT SUBQUERY	t2	unique_subquery	PRIMARY	PRIMARY	4	func	1	Using index
+DROP TABLE t1;
+End of 5.0 tests

mysql-test/t/bdb_notembedded.test

+-- source include/not_embedded.inc
+-- source include/have_bdb.inc
+
+#
+# Bug #16206: Superfluous COMMIT event in binlog when updating BDB in autocommit mode
+#
+set autocommit=1;
+
+let $VERSION=`select version()`;
+
+reset master;
+create table bug16206 (a int);
+insert into bug16206 values(1);
+start transaction;
+insert into bug16206 values(2);
+commit;
+--replace_result $VERSION VERSION
+--replace_column 1 f 2 n 5 n
+show binlog events;
+drop table bug16206;
+
+reset master;
+create table bug16206 (a int) engine=         bdb;
+insert into bug16206 values(0);
+insert into bug16206 values(1);
+start transaction;
+insert into bug16206 values(2);
+commit;
+insert into bug16206 values(3);
+--replace_result $VERSION VERSION
+--replace_column 1 f 2 n 5 n
+show binlog events;
+drop table bug16206;
+
+set autocommit=0;
+
+
+--echo End of 5.0 tests

mysql-test/t/subselect3.test

@@ -531,3 +531,21 @@ SELECT a, MAX(b),
 
 
 DROP TABLE t1, t2;
+
+#
+# Bug #22855: Optimizer doesn't rewrite NOT IN subselects to a correlated 
+# subquery 
+#
+CREATE TABLE t1 (a INT NOT NULL);
+INSERT INTO t1 VALUES (1),(-1), (65),(66);
+
+CREATE TABLE t2 (a INT UNSIGNED NOT NULL PRIMARY KEY);
+INSERT INTO t2 VALUES (65),(66);
+
+SELECT a FROM t1 WHERE a NOT IN (65,66);
+SELECT a FROM t1 WHERE a NOT IN (SELECT a FROM t2);
+EXPLAIN SELECT a FROM t1 WHERE a NOT IN (SELECT a FROM t2);
+
+DROP TABLE t1;
+
+--echo End of 5.0 tests

sql/item_subselect.cc

@@ -775,6 +775,11 @@ bool Item_in_subselect::val_bool()
   if (exec())
   {
     reset();
+    /* 
+      Must mark the IN predicate as NULL so as to make sure an enclosing NOT
+      predicate will return FALSE. See the comments in 
+      subselect_uniquesubquery_engine::copy_ref_key for further details.
+    */
     null_value= 1;
     return 0;
   }
@@ -1919,10 +1924,38 @@ int subselect_uniquesubquery_engine::scan_table()
 
   DESCRIPTION
     Copy ref key and check for null parts in it.
+    Depending on the nullability and conversion problems this function
+    recognizes and processes the following states :
+      1. Partial match on top level. This means IN has a value of FALSE
+         regardless of the data in the subquery table.
+         Detected by finding a NULL in the left IN operand of a top level
+         expression.
+         We may actually skip reading the subquery, so return TRUE to skip
+         the table scan in subselect_uniquesubquery_engine::exec and make
+         the value of the IN predicate a NULL (that is equal to FALSE on
+         top level).
+      2. No exact match when IN is nested inside another predicate.
+         Detected by finding a NULL in the left IN operand when IN is not
+         a top level predicate.
+         We cannot have an exact match. But we must proceed further with a
+         table scan to find out if it's a partial match (and IN has a value
+         of NULL) or no match (and IN has a value of FALSE).
+         So we return FALSE to continue with the scan and see if there are
+         any record that would constitute a partial match (as we cannot
+         determine that from the index).
+      3. Error converting the left IN operand to the column type of the
+         right IN operand. This counts as no match (and IN has the value of
+         FALSE). We mark the subquery table cursor as having no more rows
+         (to ensure that the processing that follows will not find a match)
+         and return FALSE, so IN is not treated as returning NULL.
+
 
   RETURN
-    FALSE - ok, index lookup key without keys copied.
-    TRUE  - an error occured while copying the key
+    FALSE - The value of the IN predicate is not known. Proceed to find the
+            value of the IN predicate using the determined values of
+            null_keypart and table->status.
+    TRUE  - IN predicate has a value of NULL. Stop the processing right there
+            and return NULL to the outer predicates.
 */
 
 bool subselect_uniquesubquery_engine::copy_ref_key()
@@ -1942,13 +1975,37 @@ bool subselect_uniquesubquery_engine::copy_ref_key()
       function.
     */
     null_keypart= (*copy)->null_key;
+    if (null_keypart)
+    {
       bool top_level= ((Item_in_subselect *) item)->is_top_level_item();
-    if (null_keypart && !top_level)
+      if (top_level)
+      {
+        /* Partial match on top level */
+        DBUG_RETURN(1);
+      }
+      else
+      {
+        /* No exact match when IN is nested inside another predicate */
         break;
-    if ((tab->ref.key_err) & 1 || (null_keypart && top_level))
+      }
+    }
+
+    /*
+      Check if the error is equal to STORE_KEY_FATAL. This is not expressed 
+      using the store_key::store_key_result enum because ref.key_err is a 
+      boolean and we want to detect both TRUE and STORE_KEY_FATAL from the 
+      space of the union of the values of [TRUE, FALSE] and 
+      store_key::store_key_result.  
+      TODO: fix the variable an return types.
+    */
+    if (tab->ref.key_err & 1)
     {
+      /*
+       Error converting the left IN operand to the column type of the right
+       IN operand. 
+      */
       tab->table->status= STATUS_NOT_FOUND;
-      DBUG_RETURN(1);
+      break;
     }
   }
   DBUG_RETURN(0);
@@ -1991,10 +2048,20 @@ int subselect_uniquesubquery_engine::exec()
   int error;
   TABLE *table= tab->table;
   empty_result_set= TRUE;
+  table->status= 0;
  
   /* TODO: change to use of 'full_scan' here? */
   if (copy_ref_key())
     DBUG_RETURN(1);
+  if (table->status)
+  {
+    /* 
+      We know that there will be no rows even if we scan. 
+      Can be set in copy_ref_key.
+    */
+    ((Item_in_subselect *) item)->value= 0;
+    DBUG_RETURN(0);
+  }
 
   if (null_keypart)
     DBUG_RETURN(scan_table());