MDEV-22534 Decorrelate IN subquery

Transform

in (select inner_col' from inner_table where inner_col = outer_col)

to

, outer_col in (select inner_col', inner_col from inner_table)

Achieved by implementing Item_in_subselect::exists2in_processor(),
accompanied with comprehensive test coverage. Factored out common code
between the two transformations.

Caveat:

- Cannot recognise bad item mismatch in equalities that causes
  materialization to not materialize down the road

mysql-test/main/explain_json.result

@@ -906,7 +906,7 @@ EXPLAIN
           "buffer_type": "flat",
           "buffer_size": "141",
           "join_type": "BNL",
-          "attached_condition": "t1.b = t2.b and t1.a = t2.a"
+          "attached_condition": "t1.a = t2.a and t1.b = t2.b"
         }
       }
     ]
@@ -921,7 +921,8 @@ explain
 select * from t2 where t2.a in ( select a from t1 where t1.b=t2.b);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	PRIMARY	t2	ALL	NULL	NULL	NULL	NULL	10	
-1	PRIMARY	t1	ALL	NULL	NULL	NULL	NULL	10	Using where; Start temporary; End temporary; Using join buffer (flat, BNL join)
+1	PRIMARY	<subquery2>	eq_ref	distinct_key	distinct_key	8	func,func	1	
+2	MATERIALIZED	t1	ALL	NULL	NULL	NULL	NULL	10	
 explain format=json
 select * from t2 where t2.a in ( select a from t1 where t1.b=t2.b);
 EXPLAIN
@@ -929,6 +930,7 @@ EXPLAIN
   "query_block": {
     "select_id": 1,
     "cost": "COST_REPLACED",
+    "const_condition": "1",
     "nested_loop": [
       {
         "table": {
@@ -941,24 +943,35 @@ EXPLAIN
         }
       },
       {
-        "duplicates_removal": [
-          {
-            "block-nl-join": {
-              "table": {
-                "table_name": "t1",
-                "access_type": "ALL",
-                "loops": 10,
-                "rows": 10,
-                "cost": "COST_REPLACED",
-                "filtered": 10
-              },
-              "buffer_type": "flat",
-              "buffer_size": "206",
-              "join_type": "BNL",
-              "attached_condition": "t1.b = t2.b and t1.a = t2.a"
+        "table": {
+          "table_name": "<subquery2>",
+          "access_type": "eq_ref",
+          "possible_keys": ["distinct_key"],
+          "key": "distinct_key",
+          "key_length": "8",
+          "used_key_parts": ["a", "b"],
+          "ref": ["func", "func"],
+          "rows": 1,
+          "filtered": 100,
+          "materialized": {
+            "unique": 1,
+            "query_block": {
+              "select_id": 2,
+              "nested_loop": [
+                {
+                  "table": {
+                    "table_name": "t1",
+                    "access_type": "ALL",
+                    "loops": 1,
+                    "rows": 10,
+                    "cost": "COST_REPLACED",
+                    "filtered": 100
+                  }
+                }
+              ]
             }
           }
-        ]
+        }
       }
     ]
   }

mysql-test/main/innodb_ext_key.result

@@ -384,15 +384,16 @@ EXPLAIN
 SELECT a FROM t1 AS t, t2 as t2_out 
 WHERE t2_out.c = t.a AND t.b IN (SELECT b FROM t1, t2 WHERE b = t.b);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
-1	PRIMARY	t1	index	b	b	7	NULL	10	Using index; Start temporary
-1	PRIMARY	t	ref	a,b	b	3	test.t1.b	2	Using index
-1	PRIMARY	t2	index	NULL	PRIMARY	4	NULL	11	Using index; End temporary; Using join buffer (flat, BNL join)
+1	PRIMARY	t	index	a,b	b	7	NULL	10	Using index
 1	PRIMARY	t2_out	eq_ref	PRIMARY	PRIMARY	4	test.t.a	1	Using index
+1	PRIMARY	<subquery2>	eq_ref	distinct_key	distinct_key	8	func,func	1	
+2	MATERIALIZED	t1	index	b	b	7	NULL	10	Using index
+2	MATERIALIZED	t2	index	NULL	PRIMARY	4	NULL	11	Using index; Using join buffer (flat, BNL join)
 SELECT a FROM t1 AS t, t2 as t2_out 
 WHERE t2_out.c = t.a AND t.b IN (SELECT b FROM t1, t2 WHERE b = t.b);
 a
 24
-Last_query_cost	0.120558
+Last_query_cost	0.083937
 DROP TABLE t1,t2;
 #
 # LP Bug #923236: hash join + extended_keys = on 

mysql-test/main/rowid_filter_innodb.result

@@ -3472,11 +3472,11 @@ WHERE t1.c1 NOT IN (SELECT t2.c1 FROM t2, t1 AS a1
 WHERE t2.i1 = t1.pk AND t2.i1 BETWEEN 3 AND 5);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
 1	PRIMARY	t1	ALL	NULL	NULL	NULL	NULL	60	100.00	Using where
-2	DEPENDENT SUBQUERY	t2	ref|filter	c1,i1	c1|i1	3|5	func	38 (25%)	25.00	Using where; Full scan on NULL key; Using rowid filter
-2	DEPENDENT SUBQUERY	a1	ALL	NULL	NULL	NULL	NULL	60	100.00	Using join buffer (flat, BNL join)
+2	MATERIALIZED	t2	ALL	c1,i1	NULL	NULL	NULL	192	25.00	Using where
+2	MATERIALIZED	a1	ALL	NULL	NULL	NULL	NULL	60	100.00	Using join buffer (flat, BNL join)
 Warnings:
 Note	1276	Field or reference 'test.t1.pk' of SELECT #2 was resolved in SELECT #1
-Note	1003	/* select#1 */ select `test`.`t1`.`pk` AS `pk`,`test`.`t1`.`c1` AS `c1` from `test`.`t1` where !<expr_cache><`test`.`t1`.`c1`,`test`.`t1`.`pk`>(<in_optimizer>(`test`.`t1`.`c1`,<exists>(/* select#2 */ select `test`.`t2`.`c1` from `test`.`t2` join `test`.`t1` `a1` where `test`.`t2`.`i1` = `test`.`t1`.`pk` and `test`.`t2`.`i1` between 3 and 5 and trigcond(<cache>(`test`.`t1`.`c1`) = `test`.`t2`.`c1`))))
+Note	1003	/* select#1 */ select `test`.`t1`.`pk` AS `pk`,`test`.`t1`.`c1` AS `c1` from `test`.`t1` where !<expr_cache><`test`.`t1`.`c1`,`test`.`t1`.`pk`>(<in_optimizer>((`test`.`t1`.`c1`,`test`.`t1`.`pk`),(`test`.`t1`.`c1`,`test`.`t1`.`pk`) in ( <materialize> (/* select#2 */ select `test`.`t2`.`c1`,`test`.`t2`.`i1` from `test`.`t2` join `test`.`t1` `a1` where `test`.`t2`.`i1` is not null and `test`.`t2`.`i1` between 3 and 5 ), <primary_index_lookup>(`test`.`t1`.`c1` in <temporary table> on distinct_key where `test`.`t1`.`c1` = `<subquery2>`.`c1` and `test`.`t1`.`pk` = `<subquery2>`.`i1`))))
 SELECT * FROM t1
 WHERE t1.c1 NOT IN (SELECT t2.c1 FROM t2, t1 AS a1
 WHERE t2.i1 = t1.pk AND t2.i1 BETWEEN 3 AND 5);

mysql-test/main/subselect4.result

@@ -225,7 +225,7 @@ EXPLAIN
 SELECT * FROM t1 WHERE NULL NOT IN (SELECT t2a.i FROM t2a WHERE t2a.pk = t1.pk);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	PRIMARY	t1	ALL	NULL	NULL	NULL	NULL	4	Using where
-2	DEPENDENT SUBQUERY	t2a	unique_subquery	PRIMARY	PRIMARY	8	const,test.t1.pk	1	Using index; Using where; Full scan on NULL key
+2	DEPENDENT SUBQUERY	t2a	unique_subquery	PRIMARY	PRIMARY	8	const,func	1	Using index; Using where; Full scan on NULL key
 SELECT * FROM t1 WHERE NULL NOT IN (SELECT t2a.i FROM t2a WHERE t2a.pk = t1.pk);
 pk	i
 SELECT * FROM t1 WHERE 1+NULL NOT IN (SELECT t2a.i FROM t2a WHERE t2a.pk = t1.pk);
@@ -265,7 +265,7 @@ EXPLAIN
 SELECT * FROM t1 WHERE NULL NOT IN (SELECT t2c.i FROM t2c WHERE t2c.pk = t1.pk);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	PRIMARY	t1	ALL	NULL	NULL	NULL	NULL	4	Using where
-2	DEPENDENT SUBQUERY	t2c	index_subquery	it2c	it2c	8	const,test.t1.pk	1	Using index; Using where; Full scan on NULL key
+2	DEPENDENT SUBQUERY	t2c	index_subquery	it2c	it2c	8	const,func	1	Using index; Using where; Full scan on NULL key
 SELECT * FROM t1 WHERE NULL NOT IN (SELECT t2c.i FROM t2c WHERE t2c.pk = t1.pk);
 pk	i
 SELECT * FROM t1 WHERE NULL     IN (SELECT t2c.i FROM t2c WHERE t2c.pk = t1.pk) IS UNKNOWN;
@@ -303,7 +303,7 @@ EXPLAIN
 SELECT * FROM t1 WHERE (NULL, 1) NOT IN (SELECT t2a.i, t2a.pk FROM t2a WHERE t2a.pk = t1.pk);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	PRIMARY	t1	ALL	NULL	NULL	NULL	NULL	4	Using where
-2	DEPENDENT SUBQUERY	t2a	unique_subquery	PRIMARY	PRIMARY	8	const,test.t1.pk	1	Using index; Using where; Full scan on NULL key
+2	DEPENDENT SUBQUERY	t2a	const	PRIMARY	PRIMARY	8	const,const	1	Using where; Using index; Full scan on NULL key
 SELECT * FROM t1 WHERE (NULL, 1) NOT IN (SELECT t2a.i, t2a.pk FROM t2a WHERE t2a.pk = t1.pk);
 pk	i
 0	10
@@ -335,7 +335,7 @@ EXPLAIN
 SELECT * FROM t1 WHERE (NULL, 1) NOT IN (SELECT t2c.i, t2c.pk FROM t2c WHERE t2c.pk = t1.pk);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	PRIMARY	t1	ALL	NULL	NULL	NULL	NULL	4	Using where
-2	DEPENDENT SUBQUERY	t2c	index_subquery	it2c	it2c	8	const,test.t1.pk	1	Using index; Using where; Full scan on NULL key
+2	DEPENDENT SUBQUERY	t2c	index_subquery	it2c	it2c	8	const,const	0	Using index; Using where; Full scan on NULL key
 SELECT * FROM t1 WHERE (NULL, 1) NOT IN (SELECT t2c.i, t2c.pk FROM t2c WHERE t2c.pk = t1.pk);
 pk	i
 0	10

mysql-test/main/subselect_extra_no_semijoin.result

@@ -454,7 +454,7 @@ WHERE t2.c = v1.c AND t2.c = v1.b AND v1.b = t3.c);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	PRIMARY	t3	ALL	NULL	NULL	NULL	NULL	2	Using where
 2	DEPENDENT SUBQUERY	t2	system	NULL	NULL	NULL	NULL	1	
-2	DEPENDENT SUBQUERY	<derived3>	ref	key1	key1	8	const,const	0	Using where
+2	DEPENDENT SUBQUERY	<derived3>	ref	key0	key0	8	const,const	0	Using where
 3	DERIVED	t1	ALL	NULL	NULL	NULL	NULL	3	Using where
 SELECT * FROM t3 
 WHERE t3.b IN (SELECT v1.b FROM  v1, t2 
@@ -476,7 +476,7 @@ EXPLAIN
 SELECT * FROM t1 WHERE t1.b IN (SELECT v2.a FROM v2 WHERE v2.b = t1.a);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	PRIMARY	t1	ALL	NULL	NULL	NULL	NULL	2	Using where
-2	DEPENDENT SUBQUERY	<derived3>	ref	key0	key0	5	test.t1.a	1	Using where
+2	DEPENDENT SUBQUERY	<derived3>	index_subquery	key0	key0	10	func,func	1	Using where
 3	DERIVED	t2	ALL	NULL	NULL	NULL	NULL	2	
 SELECT * FROM t1 WHERE t1.b IN (SELECT v2.a FROM v2 WHERE v2.b = t1.a);
 a	b

mysql-test/main/subselect_mat.result

@@ -387,10 +387,10 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
 6	MATERIALIZED	t2i	index	it2i2	it2i3	18	NULL	5	100.00	Using where; Using index
 2	DEPENDENT SUBQUERY	t2	ALL	NULL	NULL	NULL	NULL	5	100.00	Using where
 4	MATERIALIZED	t3b	ALL	NULL	NULL	NULL	NULL	4	100.00	Using where
-3	DEPENDENT SUBQUERY	t3a	ALL	NULL	NULL	NULL	NULL	4	100.00	Using where
+3	MATERIALIZED	t3a	ALL	NULL	NULL	NULL	NULL	4	100.00	
 Warnings:
 Note	1276	Field or reference 'test.t1.a1' of SELECT #3 was resolved in SELECT #1
-Note	1003	/* select#1 */ select `test`.`t1`.`a1` AS `a1`,`test`.`t1`.`a2` AS `a2` from `test`.`t1` where <expr_cache><`test`.`t1`.`a1`,`test`.`t1`.`a2`>(<in_optimizer>((`test`.`t1`.`a1`,`test`.`t1`.`a2`),<exists>(/* select#2 */ select `test`.`t2`.`b1`,`test`.`t2`.`b2` from `test`.`t2` where (<expr_cache><`test`.`t2`.`b2`,`test`.`t1`.`a1`>(<in_optimizer>(`test`.`t2`.`b2`,<exists>(/* select#3 */ select `test`.`t3a`.`c2` from `test`.`t3` `t3a` where `test`.`t3a`.`c1` = `test`.`t1`.`a1` and <cache>(`test`.`t2`.`b2`) = `test`.`t3a`.`c2`))) or <expr_cache><`test`.`t2`.`b2`>(<in_optimizer>(`test`.`t2`.`b2`,`test`.`t2`.`b2` in ( <materialize> (/* select#4 */ select `test`.`t3b`.`c2` from `test`.`t3` `t3b` where `test`.`t3b`.`c2` like '%03' ), <primary_index_lookup>(`test`.`t2`.`b2` in <temporary table> on distinct_key where `test`.`t2`.`b2` = `<subquery4>`.`c2`))))) and <cache>(`test`.`t1`.`a1`) = `test`.`t2`.`b1` and <cache>(`test`.`t1`.`a2`) = `test`.`t2`.`b2`))) and <expr_cache><`test`.`t1`.`a1`,`test`.`t1`.`a2`>(<in_optimizer>((`test`.`t1`.`a1`,`test`.`t1`.`a2`),(`test`.`t1`.`a1`,`test`.`t1`.`a2`) in ( <materialize> (/* select#5 */ select `test`.`t3c`.`c1`,`test`.`t3c`.`c2` from `test`.`t3` `t3c` where <expr_cache><`test`.`t3c`.`c1`,`test`.`t3c`.`c2`>(<in_optimizer>((`test`.`t3c`.`c1`,`test`.`t3c`.`c2`),(`test`.`t3c`.`c1`,`test`.`t3c`.`c2`) in ( <materialize> (/* select#6 */ select `test`.`t2i`.`b1`,`test`.`t2i`.`b2` from `test`.`t2i` where `test`.`t2i`.`b2` > '0' ), <primary_index_lookup>(`test`.`t3c`.`c1` in <temporary table> on distinct_key where `test`.`t3c`.`c1` = `<subquery6>`.`b1` and `test`.`t3c`.`c2` = `<subquery6>`.`b2`)))) ), <primary_index_lookup>(`test`.`t1`.`a1` in <temporary table> on distinct_key where `test`.`t1`.`a1` = `<subquery5>`.`c1` and `test`.`t1`.`a2` = `<subquery5>`.`c2`))))
+Note	1003	/* select#1 */ select `test`.`t1`.`a1` AS `a1`,`test`.`t1`.`a2` AS `a2` from `test`.`t1` where <expr_cache><`test`.`t1`.`a1`,`test`.`t1`.`a2`>(<in_optimizer>((`test`.`t1`.`a1`,`test`.`t1`.`a2`),<exists>(/* select#2 */ select `test`.`t2`.`b1`,`test`.`t2`.`b2` from `test`.`t2` where (<expr_cache><`test`.`t2`.`b2`,`test`.`t1`.`a1`>(<in_optimizer>((`test`.`t2`.`b2`,`test`.`t1`.`a1`),(`test`.`t2`.`b2`,`test`.`t1`.`a1`) in ( <materialize> (/* select#3 */ select `test`.`t3a`.`c2`,`test`.`t3a`.`c1` from `test`.`t3` `t3a` where 1 ), <primary_index_lookup>(`test`.`t2`.`b2` in <temporary table> on distinct_key where `test`.`t2`.`b2` = `<subquery3>`.`c2` and `test`.`t1`.`a1` = `<subquery3>`.`c1`)))) or <expr_cache><`test`.`t2`.`b2`>(<in_optimizer>(`test`.`t2`.`b2`,`test`.`t2`.`b2` in ( <materialize> (/* select#4 */ select `test`.`t3b`.`c2` from `test`.`t3` `t3b` where `test`.`t3b`.`c2` like '%03' ), <primary_index_lookup>(`test`.`t2`.`b2` in <temporary table> on distinct_key where `test`.`t2`.`b2` = `<subquery4>`.`c2`))))) and <cache>(`test`.`t1`.`a1`) = `test`.`t2`.`b1` and <cache>(`test`.`t1`.`a2`) = `test`.`t2`.`b2`))) and <expr_cache><`test`.`t1`.`a1`,`test`.`t1`.`a2`>(<in_optimizer>((`test`.`t1`.`a1`,`test`.`t1`.`a2`),(`test`.`t1`.`a1`,`test`.`t1`.`a2`) in ( <materialize> (/* select#5 */ select `test`.`t3c`.`c1`,`test`.`t3c`.`c2` from `test`.`t3` `t3c` where <expr_cache><`test`.`t3c`.`c1`,`test`.`t3c`.`c2`>(<in_optimizer>((`test`.`t3c`.`c1`,`test`.`t3c`.`c2`),(`test`.`t3c`.`c1`,`test`.`t3c`.`c2`) in ( <materialize> (/* select#6 */ select `test`.`t2i`.`b1`,`test`.`t2i`.`b2` from `test`.`t2i` where `test`.`t2i`.`b2` > '0' ), <primary_index_lookup>(`test`.`t3c`.`c1` in <temporary table> on distinct_key where `test`.`t3c`.`c1` = `<subquery6>`.`b1` and `test`.`t3c`.`c2` = `<subquery6>`.`b2`)))) ), <primary_index_lookup>(`test`.`t1`.`a1` in <temporary table> on distinct_key where `test`.`t1`.`a1` = `<subquery5>`.`c1` and `test`.`t1`.`a2` = `<subquery5>`.`c2`))))
 select * from t1
 where (a1, a2) in (select b1, b2 from t2
 where b2 in (select c2 from t3 t3a where c1 = a1) or
@@ -520,11 +520,11 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
 6	DEPENDENT SUBQUERY	t2i	index_subquery	it2i1,it2i2,it2i3	it2i3	18	func,func	1	100.00	Using index; Using where
 2	DEPENDENT SUBQUERY	t2	ALL	NULL	NULL	NULL	NULL	5	100.00	Using where
 4	MATERIALIZED	t3b	ALL	NULL	NULL	NULL	NULL	4	100.00	Using where
-3	DEPENDENT SUBQUERY	t3a	ALL	NULL	NULL	NULL	NULL	4	100.00	Using where
+3	MATERIALIZED	t3a	ALL	NULL	NULL	NULL	NULL	4	100.00	
 Warnings:
 Note	1276	Field or reference 'test.t1.a1' of SELECT #3 was resolved in SELECT #1
 Note	1276	Field or reference 'test.t1.a2' of SELECT #6 was resolved in SELECT #1
-Note	1003	/* select#1 */ select `test`.`t1`.`a1` AS `a1`,`test`.`t1`.`a2` AS `a2` from `test`.`t1` where <expr_cache><`test`.`t1`.`a1`,`test`.`t1`.`a2`>(<in_optimizer>((`test`.`t1`.`a1`,`test`.`t1`.`a2`),<exists>(/* select#2 */ select `test`.`t2`.`b1`,`test`.`t2`.`b2` from `test`.`t2` where (<expr_cache><`test`.`t2`.`b2`,`test`.`t1`.`a1`>(<in_optimizer>(`test`.`t2`.`b2`,<exists>(/* select#3 */ select `test`.`t3a`.`c2` from `test`.`t3` `t3a` where `test`.`t3a`.`c1` = `test`.`t1`.`a1` and <cache>(`test`.`t2`.`b2`) = `test`.`t3a`.`c2`))) or <expr_cache><`test`.`t2`.`b2`>(<in_optimizer>(`test`.`t2`.`b2`,`test`.`t2`.`b2` in ( <materialize> (/* select#4 */ select `test`.`t3b`.`c2` from `test`.`t3` `t3b` where `test`.`t3b`.`c2` like '%03' ), <primary_index_lookup>(`test`.`t2`.`b2` in <temporary table> on distinct_key where `test`.`t2`.`b2` = `<subquery4>`.`c2`))))) and <cache>(`test`.`t1`.`a1`) = `test`.`t2`.`b1` and <cache>(`test`.`t1`.`a2`) = `test`.`t2`.`b2`))) and <expr_cache><`test`.`t1`.`a1`,`test`.`t1`.`a2`>(<in_optimizer>((`test`.`t1`.`a1`,`test`.`t1`.`a2`),<exists>(/* select#5 */ select `test`.`t3c`.`c1`,`test`.`t3c`.`c2` from `test`.`t3` `t3c` where <expr_cache><`test`.`t3c`.`c1`,`test`.`t3c`.`c2`,`test`.`t1`.`a2`>(<in_optimizer>((`test`.`t3c`.`c1`,`test`.`t3c`.`c2`),<exists>(<index_lookup>(<cache>(`test`.`t3c`.`c1`) in t2i on it2i3 where (`test`.`t2i`.`b2` > '0' or `test`.`t2i`.`b2` = `test`.`t1`.`a2`) and <cache>(`test`.`t3c`.`c1`) = `test`.`t2i`.`b1` and <cache>(`test`.`t3c`.`c2`) = `test`.`t2i`.`b2`)))) and <cache>(`test`.`t1`.`a1`) = `test`.`t3c`.`c1` and <cache>(`test`.`t1`.`a2`) = `test`.`t3c`.`c2`)))
+Note	1003	/* select#1 */ select `test`.`t1`.`a1` AS `a1`,`test`.`t1`.`a2` AS `a2` from `test`.`t1` where <expr_cache><`test`.`t1`.`a1`,`test`.`t1`.`a2`>(<in_optimizer>((`test`.`t1`.`a1`,`test`.`t1`.`a2`),<exists>(/* select#2 */ select `test`.`t2`.`b1`,`test`.`t2`.`b2` from `test`.`t2` where (<expr_cache><`test`.`t2`.`b2`,`test`.`t1`.`a1`>(<in_optimizer>((`test`.`t2`.`b2`,`test`.`t1`.`a1`),(`test`.`t2`.`b2`,`test`.`t1`.`a1`) in ( <materialize> (/* select#3 */ select `test`.`t3a`.`c2`,`test`.`t3a`.`c1` from `test`.`t3` `t3a` where 1 ), <primary_index_lookup>(`test`.`t2`.`b2` in <temporary table> on distinct_key where `test`.`t2`.`b2` = `<subquery3>`.`c2` and `test`.`t1`.`a1` = `<subquery3>`.`c1`)))) or <expr_cache><`test`.`t2`.`b2`>(<in_optimizer>(`test`.`t2`.`b2`,`test`.`t2`.`b2` in ( <materialize> (/* select#4 */ select `test`.`t3b`.`c2` from `test`.`t3` `t3b` where `test`.`t3b`.`c2` like '%03' ), <primary_index_lookup>(`test`.`t2`.`b2` in <temporary table> on distinct_key where `test`.`t2`.`b2` = `<subquery4>`.`c2`))))) and <cache>(`test`.`t1`.`a1`) = `test`.`t2`.`b1` and <cache>(`test`.`t1`.`a2`) = `test`.`t2`.`b2`))) and <expr_cache><`test`.`t1`.`a1`,`test`.`t1`.`a2`>(<in_optimizer>((`test`.`t1`.`a1`,`test`.`t1`.`a2`),<exists>(/* select#5 */ select `test`.`t3c`.`c1`,`test`.`t3c`.`c2` from `test`.`t3` `t3c` where <expr_cache><`test`.`t3c`.`c1`,`test`.`t3c`.`c2`,`test`.`t1`.`a2`>(<in_optimizer>((`test`.`t3c`.`c1`,`test`.`t3c`.`c2`),<exists>(<index_lookup>(<cache>(`test`.`t3c`.`c1`) in t2i on it2i3 where (`test`.`t2i`.`b2` > '0' or `test`.`t2i`.`b2` = `test`.`t1`.`a2`) and <cache>(`test`.`t3c`.`c1`) = `test`.`t2i`.`b1` and <cache>(`test`.`t3c`.`c2`) = `test`.`t2i`.`b2`)))) and <cache>(`test`.`t1`.`a1`) = `test`.`t3c`.`c1` and <cache>(`test`.`t1`.`a2`) = `test`.`t3c`.`c2`)))
 explain extended
 select * from t1 where (a1, a2) in (select '1 - 01', '2 - 01');
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra

mysql-test/main/subselect_mat_cost_bugs.result

@@ -446,8 +446,8 @@ SELECT i2 FROM t2 RIGHT JOIN t3 ON (c3 = c2) WHERE pk3 = i1
 );
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	PRIMARY	t1	system	NULL	NULL	NULL	NULL	1	
-2	DEPENDENT SUBQUERY	t3	const	PRIMARY	PRIMARY	4	const	1	
-2	DEPENDENT SUBQUERY	t2	index	i2	i2	11	NULL	2	Using where; Using index
+2	DEPENDENT SUBQUERY	t3	eq_ref	PRIMARY	PRIMARY	4	func	1	
+2	DEPENDENT SUBQUERY	t2	index	NULL	i2	11	NULL	2	Using where; Using index; Using join buffer (flat, BNL join)
 DROP TABLE t1,t2,t3;
 #
 # MDEV-7599: in-to-exists chosen after min/max optimization

mysql-test/main/subselect_sj2.result

@@ -466,7 +466,7 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
 1	PRIMARY	t1	ref	a	a	5	test.t0.a	1	100.00	Using where; FirstMatch(t2)
 Warnings:
 Note	1276	Field or reference 'test.t0.a' of SELECT #2 was resolved in SELECT #1
-Note	1003	select `test`.`t0`.`a` AS `a` from `test`.`t2` semi join (`test`.`t1`) join `test`.`t0` where `test`.`t2`.`a` = `test`.`t0`.`a` and `test`.`t1`.`a` = `test`.`t0`.`a` and `test`.`t1`.`b` = `test`.`t2`.`b`
+Note	1003	select `test`.`t0`.`a` AS `a` from `test`.`t2` semi join (`test`.`t1`) join `test`.`t0` where `test`.`t1`.`b` = `test`.`t2`.`b` and `test`.`t2`.`a` = `test`.`t0`.`a` and `test`.`t1`.`a` = `test`.`t0`.`a`
 select * from t0
 where t0.a in ( select t1.a from t1,t2 where t2.a=t0.a and
 t1.b=t2.b);

mysql-test/main/subselect_sj2_jcl6.result

@@ -472,7 +472,7 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
 1	PRIMARY	t1	ref	a	a	5	test.t0.a	1	100.00	Using where; FirstMatch(t2); Using join buffer (incremental, BKA join); Key-ordered Rowid-ordered scan
 Warnings:
 Note	1276	Field or reference 'test.t0.a' of SELECT #2 was resolved in SELECT #1
-Note	1003	select `test`.`t0`.`a` AS `a` from `test`.`t2` semi join (`test`.`t1`) join `test`.`t0` where `test`.`t2`.`a` = `test`.`t0`.`a` and `test`.`t1`.`a` = `test`.`t0`.`a` and `test`.`t1`.`b` = `test`.`t2`.`b`
+Note	1003	select `test`.`t0`.`a` AS `a` from `test`.`t2` semi join (`test`.`t1`) join `test`.`t0` where `test`.`t1`.`b` = `test`.`t2`.`b` and `test`.`t2`.`a` = `test`.`t0`.`a` and `test`.`t1`.`a` = `test`.`t0`.`a`
 select * from t0
 where t0.a in ( select t1.a from t1,t2 where t2.a=t0.a and
 t1.b=t2.b);

mysql-test/main/subselect_sj2_mat.result

@@ -468,7 +468,7 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
 1	PRIMARY	t1	ref	a	a	5	test.t0.a	1	100.00	Using where; FirstMatch(t2)
 Warnings:
 Note	1276	Field or reference 'test.t0.a' of SELECT #2 was resolved in SELECT #1
-Note	1003	select `test`.`t0`.`a` AS `a` from `test`.`t2` semi join (`test`.`t1`) join `test`.`t0` where `test`.`t2`.`a` = `test`.`t0`.`a` and `test`.`t1`.`a` = `test`.`t0`.`a` and `test`.`t1`.`b` = `test`.`t2`.`b`
+Note	1003	select `test`.`t0`.`a` AS `a` from `test`.`t2` semi join (`test`.`t1`) join `test`.`t0` where `test`.`t1`.`b` = `test`.`t2`.`b` and `test`.`t2`.`a` = `test`.`t0`.`a` and `test`.`t1`.`a` = `test`.`t0`.`a`
 select * from t0
 where t0.a in ( select t1.a from t1,t2 where t2.a=t0.a and
 t1.b=t2.b);

mysql-test/main/update_use_source.result

@@ -504,11 +504,13 @@ test.t1	analyze	status	OK
 explain select * from t1 where c2 in (select distinct a.c2 from t1 a where t1.c1=a.c1);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	PRIMARY	t1	ALL	NULL	NULL	NULL	NULL	32	
-1	PRIMARY	a	ALL	NULL	NULL	NULL	NULL	32	Using where; FirstMatch(t1); Using join buffer (flat, BNL join)
+1	PRIMARY	<subquery2>	eq_ref	distinct_key	distinct_key	8	func,func	1	
+2	MATERIALIZED	a	ALL	NULL	NULL	NULL	NULL	32	
 explain update t1 set c3=c3+110 where c2 in (select distinct a.c2 from t1 a where t1.c1=a.c1);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	PRIMARY	t1	ALL	NULL	NULL	NULL	NULL	32	
-1	PRIMARY	a	ALL	NULL	NULL	NULL	NULL	32	Using where; FirstMatch(t1)
+1	PRIMARY	<subquery2>	eq_ref	distinct_key	distinct_key	8	func,func	1	
+2	MATERIALIZED	a	ALL	NULL	NULL	NULL	NULL	32	
 update t1 set c3=c3+110 where c2 in (select distinct a.c2 from t1 a where t1.c1=a.c1);
 affected rows: 32
 info: Rows matched: 32  Changed: 32  Warnings: 0
@@ -1777,11 +1779,13 @@ test.t1	analyze	status	OK
 explain select * from t1 where c2 in (select distinct a.c2 from t1 a where t1.c1=a.c1);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	PRIMARY	t1	ALL	NULL	NULL	NULL	NULL	32	
-1	PRIMARY	a	ALL	NULL	NULL	NULL	NULL	32	Using where; FirstMatch(t1); Using join buffer (flat, BNL join)
+1	PRIMARY	<subquery2>	eq_ref	distinct_key	distinct_key	8	func,func	1	
+2	MATERIALIZED	a	ALL	NULL	NULL	NULL	NULL	32	
 explain update t1 set c3=c3+110 where c2 in (select distinct a.c2 from t1 a where t1.c1=a.c1);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	PRIMARY	t1	ALL	NULL	NULL	NULL	NULL	32	
-1	PRIMARY	a	ALL	NULL	NULL	NULL	NULL	32	Using where; FirstMatch(t1)
+1	PRIMARY	<subquery2>	eq_ref	distinct_key	distinct_key	8	func,func	1	
+2	MATERIALIZED	a	ALL	NULL	NULL	NULL	NULL	32	
 update t1 set c3=c3+110 where c2 in (select distinct a.c2 from t1 a where t1.c1=a.c1);
 affected rows: 32
 info: Rows matched: 32  Changed: 32  Warnings: 0
@@ -3051,11 +3055,13 @@ test.t1	analyze	status	OK
 explain select * from t1 where c2 in (select distinct a.c2 from t1 a where t1.c1=a.c1);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	PRIMARY	t1	ALL	NULL	NULL	NULL	NULL	32	
-1	PRIMARY	a	ALL	NULL	NULL	NULL	NULL	32	Using where; FirstMatch(t1); Using join buffer (flat, BNL join)
+1	PRIMARY	<subquery2>	eq_ref	distinct_key	distinct_key	8	func,func	1	
+2	MATERIALIZED	a	ALL	NULL	NULL	NULL	NULL	32	
 explain update t1 set c3=c3+110 where c2 in (select distinct a.c2 from t1 a where t1.c1=a.c1);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	PRIMARY	t1	ALL	NULL	NULL	NULL	NULL	32	
-1	PRIMARY	a	ALL	NULL	NULL	NULL	NULL	32	Using where; FirstMatch(t1)
+1	PRIMARY	<subquery2>	eq_ref	distinct_key	distinct_key	8	func,func	1	
+2	MATERIALIZED	a	ALL	NULL	NULL	NULL	NULL	32	
 update t1 set c3=c3+110 where c2 in (select distinct a.c2 from t1 a where t1.c1=a.c1);
 affected rows: 32
 info: Rows matched: 32  Changed: 32  Warnings: 0
@@ -4324,11 +4330,13 @@ test.t1	analyze	status	OK
 explain select * from t1 where c2 in (select distinct a.c2 from t1 a where t1.c1=a.c1);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	PRIMARY	t1	ALL	NULL	NULL	NULL	NULL	32	
-1	PRIMARY	a	ALL	NULL	NULL	NULL	NULL	32	Using where; FirstMatch(t1); Using join buffer (flat, BNL join)
+1	PRIMARY	<subquery2>	eq_ref	distinct_key	distinct_key	8	func,func	1	
+2	MATERIALIZED	a	ALL	NULL	NULL	NULL	NULL	32	
 explain update t1 set c3=c3+110 where c2 in (select distinct a.c2 from t1 a where t1.c1=a.c1);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	PRIMARY	t1	ALL	NULL	NULL	NULL	NULL	32	
-1	PRIMARY	a	ALL	NULL	NULL	NULL	NULL	32	Using where; FirstMatch(t1)
+1	PRIMARY	<subquery2>	eq_ref	distinct_key	distinct_key	8	func,func	1	
+2	MATERIALIZED	a	ALL	NULL	NULL	NULL	NULL	32	
 update t1 set c3=c3+110 where c2 in (select distinct a.c2 from t1 a where t1.c1=a.c1);
 affected rows: 32
 info: Rows matched: 32  Changed: 32  Warnings: 0
@@ -5598,11 +5606,13 @@ test.t1	analyze	status	OK
 explain select * from t1 where c2 in (select distinct a.c2 from t1 a where t1.c1=a.c1);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	PRIMARY	t1	ALL	NULL	NULL	NULL	NULL	32	
-1	PRIMARY	a	ALL	NULL	NULL	NULL	NULL	32	Using where; FirstMatch(t1); Using join buffer (flat, BNL join)
+1	PRIMARY	<subquery2>	eq_ref	distinct_key	distinct_key	8	func,func	1	
+2	MATERIALIZED	a	ALL	NULL	NULL	NULL	NULL	32	
 explain update t1 set c3=c3+110 where c2 in (select distinct a.c2 from t1 a where t1.c1=a.c1);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	PRIMARY	t1	ALL	NULL	NULL	NULL	NULL	32	
-1	PRIMARY	a	ALL	NULL	NULL	NULL	NULL	32	Using where; FirstMatch(t1)
+1	PRIMARY	<subquery2>	eq_ref	distinct_key	distinct_key	8	func,func	1	
+2	MATERIALIZED	a	ALL	NULL	NULL	NULL	NULL	32	
 update t1 set c3=c3+110 where c2 in (select distinct a.c2 from t1 a where t1.c1=a.c1);
 affected rows: 32
 info: Rows matched: 32  Changed: 32  Warnings: 0
@@ -6871,11 +6881,13 @@ test.t1	analyze	status	OK
 explain select * from t1 where c2 in (select distinct a.c2 from t1 a where t1.c1=a.c1);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	PRIMARY	t1	ALL	NULL	NULL	NULL	NULL	32	
-1	PRIMARY	a	ALL	NULL	NULL	NULL	NULL	32	Using where; FirstMatch(t1); Using join buffer (flat, BNL join)
+1	PRIMARY	<subquery2>	eq_ref	distinct_key	distinct_key	8	func,func	1	
+2	MATERIALIZED	a	ALL	NULL	NULL	NULL	NULL	32	
 explain update t1 set c3=c3+110 where c2 in (select distinct a.c2 from t1 a where t1.c1=a.c1);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	PRIMARY	t1	ALL	NULL	NULL	NULL	NULL	32	
-1	PRIMARY	a	ALL	NULL	NULL	NULL	NULL	32	Using where; FirstMatch(t1)
+1	PRIMARY	<subquery2>	eq_ref	distinct_key	distinct_key	8	func,func	1	
+2	MATERIALIZED	a	ALL	NULL	NULL	NULL	NULL	32	
 update t1 set c3=c3+110 where c2 in (select distinct a.c2 from t1 a where t1.c1=a.c1);
 affected rows: 32
 info: Rows matched: 32  Changed: 32  Warnings: 0
@@ -8145,11 +8157,13 @@ test.t1	analyze	note	The storage engine for the table doesn't support analyze
 explain select * from t1 where c2 in (select distinct a.c2 from t1 a where t1.c1=a.c1);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	PRIMARY	t1	ALL	NULL	NULL	NULL	NULL	32	
-1	PRIMARY	a	ALL	NULL	NULL	NULL	NULL	32	Using where; FirstMatch(t1); Using join buffer (flat, BNL join)
+1	PRIMARY	<subquery2>	eq_ref	distinct_key	distinct_key	8	func,func	1	
+2	MATERIALIZED	a	ALL	NULL	NULL	NULL	NULL	32	
 explain update t1 set c3=c3+110 where c2 in (select distinct a.c2 from t1 a where t1.c1=a.c1);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	PRIMARY	t1	ALL	NULL	NULL	NULL	NULL	32	
-1	PRIMARY	a	ALL	NULL	NULL	NULL	NULL	32	Using where; FirstMatch(t1)
+1	PRIMARY	<subquery2>	eq_ref	distinct_key	distinct_key	8	func,func	1	
+2	MATERIALIZED	a	ALL	NULL	NULL	NULL	NULL	32	
 update t1 set c3=c3+110 where c2 in (select distinct a.c2 from t1 a where t1.c1=a.c1);
 affected rows: 32
 info: Rows matched: 32  Changed: 32  Warnings: 0
@@ -9410,11 +9424,13 @@ test.t1	analyze	note	The storage engine for the table doesn't support analyze
 explain select * from t1 where c2 in (select distinct a.c2 from t1 a where t1.c1=a.c1);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	PRIMARY	t1	ALL	NULL	NULL	NULL	NULL	32	
-1	PRIMARY	a	ALL	NULL	NULL	NULL	NULL	32	Using where; FirstMatch(t1); Using join buffer (flat, BNL join)
+1	PRIMARY	<subquery2>	eq_ref	distinct_key	distinct_key	8	func,func	1	
+2	MATERIALIZED	a	ALL	NULL	NULL	NULL	NULL	32	
 explain update t1 set c3=c3+110 where c2 in (select distinct a.c2 from t1 a where t1.c1=a.c1);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	PRIMARY	t1	ALL	NULL	NULL	NULL	NULL	32	
-1	PRIMARY	a	ALL	NULL	NULL	NULL	NULL	32	Using where; FirstMatch(t1)
+1	PRIMARY	<subquery2>	eq_ref	distinct_key	distinct_key	8	func,func	1	
+2	MATERIALIZED	a	ALL	NULL	NULL	NULL	NULL	32	
 update t1 set c3=c3+110 where c2 in (select distinct a.c2 from t1 a where t1.c1=a.c1);
 affected rows: 32
 info: Rows matched: 32  Changed: 32  Warnings: 0

sql/item_subselect.h

@@ -365,6 +365,13 @@ class Item_maxmin_subselect :public Item_singlerow_subselect
   void no_rows_in_result() override;
 };
 
+typedef struct st_eq_field_outer
+{
+  Item **eq_ref;
+  Item_ident *local_field;
+  Item *outer_exp;
+} EQ_FIELD_OUTER;
+
 /* exists subselect */
 
 class Item_exists_subselect :public Item_subselect
@@ -375,7 +382,12 @@ class Item_exists_subselect :public Item_subselect
 
   void init_length_and_dec();
   bool select_prepare_to_be_in();
-
+  bool exists2in_prepare(THD *thd, Dynamic_array<EQ_FIELD_OUTER> &eqs,
+                         bool *will_be_correlated);
+  bool exists2in_create_or_update_in(THD *thd,
+                                     const Dynamic_array<EQ_FIELD_OUTER> &eqs,
+                                     Item** left_exp_ref);
+  bool exists2in_and_is_not_nulls(uint offset, Item **left_exp_ref);
 public:
   /*
     Used by subquery optimizations to keep track about in which clause this
@@ -616,6 +628,14 @@ class Item_in_subselect :public Item_exists_subselect
     SET to TRUE if IN subquery is converted from an IN predicate
   */
   bool converted_from_in_predicate;
+  
+  /*
+    -1 means "do nothing"
+    Other value means that we can assume that left_expr->element_index(i) 
+    cannot be NULL (because decorrelation code has added IS NOT NULL before
+    the subquery predicate).
+  */
+  int  not_nulls_after;
 
   Item_in_subselect(THD *thd_arg, Item * left_expr, st_select_lex *select_lex);
   Item_in_subselect(THD *thd_arg):
@@ -623,7 +643,8 @@ class Item_in_subselect :public Item_exists_subselect
     in_strategy(SUBS_NOT_TRANSFORMED),
     pushed_cond_guards(NULL), func(NULL), do_not_convert_to_sj(FALSE),
     is_jtbm_merged(FALSE), is_jtbm_const_tab(FALSE), upper_item(0),
-    converted_from_in_predicate(FALSE) {}
+    converted_from_in_predicate(FALSE),
+    not_nulls_after(-1) {}
   void cleanup() override;
   subs_type substype() override { return IN_SUBS; }
   void reset() override
@@ -757,10 +778,7 @@ class Item_in_subselect :public Item_exists_subselect
            Item_subselect::walk(processor, walk_subquery, arg);
   }
 
-  bool exists2in_processor(void *opt_arg __attribute__((unused))) override
-  {
-    return 0;
-  };
+  bool exists2in_processor(void *opt_arg) override;
 
   bool pushdown_cond_for_in_subquery(THD *thd, Item *cond);
 
@@ -804,6 +822,7 @@ class Item_allany_subselect :public Item_in_subselect
   bool is_maxmin_applicable(JOIN *join);
   bool transform_into_max_min(JOIN *join);
   void no_rows_in_result();
+  bool exists2in_processor(void *arg) override { return FALSE; }
 };
 
 

sql/opt_subselect.h

@@ -431,3 +431,6 @@ void get_delayed_table_estimates(TABLE *table,
 
 enum_nested_loop_state join_tab_execution_startup(JOIN_TAB *tab);
 
+bool is_materialization_applicable(THD *thd, Item_in_subselect *in_subs,
+                                   st_select_lex *child_select);
+