Fixed LP bug #902356.

A memory overwrite in the function test_if_skip_sort_order()
could cause a crash for some queries with subqueries.

mysql-test/r/subselect2.result

@@ -146,4 +146,37 @@ and t2.a='1' AND t1.a=t3.b) > 0;
 a
 2
 DROP TABLE t1,t2,t3;
+#
+# Bug #902356: DISTINCT in materialized subquery 
+#
+CREATE TABLE t1 (pk int PRIMARY KEY, a int, KEY(a)) ENGINE=InnoDB;
+INSERT INTO t1 VALUES (0, 4), (8, 6);
+CREATE TABLE t2 (pk int PRIMARY KEY, a int, KEY(a)) ENGINE=InnoDB;
+INSERT INTO t2 VALUES (0, 4), (8, 6);
+CREATE TABLE t3 (b INT, KEY(b));
+INSERT INTO t3 VALUES (7), (0), (4), (2);
+CREATE VIEW v1 AS SELECT * FROM t1;
+SET @tmp_optimizer_switch=@@optimizer_switch;
+SET optimizer_switch='materialization=on,in_to_exists=on';
+EXPLAIN
+SELECT * FROM t2,t3 WHERE (2,9) IN (SELECT DISTINCT a,pk FROM t1) OR a = b;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	t2	index	a	a	5	NULL	2	Using index
+1	PRIMARY	t3	index	b	b	5	NULL	4	Using where; Using index; Using join buffer (flat, BNL join)
+2	MATERIALIZED	t1	index	PRIMARY,a	a	5	NULL	2	Using index
+SELECT * FROM t2,t3 WHERE (2,9) IN (SELECT DISTINCT a,pk FROM t1) OR a = b;
+pk	a	b
+0	4	4
+EXPLAIN
+SELECT * FROM t2,t3 WHERE (2,9) IN (SELECT DISTINCT a,pk FROM v1) OR a = b;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	t2	index	a	a	5	NULL	2	Using index
+1	PRIMARY	t3	index	b	b	5	NULL	4	Using where; Using index; Using join buffer (flat, BNL join)
+2	MATERIALIZED	t1	index	PRIMARY,a	a	5	NULL	2	Using index
+SELECT * FROM t2,t3 WHERE (2,9) IN (SELECT DISTINCT a,pk FROM v1) OR a = b;
+pk	a	b
+0	4	4
+SET optimizer_switch=@tmp_optimizer_switch;
+DROP VIEW v1;
+DROP TABLE t1,t2,t3;
 set optimizer_switch=@subselect2_test_tmp;

mysql-test/t/subselect2.test

@@ -172,5 +172,36 @@ SELECT t1.* FROM t1 WHERE (SELECT COUNT(*) FROM t3,t2 WHERE t3.c=t2.a
 
 DROP TABLE t1,t2,t3;
 
+--echo #
+--echo # Bug #902356: DISTINCT in materialized subquery 
+--echo #
+
+CREATE TABLE t1 (pk int PRIMARY KEY, a int, KEY(a)) ENGINE=InnoDB;
+INSERT INTO t1 VALUES (0, 4), (8, 6);
+
+CREATE TABLE t2 (pk int PRIMARY KEY, a int, KEY(a)) ENGINE=InnoDB;
+INSERT INTO t2 VALUES (0, 4), (8, 6);
+
+CREATE TABLE t3 (b INT, KEY(b));
+INSERT INTO t3 VALUES (7), (0), (4), (2);
+
+CREATE VIEW v1 AS SELECT * FROM t1;
+
+SET @tmp_optimizer_switch=@@optimizer_switch;
+SET optimizer_switch='materialization=on,in_to_exists=on';
+
+EXPLAIN
+SELECT * FROM t2,t3 WHERE (2,9) IN (SELECT DISTINCT a,pk FROM t1) OR a = b;
+SELECT * FROM t2,t3 WHERE (2,9) IN (SELECT DISTINCT a,pk FROM t1) OR a = b;
+
+EXPLAIN
+SELECT * FROM t2,t3 WHERE (2,9) IN (SELECT DISTINCT a,pk FROM v1) OR a = b;
+SELECT * FROM t2,t3 WHERE (2,9) IN (SELECT DISTINCT a,pk FROM v1) OR a = b;
+
+SET optimizer_switch=@tmp_optimizer_switch;
+
+DROP VIEW v1;
+DROP TABLE t1,t2,t3;
+
 set optimizer_switch=@subselect2_test_tmp;
 

sql/sql_select.cc

@@ -17530,8 +17530,8 @@ find_field_in_item_list (Field *field, void *data)
 
   while ((item= li++))
   {
-    if (item->type() == Item::FIELD_ITEM &&
-        ((Item_field*) item)->field->eq(field))
+    if (item->real_item()->type() == Item::FIELD_ITEM &&
+	((Item_field*) (item->real_item()))->field->eq(field))
     {
       part_found= 1;
       break;
@@ -17827,7 +17827,8 @@ test_if_skip_sort_order(JOIN_TAB *tab,ORDER *order,ha_rows select_limit_arg,
              uint used_pk_parts= 0;
              if (used_key_parts > used_index_parts)
                used_pk_parts= used_key_parts-used_index_parts;
-             rec_per_key= keyinfo->rec_per_key[used_key_parts-1];
+             rec_per_key= used_key_parts ?
+	                  keyinfo->rec_per_key[used_key_parts-1] : 1;
              /* Take into account the selectivity of the used pk prefix */
              if (used_pk_parts)
 	     {