Fixed bug mdev-277 as part of the fix for lp:944706

The cause for this bug is that the method JOIN::get_examined_rows iterates over all
JOIN_TABs of the join assuming they are just a sequence. In the query above, the
innermost subquery is merged into its parent query. When we call
JOIN::get_examined_rows for the second-level subquery, the iteration that
assumes sequential order of join tabs goes outside the join_tab array and calls
the method JOIN_TAB::get_examined_rows on uninitialized memory. 

The fix is to iterate over JOIN_TABs in a way that takes into account the nested
semi-join structure of JOIN_TABs. In particular iterate as select_describe.

mysql-test/r/subselect.result

@@ -6594,5 +6594,26 @@ WHERE alias1.a = alias2.a OR ('Moscow') IN ( SELECT a FROM t1 );
 MAX( alias2.a )
 Arden-Arcade
 drop table t1;
+#
+# MDEV-277 CHEAP SQ: Server crashes in st_join_table::get_examined_rows
+# with semijoin+materialization, IN and = subqueries
+#
+CREATE TABLE t1 (a1 INT);
+INSERT INTO t1 VALUES (4),(6);
+CREATE TABLE t2 (b1 INT);
+INSERT INTO t2 VALUES (1),(7);
+EXPLAIN
+SELECT * FROM t1
+WHERE a1 = (SELECT COUNT(*) FROM t1 WHERE a1 IN (SELECT a1 FROM t1, t2));
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	t1	ALL	NULL	NULL	NULL	NULL	2	Using where
+2	SUBQUERY	t1	ALL	NULL	NULL	NULL	NULL	2	
+2	SUBQUERY	<subquery3>	eq_ref	distinct_key	distinct_key	4	func	1	
+3	MATERIALIZED	t1	ALL	NULL	NULL	NULL	NULL	2	
+3	MATERIALIZED	t2	ALL	NULL	NULL	NULL	NULL	2	Using join buffer (flat, BNL join)
+SELECT * FROM t1
+WHERE a1 = (SELECT COUNT(*) FROM t1 WHERE a1 IN (SELECT a1 FROM t1, t2));
+a1
+drop table t1, t2;
 # return optimizer switch changed in the beginning of this test
 set optimizer_switch=@subselect_tmp;

mysql-test/r/subselect_no_mat.result

@@ -6593,6 +6593,26 @@ WHERE alias1.a = alias2.a OR ('Moscow') IN ( SELECT a FROM t1 );
 MAX( alias2.a )
 Arden-Arcade
 drop table t1;
+#
+# MDEV-277 CHEAP SQ: Server crashes in st_join_table::get_examined_rows
+# with semijoin+materialization, IN and = subqueries
+#
+CREATE TABLE t1 (a1 INT);
+INSERT INTO t1 VALUES (4),(6);
+CREATE TABLE t2 (b1 INT);
+INSERT INTO t2 VALUES (1),(7);
+EXPLAIN
+SELECT * FROM t1
+WHERE a1 = (SELECT COUNT(*) FROM t1 WHERE a1 IN (SELECT a1 FROM t1, t2));
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	t1	ALL	NULL	NULL	NULL	NULL	2	Using where
+2	SUBQUERY	t1	ALL	NULL	NULL	NULL	NULL	2	
+2	SUBQUERY	t1	ALL	NULL	NULL	NULL	NULL	2	Using where; Start temporary; Using join buffer (flat, BNL join)
+2	SUBQUERY	t2	ALL	NULL	NULL	NULL	NULL	2	End temporary; Using join buffer (flat, BNL join)
+SELECT * FROM t1
+WHERE a1 = (SELECT COUNT(*) FROM t1 WHERE a1 IN (SELECT a1 FROM t1, t2));
+a1
+drop table t1, t2;
 # return optimizer switch changed in the beginning of this test
 set optimizer_switch=@subselect_tmp;
 set optimizer_switch=default;

mysql-test/r/subselect_no_opts.result

@@ -6589,6 +6589,27 @@ WHERE alias1.a = alias2.a OR ('Moscow') IN ( SELECT a FROM t1 );
 MAX( alias2.a )
 Arden-Arcade
 drop table t1;
+#
+# MDEV-277 CHEAP SQ: Server crashes in st_join_table::get_examined_rows
+# with semijoin+materialization, IN and = subqueries
+#
+CREATE TABLE t1 (a1 INT);
+INSERT INTO t1 VALUES (4),(6);
+CREATE TABLE t2 (b1 INT);
+INSERT INTO t2 VALUES (1),(7);
+EXPLAIN
+SELECT * FROM t1
+WHERE a1 = (SELECT COUNT(*) FROM t1 WHERE a1 IN (SELECT a1 FROM t1, t2));
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	t1	ALL	NULL	NULL	NULL	NULL	2	Using where
+2	SUBQUERY	t1	ALL	NULL	NULL	NULL	NULL	2	
+2	SUBQUERY	<subquery3>	eq_ref	distinct_key	distinct_key	4	func	1	
+3	MATERIALIZED	t1	ALL	NULL	NULL	NULL	NULL	2	
+3	MATERIALIZED	t2	ALL	NULL	NULL	NULL	NULL	2	Using join buffer (flat, BNL join)
+SELECT * FROM t1
+WHERE a1 = (SELECT COUNT(*) FROM t1 WHERE a1 IN (SELECT a1 FROM t1, t2));
+a1
+drop table t1, t2;
 # return optimizer switch changed in the beginning of this test
 set optimizer_switch=@subselect_tmp;
 set @optimizer_switch_for_subselect_test=null;

mysql-test/r/subselect_no_scache.result

@@ -6600,6 +6600,27 @@ WHERE alias1.a = alias2.a OR ('Moscow') IN ( SELECT a FROM t1 );
 MAX( alias2.a )
 Arden-Arcade
 drop table t1;
+#
+# MDEV-277 CHEAP SQ: Server crashes in st_join_table::get_examined_rows
+# with semijoin+materialization, IN and = subqueries
+#
+CREATE TABLE t1 (a1 INT);
+INSERT INTO t1 VALUES (4),(6);
+CREATE TABLE t2 (b1 INT);
+INSERT INTO t2 VALUES (1),(7);
+EXPLAIN
+SELECT * FROM t1
+WHERE a1 = (SELECT COUNT(*) FROM t1 WHERE a1 IN (SELECT a1 FROM t1, t2));
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	t1	ALL	NULL	NULL	NULL	NULL	2	Using where
+2	SUBQUERY	t1	ALL	NULL	NULL	NULL	NULL	2	
+2	SUBQUERY	<subquery3>	eq_ref	distinct_key	distinct_key	4	func	1	
+3	MATERIALIZED	t1	ALL	NULL	NULL	NULL	NULL	2	
+3	MATERIALIZED	t2	ALL	NULL	NULL	NULL	NULL	2	Using join buffer (flat, BNL join)
+SELECT * FROM t1
+WHERE a1 = (SELECT COUNT(*) FROM t1 WHERE a1 IN (SELECT a1 FROM t1, t2));
+a1
+drop table t1, t2;
 # return optimizer switch changed in the beginning of this test
 set optimizer_switch=@subselect_tmp;
 set optimizer_switch=default;

mysql-test/r/subselect_no_semijoin.result

@@ -6589,6 +6589,27 @@ WHERE alias1.a = alias2.a OR ('Moscow') IN ( SELECT a FROM t1 );
 MAX( alias2.a )
 Arden-Arcade
 drop table t1;
+#
+# MDEV-277 CHEAP SQ: Server crashes in st_join_table::get_examined_rows
+# with semijoin+materialization, IN and = subqueries
+#
+CREATE TABLE t1 (a1 INT);
+INSERT INTO t1 VALUES (4),(6);
+CREATE TABLE t2 (b1 INT);
+INSERT INTO t2 VALUES (1),(7);
+EXPLAIN
+SELECT * FROM t1
+WHERE a1 = (SELECT COUNT(*) FROM t1 WHERE a1 IN (SELECT a1 FROM t1, t2));
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	t1	ALL	NULL	NULL	NULL	NULL	2	Using where
+2	SUBQUERY	t1	ALL	NULL	NULL	NULL	NULL	2	
+2	SUBQUERY	<subquery3>	eq_ref	distinct_key	distinct_key	4	func	1	
+3	MATERIALIZED	t1	ALL	NULL	NULL	NULL	NULL	2	
+3	MATERIALIZED	t2	ALL	NULL	NULL	NULL	NULL	2	Using join buffer (flat, BNL join)
+SELECT * FROM t1
+WHERE a1 = (SELECT COUNT(*) FROM t1 WHERE a1 IN (SELECT a1 FROM t1, t2));
+a1
+drop table t1, t2;
 # return optimizer switch changed in the beginning of this test
 set optimizer_switch=@subselect_tmp;
 set @optimizer_switch_for_subselect_test=null;

mysql-test/t/subselect.test

@@ -5561,5 +5561,24 @@ WHERE alias1.a = alias2.a OR ('Moscow') IN ( SELECT a FROM t1 );
 
 drop table t1;
 
+--echo #
+--echo # MDEV-277 CHEAP SQ: Server crashes in st_join_table::get_examined_rows
+--echo # with semijoin+materialization, IN and = subqueries
+--echo #
+
+CREATE TABLE t1 (a1 INT);
+INSERT INTO t1 VALUES (4),(6);
+CREATE TABLE t2 (b1 INT);
+INSERT INTO t2 VALUES (1),(7);
+
+EXPLAIN
+SELECT * FROM t1
+WHERE a1 = (SELECT COUNT(*) FROM t1 WHERE a1 IN (SELECT a1 FROM t1, t2));
+
+SELECT * FROM t1
+WHERE a1 = (SELECT COUNT(*) FROM t1 WHERE a1 IN (SELECT a1 FROM t1, t2));
+
+drop table t1, t2;
+
 --echo # return optimizer switch changed in the beginning of this test
 set optimizer_switch=@subselect_tmp;

sql/sql_select.cc

@@ -270,6 +270,8 @@ Item_equal *find_item_equal(COND_EQUAL *cond_equal, Field *field,
                             bool *inherited_fl);
 JOIN_TAB *first_depth_first_tab(JOIN* join);
 JOIN_TAB *next_depth_first_tab(JOIN* join, JOIN_TAB* tab);
+JOIN_TAB *first_breadth_first_tab(JOIN *join);
+JOIN_TAB *next_breadth_first_tab(JOIN *join, JOIN_TAB *tab);
 
 /**
   This handles SELECT with and without UNION.
@@ -6615,19 +6617,18 @@ void JOIN::get_prefix_cost_and_fanout(uint n_tables,
 
 double JOIN::get_examined_rows()
 {
-  /* Each constant table examines one row, and the result is at most one row. */
-  ha_rows examined_rows= const_tables;
-  uint i= const_tables;
+  ha_rows examined_rows;
   double prev_fanout= 1;
+  JOIN_TAB *tab= first_breadth_first_tab(this);
+  JOIN_TAB *prev_tab= tab;
 
-  if (table_count == const_tables)
-    return examined_rows;
+  examined_rows= tab->get_examined_rows();
 
-  examined_rows+= join_tab[i++].get_examined_rows();
-  for (; i < table_count ; i++)
+  while ((tab= next_breadth_first_tab(this, tab)))
   {
-    prev_fanout *= best_positions[i-1].records_read;
-    examined_rows+= join_tab[i].get_examined_rows() * prev_fanout;
+    prev_fanout *= prev_tab->records_read;
+    examined_rows+= tab->get_examined_rows() * prev_fanout;
+    prev_tab= tab;
   }
   return examined_rows;
 }