BUG#860300: Second crash with get_fanout_with_deps() with semijoin + materialization

- Make get_post_group_estimate() take into account semi-join materialization nests.

mysql-test/r/subselect_mat.result

@@ -1657,6 +1657,24 @@ a
 0
 DROP TABLE t2,t3,t4,t5;
 set optimizer_switch=@subselect_sj_mat_tmp;
+#
+# BUG#860300: Second crash with get_fanout_with_deps() with semijoin + materialization
+#
+set @tmp_860300=@@optimizer_switch;
+set optimizer_switch='semijoin=on,materialization=on,loosescan=off,firstmatch=off';
+CREATE TABLE t1 (f2 int);
+INSERT INTO t1 VALUES (9),(6);
+CREATE TABLE t3 (f4 int);
+CREATE TABLE t4 (f6 varchar(1));
+SELECT *
+FROM t3
+WHERE 'h' IN (SELECT f6 
+FROM t4
+WHERE 5 IN (SELECT f2 FROM t1)
+GROUP BY t4.f6);
+f4
+DROP TABLE t1,t3,t4;
+set optimizer_switch=@tmp_860300;
 set @subselect_mat_test_optimizer_switch_value=null;
 set @@optimizer_switch='materialization=on,in_to_exists=off,semijoin=off';
 set optimizer_switch='mrr=on,mrr_sort_keys=on,index_condition_pushdown=on';

mysql-test/r/subselect_sj_mat.result

@@ -1697,3 +1697,21 @@ a
 0
 DROP TABLE t2,t3,t4,t5;
 set optimizer_switch=@subselect_sj_mat_tmp;
+#
+# BUG#860300: Second crash with get_fanout_with_deps() with semijoin + materialization
+#
+set @tmp_860300=@@optimizer_switch;
+set optimizer_switch='semijoin=on,materialization=on,loosescan=off,firstmatch=off';
+CREATE TABLE t1 (f2 int);
+INSERT INTO t1 VALUES (9),(6);
+CREATE TABLE t3 (f4 int);
+CREATE TABLE t4 (f6 varchar(1));
+SELECT *
+FROM t3
+WHERE 'h' IN (SELECT f6 
+FROM t4
+WHERE 5 IN (SELECT f2 FROM t1)
+GROUP BY t4.f6);
+f4
+DROP TABLE t1,t3,t4;
+set optimizer_switch=@tmp_860300;

mysql-test/t/subselect_sj_mat.test

@@ -1351,3 +1351,22 @@ DROP TABLE t2,t3,t4,t5;
 
 set optimizer_switch=@subselect_sj_mat_tmp;
 
+--echo #
+--echo # BUG#860300: Second crash with get_fanout_with_deps() with semijoin + materialization
+--echo #
+set @tmp_860300=@@optimizer_switch;
+set optimizer_switch='semijoin=on,materialization=on,loosescan=off,firstmatch=off';
+CREATE TABLE t1 (f2 int);
+INSERT INTO t1 VALUES (9),(6);
+CREATE TABLE t3 (f4 int);
+CREATE TABLE t4 (f6 varchar(1));
+
+SELECT *
+FROM t3
+WHERE 'h' IN (SELECT f6 
+              FROM t4
+              WHERE 5 IN (SELECT f2 FROM t1)
+              GROUP BY t4.f6);
+DROP TABLE t1,t3,t4;
+set optimizer_switch=@tmp_860300;
+

sql/item_subselect.cc

@@ -4394,7 +4394,13 @@ double get_fanout_with_deps(JOIN *join, table_map tset)
   for (JOIN_TAB *tab= first_top_level_tab(join, WITHOUT_CONST_TABLES); tab;
        tab= next_top_level_tab(join, tab))
   {
-    if ((tab->table->map & checked_deps) && !tab->emb_sj_nest && 
+    /* 
+      Ignore SJM nests. They have tab->table==NULL. There is no point to walk
+      inside them, because GROUP BY clause cannot refer to tables from within
+      subquery.
+    */
+    if (!tab->is_sjm_nest() && (tab->table->map & checked_deps) && 
+        !tab->emb_sj_nest && 
         tab->records_read != 0)
     {
       fanout *= rows2double(tab->records_read);