Fixed bug mdev-5135.

The patch for bug mdev-5105 incorrectly counted conditions in nested joins.

mysql-test/r/subselect_sj2.result

@@ -1097,5 +1097,20 @@ COUNT(*)
 3724
 set optimizer_prune_level=@tmp_951283;
 DROP TABLE t1,t2;
+#
+# Bug mdev-5135: crash on semijoin with nested outer joins
+#
+CREATE TABLE t1 (i1 int) ENGINE=MyISAM;
+INSERT INTO t1 VALUES (1),(2);
+CREATE TABLE t2 (i2 int, INDEX(i2)) ENGINE=MyISAM;
+CREATE TABLE t3 (i3 int, c varchar(1), INDEX(i3), INDEX(c)) ENGINE=MyISAM;
+INSERT INTO t3 VALUES (3,'x'),(4,'y');
+SELECT * FROM t1 WHERE ( 1, 1 ) IN ( 
+SELECT i2, i2 FROM t2 LEFT OUTER JOIN ( 
+t3 AS t3a INNER JOIN t3 AS t3b ON ( t3a.i3 = t3b.i3 ) 
+) ON ( t3a.c = t3b.c ) 
+);
+i1
+DROP TABLE t1,t2,t3;
 # This must be the last in the file:
 set optimizer_switch=@subselect_sj2_tmp;

mysql-test/r/subselect_sj2_jcl6.result

@@ -1112,6 +1112,21 @@ COUNT(*)
 3724
 set optimizer_prune_level=@tmp_951283;
 DROP TABLE t1,t2;
+#
+# Bug mdev-5135: crash on semijoin with nested outer joins
+#
+CREATE TABLE t1 (i1 int) ENGINE=MyISAM;
+INSERT INTO t1 VALUES (1),(2);
+CREATE TABLE t2 (i2 int, INDEX(i2)) ENGINE=MyISAM;
+CREATE TABLE t3 (i3 int, c varchar(1), INDEX(i3), INDEX(c)) ENGINE=MyISAM;
+INSERT INTO t3 VALUES (3,'x'),(4,'y');
+SELECT * FROM t1 WHERE ( 1, 1 ) IN ( 
+SELECT i2, i2 FROM t2 LEFT OUTER JOIN ( 
+t3 AS t3a INNER JOIN t3 AS t3b ON ( t3a.i3 = t3b.i3 ) 
+) ON ( t3a.c = t3b.c ) 
+);
+i1
+DROP TABLE t1,t2,t3;
 # This must be the last in the file:
 set optimizer_switch=@subselect_sj2_tmp;
 #

mysql-test/r/subselect_sj2_mat.result

@@ -1099,6 +1099,21 @@ COUNT(*)
 3724
 set optimizer_prune_level=@tmp_951283;
 DROP TABLE t1,t2;
+#
+# Bug mdev-5135: crash on semijoin with nested outer joins
+#
+CREATE TABLE t1 (i1 int) ENGINE=MyISAM;
+INSERT INTO t1 VALUES (1),(2);
+CREATE TABLE t2 (i2 int, INDEX(i2)) ENGINE=MyISAM;
+CREATE TABLE t3 (i3 int, c varchar(1), INDEX(i3), INDEX(c)) ENGINE=MyISAM;
+INSERT INTO t3 VALUES (3,'x'),(4,'y');
+SELECT * FROM t1 WHERE ( 1, 1 ) IN ( 
+SELECT i2, i2 FROM t2 LEFT OUTER JOIN ( 
+t3 AS t3a INNER JOIN t3 AS t3b ON ( t3a.i3 = t3b.i3 ) 
+) ON ( t3a.c = t3b.c ) 
+);
+i1
+DROP TABLE t1,t2,t3;
 # This must be the last in the file:
 set optimizer_switch=@subselect_sj2_tmp;
 set optimizer_switch=default;

mysql-test/t/subselect_sj2.test

@@ -1229,5 +1229,25 @@ WHERE alias3.d IN (
 set optimizer_prune_level=@tmp_951283;
 DROP TABLE t1,t2;
 
+--echo #
+--echo # Bug mdev-5135: crash on semijoin with nested outer joins
+--echo #
+
+CREATE TABLE t1 (i1 int) ENGINE=MyISAM;
+INSERT INTO t1 VALUES (1),(2);
+
+CREATE TABLE t2 (i2 int, INDEX(i2)) ENGINE=MyISAM;
+
+CREATE TABLE t3 (i3 int, c varchar(1), INDEX(i3), INDEX(c)) ENGINE=MyISAM;
+INSERT INTO t3 VALUES (3,'x'),(4,'y');
+
+SELECT * FROM t1 WHERE ( 1, 1 ) IN ( 
+  SELECT i2, i2 FROM t2 LEFT OUTER JOIN ( 
+    t3 AS t3a INNER JOIN t3 AS t3b ON ( t3a.i3 = t3b.i3 ) 
+  ) ON ( t3a.c = t3b.c ) 
+);
+
+DROP TABLE t1,t2,t3;
+
 --echo # This must be the last in the file:
 set optimizer_switch=@subselect_sj2_tmp;

sql/sql_select.cc

@@ -4703,6 +4703,33 @@ static void add_key_fields_for_nj(JOIN *join, TABLE_LIST *nested_join_table,
 }
 
 
+void count_cond_for_nj(SELECT_LEX *sel, TABLE_LIST *nested_join_table)
+{
+  List_iterator<TABLE_LIST> li(nested_join_table->nested_join->join_list);
+  List_iterator<TABLE_LIST> li2(nested_join_table->nested_join->join_list);
+  bool have_another = FALSE;
+  TABLE_LIST *table;
+
+  while ((table= li++) || (have_another && (li=li2, have_another=FALSE,
+                                            (table= li++))))
+  if (table->nested_join)
+  {
+    if (!table->on_expr)
+    {
+      /* It's a semi-join nest. Walk into it as if it wasn't a nest */
+      have_another= TRUE;
+      li2= li;
+      li= List_iterator<TABLE_LIST>(table->nested_join->join_list); 
+    }
+    else
+      count_cond_for_nj(sel, table); 
+  }
+  if (nested_join_table->on_expr)
+    nested_join_table->on_expr->walk(&Item::count_sargable_conds, 
+                                     0, (uchar*) sel);
+    
+}
+
 /**
   Update keyuse array with all possible keys we can use to fetch rows.
   
@@ -4751,12 +4778,7 @@ update_ref_and_keys(THD *thd, DYNAMIC_ARRAY *keyuse,JOIN_TAB *join_tab,
     while ((table= li++))
     {
       if (table->nested_join)
-      {
-        if (table->on_expr)
-          table->on_expr->walk(&Item::count_sargable_conds, 0, (uchar*) sel);
-        if (table->sj_on_expr)
-          table->sj_on_expr->walk(&Item::count_sargable_conds, 0, (uchar*) sel);
-      }
+        count_cond_for_nj(sel, table);
     }
   }