lp:802965: Crash in do_copy_not_null with semijoin=on in maria-5.3

- The crash was because a NOT NULL table column inside the subquery was considered NULLable
  because the code thought it was on the inner side of an outer join nest.
- Fixed by making correct distinction between tables inside outer join nests and inside semi-join nests.

mysql-test/r/subselect_sj.result

@@ -1358,7 +1358,6 @@ f10	f11
 0	0
 set optimizer_switch=@tmp_751484;
 drop table t1, t3;
-#
 # BUG#795530 Wrong result with subquery semijoin materialization and outer join 
 #  Simplified testcase that uses DuplicateElimination
 #
@@ -1515,4 +1514,24 @@ a
 2
 set @@optimizer_switch=@tmp_20110622;
 drop table t0, t1, t2, t3;
+#
+# BUG#802965: Crash in do_copy_not_null with semijoin=on in maria-5.3
+# 
+set @save_802965= @@optimizer_switch;
+set optimizer_switch='semijoin=on,materialization=off,firstmatch=off,loosescan=off';
+CREATE TABLE t2 ( f1 int NOT NULL , PRIMARY KEY (f1)) ;
+INSERT IGNORE INTO t2 VALUES (19),(20);
+CREATE TABLE t1 ( f1 int NOT NULL , PRIMARY KEY (f1)) ;
+INSERT IGNORE INTO t1 VALUES (21),(22),(23),(24);
+SELECT *
+FROM t2 , t1
+WHERE t2.f1 IN
+(
+SELECT SQ1_alias1.f1
+FROM t1 AS SQ1_alias1 LEFT JOIN t2 AS SQ1_alias2 JOIN t2 AS SQ1_alias3 ON SQ1_alias3.f1 ON SQ1_alias3.f1
+)
+AND t1.f1 = t2.f1 ;
+f1	f1
+DROP TABLE t1, t2;
+set optimizer_switch=@save_802965;
 set @@optimizer_switch=@save_optimizer_switch;

mysql-test/r/subselect_sj_jcl6.result

@@ -1366,7 +1366,6 @@ f10	f11
 0	0
 set optimizer_switch=@tmp_751484;
 drop table t1, t3;
-#
 # BUG#795530 Wrong result with subquery semijoin materialization and outer join 
 #  Simplified testcase that uses DuplicateElimination
 #
@@ -1523,6 +1522,26 @@ a
 2
 set @@optimizer_switch=@tmp_20110622;
 drop table t0, t1, t2, t3;
+#
+# BUG#802965: Crash in do_copy_not_null with semijoin=on in maria-5.3
+# 
+set @save_802965= @@optimizer_switch;
+set optimizer_switch='semijoin=on,materialization=off,firstmatch=off,loosescan=off';
+CREATE TABLE t2 ( f1 int NOT NULL , PRIMARY KEY (f1)) ;
+INSERT IGNORE INTO t2 VALUES (19),(20);
+CREATE TABLE t1 ( f1 int NOT NULL , PRIMARY KEY (f1)) ;
+INSERT IGNORE INTO t1 VALUES (21),(22),(23),(24);
+SELECT *
+FROM t2 , t1
+WHERE t2.f1 IN
+(
+SELECT SQ1_alias1.f1
+FROM t1 AS SQ1_alias1 LEFT JOIN t2 AS SQ1_alias2 JOIN t2 AS SQ1_alias3 ON SQ1_alias3.f1 ON SQ1_alias3.f1
+)
+AND t1.f1 = t2.f1 ;
+f1	f1
+DROP TABLE t1, t2;
+set optimizer_switch=@save_802965;
 set @@optimizer_switch=@save_optimizer_switch;
 #
 # BUG#49129: Wrong result with IN-subquery with join_cache_level=6 and firstmatch=off

mysql-test/t/subselect_sj.test

@@ -1369,6 +1369,29 @@ set @@optimizer_switch=@tmp_20110622;
 
 drop table t0, t1, t2, t3;
 
+--echo #
+--echo # BUG#802965: Crash in do_copy_not_null with semijoin=on in maria-5.3
+--echo # 
+set @save_802965= @@optimizer_switch;
+set optimizer_switch='semijoin=on,materialization=off,firstmatch=off,loosescan=off';
+
+CREATE TABLE t2 ( f1 int NOT NULL , PRIMARY KEY (f1)) ;
+INSERT IGNORE INTO t2 VALUES (19),(20);
+
+CREATE TABLE t1 ( f1 int NOT NULL , PRIMARY KEY (f1)) ;
+INSERT IGNORE INTO t1 VALUES (21),(22),(23),(24);
+
+SELECT *
+FROM t2 , t1
+WHERE t2.f1 IN
+(
+  SELECT SQ1_alias1.f1
+  FROM t1 AS SQ1_alias1 LEFT JOIN t2 AS SQ1_alias2 JOIN t2 AS SQ1_alias3 ON SQ1_alias3.f1 ON SQ1_alias3.f1
+)
+AND t1.f1 = t2.f1 ;
+
+DROP TABLE t1, t2;
+set optimizer_switch=@save_802965;
 
 # The following command must be the last one the file 
 set @@optimizer_switch=@save_optimizer_switch;

sql/sql_select.cc

@@ -3060,12 +3060,23 @@ make_join_statistics(JOIN *join, List<TABLE_LIST> &tables_list,
         s->embedding_map|= embedding->nested_join->nj_map;
       continue;
     }
-    if (embedding && !(embedding->sj_on_expr && ! embedding->embedding))
+    if (embedding)
     {
       /* s belongs to a nested join, maybe to several embedded joins */
       s->embedding_map= 0;
+      bool inside_an_outer_join= FALSE;
       do
       {
+        /* 
+          If this is a semi-join nest, skip it, and proceed upwards. Maybe
+          we're in some outer join nest
+        */
+        if (embedding->sj_on_expr)
+        {
+          embedding= embedding->embedding;
+          continue;
+        }
+        inside_an_outer_join= TRUE;
         NESTED_JOIN *nested_join= embedding->nested_join;
         s->embedding_map|=nested_join->nj_map;
         s->dependent|= embedding->dep_tables;
@@ -3073,6 +3084,7 @@ make_join_statistics(JOIN *join, List<TABLE_LIST> &tables_list,
         outer_join|= nested_join->used_tables;
       }
       while (embedding);
+      if (inside_an_outer_join)
         continue;
     }
 #ifdef WITH_PARTITION_STORAGE_ENGINE