Fixed the bug mdev-13135.

For each SELECT the list sj_nests is built by the
function simplify_joins() when scanning different
join nests. This function may be called several
times for the same join nest. That's why before
adding a new member to sj_nests it is necessary
to check if it's already in the list.
The code of simplify_joins() lacked this check and
as a result it could cause memory overwright for
some queries.

mysql-test/r/subselect_mat_cost_bugs.result

@@ -502,3 +502,20 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	PRIMARY	t1	index	idx	idx	5	NULL	5	Using where; Using index
 2	DEPENDENT SUBQUERY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Select tables optimized away
 drop table t1;
+#
+# MDEV-13135: subquery with ON expression subject to
+#             semi-join optimizations
+#
+CREATE TABLE t1 (a INT);
+CREATE ALGORITHM=MERGE VIEW v1 AS SELECT a AS v_a FROM t1;
+INSERT INTO t1 VALUES (1),(3);
+CREATE TABLE t2 (b INT, KEY(b));
+INSERT INTO t2 VALUES (3),(4);
+SELECT * FROM t1 WHERE a NOT IN (
+SELECT b FROM t2 INNER JOIN v1 ON (b IN ( SELECT a FROM t1 )) 
+WHERE v_a = b
+);
+a
+1
+DROP VIEW v1;
+DROP TABLE t1,t2;

mysql-test/t/subselect_mat_cost_bugs.test

@@ -522,4 +522,23 @@ select * from t1 where a in (select max(a) from t1 where a < 4) or a > 5;
 
 drop table t1;
 
+--echo #
+--echo # MDEV-13135: subquery with ON expression subject to
+--echo #             semi-join optimizations
+--echo #
+
+CREATE TABLE t1 (a INT);
+CREATE ALGORITHM=MERGE VIEW v1 AS SELECT a AS v_a FROM t1;
+INSERT INTO t1 VALUES (1),(3);
+
+CREATE TABLE t2 (b INT, KEY(b));
+INSERT INTO t2 VALUES (3),(4);
+
+SELECT * FROM t1 WHERE a NOT IN (
+  SELECT b FROM t2 INNER JOIN v1 ON (b IN ( SELECT a FROM t1 )) 
+    WHERE v_a = b
+);
+
+DROP VIEW v1;
+DROP TABLE t1,t2;
 

sql/sql_select.cc

@@ -13349,10 +13349,23 @@ simplify_joins(JOIN *join, List<TABLE_LIST> *join_list, COND *conds, bool top,
     nested_join= table->nested_join;
     if (table->sj_on_expr && !in_sj)
     {
-       /*
-         If this is a semi-join that is not contained within another semi-join, 
-         leave it intact (otherwise it is flattened)
-       */
+      /*
+        If this is a semi-join that is not contained within another semi-join
+        leave it intact (otherwise it is flattened)
+      */
+      /*
+        Make sure that any semi-join appear in
+        the join->select_lex->sj_nests list only once
+      */
+      List_iterator_fast<TABLE_LIST> sj_it(join->select_lex->sj_nests);
+      TABLE_LIST *sj_nest;
+      while ((sj_nest= sj_it++))
+      {
+        if (table == sj_nest)
+          break;
+      }
+      if (sj_nest)
+        continue;
       join->select_lex->sj_nests.push_back(table);
 
       /*