BUG#834739: Wrong result with 3-way inner join, LooseScan,multipart keys

- Don't use join buffering for tables that are within ranges that are 
  covered by LooseScan strategy.

mysql-test/r/subselect_sj.result

-drop table if exists t0, t1, t2, t3, t4, t10, t11, t12;
+drop table if exists t0, t1, t2, t3, t4, t5, t10, t11, t12;
 drop view if exists v1, v2, v3, v4;
 drop procedure if exists p1;
 set @subselect_sj_tmp= @@optimizer_switch;
@@ -1772,4 +1772,43 @@ b	a
 5	6
 DROP TABLE t1, t2, t3;
 set @@optimizer_switch= @tmp835758;
+#
+# BUG#834739: Wrong result with 3-way inner join, LooseScan,multipart keys  
+#
+set @tmp834739=@@optimizer_switch;
+set optimizer_switch='semijoin=on,loosescan=on,materialization=off,firstmatch=off';
+CREATE TABLE t2 ( b int, c int, KEY (b)) ;
+INSERT INTO t2 VALUES (1,0),(1,0),(9,0),(1,0),(5,0);
+CREATE TABLE t3 ( a int);
+INSERT INTO t3 VALUES (0),(0),(0),(0),(0),(0),(0),(0),(0),(0),(0),(0),(0),(0),(0);
+CREATE TABLE t4 ( a int);
+INSERT INTO t4 VALUES (0),(0),(0);
+CREATE TABLE t5 ( b int, a int , KEY (a,b)) ;
+INSERT INTO t5 VALUES (7,0),(9,0);
+explain
+SELECT * FROM t3 WHERE t3.a IN (SELECT t5.a FROM t2, t4, t5 WHERE t2.c = t5.a AND t2.b = t5.b);
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	t5	index	a	a	10	NULL	2	Using where; Using index; LooseScan
+1	PRIMARY	t4	ALL	NULL	NULL	NULL	NULL	3	
+1	PRIMARY	t2	ref	b	b	5	test.t5.b	2	Using where; FirstMatch(t5)
+1	PRIMARY	t3	ALL	NULL	NULL	NULL	NULL	15	Using where; Using join buffer (flat, BNL join)
+SELECT * FROM t3 WHERE t3.a IN (SELECT t5.a FROM t2, t4, t5 WHERE t2.c = t5.a AND t2.b = t5.b);
+a
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+DROP TABLE t2, t3, t4, t5;
+set @@optimizer_switch=@tmp834739;
 set optimizer_switch=@subselect_sj_tmp;

mysql-test/r/subselect_sj_jcl6.result

@@ -9,7 +9,7 @@ set join_cache_level=6;
 show variables like 'join_cache_level';
 Variable_name	Value
 join_cache_level	6
-drop table if exists t0, t1, t2, t3, t4, t10, t11, t12;
+drop table if exists t0, t1, t2, t3, t4, t5, t10, t11, t12;
 drop view if exists v1, v2, v3, v4;
 drop procedure if exists p1;
 set @subselect_sj_tmp= @@optimizer_switch;
@@ -1783,6 +1783,45 @@ b	a
 5	6
 DROP TABLE t1, t2, t3;
 set @@optimizer_switch= @tmp835758;
+#
+# BUG#834739: Wrong result with 3-way inner join, LooseScan,multipart keys  
+#
+set @tmp834739=@@optimizer_switch;
+set optimizer_switch='semijoin=on,loosescan=on,materialization=off,firstmatch=off';
+CREATE TABLE t2 ( b int, c int, KEY (b)) ;
+INSERT INTO t2 VALUES (1,0),(1,0),(9,0),(1,0),(5,0);
+CREATE TABLE t3 ( a int);
+INSERT INTO t3 VALUES (0),(0),(0),(0),(0),(0),(0),(0),(0),(0),(0),(0),(0),(0),(0);
+CREATE TABLE t4 ( a int);
+INSERT INTO t4 VALUES (0),(0),(0);
+CREATE TABLE t5 ( b int, a int , KEY (a,b)) ;
+INSERT INTO t5 VALUES (7,0),(9,0);
+explain
+SELECT * FROM t3 WHERE t3.a IN (SELECT t5.a FROM t2, t4, t5 WHERE t2.c = t5.a AND t2.b = t5.b);
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	t5	index	a	a	10	NULL	2	Using where; Using index; LooseScan
+1	PRIMARY	t4	ALL	NULL	NULL	NULL	NULL	3	
+1	PRIMARY	t2	ref	b	b	5	test.t5.b	2	Using where; FirstMatch(t5)
+1	PRIMARY	t3	ALL	NULL	NULL	NULL	NULL	15	Using where; Using join buffer (flat, BNL join)
+SELECT * FROM t3 WHERE t3.a IN (SELECT t5.a FROM t2, t4, t5 WHERE t2.c = t5.a AND t2.b = t5.b);
+a
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+DROP TABLE t2, t3, t4, t5;
+set @@optimizer_switch=@tmp834739;
 set optimizer_switch=@subselect_sj_tmp;
 #
 # BUG#49129: Wrong result with IN-subquery with join_cache_level=6 and firstmatch=off

mysql-test/t/subselect_sj.test

@@ -2,7 +2,7 @@
 # Nested Loops semi-join subquery evaluation tests
 #
 --disable_warnings
-drop table if exists t0, t1, t2, t3, t4, t10, t11, t12;
+drop table if exists t0, t1, t2, t3, t4, t5, t10, t11, t12;
 drop view if exists v1, v2, v3, v4;
 drop procedure if exists p1;
 --enable_warnings
@@ -1618,5 +1618,29 @@ SELECT * FROM t1, t2 WHERE (t2.a , t1.b) IN (SELECT a, b FROM t3);
 DROP TABLE t1, t2, t3;
 set @@optimizer_switch= @tmp835758;
 
+--echo #
+--echo # BUG#834739: Wrong result with 3-way inner join, LooseScan,multipart keys  
+--echo #
+set @tmp834739=@@optimizer_switch;
+set optimizer_switch='semijoin=on,loosescan=on,materialization=off,firstmatch=off';
+CREATE TABLE t2 ( b int, c int, KEY (b)) ;
+INSERT INTO t2 VALUES (1,0),(1,0),(9,0),(1,0),(5,0);
+
+CREATE TABLE t3 ( a int);
+INSERT INTO t3 VALUES (0),(0),(0),(0),(0),(0),(0),(0),(0),(0),(0),(0),(0),(0),(0);
+
+CREATE TABLE t4 ( a int);
+INSERT INTO t4 VALUES (0),(0),(0);
+
+CREATE TABLE t5 ( b int, a int , KEY (a,b)) ;
+INSERT INTO t5 VALUES (7,0),(9,0);
+
+explain
+SELECT * FROM t3 WHERE t3.a IN (SELECT t5.a FROM t2, t4, t5 WHERE t2.c = t5.a AND t2.b = t5.b);
+SELECT * FROM t3 WHERE t3.a IN (SELECT t5.a FROM t2, t4, t5 WHERE t2.c = t5.a AND t2.b = t5.b);
+
+DROP TABLE t2, t3, t4, t5;
+set @@optimizer_switch=@tmp834739;
+
 # The following command must be the last one the file 
 set optimizer_switch=@subselect_sj_tmp;

sql/opt_subselect.cc

@@ -3827,6 +3827,8 @@ int setup_semijoin_dups_elimination(JOIN *join, ulonglong options,
       {
         /* We jump from the last table to the first one */
         tab->loosescan_match_tab= tab + pos->n_sj_tables - 1;
+        for (uint j= i; j < pos->n_sj_tables; j++)
+          join->join_tab[j].inside_loosescan_range= TRUE;
 
         /* Calculate key length */
         keylen= 0;

sql/sql_select.cc

@@ -7100,6 +7100,7 @@ get_best_combination(JOIN *join)
       goto loop_end;					// Handled in make_join_stat..
 
     j->loosescan_match_tab= NULL;  //non-nulls will be set later
+    j->inside_loosescan_range= FALSE;
     j->ref.key = -1;
     j->ref.key_parts=0;
 
@@ -8985,6 +8986,15 @@ uint check_join_cache_usage(JOIN_TAB *tab,
   if (tab->use_quick == 2)
     goto no_join_cache;
   
+  /*
+    Don't use join cache if we're inside a join tab range covered by LooseScan
+    strategy (TODO: LooseScan is very similar to FirstMatch so theoretically it 
+    should be possible to use join buffering in the same way we're using it for
+    multi-table firstmatch ranges).
+  */
+  if (tab->inside_loosescan_range)
+    goto no_join_cache;
+
   if (tab->is_inner_table_of_semi_join_with_first_match() &&
       !join->allowed_semijoin_with_cache)
     goto no_join_cache;

sql/sql_select.h

@@ -348,6 +348,9 @@ typedef struct st_join_table {
   */
   struct st_join_table *loosescan_match_tab;
   
+  /* TRUE <=> we are inside LooseScan range */
+  bool inside_loosescan_range;
+
   /* Buffer to save index tuple to be able to skip duplicates */
   uchar *loosescan_buf;