BUG#962667: Assertion `0' failed in QUICK_INDEX_SORT_SELECT::need_sorted_output()

- The problem was that
  = we've picked a LooseScan that used full index scan (tab->type==JT_ALL) on certain index.
  = there was also a quick select (tab->quick!=NULL), that used other indexes.
  = some old code assumes that (tab->type==JT_ALL && tab->quick) -> means that the
    quick select should be used, which is not true.
Fixed by discarding the quick select as soon as we know we're using LooseScan
without using the quick select.

mysql-test/r/subselect_sj2.result

@@ -903,5 +903,42 @@ a	b	c
 3	1	1
 4	1	1
 DROP TABLE t1,t2;
+# 
+# BUG#962667: Assertion `0' failed in QUICK_INDEX_SORT_SELECT::need_sorted_output() 
+#   with index_merge+index_merge_sort_union+loosescan+semijoin
+#
+CREATE TABLE t1 (
+a INT, b VARCHAR(1), c INT,
+KEY(a), KEY(b)
+) ENGINE=InnoDB;
+INSERT INTO t1 VALUES
+(1,'v',9),(2,'v',8),(3,'c',7),(4,'m',6),(5,'x',5),
+(6,'i',4),(7,'e',3),(8,'p',2),(9,'s',1),(10,'j',9),
+(11,'z',8),(12,'c',7),(13,'a',6),(14,'q',5),(15,'y',4),
+(16,'n',3),(17,'r',2),(18,'v',1),(19,'p',0);
+CREATE TABLE t2 (
+pk INT, d VARCHAR(1), e INT,
+PRIMARY KEY(pk), KEY(d,e)
+) ENGINE=InnoDB;
+INSERT INTO t2 VALUES
+(1,'x',1),(2,'d',2),(3,'r',3),(4,'f',4),(5,'y',5),
+(6,'u',6),(7,'m',7),(8,'k',8),(9,'o',9),(10,'w',1),
+(11,'m',2),(12,'q',3),(13,'m',4),(14,'d',5),
+(15,'g',6),(16,'x',7),(17,'f',8);
+explain
+SELECT * FROM t1 WHERE b IN (
+SELECT d FROM t2, t1
+WHERE a = d AND ( pk < 2 OR d = 'z' )
+);
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	t2	index	PRIMARY,d	d	9	NULL	17	Using where; Using index; LooseScan
+1	PRIMARY	t1	ref	a	a	5	test.t2.d	1	Using where; Using index; FirstMatch(t2)
+1	PRIMARY	t1	ref	b	b	4	test.t2.d	1	
+SELECT * FROM t1 WHERE b IN (
+SELECT d FROM t2, t1
+WHERE a = d AND ( pk < 2 OR d = 'z' )
+);
+a	b	c
+DROP TABLE t1, t2;
 # This must be the last in the file:
 set optimizer_switch=@subselect_sj2_tmp;

mysql-test/r/subselect_sj2_jcl6.result

@@ -917,6 +917,43 @@ a	b	c
 3	1	1
 4	1	1
 DROP TABLE t1,t2;
+# 
+# BUG#962667: Assertion `0' failed in QUICK_INDEX_SORT_SELECT::need_sorted_output() 
+#   with index_merge+index_merge_sort_union+loosescan+semijoin
+#
+CREATE TABLE t1 (
+a INT, b VARCHAR(1), c INT,
+KEY(a), KEY(b)
+) ENGINE=InnoDB;
+INSERT INTO t1 VALUES
+(1,'v',9),(2,'v',8),(3,'c',7),(4,'m',6),(5,'x',5),
+(6,'i',4),(7,'e',3),(8,'p',2),(9,'s',1),(10,'j',9),
+(11,'z',8),(12,'c',7),(13,'a',6),(14,'q',5),(15,'y',4),
+(16,'n',3),(17,'r',2),(18,'v',1),(19,'p',0);
+CREATE TABLE t2 (
+pk INT, d VARCHAR(1), e INT,
+PRIMARY KEY(pk), KEY(d,e)
+) ENGINE=InnoDB;
+INSERT INTO t2 VALUES
+(1,'x',1),(2,'d',2),(3,'r',3),(4,'f',4),(5,'y',5),
+(6,'u',6),(7,'m',7),(8,'k',8),(9,'o',9),(10,'w',1),
+(11,'m',2),(12,'q',3),(13,'m',4),(14,'d',5),
+(15,'g',6),(16,'x',7),(17,'f',8);
+explain
+SELECT * FROM t1 WHERE b IN (
+SELECT d FROM t2, t1
+WHERE a = d AND ( pk < 2 OR d = 'z' )
+);
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	t2	index	PRIMARY,d	d	9	NULL	17	Using where; Using index; LooseScan
+1	PRIMARY	t1	ref	a	a	5	test.t2.d	1	Using where; Using index; FirstMatch(t2)
+1	PRIMARY	t1	ref	b	b	4	test.t2.d	1	Using join buffer (flat, BKA join); Key-ordered Rowid-ordered scan
+SELECT * FROM t1 WHERE b IN (
+SELECT d FROM t2, t1
+WHERE a = d AND ( pk < 2 OR d = 'z' )
+);
+a	b	c
+DROP TABLE t1, t2;
 # This must be the last in the file:
 set optimizer_switch=@subselect_sj2_tmp;
 #

mysql-test/r/subselect_sj2_mat.result

@@ -905,6 +905,43 @@ a	b	c
 3	1	1
 4	1	1
 DROP TABLE t1,t2;
+# 
+# BUG#962667: Assertion `0' failed in QUICK_INDEX_SORT_SELECT::need_sorted_output() 
+#   with index_merge+index_merge_sort_union+loosescan+semijoin
+#
+CREATE TABLE t1 (
+a INT, b VARCHAR(1), c INT,
+KEY(a), KEY(b)
+) ENGINE=InnoDB;
+INSERT INTO t1 VALUES
+(1,'v',9),(2,'v',8),(3,'c',7),(4,'m',6),(5,'x',5),
+(6,'i',4),(7,'e',3),(8,'p',2),(9,'s',1),(10,'j',9),
+(11,'z',8),(12,'c',7),(13,'a',6),(14,'q',5),(15,'y',4),
+(16,'n',3),(17,'r',2),(18,'v',1),(19,'p',0);
+CREATE TABLE t2 (
+pk INT, d VARCHAR(1), e INT,
+PRIMARY KEY(pk), KEY(d,e)
+) ENGINE=InnoDB;
+INSERT INTO t2 VALUES
+(1,'x',1),(2,'d',2),(3,'r',3),(4,'f',4),(5,'y',5),
+(6,'u',6),(7,'m',7),(8,'k',8),(9,'o',9),(10,'w',1),
+(11,'m',2),(12,'q',3),(13,'m',4),(14,'d',5),
+(15,'g',6),(16,'x',7),(17,'f',8);
+explain
+SELECT * FROM t1 WHERE b IN (
+SELECT d FROM t2, t1
+WHERE a = d AND ( pk < 2 OR d = 'z' )
+);
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	t2	index	PRIMARY,d	d	9	NULL	17	Using where; Using index; LooseScan
+1	PRIMARY	t1	ref	a	a	5	test.t2.d	1	Using where; Using index; FirstMatch(t2)
+1	PRIMARY	t1	ref	b	b	4	test.t2.d	1	
+SELECT * FROM t1 WHERE b IN (
+SELECT d FROM t2, t1
+WHERE a = d AND ( pk < 2 OR d = 'z' )
+);
+a	b	c
+DROP TABLE t1, t2;
 # This must be the last in the file:
 set optimizer_switch=@subselect_sj2_tmp;
 set optimizer_switch=default;

mysql-test/t/subselect_sj2.test

@@ -1090,5 +1090,43 @@ SELECT * FROM t1, t2 WHERE c IN (SELECT c FROM t1, t2 WHERE a = b);
 
 DROP TABLE t1,t2;
 
+--echo # 
+--echo # BUG#962667: Assertion `0' failed in QUICK_INDEX_SORT_SELECT::need_sorted_output() 
+--echo #   with index_merge+index_merge_sort_union+loosescan+semijoin
+--echo #
+CREATE TABLE t1 (
+  a INT, b VARCHAR(1), c INT,
+  KEY(a), KEY(b)
+) ENGINE=InnoDB;
+
+INSERT INTO t1 VALUES
+(1,'v',9),(2,'v',8),(3,'c',7),(4,'m',6),(5,'x',5),
+(6,'i',4),(7,'e',3),(8,'p',2),(9,'s',1),(10,'j',9),
+(11,'z',8),(12,'c',7),(13,'a',6),(14,'q',5),(15,'y',4),
+(16,'n',3),(17,'r',2),(18,'v',1),(19,'p',0);
+
+CREATE TABLE t2 (
+  pk INT, d VARCHAR(1), e INT,
+  PRIMARY KEY(pk), KEY(d,e)
+) ENGINE=InnoDB;
+
+INSERT INTO t2 VALUES
+(1,'x',1),(2,'d',2),(3,'r',3),(4,'f',4),(5,'y',5),
+(6,'u',6),(7,'m',7),(8,'k',8),(9,'o',9),(10,'w',1),
+(11,'m',2),(12,'q',3),(13,'m',4),(14,'d',5),
+(15,'g',6),(16,'x',7),(17,'f',8);
+
+explain
+SELECT * FROM t1 WHERE b IN (
+  SELECT d FROM t2, t1
+  WHERE a = d AND ( pk < 2 OR d = 'z' )
+);
+SELECT * FROM t1 WHERE b IN (
+  SELECT d FROM t2, t1
+  WHERE a = d AND ( pk < 2 OR d = 'z' )
+);
+
+DROP TABLE t1, t2;
+
 --echo # This must be the last in the file:
 set optimizer_switch=@subselect_sj2_tmp;

sql/opt_subselect.cc

@@ -3083,7 +3083,22 @@ void fix_semijoin_strategies_for_picked_join_order(JOIN *join)
                             record_count, join->best_positions + idx,
                             &loose_scan_pos);
            if (idx==first)
+           {
              join->best_positions[idx]= loose_scan_pos;
+             /*
+               If LooseScan is based on ref access (including the "degenerate"
+               one with 0 key parts), we should use full index scan.
+
+               Unfortunately, lots of code assumes that if tab->type==JT_ALL && 
+               tab->quick!=NULL, then quick select should be used. The only
+               simple way to fix this is to remove the quick select:
+             */
+             if (join->best_positions[idx].key)
+             {
+               delete join->best_positions[idx].table->quick;
+               join->best_positions[idx].table->quick= NULL;
+             }
+           }
         }
         rem_tables &= ~join->best_positions[idx].table->table->map;
         record_count *= join->best_positions[idx].records_read;