Introduced an optimizer_switch for ORDER BY LIMIT

mysql-test/main/sort_nest.test

 --source include/have_sequence.inc
 
-set use_sort_nest=1;
+set optimizer_switch='cost_based_order_by_limit=on';
 
 CREATE TABLE t0 (a int);
 INSERT INTO t0 SELECT seq-1 from seq_1_to_10;
@@ -594,7 +594,6 @@ WHERE t1.a=t2.a and t1.b= 4
 ORDER BY t1.b DESC
 LIMIT 5;
 
-set use_sort_nest=1;
 eval EXPLAIN $query;
 eval $query;
 
@@ -627,13 +626,9 @@ WHERE t1.a=t2.b and t2.a=t3.a
 ORDER BY abs(t3.a+t1.b) DESC
 LIMIT 5;
 
-set use_sort_nest=1;
 eval EXPLAIN $query;
 eval $query;
 
-set use_sort_nest=0;
-eval $query;
-
 --echo #
 --echo # No sort nest where ORDER BY item are expensive to compute like 
 --echo # stored functions, subqueries etc
@@ -654,12 +649,6 @@ WHERE t1.a=t2.b and t2.a=t3.a
 ORDER BY f1(t3.a+t1.b) DESC
 LIMIT 5;
 
-set use_sort_nest=1;
-eval EXPLAIN $query;
-eval EXPLAIN FORMAT=JSON $query;
-eval $query;
-
-set use_sort_nest=0;
 eval EXPLAIN $query;
 eval $query;
 
@@ -676,12 +665,7 @@ WHERE t1.a=t2.b AND t2.a=t3.a
 ORDER BY row_number() OVER (ORDER BY t1.a) DESC
 LIMIT 5;
 
-set use_sort_nest=1;
 eval EXPLAIN $query;
-eval EXPLAIN FORMAT=JSON $query;
-eval $query;
-
-set use_sort_nest=0;
 eval $query;
 
 --echo #
@@ -700,13 +684,9 @@ t1.b = (select t2.b from t2,t3,t4
         where t2.a=t1.a and t3.b=t4.b
         ORDER BY t2.c,t3.c limit 1);
 
-set use_sort_nest=1;
 eval EXPLAIN $query;
 eval $query;
 
-set use_sort_nest= 0;
-eval $query;
-
 --echo #
 --echo # Independent subqueries can have a sort-nest
 --echo #
@@ -719,14 +699,9 @@ t1.b = (select t2.b from t2,t3,t4
         where t2.a=t3.a and t3.b=t4.b
         ORDER BY t2.c,t3.c limit 1);
 
-set use_sort_nest=1;
-
 eval EXPLAIN $query;
 eval $query;
 
-set use_sort_nest= 0;
-eval $query;
-
 --echo #
 --echo # Sort nest inside a derived table
 --echo #
@@ -739,13 +714,9 @@ FROM (SELECT t1.a as t, t2.b as u, t1.b as v, t3.a as x
       ORDER BY t2.b DESC, t1.b DESC
       LIMIT 5)q;
 
-set use_sort_nest=1;
 eval EXPLAIN $query;
 eval $query;
 
-set use_sort_nest=0;
-eval $query;
-
 --echo #
 --echo # sort-nest(t2,t1) and independent subquery in the SELECT list
 --echo #
@@ -758,7 +729,6 @@ WHERE t1.a = t2.a
 ORDER BY t2.b DESC, t1.b DESC
 LIMIT 5;
 
-set use_sort_nest=1;
 eval EXPLAIN $query;
 eval $query;
 
@@ -774,13 +744,9 @@ WHERE t1.a = t2.a and t2.c=t3.c
 ORDER BY t2.b DESC, t1.b DESC
 LIMIT 5;
 
-set use_sort_nest=1;
 eval EXPLAIN $query;
 eval $query;
 
-set use_sort_nest= 0;
-eval $query;
-
 --echo #
 --echo # Dependent Subquery attached outside the sort-nest, references inside
 --echo # subquery changed to sort-nest field items
@@ -794,14 +760,10 @@ t1.b=(SELECT A.a FROM t3 A WHERE A.b > 1 and t3.b=t1.b limit 1)
 ORDER BY t2.b DESC, t1.b DESC
 LIMIT 5;
 
-set use_sort_nest=1;
 eval EXPLAIN $query;
 eval EXPLAIN FORMAT=JSON $query;
 eval $query;
 
-set use_sort_nest= 0;
-eval $query;
-
 --echo #
 --echo # sort-nest(t2,t1) and independent subquery in the ORDER BY clause
 --echo #
@@ -816,13 +778,9 @@ ORDER BY t2.b, t1.b,
          (SELECT A.a FROM t3 A WHERE A.a < 5 limit 1)
 LIMIT 5;
 
-set use_sort_nest=1;
 eval EXPLAIN $query;
 eval $query;
 
-set use_sort_nest= 0;
-eval $query;
-
 --echo #
 --echo # sort-nest(t2,t1) and dependent subquery in the ORDER BY clause
 --echo #
@@ -837,13 +795,9 @@ ORDER BY t2.b, t1.b,
          (SELECT A.a FROM t3 A WHERE A.a = t1.c limit 1)
 LIMIT 5;
 
-set use_sort_nest=1;
 eval EXPLAIN $query;
 eval $query;
 
-set use_sort_nest= 0;
-eval $query;
-
 drop table t0,t1,t2,t3,t4;
 
 CREATE TABLE t1 (a int, b int, c int, KEY a_b (a,b), KEY a_c (a,c));
@@ -858,12 +812,9 @@ insert into t1 values (1,7,2), (1,8,2), (1,9,2), (1,10,2), (1,11,2), (1,12,2);
 --echo #
 
 let $query= select a,b,c from t1 where a=1 and c=2 order by b limit 5;
-set use_sort_nest= 1;
 eval $query;
 eval explain $query;
 
-set use_sort_nest= 0;
-eval explain $query;
 drop table t1;
 
 --echo #
@@ -880,18 +831,14 @@ insert into t1 values (1,1,2);
 
 --echo #
 --echo # index key a_b, no need for filesort
---echo # Also index condition pushdown is used with use_sort_nest=1
+--echo # Also index condition pushdown is used here
 --echo #
 
 let $query= select a,b from t1 where a=1 and c=2 order by b limit 10;
 
-set use_sort_nest=1;
 eval $query;
 eval explain $query;
 
-set use_sort_nest=0;
-eval explain $query;
-
 drop table t1;
 
 CREATE TABLE t1(
@@ -908,26 +855,18 @@ INSERT INTO t1 VALUES (1,'a'), (2,'b'), (3,'c'), (4,'d');
 
 let $query= SELECT * FROM t1 WHERE a BETWEEN 1 and 2 ORDER BY a LIMIT 2;
 
-set use_sort_nest= 1;
 eval $query;
 eval EXPLAIN $query;
 
-set use_sort_nest= 0;
-eval EXPLAIN $query;
-
 --echo #
 --echo # Should not use index condition as ORDER by DESC is used
 --echo #
 
 let $query= SELECT * FROM t1 WHERE a BETWEEN 1 and 2 ORDER BY a DESC LIMIT 2;
 
-set use_sort_nest= 1;
 eval EXPLAIN $query;
 eval $query;
 
-set use_sort_nest= 0;
-eval EXPLAIN $query;
-
 drop table t1;
 
 create table t1(a int, b int, c int, key(a), key a_b(a,b)); # 10 rows
@@ -954,13 +893,9 @@ WHERE
   t1.b=t2.b
 ORDER BY t1.b LIMIT 10;
 
-set use_sort_nest= 1;
 eval $query;
 eval EXPLAIN $query;
 
-set use_sort_nest= 0;
-eval EXPLAIN $query;
-
 --echo #
 --echo #  Using ref access
 --echo #
@@ -971,19 +906,14 @@ WHERE
   t1.b=t2.b
 ORDER BY t1.b LIMIT 10;
 
-set use_sort_nest= 1;
 eval $query;
 eval EXPLAIN $query;
 
-set use_sort_nest= 0;
-eval EXPLAIN $query;
-
 drop table t1,t2;
 
 
 --echo # TESTS with INDEX HINTS
 
-set use_sort_nest=1;
 create table t0 (a int);
 insert into t0 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9);
 create table t1 (a int, b int,c int, key idx1(a), key idx2(a,b), key idx3(c));
@@ -998,25 +928,17 @@ analyze table t1 persistent for all;
 --echo #
 
 let $query= SELECT * from t1 where b > 0 order by t1.a limit 2;
-set use_sort_nest=1;
 eval $query;
 eval EXPLAIN $query;
 
-set use_sort_nest=0;
-eval EXPLAIN $query;
-
 --echo #
 --echo # Index idx2 to be used for index scan(USE INDEX is used)
 --echo #
 
 let $query= SELECT * from t1 USE INDEX(idx2) WHERE b > 0 ORDER BY t1.a LIMIT 2;
-set use_sort_nest=1;
 eval $query;
 eval EXPLAIN $query;
 
-set use_sort_nest=0;
-eval EXPLAIN $query;
-
 --echo #
 --echo # Index idx2 to be used for index scan(USE INDEX for ORDER BY  is used)
 --echo #
@@ -1026,13 +948,9 @@ SELECT * from t1 USE INDEX FOR ORDER BY(idx2)
 WHERE b > 0
 ORDER BY t1.a LIMIT 2;
 
-set use_sort_nest=1;
 eval $query;
 eval EXPLAIN $query;
 
-set use_sort_nest=0;
-eval EXPLAIN $query;
-
 --echo #
 --echo # Use Filesort as idx3 does not resolve ORDER BY clause
 --echo #
@@ -1042,13 +960,9 @@ SELECT * from t1 USE INDEX FOR ORDER BY(idx3)
 WHERE b > 0
 ORDER BY t1.a LIMIT 2;
 
-set use_sort_nest=1;
 eval $query;
 eval EXPLAIN $query;
 
-set use_sort_nest=0;
-eval EXPLAIN $query;
-
 --echo #
 --echo # Using index idx2 as idx1 is ignored
 --echo #
@@ -1058,13 +972,9 @@ SELECT * from t1 IGNORE INDEX(idx1)
 WHERE b > 0
 ORDER BY t1.a LIMIT 2;
 
-set use_sort_nest=1;
 eval $query;
 eval EXPLAIN $query;
 
-set use_sort_nest=0;
-eval EXPLAIN $query;
-
 --echo #
 --echo # Use index idx2 for sorting, it is forced here
 --echo #
@@ -1074,13 +984,9 @@ SELECT * from t1 FORCE INDEX(idx2)
 WHERE b > 0
 ORDER BY t1.a LIMIT 2;
 
-set use_sort_nest=1;
 eval $query;
 eval EXPLAIN $query;
 
-set use_sort_nest=0;
-eval EXPLAIN $query;
-
 --echo #
 --echo # Use FILESORT as idx3 cannot resolve ORDER BY clause
 --echo #
@@ -1090,13 +996,9 @@ SELECT * from t1 FORCE INDEX FOR ORDER BY(idx3)
 WHERE b > 0
 ORDER BY t1.a LIMIT 2;
 
-set use_sort_nest=1;
 eval $query;
 eval EXPLAIN $query;
 
-set use_sort_nest=0;
-eval EXPLAIN $query;
-
 drop table t0,t1;
 
 --echo #
@@ -1138,14 +1040,10 @@ FROM t1, t2, t2 ot
 ORDER BY t2.b DESC ,t1.b DESC
 LIMIT 5;
 
-set use_sort_nest=1;
 eval EXPLAIN $query;
 eval EXPLAIN FORMAT=JSON $query;
 eval $query;
 
-set use_sort_nest=0;
-eval $query;
-
 --echo #
 --echo # SJM Lookup with sort-nest, where SJM lookup table is outside the
 --echo # sort-nest
@@ -1160,14 +1058,10 @@ FROM t1, t2, t2 ot
 ORDER BY t2.b DESC ,t1.b DESC
 LIMIT 5;
 
-set use_sort_nest=1;
 eval EXPLAIN $query;
 eval EXPLAIN FORMAT=JSON $query;
 eval $query;
 
-set use_sort_nest=0;
-eval $query;
-
 drop table t0,t1,t2,t3,t4;
 
 
@@ -1205,15 +1099,9 @@ FROM t1, t2
 ORDER BY t2.b DESC, t1.b DESC
 LIMIT 5;
 
-set use_sort_nest=1;
-eval EXPLAIN $query;
-eval $query;
-
-set use_sort_nest=0;
 eval EXPLAIN $query;
 eval $query;
 
-
 --echo #
 --echo # Duplicate weedout
 --echo #
@@ -1229,11 +1117,6 @@ FROM t1, t2
 ORDER BY t2.b DESC, t1.b DESC
 LIMIT 5;
 
-set use_sort_nest=1;
-eval EXPLAIN $query;
-eval $query;
-
-set use_sort_nest=0;
 eval EXPLAIN $query;
 eval $query;
 
@@ -1262,14 +1145,10 @@ t1.a IN (SELECT max(t3.a) FROM t3 GROUP BY t3.b)
 ORDER BY t2.a DESC,t1.a DESC
 LIMIT 5;
 
-set use_sort_nest=1;
 eval EXPLAIN $query;
 eval EXPLAIN FORMAT=JSON $query;
 eval $query;
 
-set use_sort_nest=0;
-eval $query;
-
 --echo <subquery2> inside the sort-nest
 
 let $query=
@@ -1279,12 +1158,8 @@ t3.a IN (SELECT max(t1.a) FROM t1 GROUP BY t1.b)
 ORDER BY t3.a DESC,t2.a DESC
 LIMIT 5;
 
-set use_sort_nest=1;
 eval EXPLAIN $query;
 eval EXPLAIN FORMAT=JSON $query;
 eval $query;
 
-set use_sort_nest=0;
-eval $query;
-
 DROP TABLE t0,t1,t2,t3;

mysql-test/main/sort_nest_dbt3.test

@@ -10,9 +10,9 @@ analyze table lineitem persistent for all;
 analyze table nation   persistent for all;
 --enable_query_log
 
---echo # done to avoid filter for now
+--echo # done use filter for now
 set optimizer_switch='rowid_filter=off';
-set use_sort_nest=1;
+set optimizer_switch='cost_based_order_by_limit=on';
 
 --echo #
 --echo # USING INDEXES FOR ORDERING
@@ -32,11 +32,6 @@ let $query= SELECT
             ORDER BY o_orderDATE DESC 
             LIMIT 5;
 
-set use_sort_nest=1;
-eval explain $query;
-eval $query;
-
-set use_sort_nest=0;
 eval explain $query;
 eval $query;
 
@@ -57,18 +52,11 @@ let $query= SELECT
             ORDER BY o_orderDATE, o_custkey DESC 
             LIMIT 5;
 
-set use_sort_nest=1;
-eval explain $query;
-eval $query;
-
-set use_sort_nest=0;
 eval explain $query;
 eval $query;
 
 alter table orders drop key o_orderDate;
 
-###############################################################################
-
 --echo #
 --echo # USING FILESORT
 --echo #
@@ -87,11 +75,6 @@ let $query= SELECT
             ORDER BY o_totalprice DESC 
             LIMIT 10;
 
-set use_sort_nest=1;
-eval explain $query;
-eval $query;
-
-set use_sort_nest=0;
 eval explain $query;
 eval $query;
 
@@ -100,8 +83,6 @@ eval $query;
 --echo # Filesort on first table (lineitem)
 --echo #
 
-show create table orders;
-
 let $query= SELECT
               l_extendedprice, o_orderkey, o_totalprice, l_shipDATE
             FROM
@@ -112,17 +93,9 @@ let $query= SELECT
             ORDER BY l_extendedprice DESC
             LIMIT 10;
 
-set use_sort_nest=1;
 eval explain $query;
 eval $query;
 
-set use_sort_nest=0;
-eval explain $query;
-eval $query;
-
-###############################################################################
-
-
 --echo #
 --echo # Using Filesort with Sort Nest
 --echo #
@@ -141,16 +114,9 @@ let $query= SELECT
             ORDER BY o_orderDATE DESC, c_acctbal DESC
             LIMIT 10;
 
-set use_sort_nest=1;
-eval explain $query;
-eval $query;
-
-set use_sort_nest=0;
 eval explain $query;
 eval $query;
 
-
-
 --echo #
 --echo # Sort-nest on table (lineitem, orders)
 --echo #
@@ -171,16 +137,9 @@ let $query= SELECT
             LIMIT 20;
 
 
-set use_sort_nest=1;
-eval explain $query;
-eval $query;
-
-set use_sort_nest=0;
 eval explain $query;
 eval $query;
 
---echo ########################################################################
-
 --echo #
 --echo # Sort-nest on table (customer, orders, lineitem)
 --echo #
@@ -204,13 +163,7 @@ let $query= SELECT
             LIMIT 20;
 
 
-set use_sort_nest=1;
 eval explain $query;
 eval $query;
 
-set use_sort_nest=0;
-eval explain $query;
-eval $query;
-
-
 drop database dbt3;

sql/sql_priv.h

@@ -230,6 +230,7 @@
 #define OPTIMIZER_SWITCH_USE_ROWID_FILTER          (1ULL << 33)
 #define OPTIMIZER_SWITCH_COND_PUSHDOWN_FROM_HAVING (1ULL << 34)
 #define OPTIMIZER_SWITCH_NOT_NULL_RANGE_SCAN       (1ULL << 35)
+#define OPTIMIZER_SWITCH_COST_BASED_ORDER_BY_LIMIT (1ULL << 36)
 
 #define OPTIMIZER_SWITCH_DEFAULT   (OPTIMIZER_SWITCH_INDEX_MERGE | \
                                     OPTIMIZER_SWITCH_INDEX_MERGE_UNION | \

sql/sql_sort_nest.cc

@@ -1407,7 +1407,8 @@ void JOIN::set_fraction_output_for_nest()
 
 bool JOIN::sort_nest_allowed()
 {
-  return thd->variables.use_sort_nest && order &&
+  return optimizer_flag(thd, OPTIMIZER_SWITCH_COST_BASED_ORDER_BY_LIMIT) &&
+         order &&
          !(const_tables == table_count ||
            (select_distinct || group_list) ||
            having  ||

sql/sys_vars.cc

@@ -2590,6 +2590,7 @@ export const char *optimizer_switch_names[]=
   "rowid_filter",
   "condition_pushdown_from_having",
   "not_null_range_scan",
+  "cost_based_order_by_limit",
   "default", 
   NullS
 };
@@ -6438,9 +6439,3 @@ static Sys_var_ulonglong Sys_max_rowid_filter_size(
        SESSION_VAR(max_rowid_filter_size), CMD_LINE(REQUIRED_ARG),
        VALID_RANGE(1024, (ulonglong)~(intptr)0), DEFAULT(128*1024),
        BLOCK_SIZE(1));
-
-static Sys_var_mybool Sys_use_sort_nest(
-       "use_sort_nest",
-       "Enable the sort nest",
-       SESSION_VAR(use_sort_nest), CMD_LINE(OPT_ARG),
-       DEFAULT(FALSE));