Post-merge fixes for rocksdb.group_min_max test

- Fix the LooseScan code to support storage engines that return
  HA_ERR_END_OF_FILE if the index scan goes out of provided range
  bounds
- Add a DBUG_EXECUTE_IF("force_group_by",...) to allow a test to
  force a LooseScan
- Adjust rocksdb.group_min_max test not to use features not present
  in MariaDB 10.2 (e.g. optimizer_trace.  In MariaDB 10.4 it's present
  but it doesn't meet the  assumptions that the test makes about it
- Adjust the test result file:
  = MariaDB doesn't support "Enhanced Loose Scan" that FB/MySQL has
  = MariaDB has different cost calculations.

sql/opt_range.cc

@@ -2428,6 +2428,7 @@ int SQL_SELECT::test_quick_select(THD *thd, key_map keys_to_use,
     KEY_PART *key_parts;
     KEY *key_info;
     PARAM param;
+    bool force_group_by = false;
 
     if (check_stack_overrun(thd, 2*STACK_MIN_SIZE + sizeof(PARAM), buff))
       DBUG_RETURN(0);                           // Fatal error flag is set
@@ -2555,15 +2556,20 @@ int SQL_SELECT::test_quick_select(THD *thd, key_map keys_to_use,
       Try to construct a QUICK_GROUP_MIN_MAX_SELECT.
       Notice that it can be constructed no matter if there is a range tree.
     */
+    DBUG_EXECUTE_IF("force_group_by", force_group_by = true; );
     group_trp= get_best_group_min_max(&param, tree, best_read_time);
     if (group_trp)
     {
       param.table->quick_condition_rows= MY_MIN(group_trp->records,
                                              head->stat_records());
-      if (group_trp->read_cost < best_read_time)
+      if (group_trp->read_cost < best_read_time || force_group_by)
       {
         best_trp= group_trp;
         best_read_time= best_trp->read_cost;
+        if (force_group_by)
+        {
+          goto force_plan;
+        }
       }
     }
 
@@ -2663,6 +2669,7 @@ int SQL_SELECT::test_quick_select(THD *thd, key_map keys_to_use,
       }
     }
 
+force_plan:
     thd->mem_root= param.old_root;
 
     /* If we got a read plan, create a quick select from it. */
@@ -11509,13 +11516,28 @@ int QUICK_RANGE_SELECT::get_next_prefix(uint prefix_length,
       DBUG_ASSERT(cur_prefix != NULL);
       result= file->ha_index_read_map(record, cur_prefix, keypart_map,
                                       HA_READ_AFTER_KEY);
-      if (result || last_range->max_keypart_map == 0)
-        DBUG_RETURN(result);
-
-      key_range previous_endpoint;
-      last_range->make_max_endpoint(&previous_endpoint, prefix_length, keypart_map);
-      if (file->compare_key(&previous_endpoint) <= 0)
-        DBUG_RETURN(0);
+      if (result || last_range->max_keypart_map == 0) {
+        /*
+          Only return if actual failure occurred. For HA_ERR_KEY_NOT_FOUND
+          or HA_ERR_END_OF_FILE, we just want to continue to reach the next
+          set of ranges. It is possible for the storage engine to return
+          HA_ERR_KEY_NOT_FOUND/HA_ERR_END_OF_FILE even when there are more
+          keys if it respects the end range set by the read_range_first call
+          below.
+        */
+        if (result != HA_ERR_KEY_NOT_FOUND && result != HA_ERR_END_OF_FILE)
+          DBUG_RETURN(result);
+      } else {
+        /*
+          For storage engines that don't respect end range, check if we've
+          moved past the current range.
+        */
+        key_range previous_endpoint;
+        last_range->make_max_endpoint(&previous_endpoint, prefix_length,
+                                      keypart_map);
+        if (file->compare_key(&previous_endpoint) <= 0)
+          DBUG_RETURN(0);
+      }
     }
 
     uint count= ranges.elements - (uint)(cur_range - (QUICK_RANGE**) ranges.buffer);

storage/rocksdb/mysql-test/rocksdb/include/group_min_max.inc

@@ -266,6 +266,17 @@ select a1,  max(c)           from t2 where a1 in ('a','b','d') group by a1,a2,b;
 
 # B) Equalities only over the non-group 'B' attributes
 # plans
+--echo #
+--echo # MariaDB: we dont have the following patch:
+--echo #
+--echo #  commit 60a92a79a3b7fde3c6efe91799e344b977c8e5c3
+--echo #  Author: Manuel Ung <mung@fb.com>
+--echo #  Date:   Thu Apr 19 23:06:27 2018 -0700
+--echo #
+--echo #   Enhance group-by loose index scan
+--echo #
+--echo #  So the following results are not very meaningful, but are still kept here
+
 explain select a1,a2,b,max(c),min(c) from t1 where (a2 = 'a') and (b = 'b') group by a1;
 explain select a1,a2,b,max(c),min(c) from t1 where (a2 = 'a' or a2 = 'b') and (b = 'b') group by a1;
 explain select a1,a2,b,max(c),min(c) from t1 where (a2 = 'a') and (b = 'b' or b = 'a') group by a1;
@@ -730,9 +741,9 @@ explain extended select a1,a2,count(a2) from t1 where (a1 > 'a') group by a1,a2,
 explain extended select sum(ord(a1)) from t1 where (a1 > 'a') group by a1,a2,b;
 
 # test multi_range_groupby flag
-set optimizer_switch = 'multi_range_groupby=off';
+#MariaDB: no support: set optimizer_switch = 'multi_range_groupby=off';
 explain select a1,a2,b,max(c),min(c) from t2 where (a2 = 'a') and (b = 'a' or b = 'b') group by a1;
-set optimizer_switch = 'default';
+#set optimizer_switch = 'default';
 explain select a1,a2,b,max(c),min(c) from t2 where (a2 = 'a') and (b = 'a' or b = 'b') group by a1;
 
 
@@ -1361,41 +1372,41 @@ drop table t1;
 eval CREATE TABLE t (a INT, b INT, KEY(a,b)) engine=$engine;
 INSERT INTO t VALUES (1,1), (2,2), (3,3), (4,4), (1,0), (3,2), (4,5);
 ANALYZE TABLE t;
-let $DEFAULT_TRACE_MEM_SIZE=1048576; # 1MB
-eval set optimizer_trace_max_mem_size=$DEFAULT_TRACE_MEM_SIZE;
-set @@session.optimizer_trace='enabled=on';
-set end_markers_in_json=on;
+# MariaDB: 10.2 doesn't have trace, yet: let $DEFAULT_TRACE_MEM_SIZE=1048576; # 1MB
+# eval set optimizer_trace_max_mem_size=$DEFAULT_TRACE_MEM_SIZE;
+# set @@session.optimizer_trace='enabled=on';
+# set end_markers_in_json=on;
 
 ANALYZE TABLE t;
 
 SELECT a, SUM(DISTINCT a), MIN(b) FROM t GROUP BY a;
 EXPLAIN SELECT a, SUM(DISTINCT a), MIN(b) FROM t GROUP BY a;
-SELECT TRACE RLIKE 'have_both_agg_distinct_and_min_max' AS OK
-  FROM INFORMATION_SCHEMA.OPTIMIZER_TRACE;
+#SELECT TRACE RLIKE 'have_both_agg_distinct_and_min_max' AS OK
+#  FROM INFORMATION_SCHEMA.OPTIMIZER_TRACE;
 
 SELECT a, SUM(DISTINCT a), MAX(b) FROM t GROUP BY a;
 EXPLAIN SELECT a, SUM(DISTINCT a), MAX(b) FROM t GROUP BY a;
-SELECT TRACE RLIKE 'have_both_agg_distinct_and_min_max' AS OK
-  FROM INFORMATION_SCHEMA.OPTIMIZER_TRACE;
+#SELECT TRACE RLIKE 'have_both_agg_distinct_and_min_max' AS OK
+#  FROM INFORMATION_SCHEMA.OPTIMIZER_TRACE;
 
 SELECT a, MAX(b) FROM t GROUP BY a HAVING SUM(DISTINCT a);
 EXPLAIN SELECT a, MAX(b) FROM t GROUP BY a HAVING SUM(DISTINCT a);
-SELECT TRACE RLIKE 'have_both_agg_distinct_and_min_max' AS OK
-  FROM INFORMATION_SCHEMA.OPTIMIZER_TRACE;
+#SELECT TRACE RLIKE 'have_both_agg_distinct_and_min_max' AS OK
+#  FROM INFORMATION_SCHEMA.OPTIMIZER_TRACE;
 
 SELECT SUM(DISTINCT a), MIN(b), MAX(b) FROM t;
 EXPLAIN SELECT SUM(DISTINCT a), MIN(b), MAX(b) FROM t;
-SELECT TRACE RLIKE 'have_both_agg_distinct_and_min_max' AS OK
-  FROM INFORMATION_SCHEMA.OPTIMIZER_TRACE;
+#SELECT TRACE RLIKE 'have_both_agg_distinct_and_min_max' AS OK
+#  FROM INFORMATION_SCHEMA.OPTIMIZER_TRACE;
 
 SELECT a, SUM(DISTINCT a), MIN(b), MAX(b) FROM t GROUP BY a;
 EXPLAIN SELECT a, SUM(DISTINCT a), MIN(b), MAX(b) FROM t GROUP BY a;
-SELECT TRACE RLIKE 'have_both_agg_distinct_and_min_max' AS OK
-  FROM INFORMATION_SCHEMA.OPTIMIZER_TRACE;
+#SELECT TRACE RLIKE 'have_both_agg_distinct_and_min_max' AS OK
+#  FROM INFORMATION_SCHEMA.OPTIMIZER_TRACE;
 
-SET optimizer_trace_max_mem_size=DEFAULT;
-SET optimizer_trace=DEFAULT;
-SET end_markers_in_json=DEFAULT;
+#SET optimizer_trace_max_mem_size=DEFAULT;
+#SET optimizer_trace=DEFAULT;
+#SET end_markers_in_json=DEFAULT;
 
 DROP TABLE t;
 

storage/rocksdb/mysql-test/rocksdb/t/group_min_max.test

 --source include/have_debug.inc
-set global debug="+d,force_group_by";
+set @debug_tmp= @@debug_dbug;
+set global debug_dbug="+d,force_group_by";
 
 let $engine=RocksDB;
 --source include/group_min_max.inc
 
-set global debug="-d,force_group_by";
+set global debug_dbug=@debug_tmp;