MDEV-9750: Quick memory exhaustion with 'extended_keys=on' ...

(Variant #2)
Do not produce SEL_ARG graphs that would yield huge numbers of ranges.
Introduce a concept of SEL_ARG graph's "weight". If we are about to
produce a graph whose "weight" exceeds the limit, remove the parts
of SEL_ARG graph that represent the biggest key parts. Do so until
the graph's is within the limit.

Variant #2: Don't call enforce_sel_arg_weight_limit() for sub-graphs,
as this has complicated semantics if the subgraph has shared
sub-sub-graphs. Instead, do pruning it only after we've constructed
the entire SEL_ARG graph.

mysql-test/main/range.result

@@ -3135,6 +3135,52 @@ drop table t1,ten,t2;
 #
 # End of 10.2 tests
 #
+#
+# MDEV-9750: Quick memory exhaustion with 'extended_keys=on'...
+#
+create table t1 (
+kp1 int,
+kp2 int,
+kp3 int,
+kp4 int,
+key key1(kp1, kp2, kp3,kp4)
+);
+insert into t1 values (1,1,1,1),(2,2,2,2),(3,3,3,3);
+set @tmp_9750=@@optimizer_trace;
+set optimizer_trace=1;
+explain select * from t1 where 
+kp1 in (1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20) and
+kp2 in (1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20) and
+kp3 in (1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20) and
+kp4 in (1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20)
+;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	index	key1	key1	20	NULL	3	Using where; Using index
+set @json= (select json_detailed(JSON_EXTRACT(trace, '$**.range_scan_alternatives'))
+from information_schema.optimizer_trace);
+# This will show 3-component ranges. 
+# The ranges were produced, but the optimizer has cut away kp4
+#  to keep the number of ranges at manageable level:
+select left(@json, 500);
+left(@json, 500)
+[
+    
+    [
+        
+        {
+            "index": "key1",
+            "ranges": 
+            [
+                "(1,1,1) <= (kp1,kp2,kp3) <= (1,1,1)",
+                "(1,1,2) <= (kp1,kp2,kp3) <= (1,1,2)",
+                "(1,1,3) <= (kp1,kp2,kp3) <= (1,1,3)",
+                "(1,1,4) <= (kp1,kp2,kp3) <= (1,1,4)",
+                "(1,1,5) <= (kp1,kp2,kp3) <= (1,1,5)",
+                "(1,1,6) <= (kp1,kp2,kp3) <= (1,1,6)",
+                "(1,1,7) <= (kp1,kp2,kp3) <= (1,1,7)",
+                "
+set optimizer_trace=@tmp_9750;
+drop table t1;
 set global innodb_stats_persistent= @innodb_stats_persistent_save;
 set global innodb_stats_persistent_sample_pages=
 @innodb_stats_persistent_sample_pages_save;

mysql-test/main/range.test

@@ -2119,6 +2119,40 @@ drop table t1,ten,t2;
 --echo # End of 10.2 tests
 --echo #
 
+--echo #
+--echo # MDEV-9750: Quick memory exhaustion with 'extended_keys=on'...
+--echo #
+
+create table t1 (
+   kp1 int,
+   kp2 int,
+   kp3 int,
+   kp4 int,
+   key key1(kp1, kp2, kp3,kp4)
+);
+
+insert into t1 values (1,1,1,1),(2,2,2,2),(3,3,3,3);
+
+# 20 * 20 * 20 *20 = 400*400 = 160,000 ranges
+set @tmp_9750=@@optimizer_trace;
+set optimizer_trace=1;
+explain select * from t1 where 
+  kp1 in (1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20) and
+  kp2 in (1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20) and
+  kp3 in (1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20) and
+  kp4 in (1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20)
+;
+
+set @json= (select json_detailed(JSON_EXTRACT(trace, '$**.range_scan_alternatives'))
+            from information_schema.optimizer_trace);
+--echo # This will show 3-component ranges. 
+--echo # The ranges were produced, but the optimizer has cut away kp4
+--echo #  to keep the number of ranges at manageable level:
+select left(@json, 500);
+
+set optimizer_trace=@tmp_9750;
+drop table t1;
+
 set global innodb_stats_persistent= @innodb_stats_persistent_save;
 set global innodb_stats_persistent_sample_pages=
              @innodb_stats_persistent_sample_pages_save;

mysql-test/main/range_mrr_icp.result

@@ -3132,6 +3132,52 @@ drop table t1,ten,t2;
 #
 # End of 10.2 tests
 #
+#
+# MDEV-9750: Quick memory exhaustion with 'extended_keys=on'...
+#
+create table t1 (
+kp1 int,
+kp2 int,
+kp3 int,
+kp4 int,
+key key1(kp1, kp2, kp3,kp4)
+);
+insert into t1 values (1,1,1,1),(2,2,2,2),(3,3,3,3);
+set @tmp_9750=@@optimizer_trace;
+set optimizer_trace=1;
+explain select * from t1 where 
+kp1 in (1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20) and
+kp2 in (1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20) and
+kp3 in (1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20) and
+kp4 in (1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20)
+;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	index	key1	key1	20	NULL	3	Using where; Using index
+set @json= (select json_detailed(JSON_EXTRACT(trace, '$**.range_scan_alternatives'))
+from information_schema.optimizer_trace);
+# This will show 3-component ranges. 
+# The ranges were produced, but the optimizer has cut away kp4
+#  to keep the number of ranges at manageable level:
+select left(@json, 500);
+left(@json, 500)
+[
+    
+    [
+        
+        {
+            "index": "key1",
+            "ranges": 
+            [
+                "(1,1,1) <= (kp1,kp2,kp3) <= (1,1,1)",
+                "(1,1,2) <= (kp1,kp2,kp3) <= (1,1,2)",
+                "(1,1,3) <= (kp1,kp2,kp3) <= (1,1,3)",
+                "(1,1,4) <= (kp1,kp2,kp3) <= (1,1,4)",
+                "(1,1,5) <= (kp1,kp2,kp3) <= (1,1,5)",
+                "(1,1,6) <= (kp1,kp2,kp3) <= (1,1,6)",
+                "(1,1,7) <= (kp1,kp2,kp3) <= (1,1,7)",
+                "
+set optimizer_trace=@tmp_9750;
+drop table t1;
 set global innodb_stats_persistent= @innodb_stats_persistent_save;
 set global innodb_stats_persistent_sample_pages=
 @innodb_stats_persistent_sample_pages_save;

sql/opt_range.h

@@ -223,6 +223,43 @@ class RANGE_OPT_PARAM;
 
     We avoid consuming too much memory by setting a limit on the number of
     SEL_ARG object we can construct during one range analysis invocation.
+
+  5. SEL_ARG GRAPH WEIGHT
+
+    A SEL_ARG graph has a property we call weight, and we define it as follows:
+
+    If the SEL_ARG graph does not have any node with multiple incoming
+    next_key_part edges, then its weight is the number of SEL_ARG objects used.
+
+    If there is a node with multiple next_key_part edges, clone the node (and
+    the nodes connected via prev/next links to it) and redirect one of the
+    incoming next_key_part to the clone. (If the node has "peer" nodes
+    connected to it via prev/next links, they will have to be cloned as well)
+
+    Repeat this until we get a graph without multiple next_key_part edges
+    coming into the same node. Then, the number of SEL_ARG objects in the
+    graph is the weight.
+
+    Example:
+
+      |  +-------+  $              $
+      \->| kp1=2 |--$--------------$-+
+         +-------+  $              $ |   +--------+
+             |      $              $  ==>| kp3=11 |
+         +-------+  $              $ |   +--------+
+         | kp1=3 |--$--------------$-+       |
+         +-------+  $              $     +--------+
+                    $              $     | kp3=14 |
+                    $              $     +--------+
+
+    Here, the weight is 2 + 2*2=6.
+
+    The rationale behind the weight is:
+    - it has the same order-of-magnitude as the number of ranges that the
+      SEL_ARG graph is describing,
+    - it is a lot easier to compute,
+    - it can be updated incrementally when performing AND/OR operations on
+      parts of the graph.
 */
 
 class SEL_ARG :public Sql_alloc
@@ -236,6 +273,9 @@ class SEL_ARG :public Sql_alloc
   /*
     The ordinal number the least significant component encountered in
     the ranges of the SEL_ARG tree (the first component has number 1)
+
+    Note: this number is currently not precise, it is an upper bound.
+    @seealso SEL_ARG::get_max_key_part()
   */
   uint16 max_part_no;
   /*
@@ -263,6 +303,14 @@ class SEL_ARG :public Sql_alloc
   enum leaf_color { BLACK,RED } color;
   enum Type { IMPOSSIBLE, MAYBE, MAYBE_KEY, KEY_RANGE } type;
 
+  /*
+    For R-B root nodes only: the graph weight, as defined above in the
+    SEL_ARG GRAPH WEIGHT section.
+  */
+  uint weight;
+  enum { MAX_WEIGHT = 32000 };
+
+  /* See RANGE_OPT_PARAM::alloced_sel_args */
   enum { MAX_SEL_ARGS = 16000 };
 
   SEL_ARG() {}
@@ -273,7 +321,7 @@ class SEL_ARG :public Sql_alloc
   SEL_ARG(enum Type type_arg)
     :min_flag(0), max_part_no(0) /* first key part means 1. 0 mean 'no parts'*/,
      elements(1),use_count(1),left(0),right(0),
-     next_key_part(0), color(BLACK), type(type_arg)
+     next_key_part(0), color(BLACK), type(type_arg), weight(1)
   {}
   /**
     returns true if a range predicate is equal. Use all_same()
@@ -287,6 +335,9 @@ class SEL_ARG :public Sql_alloc
       return true;
     return cmp_min_to_min(arg) == 0 && cmp_max_to_max(arg) == 0;
   }
+
+  uint get_max_key_part() const;
+
   /**
     returns true if all the predicates in the keypart tree are equal
   */