MDEV-33281 Make BNL() hint work for join_cache_level=0

join_cache_level=0 disables join cache buffers, but the hint
BNL() now allows to employ BNL(H) buffers for particular tables
or query blocks.

mysql-test/main/opt_hints.test

@@ -148,9 +148,9 @@ EXPLAIN EXTENDED SELECT * FROM
    WHERE t4.y = 8 AND t5.x BETWEEN 7 AND t4.y+0) AS TD;
 
 --echo # BKA & NO_BKA hint testing
-set optimizer_switch=default;
-set optimizer_switch='mrr=on,mrr_cost_based=off';
-set join_cache_level=6;
+set optimizer_switch = DEFAULT;
+set optimizer_switch = 'mrr=on,mrr_cost_based=off';
+set join_cache_level = 8;
 
 CREATE TABLE t10(a INT);
 INSERT INTO t10 VALUES (0),(1),(2),(3),(4),(5),(6),(7),(8),(9);
@@ -359,7 +359,9 @@ DEALLOCATE PREPARE stmt1;
 
 DROP TABLE t1, t2, t3, t10, t11, t12, t13;
 
---echo # BNL & NO_BNL hint testing
+--echo #
+--echo # BNL and NO_BNL hint testing
+--echo #
 
 set optimizer_switch=default;
 
@@ -387,16 +389,27 @@ EXPLAIN EXTENDED SELECT /*+ NO_BNL() */t1.* FROM t1,t2,t3;
 EXPLAIN EXTENDED SELECT /*+ NO_BNL(t2, t3) */t1.* FROM t1,t2,t3;
 EXPLAIN EXTENDED SELECT /*+ NO_BNL(t1, t3) */t1.* FROM t1,t2,t3;
 
---echo # MariaDB does not have optimizer_switch='block_nested_loop=off'
---echo # as MySQL does, so in fact we cannot disable BNL join. The cases below
---echo # test the BNL() hint, although it does not affect the execution plan
+--echo # Make sure query results are the same for any hints combination
+--sorted_result
+SELECT * FROM t1,t2,t3;
+--sorted_result
+SELECT /*+ NO_BNL(t1, t2) */* FROM t1,t2,t3;
+--sorted_result
+SELECT /*+ NO_BNL(t1, t3) */* FROM t1,t2,t3;
+--sorted_result
+SELECT /*+ NO_BNL(t2, t3) */* FROM t1,t2,t3;
+
+--echo # BNL() overrides current join_cache_level setting
+set join_cache_level = 0;
 EXPLAIN EXTENDED SELECT t1.* FROM t1,t2,t3;
 EXPLAIN EXTENDED SELECT /*+ BNL() */t1.* FROM t1,t2,t3;
-EXPLAIN EXTENDED SELECT /*+ BNL(t2, t3) */t1.* FROM t1,t2,t3;
+EXPLAIN EXTENDED SELECT /*+ BNL(t1, t2) */t1.* FROM t1,t2,t3;
 EXPLAIN EXTENDED SELECT /*+ BNL(t1, t3) */t1.* FROM t1,t2,t3;
+EXPLAIN EXTENDED SELECT /*+ BNL(t2, t3) */t1.* FROM t1,t2,t3;
 EXPLAIN EXTENDED SELECT /*+ BNL(t2) BNL(t3) */t1.* FROM t1,t2,t3;
 
 DROP TABLE t1, t2, t3;
+set join_cache_level = 8;
 
 --echo # BNL in subquery
 set optimizer_switch = DEFAULT;
@@ -408,34 +421,141 @@ INSERT INTO t2 VALUES (2), (3), (4), (5);
 INSERT INTO t3 VALUES (10,3), (20,4), (30,5);
 ANALYZE TABLE t1, t2, t3;
 
-EXPLAIN EXTENDED SELECT /*+ QB_NAME(q) */ * FROM t1 JOIN t2 ON t1.b = t2.a WHERE
-t2.a IN (SELECT /*+ QB_NAME(subq1) */ t3.b FROM t3 JOIN t1 t4 ON t3.b = t4.b);
-
-EXPLAIN EXTENDED SELECT /*+ QB_NAME(q) NO_BNL() */ * FROM t1 JOIN t2 ON t1.b = t2.a WHERE
-t2.a IN (SELECT /*+ QB_NAME(subq1) */ t3.b FROM t3 JOIN t1 t4 ON t3.b = t4.b);
-
-EXPLAIN EXTENDED SELECT /*+ QB_NAME(q) NO_BNL(t1, t2) */ * FROM t1 JOIN t2 ON t1.b = t2.a WHERE
-t2.a IN (SELECT /*+ QB_NAME(subq1) */ t3.b FROM t3 JOIN t1 t4 ON t3.b = t4.b);
-
-EXPLAIN EXTENDED SELECT /*+ QB_NAME(q) NO_BNL(@subq1) */ * FROM t1 JOIN t2 ON t1.b = t2.a WHERE
-t2.a IN (SELECT /*+ QB_NAME(subq1) */ t3.b FROM t3 JOIN t1 t4 ON t3.b = t4.b);
+EXPLAIN EXTENDED SELECT /*+ QB_NAME(q) */ * FROM t1 JOIN t2 ON t1.a = t2.a WHERE
+t2.a IN (SELECT /*+ QB_NAME(subq1) */ t3.b FROM t3 JOIN t1 t4 ON t3.a = t4.b);
+
+EXPLAIN EXTENDED SELECT /*+ QB_NAME(q) NO_BNL() */ *
+  FROM t1 JOIN t2 ON t1.a = t2.a
+  WHERE t2.a IN (SELECT /*+ QB_NAME(subq1) */ t3.b FROM t3 JOIN t1 t4 ON t3.a = t4.b);
+
+EXPLAIN EXTENDED SELECT /*+ QB_NAME(q) NO_BNL(t1, t2) */ *
+  FROM t1 JOIN t2 ON t1.a = t2.a
+  WHERE t2.a IN (SELECT /*+ QB_NAME(subq1) */ t3.b FROM t3 JOIN t1 t4 ON t3.a = t4.b);
+
+EXPLAIN EXTENDED SELECT /*+ QB_NAME(q) NO_BNL(@subq1) */ *
+  FROM t1 JOIN t2 ON t1.a = t2.a
+  WHERE t2.a IN (SELECT /*+ QB_NAME(subq1) */ t3.b FROM t3 JOIN t1 t4 ON t3.a = t4.b);
+
+EXPLAIN EXTENDED SELECT /*+ QB_NAME(q) NO_BNL(t4@subq1) */ *
+  FROM t1 JOIN t2 ON t1.a = t2.a
+  WHERE t2.a IN (SELECT /*+ QB_NAME(subq1) */ t3.b FROM t3 JOIN t1 t4 ON t3.a = t4.b);
+
+EXPLAIN EXTENDED SELECT /*+ QB_NAME(q) NO_BNL(t3@subq1,t4@subq1) */ *
+  FROM t1 JOIN t2 ON t1.a = t2.a
+  WHERE t2.a IN (SELECT /*+ QB_NAME(subq1) */ t3.b FROM t3 JOIN t1 t4 ON t3.a = t4.b);
+
+EXPLAIN EXTENDED SELECT /*+ QB_NAME(q) NO_BNL(@subq1 t3, t4) */ *
+  FROM t1 JOIN t2 ON t1.a = t2.a
+  WHERE t2.a IN (SELECT /*+ QB_NAME(subq1) */ t3.b FROM t3 JOIN t1 t4 ON t3.a = t4.b);
+
+EXPLAIN EXTENDED SELECT /*+ QB_NAME(q) */ *
+  FROM t1 JOIN t2 ON t1.a = t2.a
+  WHERE t2.a IN (SELECT /*+ QB_NAME(subq1)  NO_BNL(t3, t4) */ t3.b
+    FROM t3 JOIN t1 t4 ON t3.a = t4.b);
+
+--echo # Make sure query results are the same for any hints combination
+# --sorted_result
+SELECT * FROM t1 JOIN t2 ON t1.a = t2.a WHERE
+t2.a IN (SELECT t3.b FROM t3 JOIN t1 t4 ON t3.a = t4.b);
+# --sorted_result
+SELECT /*+ QB_NAME(q) */ * FROM t1 JOIN t2 ON t1.a = t2.a WHERE
+t2.a IN (SELECT /*+ QB_NAME(subq1) */ t3.b FROM t3 JOIN t1 t4 ON t3.a = t4.b);
+--sorted_result
+SELECT /*+ QB_NAME(q) NO_BNL() */ *
+  FROM t1 JOIN t2 ON t1.a = t2.a
+  WHERE t2.a IN (SELECT /*+ QB_NAME(subq1) */ t3.b FROM t3 JOIN t1 t4 ON t3.a = t4.b);
+--sorted_result
+SELECT /*+ QB_NAME(q) NO_BNL(t1, t2) */ *
+  FROM t1 JOIN t2 ON t1.a = t2.a
+  WHERE t2.a IN (SELECT /*+ QB_NAME(subq1) */ t3.b FROM t3 JOIN t1 t4 ON t3.a = t4.b);
+--sorted_result
+SELECT /*+ QB_NAME(q) NO_BNL(@subq1) */ *
+  FROM t1 JOIN t2 ON t1.a = t2.a
+  WHERE t2.a IN (SELECT /*+ QB_NAME(subq1) */ t3.b FROM t3 JOIN t1 t4 ON t3.a = t4.b);
+--sorted_result
+SELECT /*+ QB_NAME(q) NO_BNL(t4@subq1) */ *
+  FROM t1 JOIN t2 ON t1.a = t2.a
+  WHERE t2.a IN (SELECT /*+ QB_NAME(subq1) */ t3.b FROM t3 JOIN t1 t4 ON t3.a = t4.b);
+--sorted_result
+SELECT /*+ QB_NAME(q) NO_BNL(t3@subq1,t4@subq1) */ *
+  FROM t1 JOIN t2 ON t1.a = t2.a
+  WHERE t2.a IN (SELECT /*+ QB_NAME(subq1) */ t3.b FROM t3 JOIN t1 t4 ON t3.a = t4.b);
+--sorted_result
+SELECT /*+ QB_NAME(q) NO_BNL(@subq1 t3, t4) */ *
+  FROM t1 JOIN t2 ON t1.a = t2.a
+  WHERE t2.a IN (SELECT /*+ QB_NAME(subq1) */ t3.b FROM t3 JOIN t1 t4 ON t3.a = t4.b);
+--sorted_result
+SELECT /*+ QB_NAME(q) */ *
+  FROM t1 JOIN t2 ON t1.a = t2.a
+  WHERE t2.a IN (SELECT /*+ QB_NAME(subq1)  NO_BNL(t3, t4) */ t3.b
+    FROM t3 JOIN t1 t4 ON t3.a = t4.b);
 
-EXPLAIN EXTENDED SELECT /*+ QB_NAME(q) NO_BNL(t4@subq1) */ * FROM t1 JOIN t2 ON t1.b = t2.a WHERE
-t2.a IN (SELECT /*+ QB_NAME(subq1) */ t3.b FROM t3 JOIN t1 t4 ON t3.b = t4.b);
-
-EXPLAIN EXTENDED SELECT /*+ QB_NAME(q) NO_BNL(t3@subq1,t4@subq1) */ * FROM t1 JOIN t2 ON t1.b = t2.a WHERE
-t2.a IN (SELECT /*+ QB_NAME(subq1) */ t3.b FROM t3 JOIN t1 t4 ON t3.b = t4.b);
+DROP TABLE t1, t2, t3, t4;
 
-EXPLAIN EXTENDED SELECT /*+ QB_NAME(q) NO_BNL(@subq1 t3, t4) */ * FROM t1 JOIN t2 ON t1.b = t2.a WHERE
-t2.a IN (SELECT /*+ QB_NAME(subq1) */ t3.b FROM t3 JOIN t1 t4 ON t3.b = t4.b);
+--echo #
+--echo # BNL() and NO_BNL() for LEFT JOINs
+--echo #
 
-EXPLAIN EXTENDED SELECT /*+ QB_NAME(q) */ * FROM t1 JOIN t2 ON t1.b = t2.a WHERE
-t2.a IN (SELECT /*+ QB_NAME(subq1)  NO_BNL(t3, t4) */ t3.b FROM t3 JOIN t1 t4 ON t3.b = t4.b);
+CREATE TABLE t1 (a int);
+CREATE TABLE t2 (a int);
+CREATE TABLE t3 (a int);
+CREATE TABLE t4 (a int);
+
+INSERT INTO t1 VALUES (null), (2), (null), (1);
+
+set join_cache_level = 8;
+
+--echo # Two join buffers are employed by default (without hints):
+EXPLAIN SELECT t1.a
+  FROM t1 LEFT JOIN (t2 LEFT JOIN t3 ON t2.a) ON 0
+    WHERE t1.a OR t3.a;
+
+--echo # Disabling either of join buffers disables another one automatically due
+--echo # to join buffer employment rules:
+EXPLAIN SELECT /*+ NO_BNL(t2) */ t1.a
+  FROM t1 LEFT JOIN (t2 LEFT JOIN t3 ON t2.a) ON 0
+    WHERE t1.a OR t3.a;
+
+EXPLAIN SELECT /*+ NO_BNL(t3) */ t1.a
+  FROM t1 LEFT JOIN (t2 LEFT JOIN t3 ON t2.a) ON 0
+    WHERE t1.a OR t3.a;
+
+--echo # Three join buffers are employed here by default (without hints):
+EXPLAIN SELECT t1.a
+  FROM t1 LEFT JOIN (t2 LEFT JOIN (t3 LEFT JOIN t4 ON 1) ON t2.a) ON 0
+    WHERE t1.a OR t4.a;
+
+--echo # Disabling either of join buffers disables others automatically due
+--echo # to join buffer employment rules:
+EXPLAIN SELECT /*+ NO_BNL(t2)*/ t1.a
+  FROM t1 LEFT JOIN (t2 LEFT JOIN (t3 LEFT JOIN t4 ON 1) ON t2.a) ON 0
+    WHERE t1.a OR t4.a;
+EXPLAIN SELECT /*+ NO_BNL(t3)*/ t1.a
+  FROM t1 LEFT JOIN (t2 LEFT JOIN (t3 LEFT JOIN t4 ON 1) ON t2.a) ON 0
+    WHERE t1.a OR t4.a;
+EXPLAIN SELECT /*+ NO_BNL(t4)*/ t1.a
+  FROM t1 LEFT JOIN (t2 LEFT JOIN (t3 LEFT JOIN t4 ON 1) ON t2.a) ON 0
+    WHERE t1.a OR t4.a;
+
+set join_cache_level=0;
+--echo # It is not allowed to enable join buffers on some subset of inner tables
+--echo # of an outer join. Either all tables should use join buffers or none:
+EXPLAIN SELECT /*+ BNL(t2)*/ t1.a
+  FROM t1 LEFT JOIN (t2 LEFT JOIN (t3 LEFT JOIN t4 ON 1) ON t2.a) ON 0
+    WHERE t1.a OR t4.a;
+EXPLAIN SELECT /*+ BNL(t2, t3)*/ t1.a
+  FROM t1 LEFT JOIN (t2 LEFT JOIN (t3 LEFT JOIN t4 ON 1) ON t2.a) ON 0
+    WHERE t1.a OR t4.a;
+EXPLAIN SELECT /*+ BNL(t2, t3, t4)*/ t1.a
+  FROM t1 LEFT JOIN (t2 LEFT JOIN (t3 LEFT JOIN t4 ON 1) ON t2.a) ON 0
+    WHERE t1.a OR t4.a;
+
+DROP TABLE t1,t2,t3,t4;
 
-DROP TABLE t1, t2, t3, t4;
 
 --echo # MRR & NO_MRR hint testing
 set optimizer_switch=default;
+set join_cache_level = 8;
 
 CREATE TABLE t1
 (
@@ -750,4 +870,8 @@ SELECT
 /*+ ? bad syntax */ 1;
 
 DROP TABLE t1;
-set optimizer_switch=default;
+set optimizer_switch = DEFAULT;
+set join_cache_level = DEFAULT;
+--echo #
+--echo # End of 11.7 tests
+--echo #

sql/sql_select.cc

@@ -15143,12 +15143,19 @@ uint check_join_cache_usage(JOIN_TAB *tab,
   if (tab->no_forced_join_cache || (no_bnl_cache && no_bka_cache))
     goto no_join_cache;
 
+  if (cache_level == 0 && hint_table_state_or_fallback(join->thd,
+                                                       tab->tab_list->table,
+                                                       BNL_HINT_ENUM, false))
+  {
+    cache_level= 4; // BNL() hint present, raise join_cache_level to BNLH
+  }
+
   /*
     Don't use join cache if @@join_cache_level==0 or this table is the first
     one join suborder (either at top level or inside a bush)
   */
   if (cache_level == 0 || !prev_tab)
-    return 0;
+    goto no_join_cache;
 
   if (force_unlinked_cache && (cache_level%2 == 0))
     cache_level--;
@@ -15227,6 +15234,12 @@ uint check_join_cache_usage(JOIN_TAB *tab,
   if (tab->loosescan_match_tab || tab->bush_children)
     goto no_join_cache;
 
+  /*
+    An inner table of an outer join nest must not use join buffering if
+    the first inner table of that outer join nest does not use join buffering
+    or if the tables in the embedding outer join nest do not use join buffering.
+    Also see revise_cache_usage()
+  */
   for (JOIN_TAB *first_inner= tab->first_inner; first_inner;
        first_inner= first_inner->first_upper)
   {
@@ -15302,6 +15315,8 @@ uint check_join_cache_usage(JOIN_TAB *tab,
         tab->is_ref_for_hash_join() ||
 	((flags & HA_MRR_NO_ASSOCIATION) && cache_level <=6))
     {
+      if (no_bnl_cache)
+        goto no_join_cache;
       if (!tab->hash_join_is_possible() ||
           tab->make_scan_filter())
         goto no_join_cache;