MDEV-28217 Incorrect Join Execution When Controlling Join Buffer Size

The problem was that join_buffer_size conflicted with
join_buffer_space_limit, which caused the query to be run without join
buffer. However this caused wrong results as the optimizer assumed
that hash+join buffer would ensure that the equi-join condition
would be satisfied, and didn't check it itself.

Fixed by not using join_buffer_space_limit when
optimize_join_buffer_size=off. This matches the documentation at
https://mariadb.com/kb/en/block-based-join-algorithms

Other things:
- Removed not used variable JOIN_TAB::join_buffer_size_limit
- Give an error if we cannot allocate a join buffer. This can
  only happen if the join_buffer variables are wrongly configured or
  we are running out of memory.
  In the future, instead of returning an error, we could properly
  convert the query plan that uses BNL-H join into one that doesn't
  use join buffering:
  make sure the equi-join condition is checked where appropriate.

Reviewer: Sergei Petrunia <sergey@mariadb.com>

mysql-test/main/join_cache.result

@@ -5655,13 +5655,13 @@ EXPLAIN
 SELECT * FROM t1, t2 LEFT JOIN t3 ON t2.b=t3.b WHERE t1.a=t2.a;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	SIMPLE	t1	ALL	NULL	NULL	NULL	NULL	2	
-1	SIMPLE	t2	ALL	NULL	NULL	NULL	NULL	12	Using where
-1	SIMPLE	t3	ALL	NULL	NULL	NULL	NULL	3	Using where
+1	SIMPLE	t2	ALL	NULL	NULL	NULL	NULL	12	Using where; Using join buffer (flat, BNL join)
+1	SIMPLE	t3	ALL	NULL	NULL	NULL	NULL	3	Using where; Using join buffer (incremental, BNL join)
 SELECT * FROM t1, t2 LEFT JOIN t3 ON t2.b=t3.b WHERE t1.a=t2.a;
 a	a	b	b	c
+3	3	32	32	302
 3	3	30	30	300
 3	3	31	NULL	NULL
-3	3	32	32	302
 set join_buffer_space_limit=@save_join_buffer_space_limit;
 set join_buffer_size=@save_join_buffer_size;
 set join_cache_level=@save_join_cache_level;
@@ -6229,6 +6229,33 @@ EXPLAIN
 }
 drop table t1,t2,t3;
 # End of 10.3 tests
+#
+# MDEV-28217 Incorrect Join Execution When Controlling Join Buffer Size
+#
+CREATE TABLE t1 (i int PRIMARY KEY)engine=innodb;
+INSERT INTO t1 VALUES (1332945389);
+CREATE TABLE t2 (i int PRIMARY KEY)engine=innodb;
+INSERT INTO t2 VALUES (1180244875), (1951338178);
+SET SESSION join_buffer_size= X;
+Warnings:
+Warning	X	Truncated incorrect join_buffer_size value: 'X'
+SET SESSION join_cache_level = 4;
+SET optimizer_switch='optimize_join_buffer_size=on';
+SELECT t2.i FROM t2  LEFT  JOIN t1 ON t1.i = t2.i  WHERE t1.i;
+i
+SET optimizer_switch='optimize_join_buffer_size=off';
+SELECT t1.i,t2.i FROM t2  LEFT  JOIN t1 ON t1.i = t2.i  WHERE t1.i;
+ERROR HYX: Could not create a join buffer. Please check and adjust the value of the variables 'JOIN_BUFFER_SIZE (X)' and 'JOIN_BUFFER_SPACE_LIMIT (X)'
+SET SESSION join_buffer_size= 10000000;
+SELECT t1.i,t2.i FROM t2  LEFT  JOIN t1 ON t1.i = t2.i  WHERE t1.i;
+i	i
+SET SESSION optimizer_switch= default;
+SET SESSION join_buffer_size= default;
+SET SESSION join_cache_level= default;
+drop table t1,t2;
+#
+# End of 10.4 tests
+#
 set @@optimizer_switch=@save_optimizer_switch;
 set global innodb_stats_persistent= @innodb_stats_persistent_save;
 set global innodb_stats_persistent_sample_pages=

mysql-test/main/join_cache.test

@@ -4201,6 +4201,34 @@ drop table t1,t2,t3;
 
 --echo # End of 10.3 tests
 
+--echo #
+--echo # MDEV-28217 Incorrect Join Execution When Controlling Join Buffer Size
+--echo #
+
+CREATE TABLE t1 (i int PRIMARY KEY)engine=innodb;
+INSERT INTO t1 VALUES (1332945389);
+CREATE TABLE t2 (i int PRIMARY KEY)engine=innodb;
+INSERT INTO t2 VALUES (1180244875), (1951338178);
+--replace_regex /[0-9][0-9]+/X/
+SET SESSION join_buffer_size= 5250229460064350213;
+SET SESSION join_cache_level = 4;
+SET optimizer_switch='optimize_join_buffer_size=on';
+SELECT t2.i FROM t2  LEFT  JOIN t1 ON t1.i = t2.i  WHERE t1.i;
+SET optimizer_switch='optimize_join_buffer_size=off';
+--replace_regex /[0-9][0-9]+/X/
+--error ER_OUTOFMEMORY
+SELECT t1.i,t2.i FROM t2  LEFT  JOIN t1 ON t1.i = t2.i  WHERE t1.i;
+SET SESSION join_buffer_size= 10000000;
+SELECT t1.i,t2.i FROM t2  LEFT  JOIN t1 ON t1.i = t2.i  WHERE t1.i;
+SET SESSION optimizer_switch= default;
+SET SESSION join_buffer_size= default;
+SET SESSION join_cache_level= default;
+drop table t1,t2;
+
+--echo #
+--echo # End of 10.4 tests
+--echo #
+
 # The following command must be the last one in the file
 set @@optimizer_switch=@save_optimizer_switch;
 

mysql-test/main/join_optimizer.test

@@ -2,6 +2,8 @@
 drop table if exists t0,t1,t2,t3;
 --enable_warnings
 
+--source include/have_innodb.inc
+
 --echo #
 --echo # BUG#38049 incorrect rows estimations with references from preceding table
 --echo #

sql/sql_join_cache.cc

@@ -805,20 +805,18 @@ size_t JOIN_CACHE::get_min_join_buffer_size()
                         the estimated number of records in the partial join 
 
   DESCRIPTION
-    At the first its invocation for the cache the function calculates the
-    maximum possible size of join buffer for the cache. If the parameter
-    optimize_buff_size true then this value does not exceed the size of the
-    space needed for the estimated number of records 'max_records' in the
-    partial join that joins tables from the first one through join_tab. This
-    value is also capped off by the value of join_tab->join_buffer_size_limit,
-    if it has been set a to non-zero value, and by the value of the system
-    parameter join_buffer_size - otherwise. After the calculation of the
-    interesting size the function saves the value in the field 'max_buff_size'
-    in order to use it directly at the next  invocations of the function.
 
-  NOTES
-    Currently the value of join_tab->join_buffer_size_limit is initialized
-    to 0 and is never reset.
+    At the first its invocation for the cache the function calculates
+    the maximum possible size of join buffer for the cache. If the
+    parameter optimize_buff_size true then this value does not exceed
+    the size of the space needed for the estimated number of records
+    'max_records' in the partial join that joins tables from the first
+    one through join_tab. This value is also capped off by the value
+    of the system parameter join_buffer_size. After the calculation of
+    the interesting size the function saves the value in the field
+    'max_buff_size' in order to use it directly at the next
+    invocations of the function.
+
 
   RETURN VALUE
     The maximum possible size of the join buffer of this cache 
@@ -842,8 +840,6 @@ size_t JOIN_CACHE::get_max_join_buffer_size(bool optimize_buff_size)
     space_per_record= len;
     
     size_t limit_sz= (size_t)join->thd->variables.join_buff_size;
-    if (join_tab->join_buffer_size_limit)
-      set_if_smaller(limit_sz, join_tab->join_buffer_size_limit);
     if (!optimize_buff_size)
       max_sz= limit_sz;
     else
@@ -923,13 +919,25 @@ int JOIN_CACHE::alloc_buffer()
   curr_min_buff_space_sz+= min_buff_size;
   curr_buff_space_sz+= buff_size;
 
-  if (curr_min_buff_space_sz > join_buff_space_limit ||
-      (curr_buff_space_sz > join_buff_space_limit &&
-       (!optimize_buff_size || 
+  if (optimize_buff_size)
+  {
+    /*
+      optimize_join_buffer_size=on used. We should limit the join
+      buffer space to join_buff_space_limit if possible.
+    */
+    if (curr_min_buff_space_sz > join_buff_space_limit)
+    {
+      /*
+        Increase buffer size to minimum needed, to be able to use the
+        join buffer.
+      */
+      join_buff_space_limit= curr_min_buff_space_sz;
+    }
+    if (curr_buff_space_sz > join_buff_space_limit &&
         join->shrink_join_buffers(join_tab, curr_buff_space_sz,
-                                  join_buff_space_limit))))
-    goto fail;
-
+                                  join_buff_space_limit))
+      goto fail;                                // Fatal error
+  }
   if (for_explain_only)
     return 0;
                                
@@ -2766,7 +2774,6 @@ bool JOIN_CACHE_BKAH::save_explain_data(EXPLAIN_BKA_TYPE *explain)
 
 int JOIN_CACHE_HASHED::init(bool for_explain)
 {
-  int rc= 0;
   TABLE_REF *ref= &join_tab->ref;
 
   DBUG_ENTER("JOIN_CACHE_HASHED::init");
@@ -2776,8 +2783,21 @@ int JOIN_CACHE_HASHED::init(bool for_explain)
 
   key_length= ref->key_length;
 
-  if ((rc= JOIN_CACHE::init(for_explain)) || for_explain)
-    DBUG_RETURN (rc); 
+  if (JOIN_CACHE::init(for_explain))
+  {
+    THD *thd= join->thd;
+    const char *errmsg=
+      "Could not create a join buffer. Please check and "
+      "adjust the value of the variables 'JOIN_BUFFER_SIZE (%llu)' and "
+      "'JOIN_BUFFER_SPACE_LIMIT (%llu)'";
+    my_printf_error(ER_OUTOFMEMORY, errmsg, MYF(0),
+                    thd->variables.join_buff_size,
+                    thd->variables.join_buff_space_limit);
+    DBUG_RETURN (1);
+  }
+
+  if (for_explain)
+    DBUG_RETURN(0);
 
   if (!(key_buff= (uchar*) join->thd->alloc(key_length)))
     DBUG_RETURN(1);

sql/sql_select.h

@@ -395,7 +395,6 @@ typedef struct st_join_table {
   bool          idx_cond_fact_out;
   bool          use_join_cache;
   uint          used_join_cache_level;
-  ulong         join_buffer_size_limit;
   JOIN_CACHE	*cache;
   /*
     Index condition for BKA access join