Backport into MariaDB-5.2 the following:

WL#2771 "Block Nested Loop Join and Batched Key Access Join"

libmysqld/CMakeLists.txt

@@ -123,7 +123,8 @@ SET(LIBMYSQLD_SOURCES emb_qcache.cc libmysqld.c lib_sql.cc
            ../sql/sql_class.cc ../sql/sql_crypt.cc ../sql/sql_cursor.cc 
            ../sql/sql_db.cc ../sql/sql_delete.cc ../sql/sql_derived.cc 
            ../sql/sql_do.cc ../sql/sql_error.cc ../sql/sql_handler.cc 
-           ../sql/sql_help.cc ../sql/sql_insert.cc ../sql/sql_lex.cc 
+           ../sql/sql_help.cc ../sql/sql_insert.cc ../sql/sql_join_cache.cc
+           ../sql/sql_lex.cc 
            ../sql/sql_list.cc ../sql/sql_load.cc ../sql/sql_locale.cc 
            ../sql/sql_binlog.cc ../sql/sql_manager.cc ../sql/sql_map.cc 
            ../sql/sql_parse.cc ../sql/sql_partition.cc ../sql/sql_plugin.cc 

libmysqld/Makefile.am

@@ -63,6 +63,7 @@ sqlsources = ds_mrr.cc derror.cc field.cc field_conv.cc strfunc.cc filesort.cc \
 	sql_profile.cc \
 	sql_analyse.cc sql_base.cc sql_cache.cc sql_class.cc \
 	sql_crypt.cc sql_db.cc sql_delete.cc sql_error.cc sql_insert.cc \
+        sql_join_cache.cc \
 	sql_lex.cc sql_list.cc sql_manager.cc sql_map.cc \
 	scheduler.cc sql_connect.cc sql_parse.cc \
 	sql_prepare.cc sql_derived.cc sql_rename.cc \

mysql-test/r/index_merge_myisam.result

@@ -342,8 +342,6 @@ create table t4 (a int);
 insert into t4 values (1),(4),(3);
 set @save_join_buffer_size=@@join_buffer_size;
 set join_buffer_size= 4000;
-Warnings:
-Warning	1292	Truncated incorrect join_buffer_size value: '4000'
 explain select max(A.key1 + B.key1 + A.key2 + B.key2 + A.key3 + B.key3 + A.key4 + B.key4 + A.key5 + B.key5)
 from t0 as A force index(i1,i2), t0 as B force index (i1,i2)
 where (A.key1 < 500000 or A.key2 < 3)

mysql-test/r/join_outer.result

@@ -859,14 +859,14 @@ a1	a2
 EXPLAIN SELECT * FROM t1 LEFT JOIN t2 ON a1=0;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	SIMPLE	t1	system	NULL	NULL	NULL	NULL	1	
-1	SIMPLE	t2	ALL	NULL	NULL	NULL	NULL	2	
+1	SIMPLE	t2	ALL	NULL	NULL	NULL	NULL	2	Using where
 SELECT * FROM t1 LEFT JOIN (t2,t3) ON a1=0;
 a1	a2	a3
 1	NULL	NULL
 EXPLAIN SELECT * FROM t1 LEFT JOIN (t2,t3) ON a1=0;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	SIMPLE	t1	system	NULL	NULL	NULL	NULL	1	
-1	SIMPLE	t2	ALL	NULL	NULL	NULL	NULL	2	
+1	SIMPLE	t2	ALL	NULL	NULL	NULL	NULL	2	Using where
 1	SIMPLE	t3	ALL	NULL	NULL	NULL	NULL	2	
 SELECT * FROM t0, t1 LEFT JOIN (t2,t3) ON a1=0 WHERE a0=a1;
 a0	a1	a2	a3
@@ -875,7 +875,7 @@ EXPLAIN SELECT * FROM t0, t1 LEFT JOIN (t2,t3) ON a1=0 WHERE a0=a1;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	SIMPLE	t0	system	PRIMARY	NULL	NULL	NULL	1	
 1	SIMPLE	t1	system	PRIMARY	NULL	NULL	NULL	1	
-1	SIMPLE	t2	ALL	NULL	NULL	NULL	NULL	2	
+1	SIMPLE	t2	ALL	NULL	NULL	NULL	NULL	2	Using where
 1	SIMPLE	t3	ALL	NULL	NULL	NULL	NULL	2	
 INSERT INTO t0 VALUES (0);
 INSERT INTO t1 VALUES (0);
@@ -886,7 +886,7 @@ EXPLAIN SELECT * FROM t0, t1 LEFT JOIN (t2,t3) ON a1=5 WHERE a0=a1 AND a0=1;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	SIMPLE	t0	const	PRIMARY	PRIMARY	4	const	1	Using index
 1	SIMPLE	t1	const	PRIMARY	PRIMARY	4	const	1	Using index
-1	SIMPLE	t2	ALL	NULL	NULL	NULL	NULL	2	
+1	SIMPLE	t2	ALL	NULL	NULL	NULL	NULL	2	Using where
 1	SIMPLE	t3	ALL	NULL	NULL	NULL	NULL	2	
 drop table t1,t2;
 create table t1 (a int, b int);

mysql-test/r/table_elim.result

@@ -347,7 +347,7 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 explain select t1.a from t1 left join t2 on TRUE;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	SIMPLE	t1	ALL	NULL	NULL	NULL	NULL	4	
-1	SIMPLE	t2	index	NULL	PRIMARY	4	NULL	2	Using index
+1	SIMPLE	t2	index	NULL	PRIMARY	4	NULL	2	Using where; Using index
 explain select t1.a from t1 left join t3 on t3.pk1=t1.a and t3.pk2 IS NULL;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	SIMPLE	t1	ALL	NULL	NULL	NULL	NULL	4	

mysql-test/t/join_nested_jcl6.test

+# 
+# Run join_nested.test with BKA enabled 
+#
+
+set join_cache_level=6;
+show variables like 'join_cache_level';
+
+--source t/join_nested.test
+
+#
+# BUG#35835: queries with nested outer joins with BKA enabled
+#            
+
+CREATE TABLE t5 (a int, b int, c int, PRIMARY KEY(a), KEY b_i (b));
+CREATE TABLE t6 (a int, b int, c int, PRIMARY KEY(a), KEY b_i (b));
+CREATE TABLE t7 (a int, b int, c int, PRIMARY KEY(a), KEY b_i (b));
+CREATE TABLE t8 (a int, b int, c int, PRIMARY KEY(a), KEY b_i (b));
+
+INSERT INTO t5 VALUES (1,1,0), (2,2,0), (3,3,0);
+INSERT INTO t6 VALUES (1,2,0), (3,2,0), (6,1,0);
+INSERT INTO t7 VALUES (1,1,0), (2,2,0);
+INSERT INTO t8 VALUES (0,2,0), (1,2,0);
+
+EXPLAIN
+SELECT t5.a,t5.b,t6.a,t6.b,t7.a,t7.b,t8.a,t8.b
+  FROM t5 
+       LEFT JOIN 
+       ( 
+         (t6, t7)
+         LEFT JOIN 
+         t8
+         ON t7.b=t8.b AND t6.b < 10
+       )
+       ON t6.b >= 2 AND t5.b=t7.b AND
+          (t8.a > 0 OR t8.c IS NULL);
+
+SELECT t5.a,t5.b,t6.a,t6.b,t7.a,t7.b,t8.a,t8.b
+  FROM t5 
+       LEFT JOIN 
+       ( 
+         (t6, t7)
+         LEFT JOIN 
+         t8
+         ON t7.b=t8.b AND t6.b < 10
+       )
+       ON t6.b >= 2 AND t5.b=t7.b AND
+          (t8.a > 0 OR t8.c IS NULL);
+
+DELETE FROM t5;
+DELETE FROM t6;
+DELETE FROM t7;
+DELETE FROM t8;
+
+INSERT INTO t5 VALUES (1,3,0), (3,2,0);
+INSERT INTO t6 VALUES (3,3,0);
+INSERT INTO t7 VALUES (1,2,0);
+INSERT INTO t8 VALUES (1,1,0);
+
+EXPLAIN
+SELECT t5.a,t5.b,t6.a,t6.b,t7.a,t7.b,t8.a,t8.b
+  FROM t5 LEFT JOIN                
+       (t6 LEFT JOIN t7 ON t7.a=1, t8)
+       ON (t5.b=t8.b);
+
+SELECT t5.a,t5.b,t6.a,t6.b,t7.a,t7.b,t8.a,t8.b
+  FROM t5 LEFT JOIN                
+       (t6 LEFT JOIN t7 ON t7.a=1, t8)
+       ON (t5.b=t8.b);
+
+EXPLAIN
+SELECT t5.a,t5.b,t6.a,t6.b,t7.a,t7.b,t8.a,t8.b
+  FROM t5 LEFT JOIN                
+       (t6 LEFT JOIN t7 ON t7.b=2, t8)
+       ON (t5.b=t8.b);
+
+SELECT t5.a,t5.b,t6.a,t6.b,t7.a,t7.b,t8.a,t8.b
+  FROM t5 LEFT JOIN                
+       (t6 LEFT JOIN t7 ON t7.b=2, t8)
+       ON (t5.b=t8.b);
+
+EXPLAIN
+SELECT t5.a,t5.b,t6.a,t6.b,t7.a,t7.b,t8.a,t8.b
+  FROM t5 LEFT JOIN                
+       (t8, t6 LEFT JOIN t7 ON t7.a=1)
+       ON (t5.b=t8.b);
+
+SELECT t5.a,t5.b,t6.a,t6.b,t7.a,t7.b,t8.a,t8.b
+  FROM t5 LEFT JOIN                
+       (t8, t6 LEFT JOIN t7 ON t7.a=1)
+       ON (t5.b=t8.b);
+
+DROP TABLE t5,t6,t7,t8;
+
+set join_cache_level=default;
+show variables like 'join_cache_level';

mysql-test/t/join_outer_jcl6.test

+# 
+# Run join_outer.test with BKA enabled 
+#
+
+set join_cache_level=6;
+show variables like 'join_cache_level';
+
+--source t/join_outer.test
+
+set join_cache_level=default;
+show variables like 'join_cache_level';

mysql-test/t/select_jcl6.test

+# 
+# Run select.test with BKA enabled 
+#
+
+set join_cache_level=6;
+show variables like 'join_cache_level';
+
+--source t/select.test
+
+set join_cache_level=default;
+show variables like 'join_cache_level';

sql/CMakeLists.txt

@@ -62,9 +62,10 @@ SET (SQL_SOURCE
                sp_rcontext.cc spatial.cc sql_acl.cc sql_analyse.cc sql_base.cc 
                sql_cache.cc sql_class.cc sql_client.cc sql_crypt.cc sql_crypt.h 
                sql_cursor.cc sql_db.cc sql_delete.cc sql_derived.cc sql_do.cc 
-               sql_error.cc sql_handler.cc sql_help.cc sql_insert.cc sql_lex.cc 
-               sql_list.cc sql_load.cc sql_manager.cc sql_map.cc sql_parse.cc 
-               sql_partition.cc sql_plugin.cc sql_prepare.cc sql_rename.cc 
+               sql_error.cc sql_handler.cc sql_help.cc sql_insert.cc
+               sql_join_cache.cc sql_lex.cc sql_list.cc sql_load.cc sql_manager.cc
+               sql_map.cc sql_parse.cc  sql_partition.cc sql_plugin.cc
+               sql_prepare.cc sql_rename.cc 
                sql_repl.cc sql_select.cc sql_show.cc sql_state.c sql_string.cc 
                sql_table.cc sql_test.cc sql_trigger.cc sql_udf.cc sql_union.cc
                sql_update.cc sql_view.cc strfunc.cc table.cc thr_malloc.cc 

sql/Makefile.am

@@ -91,6 +91,7 @@ mysqld_SOURCES =	ds_mrr.cc sql_lex.cc sql_handler.cc sql_partition.cc \
 			mysqld.cc password.c hash_filo.cc hostname.cc \
 			sql_connect.cc scheduler.cc sql_parse.cc \
 			set_var.cc sql_yacc.yy \
+                        sql_join_cache.cc \
 			sql_base.cc table.cc sql_select.cc sql_insert.cc \
 			sql_profile.cc \
 			sql_prepare.cc sql_error.cc sql_locale.cc \

sql/ds_mrr.cc

@@ -990,9 +990,6 @@ void push_index_cond(JOIN_TAB *tab, uint keyno, bool other_tbls_ok)
     {
       Item *idx_remainder_cond= 0;
       tab->pre_idx_push_select_cond= tab->select_cond;
-#if 0
-      /* 
-        psergey: enable the below when we backport BKA: */
       /*
         For BKA cache we store condition to special BKA cache field
         because evaluation of the condition requires additional operations
@@ -1011,7 +1008,6 @@ void push_index_cond(JOIN_TAB *tab, uint keyno, bool other_tbls_ok)
            ~(tab->table->map | tab->join->const_table_map)))
         tab->cache_idx_cond= idx_cond;
       else
-#endif
         idx_remainder_cond= tab->table->file->idx_cond_push(keyno, idx_cond);
 
       /*

sql/field.cc

@@ -1705,13 +1705,12 @@ my_decimal *Field_str::val_decimal(my_decimal *decimal_value)
 uint Field::fill_cache_field(CACHE_FIELD *copy)
 {
   uint store_length;
-  copy->str=ptr;
-  copy->length=pack_length();
-  copy->blob_field=0;
+  copy->str= ptr;
+  copy->length= pack_length();
+  copy->field= this;  
   if (flags & BLOB_FLAG)
   {
-    copy->blob_field=(Field_blob*) this;
-    copy->strip=0;
+    copy->type= CACHE_BLOB;
     copy->length-= table->s->blob_ptr_size;
     return copy->length;
   }
@@ -1719,15 +1718,21 @@ uint Field::fill_cache_field(CACHE_FIELD *copy)
            (type() == MYSQL_TYPE_STRING && copy->length >= 4 &&
             copy->length < 256))
   {
-    copy->strip=1;				/* Remove end space */
+    copy->type= CACHE_STRIPPED;			    /* Remove end space */
     store_length= 2;
   }
+  else if (type() ==  MYSQL_TYPE_VARCHAR)
+  {
+    copy->type= pack_length()-row_pack_length() == 1 ? CACHE_VARSTR1:
+                                                      CACHE_VARSTR2;
+    store_length= 0;
+  }
   else
   {
-    copy->strip=0;
+    copy->type= 0;
     store_length= 0;
   }
-  return copy->length+ store_length;
+  return copy->length+store_length;
 }
 
 

sql/item.cc

@@ -646,6 +646,16 @@ bool Item_field::collect_item_field_processor(uchar *arg)
 }
 
 
+bool Item_field::add_field_to_set_processor(uchar *arg)
+{
+  DBUG_ENTER("Item_field::add_field_to_set_processor");
+  DBUG_PRINT("info", ("%s", field->field_name ? field->field_name : "noname"));
+  TABLE *table= (TABLE *) arg;
+  if (field->table == table)
+    bitmap_set_bit(&table->tmp_set, field->field_index);
+  DBUG_RETURN(FALSE);
+}
+
 /**
   Check if an Item_field references some field from a list of fields.
 

sql/item.h

@@ -913,6 +913,7 @@ class Item {
   virtual bool remove_fixed(uchar * arg) { fixed= 0; return 0; }
   virtual bool cleanup_processor(uchar *arg);
   virtual bool collect_item_field_processor(uchar * arg) { return 0; }
+  virtual bool add_field_to_set_processor(uchar * arg) { return 0; }
   virtual bool find_item_in_field_list_processor(uchar *arg) { return 0; }
   virtual bool change_context_processor(uchar *context) { return 0; }
   virtual bool reset_query_id_processor(uchar *query_id_arg) { return 0; }
@@ -1571,6 +1572,7 @@ class Item_field :public Item_ident
   void update_null_value();
   Item *get_tmp_table_item(THD *thd);
   bool collect_item_field_processor(uchar * arg);
+  bool add_field_to_set_processor(uchar * arg);
   bool find_item_in_field_list_processor(uchar *arg);
   bool register_field_in_read_map(uchar *arg);
   bool register_field_in_bitmap(uchar *arg);

sql/mysqld.cc

@@ -5754,7 +5754,7 @@ enum options_mysqld
   OPT_DELAYED_INSERT_LIMIT, OPT_DELAYED_QUEUE_SIZE,
   OPT_FLUSH_TIME, OPT_FT_MIN_WORD_LEN, OPT_FT_BOOLEAN_SYNTAX,
   OPT_FT_MAX_WORD_LEN, OPT_FT_QUERY_EXPANSION_LIMIT, OPT_FT_STOPWORD_FILE,
-  OPT_INTERACTIVE_TIMEOUT, OPT_JOIN_BUFF_SIZE,
+  OPT_INTERACTIVE_TIMEOUT, OPT_JOIN_BUFF_SIZE, OPT_JOIN_CACHE_LEVEL,
   OPT_KEY_BUFFER_SIZE, OPT_KEY_CACHE_BLOCK_SIZE,
   OPT_KEY_CACHE_DIVISION_LIMIT, OPT_KEY_CACHE_AGE_THRESHOLD,
   OPT_LONG_QUERY_TIME,
@@ -6804,8 +6804,13 @@ log and this option does nothing anymore.",
    "The size of the buffer that is used for full joins.",
    (uchar**) &global_system_variables.join_buff_size,
    (uchar**) &max_system_variables.join_buff_size, 0, GET_ULONG,
-   REQUIRED_ARG, 128*1024L, IO_SIZE*2+MALLOC_OVERHEAD, (longlong) ULONG_MAX,
-   MALLOC_OVERHEAD, IO_SIZE, 0},
+   REQUIRED_ARG, 128*1024L, 128+MALLOC_OVERHEAD, (longlong) ULONG_MAX,
+   MALLOC_OVERHEAD, 128, 0},
+   {"join_cache_level", OPT_JOIN_CACHE_LEVEL,
+   "Controls what join operations can be executed with join buffers. Odd numbers are used for plain join buffers while even numbers are used for linked buffers",
+   (uchar**) &global_system_variables.join_cache_level,
+   (uchar**) &max_system_variables.join_cache_level,
+   0, GET_ULONG, REQUIRED_ARG, 1, 0, 8, 0, 1, 0},
   {"keep_files_on_create", OPT_KEEP_FILES_ON_CREATE,
    "Don't overwrite stale .MYD and .MYI even if no directory is specified.",
    (uchar**) &global_system_variables.keep_files_on_create,

sql/set_var.cc

@@ -313,6 +313,8 @@ static sys_var_thd_ulong	sys_interactive_timeout(&vars, "interactive_timeout",
 						&SV::net_interactive_timeout);
 static sys_var_thd_ulong	sys_join_buffer_size(&vars, "join_buffer_size",
 					     &SV::join_buff_size);
+static sys_var_thd_ulong	sys_join_cache_level(&vars, "join_cache_level",
+					             &SV::join_cache_level);
 static sys_var_key_buffer_size	sys_key_buffer_size(&vars, "key_buffer_size");
 static sys_var_key_cache_long  sys_key_cache_block_size(&vars, "key_cache_block_size",
 						 offsetof(KEY_CACHE,

sql/sql_class.h

@@ -306,6 +306,7 @@ struct system_variables
   ulong auto_increment_increment, auto_increment_offset;
   ulong bulk_insert_buff_size;
   ulong join_buff_size;
+  ulong join_cache_level;
   ulong max_allowed_packet;
   ulong max_error_count;
   ulong max_length_for_sort_data;

storage/myisam/ha_myisam.cc

@@ -1865,6 +1865,16 @@ int ha_myisam::info(uint flag)
     stats.max_data_file_length=  misam_info.max_data_file_length;
     stats.max_index_file_length= misam_info.max_index_file_length;
     stats.create_time= (ulong) misam_info.create_time;
+    /* 
+      We want the value of stats.mrr_length_per_rec to be platform independent.
+      The size of the chunk at the end of the join buffer used for MRR needs
+      is calculated now basing on the values passed in the stats structure.
+      The remaining part of the join buffer is used for records. A different
+      number of records in the buffer results in a different number of buffer
+      refills and in a different order of records in the result set.
+    */
+    stats.mrr_length_per_rec= misam_info.reflength + 8; // 8=max(sizeof(void *))
+
     ref_length= misam_info.reflength;
     share->db_options_in_use= misam_info.options;
     stats.block_size= myisam_block_size;        /* record block size */