Fixed LP bug#672497.

Miscalculation of the minimum possible buffer size could trigger
an assert in JOIN_CACHE_HASHED::put_record when if join_buffer_size
was set to the values that is less than the length of one record to
stored in the join buffer.
It happened due to the following mistakes:
- underestimation of space needed for a key in the hash table
  (we have to take into account that hash table can have more
  buckets than the expected number of records).
- the value of maximum total length of all records stored in
  the join buffer was not saved in the field max_used_fieldlength
  by the function calc_used_field_length.

mysql-test/r/join_cache.result

@@ -5675,4 +5675,59 @@ f	5
 f	5
 SET SESSION join_cache_level = DEFAULT;
 DROP TABLE t1,t2;
+#
+# Bug #672497: 3 way join with tiny incremental join buffer with
+#              and a ref access from the first table 
+#
+CREATE TABLE t1 (
+pk int PRIMARY KEY,
+v varchar(10) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL,
+INDEX idx (v)
+);
+INSERT INTO t1 VALUES
+(1,'abcdefjhjk'), (2,'i'),(3,'abcdefjhjk'), (4,'well'), (5,'abcdefjhjk'),
+(6,'abcdefjhjk'), (7,'that');
+CREATE TABLE t2 (
+pk int PRIMARY KEY,
+i int DEFAULT NULL,
+v varchar(1000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL,
+INDEX idx (v)
+);
+INSERT INTO t2 VALUES 
+(1,6,'yes'), (2,NULL,'will'), (3,NULL,'o'), (4,NULL,'k'), (5,NULL,'she'),
+(6,-1450835968,'abcdefjhjkl'), (7,-975831040,'abcdefjhjkl'), (8,NULL,'z'),
+(10,-343932928,'t'),
+(11,6,'yes'), (12,NULL,'will'), (13,NULL,'o'), (14,NULL,'k'), (15,NULL,'she'),
+(16,-1450835968,'abcdefjhjkl'), (17,-975831040,'abcdefjhjkl'), (18,NULL,'z'),
+(19,-343932928,'t');
+CREATE TABLE t3 (
+pk int NOT NULL PRIMARY KEY,
+i int,
+v varchar(1024) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL,
+INDEX idx (v(333))
+);
+INSERT INTO t3 VALUES 
+(1,7,'abcdefjhjkl'),(2,6,'y'), (3,NULL,'to'),(4,7,'n'),(5,7,'look'), (6,NULL,'all'),
+(7,1443168256,'c'), (8,1427046400,'right'),
+(11,7,'abcdefjhjkl'), (12,6,'y'), (13,NULL,'to'), (14,7,'n'), (15,7,'look'),
+(16,NULL,'all'), (17,1443168256,'c'), (18,1427046400,'right'),
+(21,7,'abcdefjhjkl'), (22,6,'y'), (23,NULL,'to'), (24,7,'n'), (25,7,'look'),
+(26,NULL,'all'), (27,1443168256,'c'), (28,1427046400,'right'),
+(31,7,'abcdefjhjkl'), (32,6,'y'), (33,NULL,'to'), (34,7,'n'), (35,7,'look'),
+(36,NULL,'all'), (37,1443168256,'c'), (38,1427046400,'right');
+SET SESSION join_buffer_size = 192;
+SET SESSION join_cache_level = 4;
+EXPLAIN
+SELECT t3.i FROM t1,t2,t3
+WHERE t1.v = t2.v AND t3.v = t1.v AND t2.i <> 0;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	index	idx	idx	13	NULL	7	Using where; Using index
+1	SIMPLE	t2	ref	idx	idx	1003	test.t1.v	2	Using where; Using join buffer (flat, BNLH join)
+1	SIMPLE	t3	ref	idx	idx	1002	func	3	Using where; Using join buffer (incremental, BNLH join)
+SELECT t3.i FROM t1,t2,t3
+WHERE t1.v = t2.v AND t3.v = t1.v AND t2.i <> 0;
+i
+SET SESSION join_cache_level = DEFAULT;
+SET SESSION join_buffer_size = DEFAULT;
+DROP TABLE t1,t2,t3;
 set @@optimizer_switch=@save_optimizer_switch;

mysql-test/t/join_cache.test

@@ -2358,5 +2358,63 @@ SET SESSION join_cache_level = DEFAULT;
 
 DROP TABLE t1,t2;
 
+--echo #
+--echo # Bug #672497: 3 way join with tiny incremental join buffer with
+--echo #              and a ref access from the first table 
+--echo #
+
+CREATE TABLE t1 (
+  pk int PRIMARY KEY,
+  v varchar(10) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL,
+  INDEX idx (v)
+);
+INSERT INTO t1 VALUES
+  (1,'abcdefjhjk'), (2,'i'),(3,'abcdefjhjk'), (4,'well'), (5,'abcdefjhjk'),
+  (6,'abcdefjhjk'), (7,'that');
+
+CREATE TABLE t2 (
+  pk int PRIMARY KEY,
+  i int DEFAULT NULL,
+  v varchar(1000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL,
+  INDEX idx (v)
+);
+INSERT INTO t2 VALUES 
+ (1,6,'yes'), (2,NULL,'will'), (3,NULL,'o'), (4,NULL,'k'), (5,NULL,'she'),
+ (6,-1450835968,'abcdefjhjkl'), (7,-975831040,'abcdefjhjkl'), (8,NULL,'z'),
+ (10,-343932928,'t'),
+ (11,6,'yes'), (12,NULL,'will'), (13,NULL,'o'), (14,NULL,'k'), (15,NULL,'she'),
+ (16,-1450835968,'abcdefjhjkl'), (17,-975831040,'abcdefjhjkl'), (18,NULL,'z'),
+ (19,-343932928,'t');
+
+CREATE TABLE t3 (
+  pk int NOT NULL PRIMARY KEY,
+  i int,
+  v varchar(1024) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL,
+  INDEX idx (v(333))
+);
+INSERT INTO t3 VALUES 
+(1,7,'abcdefjhjkl'),(2,6,'y'), (3,NULL,'to'),(4,7,'n'),(5,7,'look'), (6,NULL,'all'),
+(7,1443168256,'c'), (8,1427046400,'right'),
+(11,7,'abcdefjhjkl'), (12,6,'y'), (13,NULL,'to'), (14,7,'n'), (15,7,'look'),
+(16,NULL,'all'), (17,1443168256,'c'), (18,1427046400,'right'),
+(21,7,'abcdefjhjkl'), (22,6,'y'), (23,NULL,'to'), (24,7,'n'), (25,7,'look'),
+(26,NULL,'all'), (27,1443168256,'c'), (28,1427046400,'right'),
+(31,7,'abcdefjhjkl'), (32,6,'y'), (33,NULL,'to'), (34,7,'n'), (35,7,'look'),
+(36,NULL,'all'), (37,1443168256,'c'), (38,1427046400,'right');
+
+SET SESSION join_buffer_size = 192;
+
+SET SESSION join_cache_level = 4;
+EXPLAIN
+SELECT t3.i FROM t1,t2,t3
+  WHERE t1.v = t2.v AND t3.v = t1.v AND t2.i <> 0;
+SELECT t3.i FROM t1,t2,t3
+  WHERE t1.v = t2.v AND t3.v = t1.v AND t2.i <> 0;
+
+SET SESSION join_cache_level = DEFAULT;
+SET SESSION join_buffer_size = DEFAULT;
+
+DROP TABLE t1,t2,t3;
+
 # this must be the last command in the file
 set @@optimizer_switch=@save_optimizer_switch;

sql/sql_join_cache.cc

@@ -2633,10 +2633,15 @@ uint JOIN_CACHE_HASHED::get_max_key_addon_space_per_record()
 {
   ulong len;
   TABLE_REF *ref= &join_tab->ref;
+  /* 
+    The total number of hash entries in the hash tables is bounded by
+    ceiling(N/0.7) where N is the maximum number of records in the buffer.
+    That's why the multiplier 2 is used in the formula below. 
+  */ 
   len= (use_emb_key ?  get_size_of_rec_offset() : ref->key_length) +
         size_of_rec_ofs +    // size of the key chain header
         size_of_rec_ofs +    // >= size of the reference to the next key 
-        size_of_rec_ofs;     // >= size of hash table entry
+        2*size_of_rec_ofs;   // >= 2*( size of hash table entry)
   return len; 
 }    
 

sql/sql_select.cc

@@ -5925,16 +5925,17 @@ void JOIN_TAB::calc_used_field_length(bool max_fl)
     rec_length+=sizeof(my_bool);
   if (max_fl)
   {
-    // TODO: to improve this estimate for max expected length if the record
+    // TODO: to improve this estimate for max expected length 
     if (blobs)
     {
       uint blob_length=(uint) (table->file->stats.mean_rec_length-
 			      (table->s->reclength-rec_length));
-      rec_length+=(uint) max(4,blob_length);
+      rec_length+=(uint) max(sizeof(void*) * blobs, blob_length);
     }
+    max_used_fieldlength= rec_length;
   } 
-  else
-    rec_length= table->file->stats.mean_rec_length;
+  else if (table->file->stats.mean_rec_length)           
+    set_if_smaller(rec_length, table->file->stats.mean_rec_length);
       
   /*
     psergey-todo: why we don't count here rowid that we might need to store