Merge work:/home/bk/mysql into hundin.mysql.fi:/my/bk/mysql

Docs/manual.texi:
  Auto merged

Docs/manual.texi

@@ -46928,6 +46928,21 @@ not yet 100% confident in this code.
 
 @node News-3.23.54, News-3.23.53, News-3.23.x, News-3.23.x
 @appendixsubsec Changes in release 3.23.54
+@itemize
+@item
+Fixed reference to freed memory when doing complicated @code{GROUP BY
+... ORDER BY} queries.  Symptom was that @code{mysqld} died in function
+@code{send_fields}.
+@item
+Allocate heap rows in smaller blocks to get better memory usage.
+@item
+Fixed memory allocation bug when storing BLOB values in internal
+temporary tables used for some (unlikely) @code{GROUP BY} queries.
+@item
+Fixed a bug in key optimizing handling where the expression
+@code{WHERE column_name = key_column_name} was calculated as true
+for @code{NULL} values.
+@end itemize
 
 @node News-3.23.53, News-3.23.52, News-3.23.54, News-3.23.x
 @appendixsubsec Changes in release 3.23.53

heap/heapdef.h

@@ -22,6 +22,17 @@
 #endif
 #include "heap.h"			/* Structs & some defines */
 
+/*
+  When allocating keys /rows in the internal block structure, do it
+  within the following boundaries.
+
+  The challenge is to find the balance between allocate as few blocks
+  as possible and keep memory consumption down.
+*/
+
+#define HP_MIN_RECORDS_IN_BLOCK 16
+#define HP_MAX_RECORDS_IN_BLOCK 8192
+
 	/* Some extern variables */
 
 extern LIST *heap_open_list,*heap_share_list;

heap/hp_open.c

@@ -157,8 +157,14 @@ static void init_block(HP_BLOCK *block, uint reclength, ulong min_records,
     max_records=1000;			/* As good as quess as anything */
   recbuffer=(uint) (reclength+sizeof(byte**)-1) & ~(sizeof(byte**)-1);
   records_in_block=max_records/10;
-  if (records_in_block < 10 && max_records)
-    records_in_block=10;
+  if (records_in_block < HP_MIN_RECORDS_IN_BLOCK && max_records)
+    records_in_block= HP_MIN_RECORDS_IN_BLOCK;
+  /*
+    Don't allocate too many rows at one time too keep memory consumption
+    done when we don't need it.
+  */
+  if (records_in_block > HP_MAX_RECORDS_IN_BLOCK)
+    records_in_block= HP_MAX_RECORDS_IN_BLOCK;
   if (!records_in_block || records_in_block*recbuffer >
       (my_default_record_cache_size-sizeof(HP_PTRS)*HP_MAX_LEVELS))
     records_in_block=(my_default_record_cache_size-sizeof(HP_PTRS)*

mysql-test/r/distinct.result

@@ -128,7 +128,7 @@ a
 1
 table	type	possible_keys	key	key_len	ref	rows	Extra
 t1	index	PRIMARY	PRIMARY	4	NULL	2	Using index; Using temporary
-t3	ref	a	a	5	t1.a	10	Using index; Distinct
+t3	ref	a	a	5	t1.a	10	where used; Using index; Distinct
 a
 1
 2

mysql-test/r/null_key.result

@@ -120,3 +120,9 @@ id	uniq_id
 4	2
 7	3
 8	4
+order_id	product_id	product_type
+order_id	product_id	product_type
+3d7ce39b5d4b3e3d22aaafe9b633de51	1206029	3
+3d7ce39b5d4b3e3d22aaafe9b633de51	5880836	3
+id	id
+id	id

mysql-test/t/null_key.test

@@ -91,3 +91,47 @@ DELETE FROM t2 WHERE uniq_id IS NULL;
 SELECT * FROM t1 ORDER BY uniq_id, id;
 SELECT * FROM t2 ORDER BY uniq_id, id;
 DROP table t1,t2;
+
+#
+# This crashed MySQL 3.23.47
+#
+
+CREATE TABLE `t1` (
+  `order_id` char(32) NOT NULL default '',
+  `product_id` char(32) NOT NULL default '',
+  `product_type` int(11) NOT NULL default '0',
+  PRIMARY KEY  (`order_id`,`product_id`,`product_type`)
+) TYPE=MyISAM;
+CREATE TABLE `t2` (
+  `order_id` char(32) NOT NULL default '',
+  `product_id` char(32) NOT NULL default '',
+  `product_type` int(11) NOT NULL default '0',
+  PRIMARY KEY  (`order_id`,`product_id`,`product_type`)
+) TYPE=MyISAM;
+INSERT INTO t1 (order_id, product_id, product_type) VALUES
+('3d7ce39b5d4b3e3d22aaafe9b633de51',1206029, 3),
+('3d7ce39b5d4b3e3d22aaafe9b633de51',5880836, 3),
+('9d9aad7764b5b2c53004348ef8d34500',2315652, 3);
+INSERT INTO t2 (order_id, product_id, product_type) VALUES
+('9d9aad7764b5b2c53004348ef8d34500',2315652, 3);
+
+select t1.* from t1
+left join t2 using(order_id, product_id, product_type)
+where t2.order_id=NULL;
+select t1.* from t1
+left join t2 using(order_id, product_id, product_type)
+where t2.order_id is NULL;
+drop table t1,t2;
+
+#
+# The last select returned wrong results in 3.23.52
+#
+
+create table t1 (id int);
+insert into t1 values (null), (0);
+create table t2 (id int);
+insert into t2 values (null);
+select * from t1, t2 where t1.id = t2.id;
+alter table t1 add key id (id);
+select * from t1, t2 where t1.id = t2.id;
+drop table t1,t2;

mysys/safemalloc.c

@@ -194,9 +194,7 @@ gptr _mymalloc (uint uSize, const char *sFile, uint uLine, myf MyFlags)
     pthread_mutex_unlock(&THR_LOCK_malloc);
 
     /* Set the memory to the aribtrary wierd value */
-#ifdef HAVE_purify
-    if (MyFlags & MY_ZEROFILL)
-#endif
+    if ((MyFlags & MY_ZEROFILL) || !sf_malloc_quick)
       bfill(&pTmp -> aData[sf_malloc_prehunc],uSize,
 	    (char) (MyFlags & MY_ZEROFILL ? 0 : ALLOC_VAL));
     /* Return a pointer to the real data */
@@ -315,7 +313,8 @@ void _myfree (gptr pPtr, const char *sFile, uint uLine, myf myflags)
 
 #ifndef HAVE_purify
   /* Mark this data as free'ed */
-  bfill(&pRec->aData[sf_malloc_prehunc],pRec->uDataSize,(pchar) FREE_VAL);
+  if (!sf_malloc_quick)
+    bfill(&pRec->aData[sf_malloc_prehunc],pRec->uDataSize,(pchar) FREE_VAL);
 #endif
   *((long*) ((char*) &pRec -> lSpecialValue+sf_malloc_prehunc)) = ~MAGICKEY;
 

sql/sql_select.cc

@@ -3319,7 +3319,7 @@ create_tmp_table(THD *thd,TMP_TABLE_PARAM *param,List<Item> &fields,
   char	*tmpname,path[FN_REFLEN];
   byte	*pos,*group_buff;
   uchar *null_flags;
-  Field **reg_field,**from_field;
+  Field **reg_field, **from_field, **blob_field;
   Copy_field *copy=0;
   KEY *keyinfo;
   KEY_PART_INFO *key_part_info;
@@ -3364,8 +3364,9 @@ create_tmp_table(THD *thd,TMP_TABLE_PARAM *param,List<Item> &fields,
   hidden_field_count=param->hidden_field_count;
   if (!my_multi_malloc(MYF(MY_WME),
 		       &table,sizeof(*table),
-		       &reg_field,sizeof(Field*)*(field_count+1),
-		       &from_field,sizeof(Field*)*field_count,
+		       &reg_field,  sizeof(Field*)*(field_count+1),
+		       &blob_field, sizeof(Field*)*(field_count+1),
+		       &from_field, sizeof(Field*)*field_count,
 		       &copy_func,sizeof(*copy_func)*(param->func_count+1),
 		       &param->keyinfo,sizeof(*param->keyinfo),
 		       &key_part_info,
@@ -3394,8 +3395,12 @@ create_tmp_table(THD *thd,TMP_TABLE_PARAM *param,List<Item> &fields,
   bzero((char*) reg_field,sizeof(Field*)*(field_count+1));
   bzero((char*) from_field,sizeof(Field*)*field_count);
   table->field=reg_field;
+  table->blob_field= (Field_blob**) blob_field;
   table->real_name=table->path=tmpname;
-  table->table_name=base_name(tmpname);
+  /*
+    This must be "" as field may refer to it after tempory table is dropped
+  */
+  table->table_name= (char*) "";
   table->reginfo.lock_type=TL_WRITE;	/* Will be updated */
   table->db_stat=HA_OPEN_KEYFILE+HA_OPEN_RNDFILE;
   table->blob_ptr_size=mi_portable_sizeof_char_ptr;
@@ -3403,7 +3408,7 @@ create_tmp_table(THD *thd,TMP_TABLE_PARAM *param,List<Item> &fields,
   table->tmp_table= TMP_TABLE;
   table->db_low_byte_first=1;			// True for HEAP and MyISAM
   table->temp_pool_slot = temp_pool_slot;
-
+  table->copy_blobs= 1;
 
   /* Calculate which type of fields we will store in the temporary table */
 
@@ -3447,7 +3452,10 @@ create_tmp_table(THD *thd,TMP_TABLE_PARAM *param,List<Item> &fields,
 	  if (!(new_field->flags & NOT_NULL_FLAG))
 	    null_count++;
 	  if (new_field->flags & BLOB_FLAG)
+	  {
+	    *blob_field++= new_field;
 	    blob_count++;
+	  }
 	  ((Item_sum*) item)->args[i]= new Item_field(new_field);
 	}
       }
@@ -3469,7 +3477,10 @@ create_tmp_table(THD *thd,TMP_TABLE_PARAM *param,List<Item> &fields,
       if (!(new_field->flags & NOT_NULL_FLAG))
 	null_count++;
       if (new_field->flags & BLOB_FLAG)
+      {
+	*blob_field++= new_field;
 	blob_count++;
+      }
       if (item->marker == 4 && item->maybe_null)
       {
 	group_null_items++;
@@ -3481,6 +3492,7 @@ create_tmp_table(THD *thd,TMP_TABLE_PARAM *param,List<Item> &fields,
       hidden_null_count=null_count;
   }
   field_count= (uint) (reg_field - table->field);
+  *blob_field= 0;				// End marker
 
   /* If result table is small; use a heap */
   if (blob_count || using_unique_constraint || group_null_items ||
@@ -3731,6 +3743,8 @@ create_tmp_table(THD *thd,TMP_TABLE_PARAM *param,List<Item> &fields,
     if (create_myisam_tmp_table(table,param,select_options))
       goto err;
   }
+  /* Set table_name for easier debugging */
+  table->table_name= base_name(tmpname);
   if (!open_tmp_table(table))
     DBUG_RETURN(table);
 
@@ -3877,6 +3891,7 @@ free_tmp_table(THD *thd, TABLE *entry)
 
   save_proc_info=thd->proc_info;
   thd->proc_info="removing tmp table";
+  free_blobs(entry);
   if (entry->db_stat && entry->file)
   {
     (void) entry->file->close();

sql/table.cc

@@ -455,6 +455,13 @@ int openfrm(const char *name, const char *alias, uint db_stat, uint prgflag,
 	      field->field_length=key_part->length;
 	    }
 	  }
+	  /*
+	    If the field can be NULL, don't optimize away the test
+	    key_part_column = expression from the WHERE clause
+	    as we need to test for NULL = NULL.
+	  */
+	  if (field->real_maybe_null())
+	    key_part->key_part_flag|= HA_PART_KEY;
 	}
 	else
 	{					// Error: shorten key