Added support for 'internal temporary tables' in HEAP tables.

Now we don't take any mutexes when creating or dropping internal HEAP tables during SELECT.
Change buffer sizes to size_t to make keycache 64 bit safe on platforms where sizeof(ulong) != sizeof(size_t)

.bzrignore

@@ -2995,3 +2995,4 @@ win/vs71cache.txt
 win/vs8cache.txt
 zlib/*.ds?
 zlib/*.vcproj
+support-files/mysqld_multi.server

include/heap.h

@@ -189,11 +189,14 @@ typedef struct st_heap_create_info
   ulonglong max_table_size;
   ulonglong auto_increment;
   my_bool with_auto_increment;
+  my_bool internal_table;
 } HP_CREATE_INFO;
 
 	/* Prototypes for heap-functions */
 
 extern HP_INFO *heap_open(const char *name, int mode);
+extern HP_INFO *heap_open_from_share(HP_SHARE *share, int mode);
+extern HP_INFO *heap_open_from_share_and_register(HP_SHARE *share, int mode);
 extern int heap_close(HP_INFO *info);
 extern int heap_write(HP_INFO *info,const uchar *buff);
 extern int heap_update(HP_INFO *info,const uchar *old,const uchar *newdata);
@@ -204,7 +207,7 @@ extern int heap_delete(HP_INFO *info,const uchar *buff);
 extern int heap_info(HP_INFO *info,HEAPINFO *x,int flag);
 extern int heap_create(const char *name, uint keys, HP_KEYDEF *keydef,
 		       uint reclength, ulong max_records, ulong min_records,
-		       HP_CREATE_INFO *create_info);
+		       HP_CREATE_INFO *create_info, HP_SHARE **share);
 extern int heap_delete_table(const char *name);
 extern void heap_drop_table(HP_INFO *info);
 extern int heap_extra(HP_INFO *info,enum ha_extra_function function);

include/keycache.h

@@ -47,7 +47,7 @@ typedef struct st_key_cache
   my_bool in_resize;             /* true during resize operation             */
   my_bool resize_in_flush;       /* true during flush of resize operation    */
   my_bool can_be_used;           /* usage of cache for read/write is allowed */
-  ulong key_cache_mem_size;      /* specified size of the cache memory       */
+  size_t key_cache_mem_size;      /* specified size of the cache memory       */
   uint key_cache_block_size;     /* size of the page buffer of a cache block */
   ulong min_warm_blocks;         /* min number of warm blocks;               */
   ulong age_threshold;           /* age threshold for hot blocks             */
@@ -107,10 +107,10 @@ typedef struct st_key_cache
 extern KEY_CACHE dflt_key_cache_var, *dflt_key_cache;
 
 extern int init_key_cache(KEY_CACHE *keycache, uint key_cache_block_size,
-			  ulong use_mem, uint division_limit,
+			  size_t use_mem, uint division_limit,
 			  uint age_threshold);
 extern int resize_key_cache(KEY_CACHE *keycache, uint key_cache_block_size,
-			    ulong use_mem, uint division_limit,
+			    size_t use_mem, uint division_limit,
 			    uint age_threshold);
 extern void change_key_cache_param(KEY_CACHE *keycache, uint division_limit,
 				   uint age_threshold);

include/my_base.h

@@ -48,6 +48,8 @@
 #define HA_OPEN_FOR_REPAIR		32	/* open even if crashed */
 #define HA_OPEN_FROM_SQL_LAYER          64
 #define HA_OPEN_MMAP                    128     /* open memory mapped */
+/* Internal temp table, used for temporary results */
+#define HA_OPEN_INTERNAL_TABLE          256
 
 	/* The following is parameter to ha_rkey() how to use key */
 

mysys/mf_keycache.c

@@ -366,10 +366,11 @@ static inline uint next_power(uint value)
 */
 
 int init_key_cache(KEY_CACHE *keycache, uint key_cache_block_size,
-		   ulong use_mem, uint division_limit,
+		   size_t use_mem, uint division_limit,
 		   uint age_threshold)
 {
-  uint blocks, hash_links, length;
+  ulong blocks, hash_links;
+  size_t length;
   int error;
   DBUG_ENTER("init_key_cache");
   DBUG_ASSERT(key_cache_block_size >= 512);
@@ -405,8 +406,8 @@ int init_key_cache(KEY_CACHE *keycache, uint key_cache_block_size,
   DBUG_PRINT("info", ("key_cache_block_size: %u",
 		      key_cache_block_size));
 
-  blocks= (uint) (use_mem / (sizeof(BLOCK_LINK) + 2 * sizeof(HASH_LINK) +
-			     sizeof(HASH_LINK*) * 5/4 + key_cache_block_size));
+  blocks= (ulong) (use_mem / (sizeof(BLOCK_LINK) + 2 * sizeof(HASH_LINK) +
+                              sizeof(HASH_LINK*) * 5/4 + key_cache_block_size));
   /* It doesn't make sense to have too few blocks (less than 8) */
   if (blocks >= 8)
   {
@@ -424,18 +425,18 @@ int init_key_cache(KEY_CACHE *keycache, uint key_cache_block_size,
 		       ALIGN_SIZE(hash_links * sizeof(HASH_LINK)) +
 		       ALIGN_SIZE(sizeof(HASH_LINK*) *
                                   keycache->hash_entries))) +
-	     ((ulong) blocks * keycache->key_cache_block_size) > use_mem)
+	     ((size_t) blocks * keycache->key_cache_block_size) > use_mem)
         blocks--;
       /* Allocate memory for cache page buffers */
       if ((keycache->block_mem=
-	   my_large_malloc((ulong) blocks * keycache->key_cache_block_size,
+	   my_large_malloc((size_t) blocks * keycache->key_cache_block_size,
 			  MYF(MY_WME))))
       {
         /*
 	  Allocate memory for blocks, hash_links and hash entries;
 	  For each block 2 hash links are allocated
         */
-        if ((keycache->block_root= (BLOCK_LINK*) my_malloc((uint) length,
+        if ((keycache->block_root= (BLOCK_LINK*) my_malloc(length,
                                                            MYF(0))))
           break;
         my_large_free(keycache->block_mem, MYF(0));
@@ -448,7 +449,7 @@ int init_key_cache(KEY_CACHE *keycache, uint key_cache_block_size,
       }
       blocks= blocks / 4*3;
     }
-    keycache->blocks_unused= (ulong) blocks;
+    keycache->blocks_unused= blocks;
     keycache->disk_blocks= (int) blocks;
     keycache->hash_links= hash_links;
     keycache->hash_root= (HASH_LINK**) ((char*) keycache->block_root +
@@ -556,7 +557,7 @@ int init_key_cache(KEY_CACHE *keycache, uint key_cache_block_size,
 */
 
 int resize_key_cache(KEY_CACHE *keycache, uint key_cache_block_size,
-		     ulong use_mem, uint division_limit,
+		     size_t use_mem, uint division_limit,
 		     uint age_threshold)
 {
   int blocks;

sql/sql_select.cc

@@ -10198,7 +10198,7 @@ static bool open_tmp_table(TABLE *table)
 {
   int error;
   if ((error=table->file->ha_open(table, table->s->table_name.str,O_RDWR,
-                                  HA_OPEN_TMP_TABLE)))
+                                  HA_OPEN_TMP_TABLE | HA_OPEN_INTERNAL_TABLE)))
   {
     table->file->print_error(error,MYF(0)); /* purecov: inspected */
     table->db_stat=0;
@@ -10436,8 +10436,7 @@ bool create_myisam_from_heap(THD *thd, TABLE *table, TMP_TABLE_PARAM *param,
 
   /* remove heap table and change to use myisam table */
   (void) table->file->ha_rnd_end();
-  (void) table->file->close();
-  (void) table->file->delete_table(table->s->table_name.str);
+  (void) table->file->close();                  // This deletes the table !
   delete table->file;
   table->file=0;
   plugin_unlock(0, table->s->db_plugin);

storage/heap/ha_heap.cc

@@ -22,7 +22,7 @@
 #include "mysql_priv.h"
 #include <mysql/plugin.h>
 #include "ha_heap.h"
-
+#include "heapdef.h"
 
 static handler *heap_create_handler(handlerton *hton,
                                     TABLE_SHARE *table, 
@@ -61,8 +61,8 @@ static handler *heap_create_handler(handlerton *hton,
 *****************************************************************************/
 
 ha_heap::ha_heap(handlerton *hton, TABLE_SHARE *table_arg)
-  :handler(hton, table_arg), file(0), records_changed(0),
-  key_stat_version(0)
+  :handler(hton, table_arg), file(0), records_changed(0), internal_table(0),
+   key_stat_version(0)
 {}
 
 
@@ -90,13 +90,25 @@ const char **ha_heap::bas_ext() const
 
 int ha_heap::open(const char *name, int mode, uint test_if_locked)
 {
-  if (!(file= heap_open(name, mode)) && my_errno == ENOENT)
+  if ((test_if_locked & HA_OPEN_INTERNAL_TABLE) ||
+      !(file= heap_open(name, mode)) && my_errno == ENOENT)
   {
     HA_CREATE_INFO create_info;
+    internal_table= test(test_if_locked & HA_OPEN_INTERNAL_TABLE);
     bzero(&create_info, sizeof(create_info));
+    file= 0;
     if (!create(name, table, &create_info))
     {
-      file= heap_open(name, mode);
+        file= internal_table ?
+          heap_open_from_share(internal_share, mode) :
+          heap_open_from_share_and_register(internal_share, mode);
+      if (!file)
+      {
+         /* Couldn't open table; Remove the newly created table */
+        pthread_mutex_lock(&THR_LOCK_heap);
+        hp_free(internal_share);
+        pthread_mutex_unlock(&THR_LOCK_heap);
+      }
       implicit_emptied= 1;
     }
   }
@@ -120,7 +132,7 @@ int ha_heap::open(const char *name, int mode, uint test_if_locked)
 
 int ha_heap::close(void)
 {
-  return heap_close(file);
+  return internal_table ? hp_close(file) : heap_close(file);
 }
 
 
@@ -542,7 +554,7 @@ int ha_heap::delete_table(const char *name)
 
 void ha_heap::drop_table(const char *name)
 {
-  heap_drop_table(file);
+  file->s->delete_on_close= 1;
   close();
 }
 
@@ -681,16 +693,16 @@ int ha_heap::create(const char *name, TABLE *table_arg,
 				  create_info->auto_increment_value - 1 : 0);
   hp_create_info.max_table_size=current_thd->variables.max_heap_table_size;
   hp_create_info.with_auto_increment= found_real_auto_increment;
+  hp_create_info.internal_table= internal_table;
   max_rows = (ha_rows) (hp_create_info.max_table_size / mem_per_row);
   error= heap_create(name,
 		     keys, keydef, share->reclength,
 		     (ulong) ((share->max_rows < max_rows &&
 			       share->max_rows) ? 
 			      share->max_rows : max_rows),
-		     (ulong) share->min_rows, &hp_create_info);
+		     (ulong) share->min_rows, &hp_create_info, &internal_share);
   my_free((uchar*) keydef, MYF(0));
-  if (file)
-    info(HA_STATUS_NO_LOCK | HA_STATUS_CONST | HA_STATUS_VARIABLE);
+  DBUG_ASSERT(file == 0);
   return (error);
 }
 

storage/heap/ha_heap.h

@@ -25,10 +25,12 @@
 class ha_heap: public handler
 {
   HP_INFO *file;
+  HP_SHARE *internal_share;
   key_map btree_keys;
   /* number of records changed since last statistics update */
   uint    records_changed;
   uint    key_stat_version;
+  my_bool internal_table;
 public:
   ha_heap(handlerton *hton, TABLE_SHARE *table);
   ~ha_heap() {}

storage/heap/heapdef.h

@@ -16,6 +16,7 @@
 /* This file is included in all heap-files */
 
 #include <my_base.h>			/* This includes global */
+C_MODE_START
 #ifdef THREAD
 #include <my_pthread.h>
 #endif
@@ -107,3 +108,4 @@ extern pthread_mutex_t THR_LOCK_heap;
 #define pthread_mutex_lock(A)
 #define pthread_mutex_unlock(A)
 #endif
+C_MODE_END

storage/heap/hp_close.c

@@ -42,7 +42,8 @@ int hp_close(register HP_INFO *info)
   }
 #endif
   info->s->changed=0;
-  heap_open_list=list_delete(heap_open_list,&info->open_list);
+  if (info->open_list.data)
+    heap_open_list=list_delete(heap_open_list,&info->open_list);
   if (!--info->s->open_count && info->s->delete_on_close)
     hp_free(info->s);				/* Table was deleted */
   my_free((uchar*) info,MYF(0));

storage/heap/hp_create.c

@@ -19,23 +19,27 @@ static int keys_compare(heap_rb_param *param, uchar *key1, uchar *key2);
 static void init_block(HP_BLOCK *block,uint reclength,ulong min_records,
 		       ulong max_records);
 
+/* Create a heap table */
+
 int heap_create(const char *name, uint keys, HP_KEYDEF *keydef,
 		uint reclength, ulong max_records, ulong min_records,
-		HP_CREATE_INFO *create_info)
+		HP_CREATE_INFO *create_info, HP_SHARE **res)
 {
   uint i, j, key_segs, max_length, length;
-  HP_SHARE *share;
+  HP_SHARE *share= 0;
   HA_KEYSEG *keyseg;
-  
   DBUG_ENTER("heap_create");
-  pthread_mutex_lock(&THR_LOCK_heap);
 
-  if ((share= hp_find_named_heap(name)) && share->open_count == 0)
+  if (!create_info->internal_table)
   {
-    hp_free(share);
-    share= NULL;
-  }
-  
+    pthread_mutex_lock(&THR_LOCK_heap);
+    if ((share= hp_find_named_heap(name)) && share->open_count == 0)
+    {
+      hp_free(share);
+      share= 0;
+    }
+  }  
+
   if (!share)
   {
     HP_KEYDEF *keyinfo;
@@ -131,10 +135,7 @@ int heap_create(const char *name, uint keys, HP_KEYDEF *keydef,
 				       keys*sizeof(HP_KEYDEF)+
 				       key_segs*sizeof(HA_KEYSEG),
 				       MYF(MY_ZEROFILL))))
-    {
-      pthread_mutex_unlock(&THR_LOCK_heap);
-      DBUG_RETURN(1);
-    }
+      goto err;
     share->keydef= (HP_KEYDEF*) (share + 1);
     share->key_stat_version= 1;
     keyseg= (HA_KEYSEG*) (share->keydef + keys);
@@ -189,20 +190,33 @@ int heap_create(const char *name, uint keys, HP_KEYDEF *keydef,
     if (!(share->name= my_strdup(name,MYF(0))))
     {
       my_free((uchar*) share,MYF(0));
-      pthread_mutex_unlock(&THR_LOCK_heap);
-      DBUG_RETURN(1);
+      goto err;
     }
 #ifdef THREAD
     thr_lock_init(&share->lock);
     VOID(pthread_mutex_init(&share->intern_lock,MY_MUTEX_INIT_FAST));
 #endif
-    share->open_list.data= (void*) share;
-    heap_share_list= list_add(heap_share_list,&share->open_list);
+    if (!create_info->internal_table)
+    {
+      share->open_list.data= (void*) share;
+      heap_share_list= list_add(heap_share_list,&share->open_list);
+    }
+    else
+      share->delete_on_close= 1;
   }
-  pthread_mutex_unlock(&THR_LOCK_heap);
+  if (!create_info->internal_table)
+    pthread_mutex_unlock(&THR_LOCK_heap);
+
+  *res= share;
   DBUG_RETURN(0);
+
+err:
+  if (!create_info->internal_table)
+    pthread_mutex_unlock(&THR_LOCK_heap);
+  DBUG_RETURN(1);
 } /* heap_create */
 
+
 static int keys_compare(heap_rb_param *param, uchar *key1, uchar *key2)
 {
   uint not_used[2];
@@ -279,7 +293,8 @@ void heap_drop_table(HP_INFO *info)
 
 void hp_free(HP_SHARE *share)
 {
-  heap_share_list= list_delete(heap_share_list, &share->open_list);
+  if (share->open_list.data)                    /* If not internal table */
+    heap_share_list= list_delete(heap_share_list, &share->open_list);
   hp_clear(share);			/* Remove blocks from memory */
 #ifdef THREAD
   thr_lock_delete(&share->lock);

storage/heap/hp_open.c

@@ -22,43 +22,34 @@
 
 #include "my_sys.h"
 
-HP_INFO *heap_open(const char *name, int mode)
+/*
+  Open heap table based on HP_SHARE structure
+  
+  NOTE
+    This doesn't register the table in the open table list.
+*/
+
+HP_INFO *heap_open_from_share(HP_SHARE *share, int mode)
 {
   HP_INFO *info;
-  HP_SHARE *share;
+  DBUG_ENTER("heap_open_from_share");
 
-  DBUG_ENTER("heap_open");
-  pthread_mutex_lock(&THR_LOCK_heap);
-  if (!(share= hp_find_named_heap(name)))
-  {
-    my_errno= ENOENT;
-    pthread_mutex_unlock(&THR_LOCK_heap);
-    DBUG_RETURN(0);
-  }
   if (!(info= (HP_INFO*) my_malloc((uint) sizeof(HP_INFO) +
 				  2 * share->max_key_length,
 				  MYF(MY_ZEROFILL))))
   {
-    pthread_mutex_unlock(&THR_LOCK_heap);
     DBUG_RETURN(0);
   }
   share->open_count++; 
 #ifdef THREAD
   thr_lock_data_init(&share->lock,&info->lock,NULL);
 #endif
-  info->open_list.data= (void*) info;
-  heap_open_list= list_add(heap_open_list,&info->open_list);
-  pthread_mutex_unlock(&THR_LOCK_heap);
-
   info->s= share;
   info->lastkey= (uchar*) (info + 1);
   info->recbuf= (uchar*) (info->lastkey + share->max_key_length);
   info->mode= mode;
   info->current_record= (ulong) ~0L;		/* No current record */
-  info->current_ptr= 0;
-  info->current_hash_ptr= 0;
   info->lastinx= info->errkey= -1;
-  info->update= 0;
 #ifndef DBUG_OFF
   info->opt_flag= READ_CHECK_USED;		/* Check when changing */
 #endif
@@ -68,7 +59,59 @@ HP_INFO *heap_open(const char *name, int mode)
   DBUG_RETURN(info);
 }
 
-	/* map name to a heap-nr. If name isn't found return 0 */
+
+/*
+  Open heap table based on HP_SHARE structure and register it
+*/
+
+HP_INFO *heap_open_from_share_and_register(HP_SHARE *share, int mode)
+{
+  HP_INFO *info;
+  DBUG_ENTER("heap_open_from_share_and_register");
+
+  pthread_mutex_lock(&THR_LOCK_heap);
+  if ((info= heap_open_from_share(share, mode)))
+  {
+    info->open_list.data= (void*) info;
+    heap_open_list= list_add(heap_open_list,&info->open_list);
+  }
+  pthread_mutex_unlock(&THR_LOCK_heap);
+  DBUG_RETURN(info);
+}
+
+
+/*
+  Open heap table based on name
+
+  NOTE
+    This register the table in the open table list. so that it can be
+    found by future heap_open() calls.
+*/
+
+HP_INFO *heap_open(const char *name, int mode)
+{
+  HP_INFO *info;
+  HP_SHARE *share;
+  DBUG_ENTER("heap_open");
+
+  pthread_mutex_lock(&THR_LOCK_heap);
+  if (!(share= hp_find_named_heap(name)))
+  {
+    my_errno= ENOENT;
+    pthread_mutex_unlock(&THR_LOCK_heap);
+    DBUG_RETURN(0);
+  }
+  if ((info= heap_open_from_share(share, mode)))
+  {
+    info->open_list.data= (void*) info;
+    heap_open_list= list_add(heap_open_list,&info->open_list);
+  }
+  pthread_mutex_unlock(&THR_LOCK_heap);
+  DBUG_RETURN(info);
+}
+
+
+/* map name to a heap-nr. If name isn't found return 0 */
 
 HP_SHARE *hp_find_named_heap(const char *name)
 {

storage/heap/hp_test1.c

@@ -37,6 +37,7 @@ int main(int argc, char **argv)
   HP_KEYDEF keyinfo[10];
   HA_KEYSEG keyseg[4];
   HP_CREATE_INFO hp_create_info;
+  HP_SHARE *tmp_share;
   MY_INIT(argv[0]);
 
   filename= "test1";
@@ -52,6 +53,7 @@ int main(int argc, char **argv)
   keyinfo[0].seg[0].start=1;
   keyinfo[0].seg[0].length=6;
   keyinfo[0].seg[0].charset= &my_charset_latin1;
+  keyinfo[0].seg[0].null_bit= 0;
   keyinfo[0].flag = HA_NOSAME;
   
   deleted=0;
@@ -59,7 +61,7 @@ int main(int argc, char **argv)
 
   printf("- Creating heap-file\n");
   if (heap_create(filename,1,keyinfo,30,(ulong) flag*100000L,10L,
-		  &hp_create_info) ||
+		  &hp_create_info, &tmp_share) ||
       !(file= heap_open(filename, 2)))
     goto err;
   printf("- Writing records:s\n");

storage/heap/hp_test2.c

@@ -59,6 +59,7 @@ int main(int argc, char *argv[])
   char record[128],record2[128],record3[128],key[10];
   const char *filename,*filename2;
   HP_INFO *file,*file2;
+  HP_SHARE *tmp_share;
   HP_KEYDEF keyinfo[MAX_KEYS];
   HA_KEYSEG keyseg[MAX_KEYS*5];
   HEAP_PTR position;
@@ -126,7 +127,7 @@ int main(int argc, char *argv[])
 
   printf("- Creating heap-file\n");
   if (heap_create(filename,keys,keyinfo,reclength,(ulong) flag*100000L, 
-		(ulong) recant/2, &hp_create_info) ||
+                  (ulong) recant/2, &hp_create_info, &tmp_share) ||
       !(file= heap_open(filename, 2)))
     goto err;
   signal(SIGINT,endprog);
@@ -562,8 +563,9 @@ int main(int argc, char *argv[])
   heap_close(file2);
 
   printf("- Creating output heap-file 2\n");
-  if (heap_create(filename2,1,keyinfo,reclength,0L,0L,&hp_create_info) ||
-      !(file2= heap_open(filename2, 2)))
+  if (heap_create(filename2, 1, keyinfo, reclength, 0L, 0L, &hp_create_info,
+                  &tmp_share) ||
+      !(file2= heap_open_from_share_and_register(tmp_share, 2)))
     goto err;
 
   printf("- Copying and removing records\n");