memory-limited tree

bulk inserts optimization: caching keys in binary tree


include/my_base.h:
  bulk inserts optimization: caching keys in binary tree
include/my_tree.h:
  memory-limited tree
include/myisam.h:
  bulk inserts optimization: caching keys in binary tree
isam/isamlog.c:
  memory-limited tree
isam/pack_isam.c:
  memory-limited tree
myisam/ft_boolean_search.c:
  memory-limited tree
myisam/ft_nlq_search.c:
  memory-limited tree
myisam/ft_parser.c:
  memory-limited tree
myisam/ft_stopwords.c:
  memory-limited tree
myisam/mi_extra.c:
  bulk inserts optimization: caching keys in binary tree
myisam/mi_open.c:
  bulk inserts optimization: caching keys in binary tree
myisam/mi_static.c:
  bulk inserts optimization: caching keys in binary tree
myisam/mi_write.c:
  bulk inserts optimization: caching keys in binary tree
myisam/myisamdef.h:
  bulk inserts optimization: caching keys in binary tree
myisam/myisamlog.c:
  memory-limited tree
myisam/myisampack.c:
  memory-limited tree
mysql-test/mysql-test-run.sh:
  memory-limited tree
mysql-test/t/alter_table.test:
  make it faster
mysys/tree.c:
  memory-limited tree
sql/ha_myisam.cc:
  bulk inserts optimization: caching keys in binary tree
sql/item_sum.cc:
  memory-limited tree
sql/mysqld.cc:
  bulk inserts optimization: caching keys in binary tree
sql/sql_analyse.h:
  memory-limited tree
sql/uniques.cc:
  memory-limited tree

include/my_base.h

@@ -91,7 +91,9 @@ enum ha_extra_function {
   HA_EXTRA_RESET_STATE,			/* Reset positions */
   HA_EXTRA_IGNORE_DUP_KEY,		/* Dup keys don't rollback everything*/
   HA_EXTRA_NO_IGNORE_DUP_KEY,
-  HA_EXTRA_DONT_USE_CURSOR_TO_UPDATE	/* Cursor will not be used for update */
+  HA_EXTRA_DONT_USE_CURSOR_TO_UPDATE,	/* Cursor will not be used for update */
+  HA_EXTRA_BULK_INSERT_BEGIN,
+  HA_EXTRA_BULK_INSERT_END
 };
 
 	/* The following is parameter to ha_panic() */

include/my_tree.h

@@ -32,6 +32,9 @@ typedef enum { left_root_right, right_root_left } TREE_WALK;
 typedef uint32 element_count;
 typedef int (*tree_walk_action)(void *,element_count,void *);
 
+typedef enum { free_init, free_free, free_end } TREE_FREE;
+typedef void (*tree_element_free)(void*, TREE_FREE, void *);
+
 #ifdef MSDOS
 typedef struct st_tree_element {
   struct st_tree_element *left,*right;
@@ -49,18 +52,18 @@ typedef struct st_tree_element {
 typedef struct st_tree {
   TREE_ELEMENT *root,null_element;
   TREE_ELEMENT **parents[MAX_TREE_HIGHT];
-  uint offset_to_key,elements_in_tree,size_of_element;
+  uint offset_to_key,elements_in_tree,size_of_element,memory_limit,allocated;
   qsort_cmp2 compare;
-  void* cmp_arg;
+  void* custom_arg;
   MEM_ROOT mem_root;
   my_bool with_delete;
-  void (*free)(void *);
+  tree_element_free free;
 } TREE;
 
 	/* Functions on whole tree */
-void init_tree(TREE *tree,uint default_alloc_size, int element_size,
-	       qsort_cmp2 compare, my_bool with_delete,
-	       void (*free_element)(void*));
+void init_tree(TREE *tree, uint default_alloc_size, uint memory_limit,
+               int size, qsort_cmp2 compare, my_bool with_delete,
+	       tree_element_free free_element, void *custom_arg);
 void delete_tree(TREE*);
 void reset_tree(TREE*);
   /* similar to delete tree, except we do not my_free() blocks in mem_root

include/myisam.h

@@ -190,6 +190,7 @@ extern uint myisam_block_size;
 extern my_bool myisam_flush,myisam_delay_key_write,myisam_single_user;
 extern my_bool myisam_concurrent_insert;
 extern my_off_t myisam_max_temp_length,myisam_max_extra_temp_length;
+extern uint myisam_bulk_insert_tree_size;
 
 	/* Prototypes for myisam-functions */
 

isam/isamlog.c

@@ -325,8 +325,8 @@ static int examine_log(my_string file_name, char **table_names)
 
   init_io_cache(&cache,file,0,READ_CACHE,start_offset,0,MYF(0));
   bzero((gptr) com_count,sizeof(com_count));
-  init_tree(&tree,0,sizeof(file_info),(qsort_cmp) file_info_compare,1,
-	    (void(*)(void*)) file_info_free);
+  init_tree(&tree,0,0,sizeof(file_info),(qsort_cmp2) file_info_compare,1,
+	    (tree_element_free) file_info_free, NULL);
   VOID(init_key_cache(KEY_CACHE_SIZE,(uint) (10*4*(IO_SIZE+MALLOC_OVERHEAD))));
 
   files_open=0; access_time=0;

isam/pack_isam.c

@@ -684,7 +684,7 @@ static HUFF_COUNTS *init_huff_count(N_INFO *info,my_off_t records)
 	  (type == FIELD_NORMAL ||
 	   type == FIELD_SKIPP_ZERO))
 	count[i].max_zero_fill= count[i].field_length;
-      init_tree(&count[i].int_tree,0,-1,(qsort_cmp) compare_tree,0,NULL);
+      init_tree(&count[i].int_tree,0,0,-1,(qsort_cmp2) compare_tree,0,NULL,NULL);
       if (records)
 	count[i].tree_pos=count[i].tree_buff =
 	  my_malloc(count[i].field_length > 1 ? tree_buff_length : 2,

myisam/ft_boolean_search.c

@@ -198,8 +198,8 @@ FT_DOCLIST *ft_boolean_search(MI_INFO *info, uint keynr, byte *query,
   aio.end=query+query_len;
   aio.total_yes=aio.total_no=0;
 
-  init_tree(&aio.dtree,0,sizeof(FT_SUPERDOC),(qsort_cmp2)&FT_SUPERDOC_cmp,0,
-            NULL);
+  init_tree(&aio.dtree,0,0,sizeof(FT_SUPERDOC),(qsort_cmp2)&FT_SUPERDOC_cmp,0,
+            NULL, NULL);
 
   if (do_boolean(&aio,0,0,0,0))
     goto err;

myisam/ft_nlq_search.c

@@ -163,8 +163,8 @@ FT_DOCLIST *ft_nlq_search(MI_INFO *info, uint keynr, byte *query,
 
   bzero(&allocated_wtree,sizeof(allocated_wtree));
 
-  init_tree(&aio.dtree,0,sizeof(FT_SUPERDOC),(qsort_cmp2)&FT_SUPERDOC_cmp,0,
-            NULL);
+  init_tree(&aio.dtree,0,0,sizeof(FT_SUPERDOC),(qsort_cmp2)&FT_SUPERDOC_cmp,0,
+            NULL, NULL);
 
   if(!(wtree=ft_parse(&allocated_wtree,query,query_len)))
     return NULL;

myisam/ft_parser.c

@@ -226,7 +226,7 @@ TREE * ft_parse(TREE *wtree, byte *doc, int doclen)
 
   if (!is_tree_inited(wtree))
   {
-    init_tree(wtree,0,sizeof(FT_WORD),(qsort_cmp2)&FT_WORD_cmp,0,NULL);
+    init_tree(wtree,0,0,sizeof(FT_WORD),(qsort_cmp2)&FT_WORD_cmp,0,NULL, NULL);
   }
 
   while (ft_simple_get_word(&doc,end,&w))

myisam/ft_stopwords.c

@@ -41,8 +41,8 @@ int ft_init_stopwords(const char **sws)
   if(!stopwords3)
   {
     if(!(stopwords3=(TREE *)my_malloc(sizeof(TREE),MYF(0)))) return -1;
-    init_tree(stopwords3,0,sizeof(FT_STOPWORD),(qsort_cmp2)&FT_STOPWORD_cmp,0,
-	      NULL);
+    init_tree(stopwords3,0,0,sizeof(FT_STOPWORD),(qsort_cmp2)&FT_STOPWORD_cmp,0,
+	      NULL, NULL);
   }
 
   if(!sws) return 0;

myisam/mi_extra.c

@@ -336,6 +336,24 @@ int mi_extra(MI_INFO *info, enum ha_extra_function function)
   case HA_EXTRA_QUICK:
     info->quick_mode=1;
     break;
+  case HA_EXTRA_BULK_INSERT_BEGIN:
+    error=_mi_init_bulk_insert(info);
+    break;
+  case HA_EXTRA_BULK_INSERT_END:
+    if (info->bulk_insert)
+    {
+      uint i;
+      for (i=0 ; i < share->base.keys ; i++)
+      {
+        if (is_tree_inited(& info->bulk_insert[i]))
+        {
+          delete_tree(& info->bulk_insert[i]);
+        }
+      }
+      my_free((void *)info->bulk_insert, MYF(0));
+      info->bulk_insert=0;
+    }
+    break;
   case HA_EXTRA_NO_ROWS:
     if (!share->state.header.uniques)
       info->opt_flag|= OPT_NO_ROWS;

myisam/mi_open.c

@@ -461,6 +461,7 @@ MI_INFO *mi_open(const char *name, int mode, uint open_flags)
     share->options|=HA_OPTION_READ_ONLY_DATA;
   info.lock_type=F_UNLCK;
   info.quick_mode=0;
+  info.bulk_insert=0;
   info.errkey= -1;
   info.page_changed=1;
   pthread_mutex_lock(&share->intern_lock);

myisam/mi_static.c

@@ -40,7 +40,7 @@ my_bool myisam_concurrent_insert=0;
 #endif
 my_off_t myisam_max_extra_temp_length= MI_MAX_TEMP_LENGTH;
 my_off_t myisam_max_temp_length= MAX_FILE_SIZE;
-
+uint myisam_bulk_insert_tree_size=8192*1024;
 
 /* read_vec[] is used for converting between P_READ_KEY.. and SEARCH_ */
 /* Position is , == , >= , <= , > , < */

myisam/mi_write.c

@@ -96,27 +96,27 @@ int mi_write(MI_INFO *info, byte *record)
   {
     if (((ulonglong) 1 << i) & share->state.key_map)
     {
-      if (share->concurrent_insert)
+      if (share->concurrent_insert && ! info->bulk_insert)
       {
 	rw_wrlock(&share->key_root_lock[i]);
 	share->keyinfo[i].version++;
       }
-      if (share->keyinfo[i].flag & HA_FULLTEXT )                  /* SerG */
-      {                                                           /* SerG */
-        if (_mi_ft_add(info,i,(char*) buff,record,filepos))       /* SerG */
-        {                                                         /* SerG */
+      if (share->keyinfo[i].flag & HA_FULLTEXT )
+      {
+        if (_mi_ft_add(info,i,(char*) buff,record,filepos))
+        {
 	  if (share->concurrent_insert)
 	    rw_unlock(&share->key_root_lock[i]);
-          DBUG_PRINT("error",("Got error: %d on write",my_errno));  /* SerG */
-          goto err;                                                 /* SerG */
-        }                                                           /* SerG */
-      }                                                             /* SerG */
-      else                                                          /* SerG */
+          DBUG_PRINT("error",("Got error: %d on write",my_errno));
+          goto err;
+        }
+      }
+      else
       {
 	uint key_length=_mi_make_key(info,i,buff,record,filepos);
 	if (_mi_ck_write(info,i,buff,key_length))
 	{
-	  if (share->concurrent_insert)
+	  if (share->concurrent_insert && ! info->bulk_insert)
 	    rw_unlock(&share->key_root_lock[i]);
 	  DBUG_PRINT("error",("Got error: %d on write",my_errno));
 	  goto err;
@@ -156,7 +156,6 @@ err:
       {
 	if (share->concurrent_insert)
 	  rw_wrlock(&share->key_root_lock[i]);
-	/* The following code block is for text searching by SerG */
 	if (share->keyinfo[i].flag & HA_FULLTEXT)
         {
           if (_mi_ft_del(info,i,(char*) buff,record,filepos))
@@ -196,11 +195,29 @@ err2:
 
 	/* Write one key to btree */
 
-int _mi_ck_write(register MI_INFO *info, uint keynr, uchar *key,
+int _mi_ck_write(MI_INFO *info, uint keynr, uchar *key, uint key_length)
+{
+  DBUG_ENTER("_mi_ck_write");
+
+  if (info->bulk_insert && is_tree_inited(& info->bulk_insert[keynr]))
+  {
+    DBUG_RETURN(_mi_ck_write_tree(info, keynr, key, key_length));
+  }
+  else
+  {
+    DBUG_RETURN(_mi_ck_write_btree(info, keynr, key, key_length));
+  }
+} /* _mi_ck_write */
+
+/**********************************************************************
+ *                Normal insert code                                  *
+ **********************************************************************/
+
+int _mi_ck_write_btree(register MI_INFO *info, uint keynr, uchar *key,
 		 uint key_length)
 {
   int error;
-  DBUG_ENTER("_mi_ck_write");
+  DBUG_ENTER("_mi_ck_write_btree");
 
   if (info->s->state.key_root[keynr] == HA_OFFSET_ERROR ||
       (error=w_search(info,info->s->keyinfo+keynr,key, key_length,
@@ -208,7 +225,7 @@ int _mi_ck_write(register MI_INFO *info, uint keynr, uchar *key,
 		      (my_off_t) 0, 1)) > 0)
     error=_mi_enlarge_root(info,keynr,key);
   DBUG_RETURN(error);
-} /* _mi_ck_write */
+} /* _mi_ck_write_btree */
 
 
 	/* Make a new root with key as only pointer */
@@ -682,3 +699,98 @@ static int _mi_balance_page(register MI_INFO *info, MI_KEYDEF *keyinfo,
 err:
   DBUG_RETURN(-1);
 } /* _mi_balance_page */
+
+/**********************************************************************
+ *                Bulk insert code                                    *
+ **********************************************************************/
+
+typedef struct {
+  MI_INFO *info;
+  uint keynr;
+} bulk_insert_param;
+
+int _mi_ck_write_tree(register MI_INFO *info, uint keynr, uchar *key,
+		 uint key_length)
+{
+  int error;
+  DBUG_ENTER("_mi_ck_write_tree");
+
+  error= tree_insert(& info->bulk_insert[keynr], key,
+         key_length + info->s->rec_reflength) ? 0 : HA_ERR_OUT_OF_MEM ;
+
+  DBUG_RETURN(error);
+} /* _mi_ck_write_tree */
+
+/* typeof(_mi_keys_compare)=qsort_cmp2 */
+static int keys_compare(bulk_insert_param *param, uchar *key1, uchar *key2)
+{
+  uint not_used;
+  return _mi_key_cmp(param->info->s->keyinfo[param->keynr].seg,
+              key1, key2, USE_WHOLE_KEY, SEARCH_SAME, &not_used);
+}
+
+static int keys_free(uchar *key, TREE_FREE mode, bulk_insert_param *param)
+{
+  /* probably I can use info->lastkey here, but I'm not sure,
+     and to be safe I'd better use local lastkey.
+     Monty, feel free to comment on this */
+  uchar lastkey[MI_MAX_KEY_BUFF];
+  uint keylen;
+  MI_KEYDEF *keyinfo;
+
+  switch (mode) {
+    case free_init:
+      if (param->info->s->concurrent_insert)
+      {
+        rw_wrlock(&param->info->s->key_root_lock[param->keynr]);
+        param->info->s->keyinfo[param->keynr].version++;
+      }
+      return 0;
+    case free_free:
+      keyinfo=param->info->s->keyinfo+param->keynr;
+      keylen=_mi_keylength(keyinfo, key);
+      memcpy(lastkey, key, keylen);
+      return _mi_ck_write_btree(param->info,param->keynr,lastkey, 
+                 keylen - param->info->s->rec_reflength);
+    case free_end:
+      if (param->info->s->concurrent_insert)
+        rw_unlock(&param->info->s->key_root_lock[param->keynr]);
+      return 0;
+  }
+  return -1;
+}
+
+int _mi_init_bulk_insert(MI_INFO *info)
+{
+  MYISAM_SHARE *share=info->s;
+  MI_KEYDEF *key=share->keyinfo;
+  bulk_insert_param *params;
+  uint i;
+
+  if (info->bulk_insert)
+    return 0;
+  
+  info->bulk_insert=(TREE *)my_malloc(
+      (sizeof(TREE)+sizeof(bulk_insert_param))*share->base.keys, MYF(0));
+
+  if (!info->bulk_insert)
+    return HA_ERR_OUT_OF_MEM;
+  
+  params=(bulk_insert_param *)(info->bulk_insert+share->base.keys);
+  
+  for (i=0 ; i < share->base.keys ; i++,key++,params++)
+  {
+    params->info=info;
+    params->keynr=i;
+    if (!(key->flag & HA_NOSAME) && share->base.auto_key != i+1
+        && test(share->state.key_map & ((ulonglong) 1 << i)))
+    {
+      init_tree(& info->bulk_insert[i], 0, myisam_bulk_insert_tree_size, 0,
+          (qsort_cmp2)keys_compare, 0,
+          (tree_element_free) keys_free, (void *)params);
+    }
+    else
+     info->bulk_insert[i].root=0; 
+  }
+  return 0;
+}

myisam/myisamdef.h

@@ -18,6 +18,7 @@
 
 #include "myisam.h"			/* Structs & some defines */
 #include "myisampack.h"			/* packing of keys */
+#include <my_tree.h>
 #ifdef THREAD
 #include <my_pthread.h>
 #include <thr_lock.h>
@@ -259,6 +260,7 @@ struct st_myisam_info {
   my_bool page_changed;		/* If info->buff can't be used for rnext */
   my_bool buff_used;		/* If info->buff has to be reread for rnext */
   my_bool use_packed_key;		/* For MYISAMMRG */
+  TREE   *bulk_insert;   /* accumulate indexfile changes between mi_write's */
   myf lock_wait;			/* is 0 or MY_DONT_WAIT */
   int (*read_record)(struct st_myisam_info*, my_off_t, byte*);
   LIST	open_list;
@@ -641,7 +643,9 @@ my_bool check_table_is_closed(const char *name, const char *where);
 int mi_open_datafile(MI_INFO *info, MYISAM_SHARE *share);
 int mi_open_keyfile(MYISAM_SHARE *share);
 
-/* Functions needed by mi_check */
+int _mi_init_bulk_insert(MI_INFO *info);
+
+    /* Functions needed by mi_check */
 void mi_check_print_error _VARARGS((MI_CHECK *param, const char *fmt,...));
 void mi_check_print_warning _VARARGS((MI_CHECK *param, const char *fmt,...));
 void mi_check_print_info _VARARGS((MI_CHECK *param, const char *fmt,...));

myisam/myisamlog.c

@@ -331,8 +331,8 @@ static int examine_log(my_string file_name, char **table_names)
 
   init_io_cache(&cache,file,0,READ_CACHE,start_offset,0,MYF(0));
   bzero((gptr) com_count,sizeof(com_count));
-  init_tree(&tree,0,sizeof(file_info),(qsort_cmp2) file_info_compare,1,
-	    (void(*)(void*)) file_info_free);
+  init_tree(&tree,0,0,sizeof(file_info),(qsort_cmp2) file_info_compare,1,
+	    (tree_element_free) file_info_free, NULL);
   VOID(init_key_cache(KEY_CACHE_SIZE,(uint) (10*4*(IO_SIZE+MALLOC_OVERHEAD))));
 
   files_open=0; access_time=0;

myisam/myisampack.c

@@ -674,7 +674,7 @@ static HUFF_COUNTS *init_huff_count(MI_INFO *info,my_off_t records)
 	  (type == FIELD_NORMAL ||
 	   type == FIELD_SKIPP_ZERO))
 	count[i].max_zero_fill= count[i].field_length;
-      init_tree(&count[i].int_tree,0,-1,(qsort_cmp2) compare_tree,0,NULL);
+      init_tree(&count[i].int_tree,0,0,-1,(qsort_cmp2) compare_tree,0,NULL,NULL);
       if (records && type != FIELD_BLOB && type != FIELD_VARCHAR)
 	count[i].tree_pos=count[i].tree_buff =
 	  my_malloc(count[i].field_length > 1 ? tree_buff_length : 2,

mysql-test/mysql-test-run.sh

@@ -744,7 +744,7 @@ run_testcase ()
 	$ECHO "Resuming Tests"
 	$ECHO ""
       else
-        pass_inc
+#       pass_inc
 	$ECHO "$RES$RES_SPACE [ skipped ]"
       fi
     fi

mysql-test/t/alter_table.test

@@ -83,7 +83,8 @@ create table t1 (n1 int not null, n2 int, n3 int, n4 float,
                 key (n3, n4, n1, n2),
                 key (n4, n1, n2, n3) );
 alter table t1 disable keys;
-let $1=10000;
+#let $1=10000;
+let $1=10;
 while ($1)
 {
  eval insert into t1 values($1,RAND()*1000,RAND()*1000,RAND());

mysys/tree.c

@@ -62,22 +62,51 @@ static void rb_delete_fixup(TREE *tree,TREE_ELEMENT ***parent);
 
 	/* The actuall code for handling binary trees */
 
-void init_tree(TREE *tree, uint default_alloc_size, int size,
-	       qsort_cmp2 compare, my_bool with_delete,
-	       void (*free_element) (void *))
+#ifndef DBUG_OFF
+
+	/* Test that the proporties for a red-black tree holds */
+
+static int test_rb_tree(TREE_ELEMENT *element)
+{
+  int count_l,count_r;
+
+  if (!element->left)
+    return 0;				/* Found end of tree */
+  if (element->colour == RED &&
+      (element->left->colour == RED || element->right->colour == RED))
+  {
+    printf("Wrong tree: Found two red in a row\n");
+    return -1;
+  }
+  count_l=test_rb_tree(element->left);
+  count_r=test_rb_tree(element->right);
+  if (count_l >= 0 && count_r >= 0)
+  {
+    if (count_l == count_r)
+      return count_l+(element->colour == BLACK);
+    printf("Wrong tree: Incorrect black-count: %d - %d\n",count_l,count_r);
+  }
+  return -1;
+}
+#endif
+
+void init_tree(TREE *tree, uint default_alloc_size, uint memory_limit,
+               int size, qsort_cmp2 compare, my_bool with_delete,
+	       tree_element_free free_element, void *custom_arg)
 {
   DBUG_ENTER("init_tree");
   DBUG_PRINT("enter",("tree: %lx  size: %d",tree,size));
 
-  if (!default_alloc_size)
-    default_alloc_size= DEFAULT_ALLOC_SIZE;
+  default_alloc_size=DEFAULT_ALLOC_SIZE;
   bzero((gptr) &tree->null_element,sizeof(tree->null_element));
   tree->root= &tree->null_element;
   tree->compare=compare;
   tree->size_of_element=size > 0 ? (uint) size : 0;
+  tree->memory_limit=memory_limit;
   tree->free=free_element;
+  tree->allocated=0;
   tree->elements_in_tree=0;
-  tree->cmp_arg = 0;
+  tree->custom_arg = custom_arg;
   tree->null_element.colour=BLACK;
   tree->null_element.left=tree->null_element.right=0;
   if (!free_element && size >= 0 &&
@@ -115,12 +144,19 @@ static void free_tree(TREE *tree, myf free_flags)
     else
     {
       if (tree->free)
+      {
+        if (tree->memory_limit)
+          (*tree->free)(NULL, free_init, tree->custom_arg);
 	delete_tree_element(tree,tree->root);
+        if (tree->memory_limit)
+          (*tree->free)(NULL, free_end, tree->custom_arg);
+      }
       free_root(&tree->mem_root, free_flags);
     }
   }
   tree->root= &tree->null_element;
   tree->elements_in_tree=0;
+  tree->allocated=0;
 
   DBUG_VOID_RETURN;
 }
@@ -142,9 +178,9 @@ static void delete_tree_element(TREE *tree, TREE_ELEMENT *element)
   if (element != &tree->null_element)
   {
     delete_tree_element(tree,element->left);
-    delete_tree_element(tree,element->right);
     if (tree->free)
-      (*tree->free)(ELEMENT_KEY(tree,element));
+      (*tree->free)(ELEMENT_KEY(tree,element), free_free, tree->custom_arg);
+    delete_tree_element(tree,element->right);
     if (tree->with_delete)
       my_free((void*) element,MYF(0));
   }
@@ -165,7 +201,7 @@ TREE_ELEMENT *tree_insert(TREE *tree, void *key, uint key_size)
   for (;;)
   {
     if (element == &tree->null_element ||
-	(cmp=(*tree->compare)(tree->cmp_arg,
+	(cmp=(*tree->compare)(tree->custom_arg,
 			      ELEMENT_KEY(tree,element),key)) == 0)
       break;
     if (cmp < 0)
@@ -179,13 +215,22 @@ TREE_ELEMENT *tree_insert(TREE *tree, void *key, uint key_size)
   }
   if (element == &tree->null_element)
   {
+    uint alloc_size=sizeof(TREE_ELEMENT)+key_size+tree->size_of_element;
+    tree->allocated+=alloc_size;
+    
+    if (tree->memory_limit && tree->elements_in_tree
+                           && tree->allocated > tree->memory_limit)
+    {
+      reset_tree(tree);
+      return tree_insert(tree, key, key_size);
+    }
+    
     key_size+=tree->size_of_element;
     if (tree->with_delete)
-      element=(TREE_ELEMENT *) my_malloc(sizeof(TREE_ELEMENT)+key_size,
-					 MYF(MY_WME));
+      element=(TREE_ELEMENT *) my_malloc(alloc_size, MYF(MY_WME));
     else
       element=(TREE_ELEMENT *)
-	alloc_root(&tree->mem_root,sizeof(TREE_ELEMENT)+key_size);
+	alloc_root(&tree->mem_root,alloc_size);
     if (!element)
       return(NULL);
     **parent=element;
@@ -209,6 +254,7 @@ TREE_ELEMENT *tree_insert(TREE *tree, void *key, uint key_size)
   }
   else
     element->count++;
+  DBUG_EXECUTE("check_tree", test_rb_tree(tree->root););
   return element;
 }
 
@@ -226,7 +272,7 @@ int tree_delete(TREE *tree, void *key)
   {
     if (element == &tree->null_element)
       return 1;				/* Was not in tree */
-    if ((cmp=(*tree->compare)(tree->cmp_arg,
+    if ((cmp=(*tree->compare)(tree->custom_arg,
 			      ELEMENT_KEY(tree,element),key)) == 0)
       break;
     if (cmp < 0)
@@ -281,7 +327,7 @@ void *tree_search(TREE *tree, void *key)
   {
     if (element == &tree->null_element)
       return (void*) 0;
-    if ((cmp=(*tree->compare)(tree->cmp_arg,
+    if ((cmp=(*tree->compare)(tree->custom_arg,
 			      ELEMENT_KEY(tree,element),key)) == 0)
       return ELEMENT_KEY(tree,element);
     if (cmp < 0)
@@ -497,33 +543,3 @@ static void rb_delete_fixup(TREE *tree, TREE_ELEMENT ***parent)
   }
   x->colour=BLACK;
 }
-
-
-#ifdef TESTING_TREES
-
-	/* Test that the proporties for a red-black tree holds */
-
-static int test_rb_tree(TREE_ELEMENT *element)
-{
-  int count_l,count_r;
-
-  if (!element->left)
-    return 0;				/* Found end of tree */
-  if (element->colour == RED &&
-      (element->left->colour == RED || element->right->colour == RED))
-  {
-    printf("Wrong tree: Found two red in a row\n");
-    return -1;
-  }
-  count_l=test_rb_tree(element->left);
-  count_r=test_rb_tree(element->right);
-  if (count_l >= 0 && count_r >= 0)
-  {
-    if (count_l == count_r)
-      return count_l+(element->colour == BLACK);
-    printf("Wrong tree: Incorrect black-count: %d - %d\n",count_l,count_r);
-  }
-  return -1;
-}
-
-#endif

sql/ha_myisam.cc

@@ -644,7 +644,10 @@ void ha_myisam::deactivate_non_unique_index(ha_rows rows)
       if (rows==HA_POS_ERROR)
         mi_extra(file, HA_EXTRA_NO_KEYS);
       else
+      {
         mi_disable_non_unique_index(file,rows);
+        mi_extra(file, HA_EXTRA_BULK_INSERT_BEGIN);
+      }
     enable_activate_all_index=1;
   }
   else
@@ -658,6 +661,8 @@ bool ha_myisam::activate_all_index(THD *thd)
   MI_CHECK param;
   MYISAM_SHARE* share = file->s;
   DBUG_ENTER("activate_all_index");
+
+  mi_extra(file, HA_EXTRA_BULK_INSERT_END);
   if (enable_activate_all_index &&
       share->state.key_map != ((ulonglong) 1L << share->base.keys)-1)
   {

sql/item_sum.cc

@@ -931,9 +931,8 @@ bool Item_sum_count_distinct::setup(THD *thd)
 	  rec_offset = table->reclength - key_len;
 	}
 
-      init_tree(&tree, min(max_heap_table_size, sortbuff_size/16),
-		key_len, compare_key, 0, 0);
-      tree.cmp_arg = cmp_arg;
+      init_tree(&tree, min(max_heap_table_size, sortbuff_size/16), 0,
+		key_len, compare_key, 0, NULL, cmp_arg);
       use_tree = 1;
 
       // the only time key_len could be 0 is if someone does

sql/mysqld.cc

@@ -2730,13 +2730,15 @@ CHANGEABLE_VAR changeable_vars[] = {
       0, 1, ~0L, 0, 1 },
   { "max_write_lock_count",    (long*) &max_write_lock_count,
       ~0L, 1, ~0L, 0, 1 },
-  { "myisam_sort_buffer_size", (long*) &myisam_sort_buffer_size,
+  { "myisam_bulk_insert_tree_size", (long*) &myisam_bulk_insert_tree_size,
       8192*1024, 4, ~0L, 0, 1 },
   { "myisam_max_extra_sort_file_size",
     (long*) &myisam_max_extra_sort_file_size,
     (long) (MI_MAX_TEMP_LENGTH/(1024L*1024L)), 0, ~0L, 0, 1 },
   { "myisam_max_sort_file_size", (long*) &myisam_max_sort_file_size,
     (long) (LONG_MAX/(1024L*1024L)), 0, ~0L, 0, 1 },
+  { "myisam_sort_buffer_size", (long*) &myisam_sort_buffer_size,
+      8192*1024, 4, ~0L, 0, 1 },
   { "net_buffer_length",       (long*) &net_buffer_length,
       16384, 1024, 1024*1024L, MALLOC_OVERHEAD, 1024 },
   { "net_retry_count",         (long*) &mysqld_net_retry_count,
@@ -2853,11 +2855,12 @@ struct show_var_st init_vars[]= {
   {"max_user_connections",    (char*) &max_user_connections,        SHOW_LONG},
   {"max_tmp_tables",          (char*) &max_tmp_tables,              SHOW_LONG},
   {"max_write_lock_count",    (char*) &max_write_lock_count,        SHOW_LONG},
-  {"myisam_recover_options",  (char*) &myisam_recover_options_str,  SHOW_CHAR_PTR},
+  {"myisam_bulk_insert_tree_size", (char*) &myisam_bulk_insert_tree_size, SHOW_INT},
   {"myisam_max_extra_sort_file_size", (char*) &myisam_max_extra_sort_file_size,
    SHOW_LONG},
   {"myisam_max_sort_file_size",(char*) &myisam_max_sort_file_size,  SHOW_LONG},
   {"myisam_sort_buffer_size", (char*) &myisam_sort_buffer_size,     SHOW_LONG},
+  {"myisam_recover_options",  (char*) &myisam_recover_options_str,  SHOW_CHAR_PTR},
   {"net_buffer_length",       (char*) &net_buffer_length,           SHOW_LONG},
   {"net_read_timeout",        (char*) &net_read_timeout,	    SHOW_LONG},
   {"net_retry_count",         (char*) &mysqld_net_retry_count,      SHOW_LONG},

sql/sql_analyse.h

@@ -114,9 +114,9 @@ public:
     max_arg(""), sum(0),
     must_be_blob(0), was_zero_fill(0),
     was_maybe_zerofill(0), can_be_still_num(1)
-    { init_tree(&tree, 0, sizeof(String), a->binary ?
+    { init_tree(&tree, 0, 0, sizeof(String), a->binary ?
 		(qsort_cmp2) stringcmp2 : (qsort_cmp2) sortcmp2,
-		0, (void (*)(void*)) free_string); };
+		0, (tree_element_free) free_string, NULL); };
 
   void	 add();
   void	 get_opt_type(String*, ha_rows);
@@ -154,8 +154,8 @@ class field_real: public field_info
 public:
   field_real(Item* a, analyse* b) :field_info(a,b),
     min_arg(0), max_arg(0),  sum(0), sum_sqr(0), max_notzero_dec_len(0)
-    { init_tree(&tree, 0, sizeof(double),
-		(qsort_cmp2) compare_double2, 0, NULL); }
+    { init_tree(&tree, 0, 0, sizeof(double),
+		(qsort_cmp2) compare_double2, 0, NULL, NULL); }
 
   void	 add();
   void	 get_opt_type(String*, ha_rows);
@@ -200,8 +200,8 @@ class field_longlong: public field_info
 public:
   field_longlong(Item* a, analyse* b) :field_info(a,b), 
     min_arg(0), max_arg(0), sum(0), sum_sqr(0)
-    { init_tree(&tree, 0, sizeof(longlong),
-		(qsort_cmp2) compare_longlong2, 0, NULL); }
+    { init_tree(&tree, 0, 0, sizeof(longlong),
+		(qsort_cmp2) compare_longlong2, 0, NULL, NULL); }
 
   void	 add();
   void	 get_opt_type(String*, ha_rows);
@@ -245,8 +245,8 @@ class field_ulonglong: public field_info
 public:
   field_ulonglong(Item* a, analyse * b) :field_info(a,b),
     min_arg(0), max_arg(0), sum(0),sum_sqr(0)
-    { init_tree(&tree, 0, sizeof(ulonglong),
-		(qsort_cmp2) compare_ulonglong2, 0, NULL); }
+    { init_tree(&tree, 0, 0, sizeof(ulonglong),
+		(qsort_cmp2) compare_ulonglong2, 0, NULL, NULL); }
   void	 add();
   void	 get_opt_type(String*, ha_rows);
   String *get_min_arg(String *s) { s->set(min_arg); return s; }

sql/uniques.cc

@@ -38,7 +38,7 @@ Unique::Unique(qsort_cmp2 comp_func, uint size, ulong max_in_memory_size_arg)
   :max_in_memory_size(max_in_memory_size_arg),elements(0)
 {
   my_b_clear(&file);
-  init_tree(&tree, max_in_memory_size / 16, size, comp_func, 0, 0);
+  init_tree(&tree, max_in_memory_size / 16, 0, size, comp_func, 0, NULL, NULL);
   /* If the following fail's the next add will also fail */
   init_dynamic_array(&file_ptrs, sizeof(BUFFPEK), 16, 16);
   max_elements= max_in_memory_size / ALIGN_SIZE(sizeof(TREE_ELEMENT)+size);