merged

include/my_handler.h

@@ -43,6 +43,13 @@ typedef struct st_HA_KEYSEG		/* Key-portion */
   { length=mi_uint2korr((key)+1); (key)+=3; } \
 }
 
+#define get_key_length_rdonly(length,key) \
+{ if ((uchar) *(key) != 255) \
+    length= ((uint) (uchar) *((key))); \
+  else \
+  { length=mi_uint2korr((key)+1); } \
+}
+
 #define get_key_pack_length(length,length_pack,key) \
 { if ((uchar) *(key) != 255) \
   { length= (uint) (uchar) *((key)++); length_pack=1; }\

include/myisam.h

@@ -342,6 +342,12 @@ typedef struct st_mi_check_param
   char *op_name;
 } MI_CHECK;
 
+typedef struct st_sort_ft_buf
+{
+  uchar *buf, *end;
+  int   count;
+  uchar lastkey[MI_MAX_KEY_BUFF];
+} SORT_FT_BUF;
 
 typedef struct st_sort_info
 {
@@ -354,7 +360,8 @@ typedef struct st_sort_info
   MI_CHECK *param;
   char *buff;
   SORT_KEY_BLOCKS *key_block,*key_block_end;
-  /* sync things*/
+  SORT_FT_BUF *ft_buf;
+  /* sync things */
   uint got_error, threads_running;
   pthread_mutex_t mutex;
   pthread_cond_t  cond;

myisam/ft_boolean_search.c

@@ -21,6 +21,7 @@
 #define FT_CORE
 #include "ftdefs.h"
 #include <queues.h>
+#include <assert.h> /* for DBUG_ASSERT() */
 
 /* search with boolean queries */
 
@@ -63,25 +64,27 @@ struct st_ftb_expr
 {
   FTB_EXPR *up;
   byte     *quot, *qend;
-  float     weight;
-  uint      flags;
   my_off_t  docid[2];             /* for index search and for scan */
+  float     weight;
   float     cur_weight;
-  int       yesses;               /* number of "yes" words matched */
-  int       nos;                  /* number of "no"  words matched */
-  int       ythresh;              /* number of "yes" words in expr */
-  int       yweaks;               /* number of "yes" words for scan only */
+  uint      flags;
+  uint      yesses;               /* number of "yes" words matched */
+  uint      nos;                  /* number of "no"  words matched */
+  uint      ythresh;              /* number of "yes" words in expr */
+  uint      yweaks;               /* number of "yes" words for scan only */
 };
 
 typedef struct st_ftb_word
 {
   FTB_EXPR  *up;
-  float     weight;
-  uint      flags;
+  MI_KEYDEF *keyinfo;
   my_off_t   docid[2];             /* for index search and for scan */
+  my_off_t   key_root;
+  float      weight;
   uint       ndepth;
-  int       len;
-  /* ... docid cache can be added here. SerG */
+  uint       flags;
+  uint       len;
+  uchar      off;
   byte       word[1];
 } FTB_WORD;
 
@@ -89,16 +92,16 @@ typedef struct st_ft_info
 {
   struct _ft_vft *please;
   MI_INFO   *info;
-  uint       keynr;
   CHARSET_INFO *charset;
-  enum { UNINITIALIZED, READY, INDEX_SEARCH, INDEX_DONE /*, SCAN*/ } state;
-  uint       with_scan;
-  my_off_t   lastpos;
   FTB_EXPR  *root;
-  QUEUE      queue;
-  TREE       no_dupes;
   FTB_WORD **list;
   MEM_ROOT   mem_root;
+  QUEUE      queue;
+  TREE       no_dupes;
+  my_off_t   lastpos;
+  uint       keynr;
+  uchar      with_scan;
+  enum { UNINITIALIZED, READY, INDEX_SEARCH, INDEX_DONE } state;
 } FTB;
 
 static int FTB_WORD_cmp(my_off_t *v, FTB_WORD *a, FTB_WORD *b)
@@ -160,6 +163,7 @@ static void _ftb_parse_query(FTB *ftb, byte **start, byte *end,
         ftbw->up=up;
         ftbw->docid[0]=ftbw->docid[1]=HA_POS_ERROR;
         ftbw->ndepth= (param.yesno<0) + depth;
+        ftbw->key_root=HA_POS_ERROR;
         memcpy(ftbw->word+1, w.pos, w.len);
         ftbw->word[0]=w.len;
         if (param.yesno > 0) up->ythresh++;
@@ -194,22 +198,98 @@ static int _ftb_no_dupes_cmp(void* not_used __attribute__((unused)),
   return CMP_NUM((*((my_off_t*)a)), (*((my_off_t*)b)));
 }
 
+/* returns 1 if the search was finished (must-word wasn't found) */
+static int _ft2_search(FTB *ftb, FTB_WORD *ftbw, my_bool init_search)
+{
+  int r;
+  uint off;
+  int subkeys;
+  MI_INFO *info=ftb->info;
+
+  if (init_search)
+  {
+    ftbw->key_root=info->s->state.key_root[ftb->keynr];
+    ftbw->keyinfo=info->s->keyinfo+ftb->keynr;
+    ftbw->off=0;
+
+    r=_mi_search(info, ftbw->keyinfo, (uchar*) ftbw->word, ftbw->len,
+                 SEARCH_FIND | SEARCH_BIGGER, ftbw->key_root);
+  }
+  else
+  {
+    r=_mi_search(info, ftbw->keyinfo, (uchar*) ftbw->word+ftbw->off,
+                   USE_WHOLE_KEY, SEARCH_BIGGER, ftbw->key_root);
+  }
+  if (!r && !ftbw->off)
+  {
+    r= mi_compare_text(ftb->charset,
+                       info->lastkey + (ftbw->flags & FTB_FLAG_TRUNC),
+                       ftbw->len     - (ftbw->flags & FTB_FLAG_TRUNC),
+              (uchar*) ftbw->word    + (ftbw->flags & FTB_FLAG_TRUNC),
+                       ftbw->len     - (ftbw->flags & FTB_FLAG_TRUNC),
+                       0);
+  }
+
+  if (r) /* not found */
+  {
+    if (!ftbw->off || !(ftbw->flags & FTB_FLAG_TRUNC))
+    {
+      ftbw->docid[0]=HA_POS_ERROR;
+      if ((ftbw->flags & FTB_FLAG_YES) && ftbw->up->up==0)
+      {
+        /*
+	  This word MUST BE present in every document returned,
+	  so we can stop the search right now
+	*/
+        ftb->state=INDEX_DONE;
+        return 1; /* search is done */
+      }
+      else
+        return 0;
+    }
+
+    /* going up to the first-level tree to continue search there */
+    _mi_dpointer(info, ftbw->word+ftbw->off+HA_FT_WLEN, ftbw->key_root);
+    ftbw->key_root=info->s->state.key_root[ftb->keynr];
+    ftbw->keyinfo=info->s->keyinfo+ftb->keynr;
+    ftbw->off=0;
+    return _ft2_search(ftb, ftbw, 0);
+  }
+
+  /* matching key found */
+  memcpy(ftbw->word+ftbw->off, info->lastkey, info->lastkey_length);
+  if (!ftbw->off && (init_search || (ftbw->flags & FTB_FLAG_TRUNC)))
+  {
+    /* going down ? */
+    get_key_full_length_rdonly(off, info->lastkey);
+    subkeys=ft_sintXkorr(info->lastkey+off);
+    if (subkeys<0)
+    {
+      /* yep, going down, to the second-level tree */
+      /* TODO here: subkey-based optimization */
+      ftbw->off=off;
+      ftbw->key_root=info->lastpos;
+      ftbw->keyinfo=& info->s->ft2_keyinfo;
+      r=_mi_search_first(info, ftbw->keyinfo, ftbw->key_root);
+      DBUG_ASSERT(r==0);  /* found something */
+      memcpy(ftbw->word+off, info->lastkey, info->lastkey_length);
+    }
+  }
+  ftbw->docid[0]=info->lastpos;
+  return 0;
+}
+
 static void _ftb_init_index_search(FT_INFO *ftb)
 {
-  int i, r;
+  int i;
   FTB_WORD   *ftbw;
   MI_INFO    *info=ftb->info;
-  MI_KEYDEF  *keyinfo;
-  my_off_t    keyroot;
 
   if ((ftb->state != READY && ftb->state !=INDEX_DONE) ||
       ftb->keynr == NO_SUCH_KEY)
     return;
   ftb->state=INDEX_SEARCH;
 
-  keyinfo=info->s->keyinfo+ftb->keynr;
-  keyroot=info->s->state.key_root[ftb->keynr];
-
   for (i=ftb->queue.elements; i; i--)
   {
     ftbw=(FTB_WORD *)(ftb->queue.root[i]);
@@ -248,35 +328,10 @@ static void _ftb_init_index_search(FT_INFO *ftb)
         }
       }
     }
-    r=_mi_search(info, keyinfo, (uchar*) ftbw->word, ftbw->len,
-                              SEARCH_FIND | SEARCH_BIGGER, keyroot);
-    if (!r)
-    {
-      r= mi_compare_text(ftb->charset,
-                         info->lastkey + (ftbw->flags&FTB_FLAG_TRUNC),
-                         ftbw->len     - (ftbw->flags&FTB_FLAG_TRUNC),
-                         (uchar*) ftbw->word    + (ftbw->flags&FTB_FLAG_TRUNC),
-                         ftbw->len     - (ftbw->flags&FTB_FLAG_TRUNC),
-                         0);
-    }
-    if (r) /* not found */
-    {
-      if (ftbw->flags&FTB_FLAG_YES && ftbw->up->up==0)
-      {
-	/*
-	  This word MUST BE present in every document returned,
-	  so we can abort the search right now
-	*/
-        ftb->state=INDEX_DONE;
+
+    if (_ft2_search(ftb, ftbw, 1))
       return;
   }
-    }
-    else
-    {
-      memcpy(ftbw->word, info->lastkey, info->lastkey_length);
-      ftbw->docid[0]=info->lastpos;
-    }
-  }
   queue_fix(& ftb->queue);
 }
 
@@ -436,10 +491,7 @@ int ft_boolean_read_next(FT_INFO *ftb, char *record)
   FTB_EXPR  *ftbe;
   FTB_WORD  *ftbw;
   MI_INFO   *info=ftb->info;
-  MI_KEYDEF *keyinfo=info->s->keyinfo+ftb->keynr;
-  my_off_t   keyroot=info->s->state.key_root[ftb->keynr];
   my_off_t   curdoc;
-  int        r;
 
   if (ftb->state != INDEX_SEARCH && ftb->state != INDEX_DONE)
     return -1;
@@ -466,34 +518,7 @@ int ft_boolean_read_next(FT_INFO *ftb, char *record)
       _ftb_climb_the_tree(ftb, ftbw, 0);
 
       /* update queue */
-      r=_mi_search(info, keyinfo, (uchar*) ftbw->word, USE_WHOLE_KEY,
-                   SEARCH_BIGGER , keyroot);
-      if (!r)
-      {
-        r= mi_compare_text(ftb->charset,
-                           info->lastkey + (ftbw->flags&FTB_FLAG_TRUNC),
-                           ftbw->len     - (ftbw->flags&FTB_FLAG_TRUNC),
-                           (uchar*) ftbw->word + (ftbw->flags&FTB_FLAG_TRUNC),
-                           ftbw->len     - (ftbw->flags&FTB_FLAG_TRUNC),
-                           0);
-      }
-      if (r) /* not found */
-      {
-        ftbw->docid[0]=HA_POS_ERROR;
-        if (ftbw->flags&FTB_FLAG_YES && ftbw->up->up==0)
-        {
-	  /*
-	    This word MUST BE present in every document returned,
-	    so we can stop the search right now
-	  */
-          ftb->state=INDEX_DONE;
-        }
-      }
-      else
-      {
-        memcpy(ftbw->word, info->lastkey, info->lastkey_length);
-        ftbw->docid[0]=info->lastpos;
-      }
+      _ft2_search(ftb, ftbw, 0);
       queue_replaced(& ftb->queue);
     }
 
@@ -505,7 +530,7 @@ int ft_boolean_read_next(FT_INFO *ftb, char *record)
       if (is_tree_inited(&ftb->no_dupes) &&
           tree_insert(&ftb->no_dupes, &curdoc, 0,
 		      ftb->no_dupes.custom_arg)->count >1)
-        /* but it managed to get past this line once */
+        /* but it managed already to get past this line once */
         continue;
 
       info->lastpos=curdoc;

myisam/ft_dump.c

@@ -56,7 +56,7 @@ static struct my_option my_long_options[] =
 
 int main(int argc,char *argv[])
 {
-  int error=0;
+  int error=0, subkeys;
   uint keylen, keylen2=0, inx, doc_cnt=0;
   float weight;
   double gws, min_gws=0, avg_gws=0;
@@ -125,7 +125,9 @@ int main(int argc,char *argv[])
       keylen=*(info->lastkey);
 
 #if HA_FT_WTYPE == HA_KEYTYPE_FLOAT
-      mi_float4get(weight,info->lastkey+keylen+1);
+      subkeys=mi_sint4korr(info->lastkey+keylen+1);
+      if (subkeys >= 0)
+        weight=*(float*)&subkeys;
 #else
 #error
 #endif
@@ -164,7 +166,10 @@ int main(int argc,char *argv[])
         }
       }
       if (dump)
+        if (subkeys>=0)
           printf("%9qx %20.7f %s\n",info->lastpos,weight,buf);
+        else
+          printf("%9qx => %17d %s\n",info->lastpos,-subkeys,buf);
 
       if(verbose && (total%HOW_OFTEN_TO_WRITE)==0)
         printf("%10ld\r",total);

myisam/ft_nlq_search.c

@@ -42,8 +42,6 @@ typedef struct st_all_in_one
   uint	      keynr;
   CHARSET_INFO *charset;
   uchar      *keybuff;
-  MI_KEYDEF  *keyinfo;
-  my_off_t    key_root;
   TREE	      dtree;
 } ALL_IN_ONE;
 
@@ -66,13 +64,14 @@ static int FT_SUPERDOC_cmp(void* cmp_arg __attribute__((unused)),
 
 static int walk_and_match(FT_WORD *word, uint32 count, ALL_IN_ONE *aio)
 {
+  int	       subkeys;
   uint	       keylen, r, doc_cnt;
-#ifdef EVAL_RUN
-  uint	       cnt;
-  double       sum, sum2, suml;
-#endif /* EVAL_RUN */
   FT_SUPERDOC  sdoc, *sptr;
   TREE_ELEMENT *selem;
+  MI_INFO      *info=aio->info;
+  uchar        *keybuff=aio->keybuff;
+  MI_KEYDEF    *keyinfo=info->s->keyinfo+aio->keynr;
+  my_off_t     key_root=info->s->state.key_root[aio->keynr];
 #if HA_FT_WTYPE == HA_KEYTYPE_FLOAT
   float tmp_weight;
 #else
@@ -83,45 +82,45 @@ static int walk_and_match(FT_WORD *word, uint32 count, ALL_IN_ONE *aio)
 
   word->weight=LWS_FOR_QUERY;
 
-  keylen=_ft_make_key(aio->info,aio->keynr,(char*) aio->keybuff,word,0);
-#ifdef EVAL_RUN
-  keylen-=1+HA_FT_WLEN;
-#else /* EVAL_RUN */
+  keylen=_ft_make_key(info,aio->keynr,(char*) keybuff,word,0);
   keylen-=HA_FT_WLEN;
-#endif /* EVAL_RUN */
-
-#ifdef EVAL_RUN
-  sum=sum2=suml=
-#endif /* EVAL_RUN */
   doc_cnt=0;
 
-  r=_mi_search(aio->info, aio->keyinfo, aio->keybuff, keylen,
-	       SEARCH_FIND | SEARCH_PREFIX, aio->key_root);
-  aio->info->update|= HA_STATE_AKTIV;  /* for _mi_test_if_changed() */
+  r=_mi_search(info, keyinfo, keybuff, keylen, SEARCH_FIND, key_root);
+  info->update|= HA_STATE_AKTIV;              /* for _mi_test_if_changed() */
 
   while (!r)
   {
-    if (mi_compare_text(aio->charset,
-			aio->info->lastkey,keylen,
-			aio->keybuff,keylen,0))
+
+    if (keylen &&
+        mi_compare_text(aio->charset,info->lastkey,keylen, keybuff,keylen,0))
      break;
 
+    subkeys=ft_sintXkorr(info->lastkey+keylen);
+    if (subkeys<0)
+    {
+      if (doc_cnt)
+        DBUG_RETURN(1); /* index is corrupted */
+      /*
+        TODO here: unsafe optimization, should this word
+        be skipped (based on subkeys) ?
+      */
+      keybuff+=keylen;
+      keyinfo=& info->s->ft2_keyinfo;
+      key_root=info->lastpos;
+      keylen=0;
+      r=_mi_search_first(info, keyinfo, key_root);
+      continue;
+    }
 #if HA_FT_WTYPE == HA_KEYTYPE_FLOAT
-#ifdef EVAL_RUN
-    mi_float4get(tmp_weight,aio->info->lastkey+keylen+1);
-#else /* EVAL_RUN */
-    mi_float4get(tmp_weight,aio->info->lastkey+keylen);
-#endif /* EVAL_RUN */
+    tmp_weight=*(float*)&subkeys;
 #else
 #error
 #endif
-    if(tmp_weight==0) DBUG_RETURN(doc_cnt); /* stopword, doc_cnt should be 0 */
-
-#ifdef EVAL_RUN
-    cnt=*(byte *)(aio->info->lastkey+keylen);
-#endif /* EVAL_RUN */
+    if (tmp_weight==0)
+      DBUG_RETURN(doc_cnt); /* stopword, doc_cnt should be 0 */
 
-    sdoc.doc.dpos=aio->info->lastpos;
+    sdoc.doc.dpos=info->lastpos;
 
     /* saving document matched into dtree */
     if (!(selem=tree_insert(&aio->dtree, &sdoc, 0, aio->dtree.custom_arg)))
@@ -138,20 +137,13 @@ static int walk_and_match(FT_WORD *word, uint32 count, ALL_IN_ONE *aio)
     sptr->tmp_weight=tmp_weight;
 
     doc_cnt++;
-#ifdef EVAL_RUN
-    sum +=cnt;
-    sum2+=cnt*cnt;
-    suml+=cnt*log(cnt);
-#endif /* EVAL_RUN */
-
-    if (_mi_test_if_changed(aio->info) == 0)
-	r=_mi_search_next(aio->info, aio->keyinfo, aio->info->lastkey,
-			  aio->info->lastkey_length, SEARCH_BIGGER,
-			  aio->key_root);
+
+    if (_mi_test_if_changed(info) == 0)
+	r=_mi_search_next(info, keyinfo, info->lastkey, info->lastkey_length,
+                          SEARCH_BIGGER, key_root);
     else
-	r=_mi_search(aio->info, aio->keyinfo, aio->info->lastkey,
-		     aio->info->lastkey_length, SEARCH_BIGGER,
-		     aio->key_root);
+	r=_mi_search(info, keyinfo, info->lastkey, info->lastkey_length,
+                     SEARCH_BIGGER, key_root);
   }
   if (doc_cnt)
   {
@@ -201,10 +193,8 @@ FT_INFO *ft_init_nlq_search(MI_INFO *info, uint keynr, byte *query,
 
   aio.info=info;
   aio.keynr=keynr;
-  aio.keyinfo=info->s->keyinfo+keynr;
-  aio.charset=aio.keyinfo->seg->charset;
+  aio.charset=info->s->keyinfo[keynr].seg->charset;
   aio.keybuff=info->lastkey+info->s->base.max_key_length;
-  aio.key_root=info->s->state.key_root[keynr];
 
   bzero(&allocated_wtree,sizeof(allocated_wtree));
 

myisam/ft_parser.c

@@ -18,21 +18,10 @@
 
 #include "ftdefs.h"
 
-#ifdef EVAL_RUN
-#ifdef PIVOT_STAT
-ulong collstat=0;
-#endif
-#endif /* EVAL_RUN */
-
 typedef struct st_ft_docstat {
   FT_WORD *list;
   uint uniq;
   double sum;
-#ifdef EVAL_RUN
-  uint words, totlen;
-  double max, nsum, nsum2;
-#endif /* EVAL_RUN */
-
 } FT_DOCSTAT;
 
 static int FT_WORD_cmp(CHARSET_INFO* cs, FT_WORD *w1, FT_WORD *w2)
@@ -44,15 +33,7 @@ static int FT_WORD_cmp(CHARSET_INFO* cs, FT_WORD *w1, FT_WORD *w2)
 static int walk_and_copy(FT_WORD *word,uint32 count,FT_DOCSTAT *docstat)
 {
     word->weight=LWS_IN_USE;
-
-#ifdef EVAL_RUN
-    word->cnt= (uchar) count;
-    if(docstat->max < word->weight) docstat->max=word->weight;
-    docstat->words+=count;
-    docstat->totlen+=word->len;
-#endif /* EVAL_RUN */
     docstat->sum+=word->weight;
-
     memcpy_fixed((docstat->list)++,word,sizeof(FT_WORD));
     return 0;
 }
@@ -70,9 +51,6 @@ FT_WORD * ft_linearize(TREE *wtree)
   {
     docstat.list=wlist;
     docstat.uniq=wtree->elements_in_tree;
-#ifdef EVAL_RUN
-    docstat.nsum=docstat.nsum2=docstat.max=docstat.words=docstat.totlen=
-#endif /* EVAL_RUN */
     docstat.sum=0;
     tree_walk(wtree,(tree_walk_action)&walk_and_copy,&docstat,left_root_right);
   }
@@ -85,18 +63,8 @@ FT_WORD * ft_linearize(TREE *wtree)
   for (p=wlist;p->pos;p++)
   {
     p->weight=PRENORM_IN_USE;
-#ifdef EVAL_RUN
-    docstat.nsum+=p->weight;
-    docstat.nsum2+=p->weight*p->weight;
-#endif /* EVAL_RUN */
   }
 
-#ifdef EVAL_RUN
-#ifdef PIVOT_STAT
-  collstat+=PIVOT_STAT;
-#endif
-#endif /* EVAL_RUN */
-
   for (p=wlist;p->pos;p++)
   {
     p->weight/=NORM_IN_USE;

myisam/ft_static.c

@@ -26,25 +26,21 @@ const char *ft_boolean_syntax="+ -><()~*:\"\"&|";
 const HA_KEYSEG ft_keysegs[FT_SEGS]={
 {
     HA_KEYTYPE_VARTEXT,               /* type */
-    7,                                /* language (will be overwritten) */
+    63,                               /* language (will be overwritten) */
     0, 0, 0,                          /* null_bit, bit_start, bit_end */
     HA_VAR_LENGTH | HA_PACK_KEY,      /* flag */
     HA_FT_MAXLEN,                     /* length */
-#ifdef EVAL_RUN
-    HA_FT_WLEN+1,                     /* start */
-#else /* EVAL_RUN */
     HA_FT_WLEN,                       /* start */
-#endif /* EVAL_RUN */
     0,                                /* null_pos */
     NULL                              /* charset  */
   },
-#ifdef EVAL_RUN
   {
-      HA_KEYTYPE_INT8, 7, 0, 0, 0, 0, 1, HA_FT_WLEN, 0, NULL
-  },
-#endif /* EVAL_RUN */
-  {
-      HA_FT_WTYPE, 7, 0, 0, 0, HA_NO_SORT, HA_FT_WLEN, 0, 0, NULL
+/*
+  Note, this (and the last HA_KEYTYPE_END) segment should NOT
+  be packed in any way, otherwise w_search() won't be able to
+  update key entry 'in vivo'
+*/
+      HA_FT_WTYPE, 63, 0, 0, 0, HA_NO_SORT, HA_FT_WLEN, 0, 0, NULL
   }
 };
 

myisam/ft_update.c

@@ -31,7 +31,7 @@
 void _mi_ft_segiterator_init(MI_INFO *info, uint keynr, const byte *record,
 			     FT_SEG_ITERATOR *ftsi)
 {
-  ftsi->num=info->s->keyinfo[keynr].keysegs-FT_SEGS;
+  ftsi->num=info->s->keyinfo[keynr].keysegs;
   ftsi->seg=info->s->keyinfo[keynr].seg;
   ftsi->rec=record;
 }
@@ -113,7 +113,7 @@ FT_WORD * _mi_ft_parserecord(MI_INFO *info, uint keynr,
   if (_mi_ft_parse(&ptree, info, keynr, record))
     return NULL;
 
-  return ft_linearize(/*info, keynr, keybuf, */ &ptree);
+  return ft_linearize(&ptree);
 }
 
 static int _mi_ft_store(MI_INFO *info, uint keynr, byte *keybuf,
@@ -267,13 +267,7 @@ uint _ft_make_key(MI_INFO *info, uint keynr, byte *keybuf, FT_WORD *wptr,
 #error
 #endif
 
-#ifdef EVAL_RUN
-  *(buf+HA_FT_WLEN)=wptr->cnt;
-  int2store(buf+HA_FT_WLEN+1,wptr->len);
-  memcpy(buf+HA_FT_WLEN+3,wptr->pos,wptr->len);
-#else /* EVAL_RUN */
   int2store(buf+HA_FT_WLEN,wptr->len);
   memcpy(buf+HA_FT_WLEN+2,wptr->pos,wptr->len);
-#endif /* EVAL_RUN */
   return _mi_make_key(info,keynr,(uchar*) keybuf,buf,filepos);
 }

myisam/ftdefs.h

@@ -60,16 +60,6 @@
 #define NORM_SUM				   (docstat.nsum)
 #define NORM_COS			    (sqrt(docstat.nsum2))
 
-#ifdef EVAL_RUN
-/*
-extern ulong collstat;
-#define PIVOT_STAT  (docstat.uniq)
-#define PIVOT_SLOPE (0.69)
-#define PIVOT_PIVOT ((double)collstat/(info->state->records+1))
-#define NORM_PIVOT  ((1-PIVOT_SLOPE)*PIVOT_PIVOT+PIVOT_SLOPE*docstat.uniq)
-*/
-#endif /* EVAL_RUN */
-
 #define PIVOT_VAL (0.0115)
 #define NORM_PIVOT  (1+PIVOT_VAL*docstat.uniq)
 /*---------------------------------------------------------------*/
@@ -102,9 +92,6 @@ typedef struct st_ft_word {
   byte * pos;
   uint	 len;
   double weight;
-#ifdef EVAL_RUN
-  byte	 cnt;
-#endif /* EVAL_RUN */
 } FT_WORD;
 
 typedef struct st_ftb_param {

myisam/fulltext.h

@@ -21,19 +21,16 @@
 #include "myisamdef.h"
 #include "ft_global.h"
 
-/* shoudn't be def'ed when linking with mysql */
-#undef EVAL_RUN
-
 #define HA_FT_WTYPE  HA_KEYTYPE_FLOAT
 #define HA_FT_WLEN   4
-#ifdef EVAL_RUN
-#define FT_SEGS      3
-#else /* EVAL_RUN */
 #define FT_SEGS      2
-#endif /* EVAL_RUN */
+
+#define ft_sintXkorr(A)    mi_sint4korr(A)
+#define ft_intXstore(T,A)  mi_int4store(T,A)
 
 extern const HA_KEYSEG ft_keysegs[FT_SEGS];
 
 int  _mi_ft_cmp(MI_INFO *, uint, const byte *, const byte *);
 int  _mi_ft_add(MI_INFO *, uint, byte *, const byte *, my_off_t);
 int  _mi_ft_del(MI_INFO *, uint, byte *, const byte *, my_off_t);
+

myisam/mi_check.c

@@ -49,6 +49,7 @@ static int sort_key_read(MI_SORT_PARAM *sort_param,void *key);
 static int sort_ft_key_read(MI_SORT_PARAM *sort_param,void *key);
 static int sort_get_next_record(MI_SORT_PARAM *sort_param);
 static int sort_key_cmp(MI_SORT_PARAM *sort_param, const void *a,const void *b);
+static int sort_ft_key_write(MI_SORT_PARAM *sort_param, const void *a);
 static int sort_key_write(MI_SORT_PARAM *sort_param, const void *a);
 static my_off_t get_record_for_key(MI_INFO *info,MI_KEYDEF *keyinfo,
 				uchar *key);
@@ -1875,7 +1876,6 @@ int mi_repair_by_sort(MI_CHECK *param, register MI_INFO *info,
     ((param->testflag & T_CREATE_MISSING_KEYS) ? info->state->records :
      (ha_rows) (sort_info.filelength/length+1));
   sort_param.key_cmp=sort_key_cmp;
-  sort_param.key_write=sort_key_write;
   sort_param.lock_in_memory=lock_memory;
   sort_param.tmpdir=param->tmpdir;
   sort_param.sort_info=&sort_info;
@@ -1928,10 +1928,14 @@ int mi_repair_by_sort(MI_CHECK *param, register MI_INFO *info,
         (ha_rows) (sort_info.filelength/ft_max_word_len_for_sort+1);
 
       sort_param.key_read=sort_ft_key_read;
+      sort_param.key_write=sort_ft_key_write;
       sort_param.key_length+=ft_max_word_len_for_sort-HA_FT_MAXLEN;
     }
     else
+    {
       sort_param.key_read=sort_key_read;
+      sort_param.key_write=sort_key_write;
+    }
 
     if (_create_index_by_sort(&sort_param,
 			      (my_bool) (!(param->testflag & T_VERBOSE)),
@@ -1977,9 +1981,6 @@ int mi_repair_by_sort(MI_CHECK *param, register MI_INFO *info,
     else
       info->state->data_file_length=sort_param.max_pos;
 
-    /*if (flush_pending_blocks(param))
-      goto err;*/
-
     param->read_cache.file=info->dfile;		/* re-init read cache */
     reinit_io_cache(&param->read_cache,READ_CACHE,share->pack.header_length,
                     1,1);
@@ -2078,6 +2079,7 @@ int mi_repair_by_sort(MI_CHECK *param, register MI_INFO *info,
                             MYF(MY_ALLOW_ZERO_PTR));
   my_free(sort_param.record,MYF(MY_ALLOW_ZERO_PTR));
   my_free((gptr) sort_info.key_block,MYF(MY_ALLOW_ZERO_PTR));
+  my_free((gptr) sort_info.ft_buf, MYF(MY_ALLOW_ZERO_PTR));
   my_free(sort_info.buff,MYF(MY_ALLOW_ZERO_PTR));
   VOID(end_io_cache(&param->read_cache));
   info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED);
@@ -2093,7 +2095,7 @@ int mi_repair_by_sort(MI_CHECK *param, register MI_INFO *info,
   Threaded repair of table using sorting
 
   SYNOPSIS
-    mi_repair_by_sort_r()
+    mi_repair_by_sort_parallel()
     param		Repair parameters
     info		MyISAM handler to repair
     name		Name of table (for warnings)
@@ -2265,10 +2267,17 @@ int mi_repair_parallel(MI_CHECK *param, register MI_INFO *info,
     }
     if ((!(param->testflag & T_SILENT)))
       printf ("- Fixing index %d\n",key+1);
-    sort_param[i].key_read= ((sort_param[i].keyinfo->flag & HA_FULLTEXT) ?
-			     sort_ft_key_read : sort_key_read);
-    sort_param[i].key_cmp=sort_key_cmp;
+    if (sort_param[i].keyinfo->flag & HA_FULLTEXT)
+    {
+      sort_param[i].key_read=sort_ft_key_read;
+      sort_param[i].key_write=sort_ft_key_write;
+    }
+    else
+    {
+      sort_param[i].key_read=sort_key_read;
       sort_param[i].key_write=sort_key_write;
+    }
+    sort_param[i].key_cmp=sort_key_cmp;
     sort_param[i].lock_in_memory=lock_memory;
     sort_param[i].tmpdir=param->tmpdir;
     sort_param[i].sort_info=&sort_info;
@@ -2461,6 +2470,7 @@ int mi_repair_parallel(MI_CHECK *param, register MI_INFO *info,
   pthread_cond_destroy (&sort_info.cond);
   pthread_mutex_destroy(&sort_info.mutex);
 
+  my_free((gptr) sort_info.ft_buf, MYF(MY_ALLOW_ZERO_PTR));
   my_free((gptr) sort_info.key_block,MYF(MY_ALLOW_ZERO_PTR));
   my_free((gptr) sort_param,MYF(MY_ALLOW_ZERO_PTR));
   my_free(sort_info.buff,MYF(MY_ALLOW_ZERO_PTR));
@@ -3095,6 +3105,137 @@ static int sort_key_write(MI_SORT_PARAM *sort_param, const void *a)
 			  (uchar*) a, HA_OFFSET_ERROR));
 } /* sort_key_write */
 
+int sort_ft_buf_flush(MI_SORT_PARAM *sort_param)
+{
+  SORT_INFO *sort_info=sort_param->sort_info;
+  SORT_KEY_BLOCKS *key_block=sort_info->key_block;
+  MYISAM_SHARE *share=sort_info->info->s;
+  uint val_off, val_len, error;
+  SORT_FT_BUF *ft_buf=sort_info->ft_buf;
+  uchar *from, *to;
+
+  val_len=share->ft2_keyinfo.keylength;
+  get_key_full_length_rdonly(val_off, ft_buf->lastkey);
+  to=ft_buf->lastkey+val_off;
+
+  if (ft_buf->buf)
+  { /* flushing first-level tree */
+    error=sort_insert_key(sort_param,key_block,ft_buf->lastkey,HA_OFFSET_ERROR);
+    for (from=to+val_len;
+         !error && from < ft_buf->buf;
+         from+= val_len)
+    {
+      memcpy(to, from, val_len);
+      error=sort_insert_key(sort_param,key_block,ft_buf->lastkey,HA_OFFSET_ERROR);
+    }
+    return error;
+  }
+  /* flushing second-level tree keyblocks */
+  error=flush_pending_blocks(sort_param);
+  /* updating lastkey with second-level tree info */
+  ft_intXstore(ft_buf->lastkey+val_off, -ft_buf->count);
+  _mi_dpointer(sort_info->info, ft_buf->lastkey+val_off+HA_FT_WLEN,
+      share->state.key_root[sort_param->key]);
+  /* restoring first level tree data in sort_info/sort_param */
+  sort_info->key_block=sort_info->key_block_end- sort_info->param->sort_key_blocks;
+  sort_param->keyinfo=share->keyinfo+sort_param->key;
+  share->state.key_root[sort_param->key]=HA_OFFSET_ERROR;
+  /* writing lastkey in first-level tree */
+  return error ? error :
+                 sort_insert_key(sort_param,sort_info->key_block,
+                                 ft_buf->lastkey,HA_OFFSET_ERROR);
+}
+
+static int sort_ft_key_write(MI_SORT_PARAM *sort_param, const void *a)
+{
+  uint a_len, val_off, val_len, error;
+  uchar *p;
+  SORT_INFO *sort_info=sort_param->sort_info;
+  SORT_FT_BUF *ft_buf=sort_info->ft_buf;
+  SORT_KEY_BLOCKS *key_block=sort_info->key_block;
+
+  val_len=HA_FT_WLEN+sort_info->info->s->base.rec_reflength;
+  get_key_full_length_rdonly(a_len, (uchar *)a);
+
+  if (!ft_buf)
+  {
+    /*
+      use two-level tree only if key_reflength fits in rec_reflength place
+      and row format is NOT static - for _mi_dpointer not to garble offsets
+     */
+    if ((sort_info->info->s->base.key_reflength <=
+         sort_info->info->s->base.rec_reflength) &&
+        (sort_info->info->s->options &
+          (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)))
+      ft_buf=(SORT_FT_BUF *)my_malloc(sort_param->keyinfo->block_length +
+                                      sizeof(SORT_FT_BUF), MYF(MY_WME));
+
+    if (!ft_buf)
+    {
+      sort_param->key_write=sort_key_write;
+      return sort_key_write(sort_param, a);
+    }
+    sort_info->ft_buf=ft_buf;
+    goto word_init_ft_buf; /* no need to duplicate the code */
+  }
+  get_key_full_length_rdonly(val_off, ft_buf->lastkey);
+
+  if (val_off == a_len &&
+      mi_compare_text(sort_param->keyinfo->seg->charset,
+                      ((uchar *)a)+1,a_len-1,
+                      ft_buf->lastkey+1,val_off-1, 0)==0)
+  {
+    if (!ft_buf->buf) /* store in second-level tree */
+    {
+      ft_buf->count++;
+      return sort_insert_key(sort_param,key_block,
+                             ((uchar *)a)+val_off, HA_OFFSET_ERROR);
+    }
+
+    /* storing the key in the buffer. */
+    memcpy (ft_buf->buf, a+val_off, val_len);
+    ft_buf->buf+=val_len;
+    if (ft_buf->buf < ft_buf->end)
+      return 0;
+
+    /* converting to two-level tree */
+    p=ft_buf->lastkey+val_off;
+
+    while (key_block->inited)
+      key_block++;
+    sort_info->key_block=key_block;
+    sort_param->keyinfo=& sort_info->info->s->ft2_keyinfo;
+    ft_buf->count=(ft_buf->buf - p)/val_len;
+
+    /* flushing buffer to second-level tree */
+    for (error=0; !error && p < ft_buf->buf; p+= val_len)
+      error=sort_insert_key(sort_param,key_block,p,HA_OFFSET_ERROR);
+    ft_buf->buf=0;
+    return error;
+  }
+  else
+  {
+    /* flushing buffer */
+    if ((error=sort_ft_buf_flush(sort_param)))
+      return error;
+
+word_init_ft_buf:
+    a_len+=val_len;
+    memcpy(ft_buf->lastkey, a, a_len);
+    ft_buf->buf=ft_buf->lastkey+a_len;
+    ft_buf->end=ft_buf->lastkey+ (sort_param->keyinfo->block_length-32);
+    /* 32 is just a safety margin here
+       (at least max(val_len, sizeof(nod_flag)) should be there).
+       May be better performance could be achieved if we'd put
+          (sort_info->keyinfo->block_length-32)/XXX
+       instead.
+       TODO: benchmark the best value for XXX.
+    */
+
+    return 0;
+  }
+  return -1; /* impossible */
+} /* sort_ft_key_write */
 
 	/* get pointer to record from a key */
 
@@ -3254,7 +3395,7 @@ int flush_pending_blocks(MI_SORT_PARAM *sort_param)
   my_off_t filepos,key_file_length;
   SORT_KEY_BLOCKS *key_block;
   SORT_INFO *sort_info= sort_param->sort_info;
-  MI_CHECK *param=sort_info->param;
+  myf myf_rw=sort_info->param->myf_rw;
   MI_INFO *info=sort_info->info;
   MI_KEYDEF *keyinfo=sort_param->keyinfo;
   DBUG_ENTER("flush_pending_blocks");
@@ -3279,7 +3420,7 @@ int flush_pending_blocks(MI_SORT_PARAM *sort_param)
 	DBUG_RETURN(1);
     }
     else if (my_pwrite(info->s->kfile,(byte*) key_block->buff,
-		       (uint) keyinfo->block_length,filepos, param->myf_rw))
+		       (uint) keyinfo->block_length,filepos, myf_rw))
       DBUG_RETURN(1);
     DBUG_DUMP("buff",(byte*) key_block->buff,length);
     nod_flag=1;

myisam/mi_create.c

@@ -44,7 +44,7 @@ int mi_create(const char *name,uint keys,MI_KEYDEF *keydefs,
   uint fields,length,max_key_length,packed,pointer,
        key_length,info_length,key_segs,options,min_key_length_skipp,
        base_pos,varchar_count,long_varchar_count,varchar_length,
-       max_key_block_length,unique_key_parts,offset;
+       max_key_block_length,unique_key_parts,fulltext_keys,offset;
   ulong reclength, real_reclength,min_pack_length;
   char filename[FN_REFLEN],linkname[FN_REFLEN], *linkname_ptr;
   ulong pack_reclength;
@@ -223,6 +223,7 @@ int mi_create(const char *name,uint keys,MI_KEYDEF *keydefs,
     reclength+=long_varchar_count;	/* We need space for this! */
 
   max_key_length=0; tot_length=0 ; key_segs=0;
+  fulltext_keys=0;
   max_key_block_length=0;
   share.state.rec_per_key_part=rec_per_key_part;
   share.state.key_root=key_root;
@@ -249,7 +250,7 @@ int mi_create(const char *name,uint keys,MI_KEYDEF *keydefs,
       {
         /*
            called by myisamchk - i.e. table structure was taken from
-           MYI file and SPATIAL key *do has* additional sp_segs keysegs.
+           MYI file and SPATIAL key *does have* additional sp_segs keysegs.
            We'd better delete them now
         */
         keydef->keysegs-=sp_segs;
@@ -271,20 +272,11 @@ int mi_create(const char *name,uint keys,MI_KEYDEF *keydefs,
       min_key_length_skipp+=SPLEN*2*SPDIMS;
     }
     else
-    if (keydef->flag & HA_FULLTEXT)                                 /* SerG */
+    if (keydef->flag & HA_FULLTEXT)
     {
       keydef->flag=HA_FULLTEXT | HA_PACK_KEY | HA_VAR_LENGTH_KEY;
       options|=HA_OPTION_PACK_KEYS;             /* Using packed keys */
 
-      if (flags & HA_DONT_TOUCH_DATA)
-      {
-        /* called by myisamchk - i.e. table structure was taken from
-           MYI file and FULLTEXT key *do has* additional FT_SEGS keysegs.
-           We'd better delete them now
-        */
-        keydef->keysegs-=FT_SEGS;
-      }
-
       for (j=0, keyseg=keydef->seg ; (int) j < keydef->keysegs ;
 	   j++, keyseg++)
       {
@@ -295,19 +287,11 @@ int mi_create(const char *name,uint keys,MI_KEYDEF *keydefs,
           goto err;
         }
       }
-      keydef->keysegs+=FT_SEGS;
 
+      fulltext_keys++;
       key_length+= HA_FT_MAXLEN+HA_FT_WLEN;
-#ifdef EVAL_RUN
-      key_length++;
-#endif
-
       length++;                              /* At least one length byte */
       min_key_length_skipp+=HA_FT_MAXLEN;
-#if HA_FT_MAXLEN >= 255
-      min_key_length_skipp+=2;                  /* prefix may be 3 bytes */
-      length+=2;
-#endif
     }
     else
     {
@@ -473,8 +457,9 @@ int mi_create(const char *name,uint keys,MI_KEYDEF *keydefs,
     mi_get_pointer_length((tot_length + max_key_block_length * keys *
 			   MI_INDEX_BLOCK_MARGIN) / MI_MIN_KEY_BLOCK_LENGTH,
 			  3);
-  share.base.keys= share.state.header.keys = keys;
+  share.base.keys= share.state.header.keys= keys;
   share.state.header.uniques= uniques;
+  share.state.header.fulltext_keys= fulltext_keys;
   mi_int2store(share.state.header.key_parts,key_segs);
   mi_int2store(share.state.header.unique_key_parts,unique_key_parts);
 
@@ -590,21 +575,13 @@ int mi_create(const char *name,uint keys,MI_KEYDEF *keydefs,
   /* Write key and keyseg definitions */
   for (i=0 ; i < share.base.keys - uniques; i++)
   {
-    uint ft_segs=(keydefs[i].flag & HA_FULLTEXT) ? FT_SEGS : 0;
     uint sp_segs=(keydefs[i].flag & HA_SPATIAL) ? 2*SPDIMS : 0;
 
     if (mi_keydef_write(file, &keydefs[i]))
       goto err;
-    for (j=0 ; j < keydefs[i].keysegs-ft_segs-sp_segs ; j++)
+    for (j=0 ; j < keydefs[i].keysegs-sp_segs ; j++)
       if (mi_keyseg_write(file, &keydefs[i].seg[j]))
        goto err;
-    for (j=0 ; j < ft_segs ; j++)
-    {
-      HA_KEYSEG seg=ft_keysegs[j];
-      seg.language= keydefs[i].seg[0].language;
-      if (mi_keyseg_write(file, &seg))
-        goto err;
-    }
     for (j=0 ; j < sp_segs ; j++)
     {
       HA_KEYSEG sseg;

myisam/mi_delete.c

@@ -23,13 +23,13 @@
 #include <errno.h>
 #endif
 
-static int d_search(MI_INFO *info,MI_KEYDEF *keyinfo,uchar *key,
-		    uint key_length, my_off_t page, uchar *anc_buff);
+static int d_search(MI_INFO *info,MI_KEYDEF *keyinfo,uint comp_flag,
+                    uchar *key,uint key_length,my_off_t page,uchar *anc_buff);
 static int del(MI_INFO *info,MI_KEYDEF *keyinfo,uchar *key,uchar *anc_buff,
 	       my_off_t leaf_page,uchar *leaf_buff,uchar *keypos,
 	       my_off_t next_block,uchar *ret_key);
 static int underflow(MI_INFO *info,MI_KEYDEF *keyinfo,uchar *anc_buff,
-		     my_off_t leaf_page, uchar *leaf_buff,uchar *keypos);
+		     my_off_t leaf_page,uchar *leaf_buff,uchar *keypos);
 static uint remove_key(MI_KEYDEF *keyinfo,uint nod_flag,uchar *keypos,
 		       uchar *lastkey,uchar *page_end,
 		       my_off_t *next_block);
@@ -73,7 +73,6 @@ int mi_delete(MI_INFO *info,const byte *record)
     if (((ulonglong) 1 << i) & info->s->state.key_map)
     {
       info->s->keyinfo[i].version++;
-      /* The following code block is for text searching by SerG */
       if (info->s->keyinfo[i].flag & HA_FULLTEXT )
       {
         if (_mi_ft_del(info,i,(char*) old_key,record,info->lastpos))
@@ -128,19 +127,24 @@ int mi_delete(MI_INFO *info,const byte *record)
 
 int _mi_ck_delete(register MI_INFO *info, uint keynr, uchar *key,
 		  uint key_length)
+{
+  return _mi_ck_real_delete(info, info->s->keyinfo+keynr, key, key_length,
+                            &info->s->state.key_root[keynr]);
+} /* _mi_ck_delete */
+
+int _mi_ck_real_delete(register MI_INFO *info, MI_KEYDEF *keyinfo,
+                       uchar *key, uint key_length, my_off_t *root)
 {
   int error;
   uint nod_flag;
   my_off_t old_root;
   uchar *root_buff;
-  MI_KEYDEF *keyinfo;
-  DBUG_ENTER("_mi_ck_delete");
+  DBUG_ENTER("_mi_ck_real_delete");
 
-  if ((old_root=info->s->state.key_root[keynr]) == HA_OFFSET_ERROR)
+  if ((old_root=*root) == HA_OFFSET_ERROR)
   {
     DBUG_RETURN(my_errno=HA_ERR_CRASHED);
   }
-  keyinfo=info->s->keyinfo+keynr;
   if (!(root_buff= (uchar*) my_alloca((uint) keyinfo->block_length+
 				      MI_MAX_KEY_BUFF*2)))
   {
@@ -153,12 +157,15 @@ int _mi_ck_delete(register MI_INFO *info, uint keynr, uchar *key,
     error= -1;
     goto err;
   }
-  if ((error=d_search(info,keyinfo,key,key_length,old_root,root_buff)) >0)
+  if ((error=d_search(info,keyinfo,
+                      (keyinfo->flag & HA_FULLTEXT ? SEARCH_FIND
+                                                   : SEARCH_SAME),
+                       key,key_length,old_root,root_buff)) >0)
   {
     if (error == 2)
     {
       DBUG_PRINT("test",("Enlarging of root when deleting"));
-      error=_mi_enlarge_root(info,keynr,key);
+      error=_mi_enlarge_root(info,keyinfo,key,root);
     }
     else /* error == 1 */
     {
@@ -166,10 +173,9 @@ int _mi_ck_delete(register MI_INFO *info, uint keynr, uchar *key,
       {
 	error=0;
 	if (nod_flag)
-	  info->s->state.key_root[keynr]=_mi_kpos(nod_flag,
-						  root_buff+2+nod_flag);
+	  *root=_mi_kpos(nod_flag,root_buff+2+nod_flag);
 	else
-	  info->s->state.key_root[keynr]= HA_OFFSET_ERROR;
+	  *root=HA_OFFSET_ERROR;
 	if (_mi_dispose(info,keyinfo,old_root))
 	  error= -1;
       }
@@ -180,7 +186,7 @@ int _mi_ck_delete(register MI_INFO *info, uint keynr, uchar *key,
 err:
   my_afree((gptr) root_buff);
   DBUG_RETURN(error);
-} /* _mi_ck_delete */
+} /* _mi_ck_real_delete */
 
 
 	/*
@@ -192,11 +198,11 @@ int _mi_ck_delete(register MI_INFO *info, uint keynr, uchar *key,
 	*/
 
 static int d_search(register MI_INFO *info, register MI_KEYDEF *keyinfo,
-		    uchar *key, uint key_length, my_off_t page,
-		    uchar *anc_buff)
+                    uint comp_flag, uchar *key, uint key_length,
+                    my_off_t page, uchar *anc_buff)
 {
   int flag,ret_value,save_flag;
-  uint length,nod_flag;
+  uint length,nod_flag,search_key_length;
   my_bool last_key;
   uchar *leaf_buff,*keypos;
   my_off_t leaf_page,next_block;
@@ -204,9 +210,9 @@ static int d_search(register MI_INFO *info, register MI_KEYDEF *keyinfo,
   DBUG_ENTER("d_search");
   DBUG_DUMP("page",(byte*) anc_buff,mi_getint(anc_buff));
 
-  flag=(*keyinfo->bin_search)(info,keyinfo,anc_buff,key, USE_WHOLE_KEY,
-			      SEARCH_SAME,
-			      &keypos, lastkey, &last_key);
+  search_key_length= (comp_flag & SEARCH_FIND) ? key_length : USE_WHOLE_KEY;
+  flag=(*keyinfo->bin_search)(info,keyinfo,anc_buff,key, search_key_length,
+                              comp_flag, &keypos, lastkey, &last_key);
   if (flag == MI_FOUND_WRONG_KEY)
   {
     DBUG_PRINT("error",("Found wrong key"));
@@ -214,6 +220,52 @@ static int d_search(register MI_INFO *info, register MI_KEYDEF *keyinfo,
   }
   nod_flag=mi_test_if_nod(anc_buff);
 
+  if (!flag && keyinfo->flag & HA_FULLTEXT)
+  {
+    uint off;
+    int  subkeys;
+
+    get_key_full_length_rdonly(off, lastkey);
+    subkeys=ft_sintXkorr(lastkey+off);
+    comp_flag=SEARCH_SAME;
+    if (subkeys >= 0)
+    {
+      /* normal word, one-level tree structure */
+      flag=(*keyinfo->bin_search)(info,keyinfo,anc_buff,key,USE_WHOLE_KEY,
+                                  comp_flag, &keypos, lastkey, &last_key);
+      /* fall through to normal delete */
+    }
+    else
+    {
+      /* popular word. two-level tree. going down */
+      uint tmp_key_length;
+      my_off_t root;
+      uchar *kpos=keypos;
+
+      tmp_key_length=(*keyinfo->get_key)(keyinfo,nod_flag,&kpos,lastkey);
+      root=_mi_dpos(info,nod_flag,kpos);
+      if (subkeys == -1)
+      {
+        /* the last entry in sub-tree */
+        _mi_dispose(info, keyinfo, root);
+        /* fall through to normal delete */
+      }
+      else
+      {
+        keyinfo=&info->s->ft2_keyinfo;
+        kpos-=keyinfo->keylength; /* we'll modify key entry 'in vivo' */
+        key+=off;
+        ret_value=_mi_ck_real_delete(info, &info->s->ft2_keyinfo,
+            key, HA_FT_WLEN, &root);
+        _mi_dpointer(info, kpos+HA_FT_WLEN, root);
+        subkeys++;
+        ft_intXstore(kpos, subkeys);
+        if (!ret_value)
+          ret_value=_mi_write_keypage(info,keyinfo,page,anc_buff);
+        DBUG_RETURN(ret_value);
+      }
+    }
+  }
   leaf_buff=0;
   LINT_INIT(leaf_page);
   if (nod_flag)
@@ -239,7 +291,8 @@ static int d_search(register MI_INFO *info, register MI_KEYDEF *keyinfo,
       goto err;
     }
     save_flag=0;
-    ret_value=d_search(info,keyinfo,key,key_length,leaf_page,leaf_buff);
+    ret_value=d_search(info,keyinfo,comp_flag,key,key_length,
+                       leaf_page,leaf_buff);
   }
   else
   {						/* Found key */

myisam/mi_open.c

@@ -69,7 +69,7 @@ MI_INFO *mi_open(const char *name, int mode, uint open_flags)
 {
   int lock_error,kfile,open_mode,save_errno,have_rtree=0;
   uint i,j,len,errpos,head_length,base_pos,offset,info_length,keys,
-    key_parts,unique_key_parts,tmp_length,uniques;
+    key_parts,unique_key_parts,fulltext_keys,uniques;
   char name_buff[FN_REFLEN], org_name [FN_REFLEN], index_name[FN_REFLEN],
        data_name[FN_REFLEN];
   char *disk_cache,*disk_pos;
@@ -126,8 +126,7 @@ MI_INFO *mi_open(const char *name, int mode, uint open_flags)
 	  HA_OPTION_TEMP_COMPRESS_RECORD | HA_OPTION_CHECKSUM |
 	  HA_OPTION_TMP_TABLE | HA_OPTION_DELAY_KEY_WRITE))
     {
-      DBUG_PRINT("error",("wrong options: 0x%lx",
-			  share->options));
+      DBUG_PRINT("error",("wrong options: 0x%lx", share->options));
       my_errno=HA_ERR_OLD_FILE;
       goto err;
     }
@@ -162,11 +161,9 @@ MI_INFO *mi_open(const char *name, int mode, uint open_flags)
     len=mi_uint2korr(share->state.header.state_info_length);
     keys=    (uint) share->state.header.keys;
     uniques= (uint) share->state.header.uniques;
+    fulltext_keys= (uint) share->state.header.fulltext_keys;
     key_parts= mi_uint2korr(share->state.header.key_parts);
     unique_key_parts= mi_uint2korr(share->state.header.unique_key_parts);
-    tmp_length=(MI_STATE_INFO_SIZE + keys * MI_STATE_KEY_SIZE +
-		key_parts*MI_STATE_KEYSEG_SIZE +
-		share->state.header.max_block_size*MI_STATE_KEYBLOCK_SIZE);
     if (len != MI_STATE_INFO_SIZE)
     {
       DBUG_PRINT("warning",
@@ -203,6 +200,7 @@ MI_INFO *mi_open(const char *name, int mode, uint open_flags)
       goto err;
     }
 
+    key_parts+=fulltext_keys*FT_SEGS;
     if (share->base.max_key_length > MI_MAX_KEY_BUFF || keys > MI_MAX_KEY ||
 	key_parts >= MI_MAX_KEY * MI_MAX_KEY_SEG)
     {
@@ -211,7 +209,7 @@ MI_INFO *mi_open(const char *name, int mode, uint open_flags)
       goto err;
     }
 
-    /* Correct max_file_length based on length of sizeof_t */
+    /* Correct max_file_length based on length of sizeof(off_t) */
     max_data_file_length=
       (share->options & (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)) ?
       (((ulonglong) 1 << (share->base.rec_reflength*8))-1) :
@@ -290,6 +288,8 @@ MI_INFO *mi_open(const char *name, int mode, uint open_flags)
       for (i=0 ; i < keys ; i++)
       {
 	disk_pos=mi_keydef_read(disk_pos, &share->keyinfo[i]);
+        if (share->keyinfo[i].key_alg == HA_KEY_ALG_RTREE)
+          have_rtree=1;
 	set_if_smaller(share->blocksize,share->keyinfo[i].block_length);
 	share->keyinfo[i].seg=pos;
 	for (j=0 ; j < share->keyinfo[i].keysegs; j++,pos++)
@@ -312,11 +312,41 @@ MI_INFO *mi_open(const char *name, int mode, uint open_flags)
 	  uint sp_segs=SPDIMS*2;
 	  share->keyinfo[i].seg=pos-sp_segs;
 	  share->keyinfo[i].keysegs--;
-	} else if (share->keyinfo[i].flag & HA_FULLTEXT)
+	}
+        else if (share->keyinfo[i].flag & HA_FULLTEXT)
 	{
+          if (!fulltext_keys)
+          { /* 4.0 compatibility code, to be removed in 5.0 */
             share->keyinfo[i].seg=pos-FT_SEGS;
-	  share->fulltext_index=1;
+            share->keyinfo[i].keysegs-=FT_SEGS;
+            share->state.header.fulltext_keys++;
+          }
+          else
+          {
+            uint j;
+            share->keyinfo[i].seg=pos;
+            for (j=0; j < FT_SEGS; j++)
+            {
+              *pos=ft_keysegs[j];
+              pos[0].language= pos[-1].language;
+              pos[0].charset= pos[-1].charset;
+              pos++;
+            }
+          }
+          if (!share->ft2_keyinfo.seg)
+          {
+            memcpy(& share->ft2_keyinfo, & share->keyinfo[i], sizeof(MI_KEYDEF));
+            share->ft2_keyinfo.keysegs=1;
+            share->ft2_keyinfo.flag=0;
+            share->ft2_keyinfo.keylength=
+            share->ft2_keyinfo.minlength=
+            share->ft2_keyinfo.maxlength=HA_FT_WLEN+share->base.rec_reflength;
+            share->ft2_keyinfo.seg=pos-1;
+            share->ft2_keyinfo.end=pos;
+            setup_key_functions(& share->ft2_keyinfo);
+          }
 	}
+        setup_key_functions(share->keyinfo+i);
 	share->keyinfo[i].end=pos;
 	pos->type=HA_KEYTYPE_END;			/* End */
 	pos->length=share->base.rec_reflength;
@@ -349,12 +379,6 @@ MI_INFO *mi_open(const char *name, int mode, uint open_flags)
 	pos++;
       }
     }
-    for (i=0 ; i < keys ; i++)
-    {
-      if (share->keyinfo[i].key_alg == HA_KEY_ALG_RTREE)
-        have_rtree=1;
-      setup_key_functions(share->keyinfo+i);
-    }
 
     for (i=j=offset=0 ; i < share->base.fields ; i++)
     {
@@ -720,9 +744,9 @@ static void setup_key_functions(register MI_KEYDEF *keyinfo)
 }
 
 
-/***************************************************************************
-** Function to save and store the header in the index file (.MSI)
-***************************************************************************/
+/*
+   Function to save and store the header in the index file (.MYI)
+*/
 
 uint mi_state_info_write(File file, MI_STATE_INFO *state, uint pWrite)
 {

myisam/mi_rnext.c

@@ -81,7 +81,7 @@ int mi_rnext(MI_INFO *info, byte *buf, int inx)
           error=_mi_search(info,info->s->keyinfo+inx,info->lastkey,
 		     USE_WHOLE_KEY,flag, info->s->state.key_root[inx]);
         }
-        if (!error)
+        if (!error && info->s->concurrent_insert)
         {
           while (info->lastpos >= info->state->data_file_length)
           {

myisam/mi_update.c

@@ -91,7 +91,6 @@ int mi_update(register MI_INFO *info, const byte *oldrec, byte *newrec)
   {
     if (((ulonglong) 1 << i) & share->state.key_map)
     {
-      /* The following code block is for text searching by SerG */
       if (share->keyinfo[i].flag & HA_FULLTEXT )
       {
 	if (_mi_ft_cmp(info,i,oldrec, newrec))
@@ -175,7 +174,6 @@ int mi_update(register MI_INFO *info, const byte *oldrec, byte *newrec)
     {
       if (((ulonglong) 1 << i) & changed)
       {
-	/* The following code block is for text searching by SerG */
 	if (share->keyinfo[i].flag & HA_FULLTEXT)
 	{
 	  if ((flag++ && _mi_ft_del(info,i,(char*) new_key,newrec,pos)) ||

myisam/mi_write.c

@@ -38,9 +38,9 @@ static int _mi_balance_page(MI_INFO *info,MI_KEYDEF *keyinfo,uchar *key,
 static uchar *_mi_find_last_pos(MI_KEYDEF *keyinfo, uchar *page,
 				uchar *key, uint *return_key_length,
 				uchar **after_key);
-int _mi_ck_write_tree(register MI_INFO *info, uint keynr, uchar *key,
+int _mi_ck_write_tree(register MI_INFO *info, uint keynr,uchar *key,
 		      uint key_length);
-int _mi_ck_write_btree(register MI_INFO *info, uint keynr, uchar *key,
+int _mi_ck_write_btree(register MI_INFO *info, uint keynr,uchar *key,
 		       uint key_length);
 
 	/* Write new record to database */
@@ -250,11 +250,12 @@ int _mi_ck_write_btree(register MI_INFO *info, uint keynr, uchar *key,
   int error;
   uint comp_flag;
   MI_KEYDEF *keyinfo=info->s->keyinfo+keynr;
+  my_off_t  *root=&info->s->state.key_root[keynr];
   DBUG_ENTER("_mi_ck_write_btree");
 
   if (keyinfo->flag & HA_SORT_ALLOWS_SAME)
     comp_flag=SEARCH_BIGGER;			/* Put after same key */
-  else if (keyinfo->flag & HA_NOSAME)
+  else if (keyinfo->flag & (HA_NOSAME|HA_FULLTEXT))
   {
     comp_flag=SEARCH_FIND | SEARCH_UPDATE;	/* No dupplicates */
     if (keyinfo->flag & HA_NULL_ARE_EQUAL)
@@ -263,37 +264,34 @@ int _mi_ck_write_btree(register MI_INFO *info, uint keynr, uchar *key,
   else
     comp_flag=SEARCH_SAME;			/* Keys in rec-pos order */
 
-  if (info->s->state.key_root[keynr] == HA_OFFSET_ERROR ||
+  if (*root == HA_OFFSET_ERROR ||
       (error=w_search(info, keyinfo, comp_flag, key, key_length,
-		      info->s->state.key_root[keynr], (uchar *) 0, (uchar*) 0,
+		      *root, (uchar *) 0, (uchar*) 0,
 		      (my_off_t) 0, 1)) > 0)
-    error=_mi_enlarge_root(info,keynr,key);
+    error=_mi_enlarge_root(info,keyinfo,key,root);
   DBUG_RETURN(error);
 } /* _mi_ck_write_btree */
 
 
 	/* Make a new root with key as only pointer */
 
-int _mi_enlarge_root(register MI_INFO *info, uint keynr, uchar *key)
+int _mi_enlarge_root(MI_INFO *info, MI_KEYDEF *keyinfo, uchar *key,
+                     my_off_t *root)
 {
   uint t_length,nod_flag;
-  reg2 MI_KEYDEF *keyinfo;
   MI_KEY_PARAM s_temp;
   MYISAM_SHARE *share=info->s;
   DBUG_ENTER("_mi_enlarge_root");
 
-  nod_flag= (share->state.key_root[keynr] != HA_OFFSET_ERROR) ?
-    share->base.key_reflength : 0;
-  _mi_kpointer(info,info->buff+2,share->state.key_root[keynr]); /* if nod */
-  keyinfo=share->keyinfo+keynr;
+  nod_flag= (*root != HA_OFFSET_ERROR) ?  share->base.key_reflength : 0;
+  _mi_kpointer(info,info->buff+2,*root); /* if nod */
   t_length=(*keyinfo->pack_key)(keyinfo,nod_flag,(uchar*) 0,
 				(uchar*) 0, (uchar*) 0, key,&s_temp);
   mi_putint(info->buff,t_length+2+nod_flag,nod_flag);
   (*keyinfo->store_key)(keyinfo,info->buff+2+nod_flag,&s_temp);
   info->buff_used=info->page_changed=1;		/* info->buff is used */
-  if ((share->state.key_root[keynr]= _mi_new(info,keyinfo)) ==
-      HA_OFFSET_ERROR ||
-      _mi_write_keypage(info,keyinfo,share->state.key_root[keynr],info->buff))
+  if ((*root= _mi_new(info,keyinfo)) == HA_OFFSET_ERROR ||
+      _mi_write_keypage(info,keyinfo,*root,info->buff))
     DBUG_RETURN(-1);
   DBUG_RETURN(0);
 } /* _mi_enlarge_root */
@@ -333,16 +331,55 @@ static int w_search(register MI_INFO *info, register MI_KEYDEF *keyinfo,
   if (flag == 0)
   {
     uint tmp_key_length;
-    my_errno=HA_ERR_FOUND_DUPP_KEY;
 	/* get position to record with duplicated key */
     tmp_key_length=(*keyinfo->get_key)(keyinfo,nod_flag,&keypos,keybuff);
     if (tmp_key_length)
       info->dupp_key_pos=_mi_dpos(info,0,keybuff+tmp_key_length);
     else
       info->dupp_key_pos= HA_OFFSET_ERROR;
+    if (keyinfo->flag & HA_FULLTEXT)
+    {
+      uint off;
+      int  subkeys;
+
+      get_key_full_length_rdonly(off, keybuff);
+      subkeys=ft_sintXkorr(keybuff+off);
+      comp_flag=SEARCH_SAME;
+      if (subkeys >= 0)
+      {
+        /* normal word, one-level tree structure */
+        flag=(*keyinfo->bin_search)(info, keyinfo, temp_buff, key,
+                                    USE_WHOLE_KEY, comp_flag,
+                                    &keypos, keybuff, &was_last_key);
+      }
+      else
+      {
+        /* popular word. two-level tree. going down */
+        my_off_t root=info->dupp_key_pos;
+        keyinfo=&info->s->ft2_keyinfo;
+        key+=off;
+        keypos-=keyinfo->keylength; /* we'll modify key entry 'in vivo' */
+        if ((error=w_search(info, keyinfo, comp_flag, key, HA_FT_WLEN, root,
+                            (uchar *) 0, (uchar*) 0, (my_off_t) 0, 1)) > 0)
+        {
+          error=_mi_enlarge_root(info, keyinfo, key, &root);
+          _mi_dpointer(info, keypos+HA_FT_WLEN, root);
+        }
+        subkeys--; /* should there be underflow protection ? */
+        ft_intXstore(keypos, subkeys);
+        if (!error)
+          error=_mi_write_keypage(info,keyinfo,page,temp_buff);
         my_afree((byte*) temp_buff);
+        DBUG_RETURN(error);
+      }
+    }
+    else /* not HA_FULLTEXT, normal HA_NOSAME key */
+    {
+      my_afree((byte*) temp_buff);
+      my_errno=HA_ERR_FOUND_DUPP_KEY;
       DBUG_RETURN(-1);
     }
+  }
   if (flag == MI_FOUND_WRONG_KEY)
     DBUG_RETURN(-1);
   if (!was_last_key)
@@ -394,7 +431,9 @@ int _mi_insert(register MI_INFO *info, register MI_KEYDEF *keyinfo,
 #ifndef DBUG_OFF
   if (key_pos != anc_buff+2+nod_flag && (keyinfo->flag &
 					 (HA_BINARY_PACK_KEY | HA_PACK_KEY)))
+  {
     DBUG_DUMP("prev_key",(byte*) key_buff,_mi_keylength(keyinfo,key_buff));
+  }
   if (keyinfo->flag & HA_PACK_KEY)
   {
     DBUG_PRINT("test",("t_length: %d  ref_len: %d",

myisam/myisamchk.c

@@ -874,8 +874,8 @@ static int myisamchk(MI_CHECK *param, my_string filename)
   }
   else
   {
-    if (share->fulltext_index)
-      ft_init_stopwords(ft_precompiled_stopwords);       /* SerG */
+    if (share->state.header.fulltext_keys)
+      ft_init_stopwords(ft_precompiled_stopwords);
 
     if (!(param->testflag & T_READONLY))
       lock_type = F_WRLCK;			/* table is changed */

myisam/myisamdef.h

@@ -55,7 +55,8 @@ typedef struct st_mi_state_info
     uchar uniques;			/* number of UNIQUE definitions */
     uchar language;			/* Language for indexes */
     uchar max_block_size;		/* max keyblock size */
-    uchar not_used[2];			/* To align to 8 */
+    uchar fulltext_keys;
+    uchar not_used;                     /* To align to 8 */
   } header;
 
   MI_STATUS_INFO state;
@@ -154,6 +155,7 @@ typedef struct st_mi_isam_pack {
 typedef struct st_mi_isam_share {	/* Shared between opens */
   MI_STATE_INFO state;
   MI_BASE_INFO base;
+  MI_KEYDEF  ft2_keyinfo;		/* Second-level ft-key definition */
   MI_KEYDEF  *keyinfo;			/* Key definitions */
   MI_UNIQUEDEF *uniqueinfo;		/* unique definitions */
   HA_KEYSEG *keyparts;			/* key part info */
@@ -197,8 +199,7 @@ typedef struct st_mi_isam_share {	/* Shared between opens */
     global_changed,			/* If changed since open */
     not_flushed,
     temporary,delay_key_write,
-    concurrent_insert,
-    fulltext_index;
+    concurrent_insert;
 #ifdef THREAD
   THR_LOCK lock;
   pthread_mutex_t intern_lock;		/* Locking for use with _locking */
@@ -229,6 +230,8 @@ struct st_myisam_info {
   byte	*rec_buff;			/* Tempbuff for recordpack */
   uchar *int_keypos,			/* Save position for next/previous */
         *int_maxpos;			/*  -""-  */
+  uint  int_nod_flag;			/*  -""-  */
+  uint32 int_keytree_version;		/*  -""-  */
   int (*read_record)(struct st_myisam_info*, my_off_t, byte*);
   invalidator_by_filename invalidator;  /* query cache invalidator */
   ulong this_unique;			/* uniq filenumber or thread */
@@ -247,7 +250,6 @@ struct st_myisam_info {
   int  dfile;				/* The datafile */
   uint opt_flag;			/* Optim. for space/speed */
   uint update;				/* If file changed since open */
-  uint  int_nod_flag;			/*  -""-  */
   int	lastinx;			/* Last used index */
   uint	lastkey_length;			/* Length of key in lastkey */
   uint	last_rkey_length;		/* Last length in mi_rkey() */
@@ -259,7 +261,6 @@ struct st_myisam_info {
   uint	data_changed;			/* Somebody has changed data */
   uint	save_update;			/* When using KEY_READ */
   int	save_lastinx;
-  uint32 int_keytree_version;		/*  -""-  */
   LIST	open_list;
   IO_CACHE rec_cache;			/* When cacheing records */
   myf lock_wait;			/* is 0 or MY_DONT_WAIT */
@@ -372,6 +373,13 @@ typedef struct st_mi_sort_param
   { length=mi_uint2korr((key)+1)+3; (key)+=3; } \
 }
 
+#define get_key_full_length_rdonly(length,key) \
+{ if ((uchar) *(key) != 255) \
+    length= ((uint) (uchar) *((key)))+1; \
+  else \
+  { length=mi_uint2korr((key)+1)+3; } \
+}
+
 #define get_pack_length(length) ((length) >= 255 ? 3 : 1)
 
 #define MI_MIN_BLOCK_LENGTH	20	/* Because of delete-link */
@@ -395,7 +403,7 @@ typedef struct st_mi_sort_param
 #define MI_FOUND_WRONG_KEY 32738	/* Impossible value from ha_key_cmp */
 
 #define MI_MAX_KEY_BLOCK_SIZE	(MI_MAX_KEY_BLOCK_LENGTH/MI_MIN_KEY_BLOCK_LENGTH)
-#define MI_BLOCK_SIZE(key_length,data_pointer,key_pointer) ((((key_length+data_pointer+key_pointer)*4+key_pointer+2)/myisam_block_size+1)*myisam_block_size)
+#define MI_BLOCK_SIZE(key_length,data_pointer,key_pointer) (((((key_length)+(data_pointer)+(key_pointer))*4+(key_pointer)+2)/myisam_block_size+1)*myisam_block_size)
 #define MI_MAX_KEYPTR_SIZE	5        /* For calculating block lengths */
 #define MI_MIN_KEYBLOCK_LENGTH	50         /* When to split delete blocks */
 
@@ -455,7 +463,7 @@ extern int _mi_delete_static_record(MI_INFO *info);
 extern int _mi_cmp_static_record(MI_INFO *info,const byte *record);
 extern int _mi_read_rnd_static_record(MI_INFO*, byte *,my_off_t, my_bool);
 extern int _mi_ck_write(MI_INFO *info,uint keynr,uchar *key,uint length);
-extern int _mi_enlarge_root(MI_INFO *info,uint keynr,uchar *key);
+extern int _mi_enlarge_root(MI_INFO *info,MI_KEYDEF *keyinfo,uchar *key, my_off_t *root);
 extern int _mi_insert(MI_INFO *info,MI_KEYDEF *keyinfo,uchar *key,
 		      uchar *anc_buff,uchar *key_pos,uchar *key_buff,
 		      uchar *father_buff, uchar *father_keypos,
@@ -690,6 +698,7 @@ void mi_check_print_error _VARARGS((MI_CHECK *param, const char *fmt,...));
 void mi_check_print_warning _VARARGS((MI_CHECK *param, const char *fmt,...));
 void mi_check_print_info _VARARGS((MI_CHECK *param, const char *fmt,...));
 int flush_pending_blocks(MI_SORT_PARAM *param);
+int sort_ft_buf_flush(MI_SORT_PARAM *sort_param);
 int thr_write_keys(MI_SORT_PARAM *sort_param);
 #ifdef THREAD
 pthread_handler_decl(thr_find_all_keys,arg);

myisam/sort.c

@@ -71,6 +71,7 @@ static int NEAR_F merge_buffers(MI_SORT_PARAM *info,uint keys,
                                 BUFFPEK *Fb, BUFFPEK *Tb);
 static int NEAR_F merge_index(MI_SORT_PARAM *,uint,uchar **,BUFFPEK *, int,
                               IO_CACHE *);
+static int flush_ft_buf(MI_SORT_PARAM *info);
 
 static int NEAR_F write_keys_varlen(MI_SORT_PARAM *info,uchar **sort_keys,
                                     uint count, BUFFPEK *buffpek,
@@ -207,7 +208,7 @@ int _create_index_by_sort(MI_SORT_PARAM *info,my_bool no_messages,
       goto err;					/* purecov: inspected */
   }
 
-  if (flush_pending_blocks(info))
+  if (flush_ft_buf(info) || flush_pending_blocks(info))
     goto err;
 
   if (my_b_inited(&tempfile_for_exceptions))
@@ -478,7 +479,7 @@ int thr_write_keys(MI_SORT_PARAM *sort_param)
           fflush(stdout);
         }
         if (write_index(sinfo, sinfo->sort_keys, sinfo->keys) ||
-            flush_pending_blocks(sinfo))
+            flush_ft_buf(sinfo) || flush_pending_blocks(sinfo))
           got_error=1;
       }
     }
@@ -551,6 +552,7 @@ int thr_write_keys(MI_SORT_PARAM *sort_param)
       if (merge_index(sinfo, keys, (uchar **)mergebuf,
                       dynamic_element(&sinfo->buffpek,0,BUFFPEK *),
                       maxbuffer,&sinfo->tempfile) ||
+          flush_ft_buf(sinfo) ||
 	  flush_pending_blocks(sinfo))
       {
         got_error=1;
@@ -976,3 +978,16 @@ merge_index(MI_SORT_PARAM *info, uint keys, uchar **sort_keys,
   DBUG_RETURN(0);
 } /* merge_index */
 
+static int
+flush_ft_buf(MI_SORT_PARAM *info)
+{
+  int err=0;
+  if (info->sort_info->ft_buf)
+  {
+    err=sort_ft_buf_flush(info);
+    my_free((gptr)info->sort_info->ft_buf, MYF(0));
+    info->sort_info->ft_buf=0;
+  }
+  return err;
+}
+

mysql-test/r/fulltext.result

@@ -16,6 +16,8 @@ select * from t1 where MATCH(a,b) AGAINST ("indexes collections");
 a	b
 Full-text indexes	are called collections
 Only MyISAM tables	support collections
+select * from t1 where MATCH(a,b) AGAINST ("only");
+a	b
 select * from t1 where MATCH(a,b) AGAINST ("collections") UNION ALL select * from t1 where MATCH(a,b) AGAINST ("indexes");
 a	b
 Only MyISAM tables	support collections

mysql-test/t/fulltext.test

@@ -18,6 +18,7 @@ INSERT INTO t1 VALUES('MySQL has now support', 'for full-text search'),
 select * from t1 where MATCH(a,b) AGAINST ("collections");
 select * from t1 where MATCH(a,b) AGAINST ("indexes");
 select * from t1 where MATCH(a,b) AGAINST ("indexes collections");
+select * from t1 where MATCH(a,b) AGAINST ("only");
 
 # UNION of fulltext's
 select * from t1 where MATCH(a,b) AGAINST ("collections") UNION ALL select * from t1 where MATCH(a,b) AGAINST ("indexes");

mysql-test/t/fulltext2.test

+#
+# test of new fulltext search features
+#
+
+#
+# two-level tree
+#
+
+DROP TABLE IF EXISTS t1;
+CREATE TABLE t1 (
+  i int(10) unsigned not null auto_increment primary key,
+  a varchar(255) not null,
+  FULLTEXT KEY (a)
+) TYPE=MyISAM;
+
+# two-level entry, second-level tree with depth 2
+let $1=260;
+while ($1)
+{
+  eval insert t1 (a) values ('aaaxxx');
+  dec $1;
+}
+
+# two-level entry, second-level tree has only one page
+let $1=255;
+while ($1)
+{
+  eval insert t1 (a) values ('aaazzz');
+  dec $1;
+}
+
+# one-level entry (entries)
+let $1=250;
+while ($1)
+{
+  eval insert t1 (a) values ('aaayyy');
+  dec $1;
+}
+
+# converting to two-level
+repair table t1 quick;
+
+select count(*) from t1 where match a against ('aaaxxx');
+select count(*) from t1 where match a against ('aaayyy');
+select count(*) from t1 where match a against ('aaazzz');
+select count(*) from t1 where match a against ('aaaxxx' in boolean mode);
+select count(*) from t1 where match a against ('aaayyy' in boolean mode);
+select count(*) from t1 where match a against ('aaazzz' in boolean mode);
+select count(*) from t1 where match a against ('aaaxxx aaayyy aaazzz');
+select count(*) from t1 where match a against ('aaaxxx aaayyy aaazzz' in boolean mode);
+
+select count(*) from t1 where match a against ('aaax*' in boolean mode);
+select count(*) from t1 where match a against ('aaay*' in boolean mode);
+select count(*) from t1 where match a against ('aaa*' in boolean mode);
+
+# mi_write:
+
+insert t1 (a) values ('aaaxxx'),('aaayyy');
+# call to enlarge_root() below
+insert t1 (a) values ('aaazzz'),('aaazzz'),('aaazzz'),('aaazzz'),('aaazzz');
+select count(*) from t1 where match a against ('aaaxxx');
+select count(*) from t1 where match a against ('aaayyy');
+select count(*) from t1 where match a against ('aaazzz');
+
+# mi_delete
+insert t1 (a) values ('aaaxxx 000000');
+select count(*) from t1 where match a against ('000000');
+delete from t1 where match a against ('000000');
+select count(*) from t1 where match a against ('000000');
+select count(*) from t1 where match a against ('aaaxxx');
+delete from t1 where match a against ('aaazzz');
+select count(*) from t1 where match a against ('aaaxxx' in boolean mode);
+select count(*) from t1 where match a against ('aaayyy' in boolean mode);
+select count(*) from t1 where match a against ('aaazzz' in boolean mode);
+# double-check without index
+select count(*) from t1 where a = 'aaaxxx';
+select count(*) from t1 where a = 'aaayyy';
+select count(*) from t1 where a = 'aaazzz';
+
+# update
+insert t1 (a) values ('aaaxxx 000000');
+select count(*) from t1 where match a against ('000000');
+update t1 set a='aaazzz' where match a against ('000000');
+select count(*) from t1 where match a against ('aaaxxx' in boolean mode);
+select count(*) from t1 where match a against ('aaazzz' in boolean mode);
+update t1 set a='aaazzz' where a = 'aaaxxx';
+update t1 set a='aaaxxx' where a = 'aaayyy';
+select count(*) from t1 where match a against ('aaaxxx' in boolean mode);
+select count(*) from t1 where match a against ('aaayyy' in boolean mode);
+select count(*) from t1 where match a against ('aaazzz' in boolean mode);
+
+DROP TABLE IF EXISTS t1;
+

mysys/mulalloc.c

@@ -19,14 +19,16 @@
 
 /*
   Malloc many pointers at the same time
+  Only ptr1 can be free'd, and doing this will free all
+  the memory allocated. ptr2, etc all point inside big allocated
+  memory area.
 
   SYNOPSIS
     my_multi_malloc()
       myFlags              Flags
-      ... 	Multiple arguments terminated by null ptr
-
-	ptr, length
-	ptr, length
+	ptr1, length1      Multiple arguments terminated by null ptr
+	ptr2, length2      ...
+        ...
 	NULL
 */