Merge svojtovich@bk-internal.mysql.com:/home/bk/mysql-5.0

into mysql.com:/home/svoj/devel/mysql/ft-mysql-5.0

myisam/ft_boolean_search.c

@@ -68,7 +68,7 @@ struct st_ftb_expr
   my_off_t  docid[2];
   float     weight;
   float     cur_weight;
-  byte     *quot, *qend;
+  LIST     *phrase;               /* phrase words */
   uint      yesses;               /* number of "yes" words matched */
   uint      nos;                  /* number of "no"  words matched */
   uint      ythresh;              /* number of "yes" words in expr */
@@ -132,20 +132,22 @@ static int FTB_WORD_cmp_list(CHARSET_INFO *cs, FTB_WORD **a, FTB_WORD **b)
 }
 
 static void _ftb_parse_query(FTB *ftb, byte **start, byte *end,
-                      FTB_EXPR *up, uint depth)
+                      FTB_EXPR *up, uint depth, byte *up_quot)
 {
   byte        res;
   FTB_PARAM   param;
   FT_WORD     w;
   FTB_WORD   *ftbw;
   FTB_EXPR   *ftbe;
+  FT_WORD    *phrase_word;
+  LIST       *phrase_list;
   uint  extra=HA_FT_WLEN+ftb->info->s->rec_reflength; /* just a shortcut */
 
   if (ftb->state != UNINITIALIZED)
     return;
 
   param.prev=' ';
-  param.quot=up->quot;
+  param.quot= up_quot;
   while ((res=ft_get_word(ftb->charset,start,end,&w,&param)))
   {
     int   r=param.plusminus;
@@ -172,6 +174,14 @@ static void _ftb_parse_query(FTB *ftb, byte **start, byte *end,
         if (param.yesno > 0) up->ythresh++;
         queue_insert(& ftb->queue, (byte *)ftbw);
         ftb->with_scan|=(param.trunc & FTB_FLAG_TRUNC);
+      case 4: /* not indexed word (stopword or too short/long) */
+        if (! up_quot) break;
+        phrase_word= (FT_WORD *)alloc_root(&ftb->mem_root, sizeof(FT_WORD));
+        phrase_list= (LIST *)alloc_root(&ftb->mem_root, sizeof(LIST));
+        phrase_word->pos= w.pos;
+        phrase_word->len= w.len;
+        phrase_list->data= (void *)phrase_word;
+        up->phrase= list_add(up->phrase, phrase_list);
         break;
       case 2: /* left bracket */
         ftbe=(FTB_EXPR *)alloc_root(&ftb->mem_root, sizeof(FTB_EXPR));
@@ -182,13 +192,14 @@ static void _ftb_parse_query(FTB *ftb, byte **start, byte *end,
         ftbe->up=up;
         ftbe->ythresh=ftbe->yweaks=0;
         ftbe->docid[0]=ftbe->docid[1]=HA_OFFSET_ERROR;
-        if ((ftbe->quot=param.quot)) ftb->with_scan|=2;
+        ftbe->phrase= NULL;
+        if (param.quot) ftb->with_scan|=2;
         if (param.yesno > 0) up->ythresh++;
-        _ftb_parse_query(ftb, start, end, ftbe, depth+1);
+        _ftb_parse_query(ftb, start, end, ftbe, depth+1, param.quot);
         param.quot=0;
         break;
       case 3: /* right bracket */
-        if (up->quot) up->qend=param.quot;
+        if (up_quot) up->phrase= list_reverse(up->phrase);
         return;
     }
   }
@@ -410,12 +421,12 @@ FT_INFO * ft_init_boolean_search(MI_INFO *info, uint keynr, byte *query,
   ftbe->weight=1;
   ftbe->flags=FTB_FLAG_YES;
   ftbe->nos=1;
-  ftbe->quot=0;
   ftbe->up=0;
   ftbe->ythresh=ftbe->yweaks=0;
   ftbe->docid[0]=ftbe->docid[1]=HA_OFFSET_ERROR;
+  ftbe->phrase= NULL;
   ftb->root=ftbe;
-  _ftb_parse_query(ftb, &query, query+query_len, ftbe, 0);
+  _ftb_parse_query(ftb, &query, query+query_len, ftbe, 0, NULL);
   ftb->list=(FTB_WORD **)alloc_root(&ftb->mem_root,
                                      sizeof(FTB_WORD *)*ftb->queue.elements);
   memcpy(ftb->list, ftb->queue.root+1, sizeof(FTB_WORD *)*ftb->queue.elements);
@@ -431,29 +442,45 @@ FT_INFO * ft_init_boolean_search(MI_INFO *info, uint keynr, byte *query,
 }
 
 
-/* returns 1 if str0 ~= /\bstr1\b/ */
-static int _ftb_strstr(const byte *s0, const byte *e0,
-                const byte *s1, const byte *e1,
-                CHARSET_INFO *cs)
+/*
+  Checks if given buffer matches phrase list.
+
+  SYNOPSIS
+    _ftb_check_phrase()
+    s0     start of buffer
+    e0     end of buffer
+    phrase broken into list phrase
+    cs     charset info
+
+  RETURN VALUE
+    1 is returned if phrase found, 0 else.
+*/
+
+static int _ftb_check_phrase(const byte *s0, const byte *e0,
+                LIST *phrase, CHARSET_INFO *cs)
 {
-  const byte *p0= s0;
-  my_bool s_after= true_word_char(cs, s1[0]);
-  my_bool e_before= true_word_char(cs, e1[-1]);
-  uint p0_len;
-  my_match_t m[2];
+  FT_WORD h_word;
+  const byte *h_start= s0;
+  DBUG_ENTER("_ftb_strstr");
+  DBUG_ASSERT(phrase);
 
-  while (p0 < e0)
+  while (ft_simple_get_word(cs, (byte **)&h_start, e0, &h_word, FALSE))
   {
-    if (cs->coll->instr(cs, p0, e0 - p0, s1, e1 - s1, m, 2) != 2)
-      return(0);
-    if ((!s_after || p0 + m[1].beg == s0 || !true_word_char(cs, p0[m[1].beg-1])) &&
-        (!e_before || p0 + m[1].end == e0 || !true_word_char(cs, p0[m[1].end])))
-      return(1);
-    p0+= m[1].beg;
-    p0+= (p0_len= my_mbcharlen(cs, *(uchar *)p0)) ? p0_len : 1;
+    FT_WORD *n_word;
+    LIST *phrase_element= phrase;
+    const byte *h_start1= h_start;
+    for (;;)
+    {
+      n_word= (FT_WORD *)phrase_element->data;
+      if (my_strnncoll(cs, h_word.pos, h_word.len, n_word->pos, n_word->len))
+        break;
+      if (! (phrase_element= phrase_element->next))
+        DBUG_RETURN(1);
+      if (! ft_simple_get_word(cs, (byte **)&h_start1, e0, &h_word, FALSE))
+        DBUG_RETURN(0);
+    }
   }
-
-  return(0);
+  DBUG_RETURN(0);
 }
 
 
@@ -484,7 +511,7 @@ static void _ftb_climb_the_tree(FTB *ftb, FTB_WORD *ftbw, FT_SEG_ITERATOR *ftsi_
       {
         yn=ftbe->flags;
         weight=ftbe->cur_weight*ftbe->weight;
-        if (mode && ftbe->quot)
+        if (mode && ftbe->phrase)
         {
           int not_found=1;
 
@@ -493,8 +520,8 @@ static void _ftb_climb_the_tree(FTB *ftb, FTB_WORD *ftbw, FT_SEG_ITERATOR *ftsi_
           {
             if (!ftsi.pos)
               continue;
-            not_found = ! _ftb_strstr(ftsi.pos, ftsi.pos+ftsi.len,
-                                      ftbe->quot, ftbe->qend, ftb->charset);
+            not_found = ! _ftb_check_phrase(ftsi.pos, ftsi.pos+ftsi.len,
+                                      ftbe->phrase, ftb->charset);
           }
           if (not_found) break;
         } /* ftbe->quot */
@@ -642,8 +669,8 @@ float ft_boolean_find_relevance(FT_INFO *ftb, byte *record, uint length)
       continue;
 
     end=ftsi.pos+ftsi.len;
-    while (ft_simple_get_word(ftb->charset,
-                              (byte **) &ftsi.pos, (byte *) end, &word))
+    while (ft_simple_get_word(ftb->charset, (byte **) &ftsi.pos,
+                              (byte *) end, &word, TRUE))
     {
       int a, b, c;
       for (a=0, b=ftb->queue.elements, c=(a+b)/2; b-a>1; c=(a+b)/2)

myisam/ft_parser.c

@@ -93,12 +93,14 @@ my_bool ft_boolean_check_syntax_string(const byte *str)
   return 0;
 }
 
-/* returns:
- * 0 - eof
- * 1 - word found
- * 2 - left bracket
- * 3 - right bracket
- */
+/*
+  RETURN VALUE
+  0 - eof
+  1 - word found
+  2 - left bracket
+  3 - right bracket
+  4 - stopword found
+*/
 byte ft_get_word(CHARSET_INFO *cs, byte **start, byte *end,
                  FT_WORD *word, FTB_PARAM *param)
 {
@@ -161,6 +163,11 @@ byte ft_get_word(CHARSET_INFO *cs, byte **start, byte *end,
       *start=doc;
       return 1;
     }
+    else if (length) /* make sure length > 0 (if start contains spaces only) */
+    {
+      *start= doc;
+      return 4;
+    }
   }
   if (param->quot)
   {
@@ -170,18 +177,19 @@ byte ft_get_word(CHARSET_INFO *cs, byte **start, byte *end,
   return 0;
 }
 
-byte ft_simple_get_word(CHARSET_INFO *cs, byte **start, byte *end,
-                        FT_WORD *word)
+byte ft_simple_get_word(CHARSET_INFO *cs, byte **start, const byte *end,
+                        FT_WORD *word, my_bool skip_stopwords)
 {
   byte *doc= *start;
   uint mwc, length, mbl;
   DBUG_ENTER("ft_simple_get_word");
 
-  while (doc<end)
+  do
   {
-    for (;doc<end;doc++)
+    for (;; doc++)
     {
-      if (true_word_char(cs,*doc)) break;
+      if (doc >= end) DBUG_RETURN(0);
+      if (true_word_char(cs, *doc)) break;
     }
 
     mwc= length= 0;
@@ -193,13 +201,14 @@ byte ft_simple_get_word(CHARSET_INFO *cs, byte **start, byte *end,
 
     word->len= (uint)(doc-word->pos) - mwc;
 
-    if (length >= ft_min_word_len && length < ft_max_word_len &&
-        !is_stopword(word->pos, word->len))
+    if (skip_stopwords == FALSE ||
+        (length >= ft_min_word_len && length < ft_max_word_len &&
+         !is_stopword(word->pos, word->len)))
     {
       *start= doc;
       DBUG_RETURN(1);
     }
-  }
+  } while (doc < end);
   DBUG_RETURN(0);
 }
 
@@ -217,7 +226,7 @@ int ft_parse(TREE *wtree, byte *doc, int doclen, my_bool with_alloc)
   FT_WORD w;
   DBUG_ENTER("ft_parse");
 
-  while (ft_simple_get_word(wtree->custom_arg, &doc,end,&w))
+  while (ft_simple_get_word(wtree->custom_arg, &doc, end, &w, TRUE))
   {
     if (with_alloc)
     {

myisam/ft_stopwords.c

@@ -81,7 +81,7 @@ int ft_init_stopwords()
       goto err0;
     len=my_read(fd, buffer, len, MYF(MY_WME));
     end=start+len;
-    while (ft_simple_get_word(default_charset_info, &start, end, &w))
+    while (ft_simple_get_word(default_charset_info, &start, end, &w, TRUE))
     {
       if (ft_add_stopword(my_strdup_with_length(w.pos, w.len, MYF(0))))
         goto err1;

myisam/ftdefs.h

@@ -112,7 +112,8 @@ int is_stopword(char *word, uint len);
 uint _ft_make_key(MI_INFO *, uint , byte *, FT_WORD *, my_off_t);
 
 byte ft_get_word(CHARSET_INFO *, byte **, byte *, FT_WORD *, FTB_PARAM *);
-byte ft_simple_get_word(CHARSET_INFO *, byte **, byte *, FT_WORD *);
+byte ft_simple_get_word(CHARSET_INFO *, byte **, const byte *,
+                        FT_WORD *, my_bool);
 
 typedef struct _st_ft_seg_iterator {
   uint        num, len;