Commit fe43179a authored by unknown's avatar unknown

FT-NL bugfix - ignore words that appear more than in 2mio rows to avoid tree overflow.

optimization: abort search for common (>50% rows) words earlier.
parent ad68064c
...@@ -67,12 +67,9 @@ static int FT_SUPERDOC_cmp(void* cmp_arg __attribute__((unused)), ...@@ -67,12 +67,9 @@ static int FT_SUPERDOC_cmp(void* cmp_arg __attribute__((unused)),
static int walk_and_match(FT_WORD *word, uint32 count, ALL_IN_ONE *aio) static int walk_and_match(FT_WORD *word, uint32 count, ALL_IN_ONE *aio)
{ {
uint keylen, r, doc_cnt; uint keylen, r, doc_cnt;
#ifdef EVAL_RUN
uint cnt;
double sum, sum2, suml;
#endif /* EVAL_RUN */
FT_SUPERDOC sdoc, *sptr; FT_SUPERDOC sdoc, *sptr;
TREE_ELEMENT *selem; TREE_ELEMENT *selem;
double gweight=1;
#if HA_FT_WTYPE == HA_KEYTYPE_FLOAT #if HA_FT_WTYPE == HA_KEYTYPE_FLOAT
float tmp_weight; float tmp_weight;
#else #else
...@@ -84,42 +81,27 @@ static int walk_and_match(FT_WORD *word, uint32 count, ALL_IN_ONE *aio) ...@@ -84,42 +81,27 @@ static int walk_and_match(FT_WORD *word, uint32 count, ALL_IN_ONE *aio)
word->weight=LWS_FOR_QUERY; word->weight=LWS_FOR_QUERY;
keylen=_ft_make_key(aio->info,aio->keynr,(char*) aio->keybuff,word,0); keylen=_ft_make_key(aio->info,aio->keynr,(char*) aio->keybuff,word,0);
#ifdef EVAL_RUN
keylen-=1+HA_FT_WLEN;
#else /* EVAL_RUN */
keylen-=HA_FT_WLEN; keylen-=HA_FT_WLEN;
#endif /* EVAL_RUN */
#ifdef EVAL_RUN
sum=sum2=suml=
#endif /* EVAL_RUN */
doc_cnt=0; doc_cnt=0;
r=_mi_search(aio->info, aio->keyinfo, aio->keybuff, keylen, r=_mi_search(aio->info, aio->keyinfo, aio->keybuff, keylen,
SEARCH_FIND | SEARCH_PREFIX, aio->key_root); SEARCH_FIND | SEARCH_PREFIX, aio->key_root);
aio->info->update|= HA_STATE_AKTIV; /* for _mi_test_if_changed() */ aio->info->update|= HA_STATE_AKTIV; /* for _mi_test_if_changed() */
while (!r) while (!r && gweight)
{ {
if (_mi_compare_text(aio->charset, if (_mi_compare_text(aio->charset,
aio->info->lastkey,keylen, aio->info->lastkey,keylen,
aio->keybuff,keylen,0)) break; aio->keybuff,keylen,0)) break;
#if HA_FT_WTYPE == HA_KEYTYPE_FLOAT #if HA_FT_WTYPE == HA_KEYTYPE_FLOAT
#ifdef EVAL_RUN
mi_float4get(tmp_weight,aio->info->lastkey+keylen+1);
#else /* EVAL_RUN */
mi_float4get(tmp_weight,aio->info->lastkey+keylen); mi_float4get(tmp_weight,aio->info->lastkey+keylen);
#endif /* EVAL_RUN */
#else #else
#error #error
#endif #endif
if(tmp_weight==0) DBUG_RETURN(doc_cnt); /* stopword, doc_cnt should be 0 */ if(tmp_weight==0) DBUG_RETURN(doc_cnt); /* stopword, doc_cnt should be 0 */
#ifdef EVAL_RUN
cnt=*(byte *)(aio->info->lastkey+keylen);
#endif /* EVAL_RUN */
sdoc.doc.dpos=aio->info->lastpos; sdoc.doc.dpos=aio->info->lastpos;
/* saving document matched into dtree */ /* saving document matched into dtree */
...@@ -137,11 +119,10 @@ static int walk_and_match(FT_WORD *word, uint32 count, ALL_IN_ONE *aio) ...@@ -137,11 +119,10 @@ static int walk_and_match(FT_WORD *word, uint32 count, ALL_IN_ONE *aio)
sptr->tmp_weight=tmp_weight; sptr->tmp_weight=tmp_weight;
doc_cnt++; doc_cnt++;
#ifdef EVAL_RUN
sum +=cnt; gweight=word->weight*GWS_IN_USE;
sum2+=cnt*cnt; if (gweight < 0 || doc_cnt > 2000000)
suml+=cnt*log(cnt); gweight=0;
#endif /* EVAL_RUN */
if (_mi_test_if_changed(aio->info) == 0) if (_mi_test_if_changed(aio->info) == 0)
r=_mi_search_next(aio->info, aio->keyinfo, aio->info->lastkey, r=_mi_search_next(aio->info, aio->keyinfo, aio->info->lastkey,
...@@ -152,13 +133,9 @@ static int walk_and_match(FT_WORD *word, uint32 count, ALL_IN_ONE *aio) ...@@ -152,13 +133,9 @@ static int walk_and_match(FT_WORD *word, uint32 count, ALL_IN_ONE *aio)
aio->info->lastkey_length, SEARCH_BIGGER, aio->info->lastkey_length, SEARCH_BIGGER,
aio->key_root); aio->key_root);
} }
if (doc_cnt)
{
word->weight*=GWS_IN_USE;
if (word->weight < 0)
word->weight=0;
} word->weight=gweight;
DBUG_RETURN(0); DBUG_RETURN(0);
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment