diff --git a/include/mysql/plugin.h b/include/mysql/plugin.h index a47b1099eef1714ae92a2aa636cc818034a6dd97..728b62ab2f4915f4b9d62cd15478bbf03a3a3707 100644 --- a/include/mysql/plugin.h +++ b/include/mysql/plugin.h @@ -201,6 +201,17 @@ typedef struct st_mysql_ftparser_boolean_info char *quot; } MYSQL_FTPARSER_BOOLEAN_INFO; +/* + The following flag means that buffer with a string (document, word) + may be overwritten by the caller before the end of the parsing (that is + before st_mysql_ftparser::deinit() call). If one needs the string + to survive between two successive calls of the parsing function, she + needs to save a copy of it. The flag may be set by MySQL before calling + st_mysql_ftparser::parse(), or it may be set by a plugin before calling + st_mysql_ftparser_param::mysql_parse() or + st_mysql_ftparser_param::mysql_add_word(). +*/ +#define MYSQL_FTFLAGS_NEED_COPY 1 /* An argument of the full-text parser plugin. This structure is @@ -234,8 +245,10 @@ typedef struct st_mysql_ftparser_boolean_info length: Length of the document or query string, in bytes. + flags: See MYSQL_FTFLAGS_* constants above. + mode: The parsing mode. With boolean operators, with stopwords, or - nothing. See MYSQL_FTPARSER_* constants above. + nothing. See enum_ftparser_mode above. */ typedef struct st_mysql_ftparser_param @@ -250,6 +263,7 @@ typedef struct st_mysql_ftparser_param struct charset_info_st *cs; char *doc; int length; + int flags; enum enum_ftparser_mode mode; } MYSQL_FTPARSER_PARAM; diff --git a/storage/myisam/ft_boolean_search.c b/storage/myisam/ft_boolean_search.c index 940bf8e34339503d3ed74e97926544b76176e1af..33c4dc705b4716926ecb0fa4c4654430501f9a90 100644 --- a/storage/myisam/ft_boolean_search.c +++ b/storage/myisam/ft_boolean_search.c @@ -311,6 +311,7 @@ static void _ftb_parse_query(FTB *ftb, byte *query, uint len, param->cs= ftb->charset; param->doc= query; param->length= len; + param->flags= 0; param->mode= MYSQL_FTPARSER_FULL_BOOLEAN_INFO; parser->parse(param); DBUG_VOID_RETURN; @@ -665,6 +666,7 @@ static int _ftb_check_phrase(FTB *ftb, const byte *document, uint len, param->cs= ftb->charset; param->doc= (byte *)document; param->length= len; + param->flags= 0; param->mode= MYSQL_FTPARSER_WITH_STOPWORDS; parser->parse(param); DBUG_RETURN(ftb_param.match ? 1 : 0); @@ -916,6 +918,7 @@ float ft_boolean_find_relevance(FT_INFO *ftb, byte *record, uint length) param->mysql_parse= ftb_find_relevance_parse; param->mysql_add_word= ftb_find_relevance_add_word; param->mysql_ftparam= (void *)&ftb_param; + param->flags= 0; param->cs= ftb->charset; param->mode= MYSQL_FTPARSER_SIMPLE_MODE; while (_mi_ft_segiterator(&ftsi)) diff --git a/storage/myisam/ft_nlq_search.c b/storage/myisam/ft_nlq_search.c index b8207c1c3d07e3c2f5b6646cee20e7b89a9615e5..53a01003dcd99b566e857104aa8b405dd260c61d 100644 --- a/storage/myisam/ft_nlq_search.c +++ b/storage/myisam/ft_nlq_search.c @@ -235,7 +235,9 @@ FT_INFO *ft_init_nlq_search(MI_INFO *info, uint keynr, byte *query, NULL, NULL); ft_parse_init(&wtree, aio.charset); - if (ft_parse(&wtree, query, query_len, 0, parser, ftparser_param)) + ftparser_param->flags= 0; + if (ft_parse(&wtree, query, query_len, parser, ftparser_param, + &wtree.mem_root)) goto err; if (tree_walk(&wtree, (tree_walk_action)&walk_and_match, &aio, @@ -255,7 +257,9 @@ FT_INFO *ft_init_nlq_search(MI_INFO *info, uint keynr, byte *query, if (!(*info->read_record)(info,docid,record)) { info->update|= HA_STATE_AKTIV; - _mi_ft_parse(&wtree, info, keynr, record, 1, ftparser_param); + ftparser_param->flags= MYSQL_FTFLAGS_NEED_COPY; + _mi_ft_parse(&wtree, info, keynr, record, ftparser_param, + &wtree.mem_root); } } delete_queue(&best); diff --git a/storage/myisam/ft_parser.c b/storage/myisam/ft_parser.c index 7793546f3bb421602720fb0523991de77cbbcda5..38ac744d4a8326d3cf76d4a7f68b9852f840741d 100644 --- a/storage/myisam/ft_parser.c +++ b/storage/myisam/ft_parser.c @@ -24,14 +24,12 @@ typedef struct st_ft_docstat { double sum; } FT_DOCSTAT; - typedef struct st_my_ft_parser_param { - TREE *wtree; - my_bool with_alloc; + TREE *wtree; + MEM_ROOT *mem_root; } MY_FT_PARSER_PARAM; - static int FT_WORD_cmp(CHARSET_INFO* cs, FT_WORD *w1, FT_WORD *w2) { return mi_compare_text(cs, (uchar*) w1->pos, w1->len, @@ -48,14 +46,14 @@ static int walk_and_copy(FT_WORD *word,uint32 count,FT_DOCSTAT *docstat) /* transforms tree of words into the array, applying normalization */ -FT_WORD * ft_linearize(TREE *wtree) +FT_WORD * ft_linearize(TREE *wtree, MEM_ROOT *mem_root) { FT_WORD *wlist,*p; FT_DOCSTAT docstat; DBUG_ENTER("ft_linearize"); - if ((wlist=(FT_WORD *) my_malloc(sizeof(FT_WORD)* - (1+wtree->elements_in_tree),MYF(0)))) + if ((wlist=(FT_WORD *) alloc_root(mem_root, sizeof(FT_WORD)* + (1+wtree->elements_in_tree)))) { docstat.list=wlist; docstat.uniq=wtree->elements_in_tree; @@ -249,12 +247,11 @@ static int ft_add_word(MYSQL_FTPARSER_PARAM *param, MY_FT_PARSER_PARAM *ft_param=param->mysql_ftparam; DBUG_ENTER("ft_add_word"); wtree= ft_param->wtree; - if (ft_param->with_alloc) + if (param->flags & MYSQL_FTFLAGS_NEED_COPY) { byte *ptr; - /* allocating the data in the tree - to avoid mallocs and frees */ DBUG_ASSERT(wtree->with_delete == 0); - ptr= (byte *)alloc_root(&wtree->mem_root, word_len); + ptr= (byte *)alloc_root(ft_param->mem_root, word_len); memcpy(ptr, word, word_len); w.pos= ptr; } @@ -286,16 +283,16 @@ static int ft_parse_internal(MYSQL_FTPARSER_PARAM *param, } -int ft_parse(TREE *wtree, byte *doc, int doclen, my_bool with_alloc, +int ft_parse(TREE *wtree, byte *doc, int doclen, struct st_mysql_ftparser *parser, - MYSQL_FTPARSER_PARAM *param) + MYSQL_FTPARSER_PARAM *param, MEM_ROOT *mem_root) { MY_FT_PARSER_PARAM my_param; DBUG_ENTER("ft_parse"); DBUG_ASSERT(parser); my_param.wtree= wtree; - my_param.with_alloc= with_alloc; + my_param.mem_root= mem_root; param->mysql_parse= ft_parse_internal; param->mysql_add_word= ft_add_word; @@ -357,6 +354,7 @@ MYSQL_FTPARSER_PARAM *ftparser_call_initializer(MI_INFO *info, info->ftparser_param= (MYSQL_FTPARSER_PARAM *) my_malloc(MAX_PARAM_NR * sizeof(MYSQL_FTPARSER_PARAM) * info->s->ftparsers, MYF(MY_WME|MY_ZEROFILL)); + init_alloc_root(&info->ft_memroot, FTPARSER_MEMROOT_ALLOC_SIZE, 0); if (! info->ftparser_param) return 0; } @@ -388,6 +386,7 @@ MYSQL_FTPARSER_PARAM *ftparser_call_initializer(MI_INFO *info, void ftparser_call_deinitializer(MI_INFO *info) { uint i, j, keys= info->s->state.header.keys; + free_root(&info->ft_memroot, MYF(0)); if (! info->ftparser_param) return; for (i= 0; i < keys; i++) diff --git a/storage/myisam/ft_update.c b/storage/myisam/ft_update.c index f5501792dcd3c24a27c48f56e2de50e015f06561..08d605dbcc237c612a04ce6c0150e0241e53226b 100644 --- a/storage/myisam/ft_update.c +++ b/storage/myisam/ft_update.c @@ -77,7 +77,7 @@ uint _mi_ft_segiterator(register FT_SEG_ITERATOR *ftsi) { uint pack_length= (ftsi->seg->bit_start); ftsi->len= (pack_length == 1 ? (uint) *(uchar*) ftsi->pos : - uint2korr(ftsi->pos)); + uint2korr(ftsi->pos)); ftsi->pos+= pack_length; /* Skip VARCHAR length */ DBUG_RETURN(1); } @@ -95,9 +95,8 @@ uint _mi_ft_segiterator(register FT_SEG_ITERATOR *ftsi) /* parses a document i.e. calls ft_parse for every keyseg */ -uint _mi_ft_parse(TREE *parsed, MI_INFO *info, uint keynr, - const byte *record, my_bool with_alloc, - MYSQL_FTPARSER_PARAM *param) +uint _mi_ft_parse(TREE *parsed, MI_INFO *info, uint keynr, const byte *record, + MYSQL_FTPARSER_PARAM *param, MEM_ROOT *mem_root) { FT_SEG_ITERATOR ftsi; struct st_mysql_ftparser *parser; @@ -110,14 +109,14 @@ uint _mi_ft_parse(TREE *parsed, MI_INFO *info, uint keynr, while (_mi_ft_segiterator(&ftsi)) { if (ftsi.pos) - if (ft_parse(parsed, (byte *)ftsi.pos, ftsi.len, with_alloc, parser, - param)) + if (ft_parse(parsed, (byte *)ftsi.pos, ftsi.len, parser, param, mem_root)) DBUG_RETURN(1); } DBUG_RETURN(0); } -FT_WORD * _mi_ft_parserecord(MI_INFO *info, uint keynr, const byte *record) +FT_WORD *_mi_ft_parserecord(MI_INFO *info, uint keynr, const byte *record, + MEM_ROOT *mem_root) { TREE ptree; MYSQL_FTPARSER_PARAM *param; @@ -125,10 +124,11 @@ FT_WORD * _mi_ft_parserecord(MI_INFO *info, uint keynr, const byte *record) if (! (param= ftparser_call_initializer(info, keynr, 0))) DBUG_RETURN(NULL); bzero((char*) &ptree, sizeof(ptree)); - if (_mi_ft_parse(&ptree, info, keynr, record, 0, param)) + param->flags= 0; + if (_mi_ft_parse(&ptree, info, keynr, record, param, mem_root)) DBUG_RETURN(NULL); - DBUG_RETURN(ft_linearize(&ptree)); + DBUG_RETURN(ft_linearize(&ptree, mem_root)); } static int _mi_ft_store(MI_INFO *info, uint keynr, byte *keybuf, @@ -206,10 +206,11 @@ int _mi_ft_update(MI_INFO *info, uint keynr, byte *keybuf, int cmp, cmp2; DBUG_ENTER("_mi_ft_update"); - if (!(old_word=oldlist=_mi_ft_parserecord(info, keynr, oldrec))) - goto err0; - if (!(new_word=newlist=_mi_ft_parserecord(info, keynr, newrec))) - goto err1; + if (!(old_word=oldlist=_mi_ft_parserecord(info, keynr, oldrec, + &info->ft_memroot)) || + !(new_word=newlist=_mi_ft_parserecord(info, keynr, newrec, + &info->ft_memroot))) + goto err; error=0; while(old_word->pos && new_word->pos) @@ -222,13 +223,13 @@ int _mi_ft_update(MI_INFO *info, uint keynr, byte *keybuf, { key_length=_ft_make_key(info,keynr,keybuf,old_word,pos); if ((error=_mi_ck_delete(info,keynr,(uchar*) keybuf,key_length))) - goto err2; + goto err; } if (cmp > 0 || cmp2) { key_length=_ft_make_key(info,keynr,keybuf,new_word,pos); if ((error=_mi_ck_write(info,keynr,(uchar*) keybuf,key_length))) - goto err2; + goto err; } if (cmp<=0) old_word++; if (cmp>=0) new_word++; @@ -238,11 +239,8 @@ int _mi_ft_update(MI_INFO *info, uint keynr, byte *keybuf, else if (new_word->pos) error=_mi_ft_store(info,keynr,keybuf,new_word,pos); -err2: - my_free((char*) newlist,MYF(0)); -err1: - my_free((char*) oldlist,MYF(0)); -err0: +err: + free_root(&info->ft_memroot, MYF(MY_MARK_BLOCKS_FREE)); DBUG_RETURN(error); } @@ -255,12 +253,13 @@ int _mi_ft_add(MI_INFO *info, uint keynr, byte *keybuf, const byte *record, int error= -1; FT_WORD *wlist; DBUG_ENTER("_mi_ft_add"); + DBUG_PRINT("enter",("keynr: %d",keynr)); - if ((wlist=_mi_ft_parserecord(info, keynr, record))) - { + if ((wlist=_mi_ft_parserecord(info, keynr, record, &info->ft_memroot))) error=_mi_ft_store(info,keynr,keybuf,wlist,pos); - my_free((char*) wlist,MYF(0)); - } + + free_root(&info->ft_memroot, MYF(MY_MARK_BLOCKS_FREE)); + DBUG_PRINT("exit",("Return: %d",error)); DBUG_RETURN(error); } @@ -275,11 +274,10 @@ int _mi_ft_del(MI_INFO *info, uint keynr, byte *keybuf, const byte *record, DBUG_ENTER("_mi_ft_del"); DBUG_PRINT("enter",("keynr: %d",keynr)); - if ((wlist=_mi_ft_parserecord(info, keynr, record))) - { + if ((wlist=_mi_ft_parserecord(info, keynr, record, &info->ft_memroot))) error=_mi_ft_erase(info,keynr,keybuf,wlist,pos); - my_free((char*) wlist,MYF(0)); - } + + free_root(&info->ft_memroot, MYF(MY_MARK_BLOCKS_FREE)); DBUG_PRINT("exit",("Return: %d",error)); DBUG_RETURN(error); } diff --git a/storage/myisam/ftdefs.h b/storage/myisam/ftdefs.h index 2c4cfa1ffd6a2486930a7a81d481a77004711d18..108faf4f1a336ab8ceabf8f75eac72700f75e51d 100644 --- a/storage/myisam/ftdefs.h +++ b/storage/myisam/ftdefs.h @@ -30,6 +30,8 @@ #define FT_MAX_WORD_LEN_FOR_SORT 31 +#define FTPARSER_MEMROOT_ALLOC_SIZE 65536 + #define COMPILE_STOPWORDS_IN /* Interested readers may consult SMART @@ -119,12 +121,12 @@ void _mi_ft_segiterator_dummy_init(const byte *, uint, FT_SEG_ITERATOR *); uint _mi_ft_segiterator(FT_SEG_ITERATOR *); void ft_parse_init(TREE *, CHARSET_INFO *); -int ft_parse(TREE *, byte *, int, my_bool, struct st_mysql_ftparser *parser, - MYSQL_FTPARSER_PARAM *param); -FT_WORD * ft_linearize(TREE *); -FT_WORD * _mi_ft_parserecord(MI_INFO *, uint, const byte *); -uint _mi_ft_parse(TREE *, MI_INFO *, uint, const byte *, my_bool, - MYSQL_FTPARSER_PARAM *param); +int ft_parse(TREE *, byte *, int, struct st_mysql_ftparser *parser, + MYSQL_FTPARSER_PARAM *, MEM_ROOT *); +FT_WORD * ft_linearize(TREE *, MEM_ROOT *); +FT_WORD * _mi_ft_parserecord(MI_INFO *, uint, const byte *, MEM_ROOT *); +uint _mi_ft_parse(TREE *, MI_INFO *, uint, const byte *, + MYSQL_FTPARSER_PARAM *, MEM_ROOT *); FT_INFO *ft_init_nlq_search(MI_INFO *, uint, byte *, uint, uint, byte *); FT_INFO *ft_init_boolean_search(MI_INFO *, uint, byte *, uint, CHARSET_INFO *); diff --git a/storage/myisam/mi_check.c b/storage/myisam/mi_check.c index 0fa095a21db618266c8804b7aa4ebc83ebcedee1..df0d363f7c3559038982b182d914f7506069ef11 100644 --- a/storage/myisam/mi_check.c +++ b/storage/myisam/mi_check.c @@ -2117,6 +2117,7 @@ int mi_repair_by_sort(MI_CHECK *param, register MI_INFO *info, my_seek(param->read_cache.file,0L,MY_SEEK_END,MYF(0)); sort_param.wordlist=NULL; + init_alloc_root(&sort_param.wordroot, FTPARSER_MEMROOT_ALLOC_SIZE, 0); if (share->data_file_type == DYNAMIC_RECORD) length=max(share->base.min_pack_length+1,share->base.min_block_length); @@ -2200,6 +2201,7 @@ int mi_repair_by_sort(MI_CHECK *param, register MI_INFO *info, goto err; } param->calc_checksum=0; /* No need to calc glob_crc */ + free_root(&sort_param.wordroot, MYF(0)); /* Set for next loop */ sort_info.max_records= (ha_rows) info->state->records; @@ -2589,6 +2591,7 @@ int mi_repair_parallel(MI_CHECK *param, register MI_INFO *info, uint ft_max_word_len_for_sort=FT_MAX_WORD_LEN_FOR_SORT* sort_param[i].keyinfo->seg->charset->mbmaxlen; sort_param[i].key_length+=ft_max_word_len_for_sort-HA_FT_MAXBYTELEN; + init_alloc_root(&sort_param[i].wordroot, FTPARSER_MEMROOT_ALLOC_SIZE, 0); } } sort_info.total_keys=i; @@ -2810,10 +2813,11 @@ static int sort_ft_key_read(MI_SORT_PARAM *sort_param, void *key) { for (;;) { - my_free((char*) wptr, MYF(MY_ALLOW_ZERO_PTR)); + free_root(&sort_param->wordroot, MYF(MY_MARK_BLOCKS_FREE)); if ((error=sort_get_next_record(sort_param))) DBUG_RETURN(error); - if (!(wptr=_mi_ft_parserecord(info,sort_param->key,sort_param->record))) + if (!(wptr=_mi_ft_parserecord(info,sort_param->key,sort_param->record, + &sort_param->wordroot))) DBUG_RETURN(1); if (wptr->pos) break; @@ -2837,7 +2841,7 @@ static int sort_ft_key_read(MI_SORT_PARAM *sort_param, void *key) #endif if (!wptr->pos) { - my_free((char*) sort_param->wordlist, MYF(0)); + free_root(&sort_param->wordroot, MYF(MY_MARK_BLOCKS_FREE)); sort_param->wordlist=0; error=sort_write_record(sort_param); } diff --git a/storage/myisam/myisamdef.h b/storage/myisam/myisamdef.h index 0b450de9c03b60a5344ce79b768d95b857acd1b3..a83fbe8f759d29194fe40b1b63265cf02c297e6b 100644 --- a/storage/myisam/myisamdef.h +++ b/storage/myisam/myisamdef.h @@ -235,13 +235,14 @@ struct st_myisam_info { /* accumulate indexfile changes between write's */ TREE *bulk_insert; DYNAMIC_ARRAY *ft1_to_ft2; /* used only in ft1->ft2 conversion */ - MYSQL_FTPARSER_PARAM *ftparser_param; /* share info between init/deinit */ - char *filename; /* parameter to open filename */ - uchar *buff, /* Temp area for key */ - *lastkey,*lastkey2; /* Last used search key */ - uchar *first_mbr_key; /* Searhed spatial key */ - byte *rec_buff; /* Tempbuff for recordpack */ - uchar *int_keypos, /* Save position for next/previous */ + MEM_ROOT ft_memroot; /* used by the parser */ + MYSQL_FTPARSER_PARAM *ftparser_param; /* share info between init/deinit */ + char *filename; /* parameter to open filename */ + uchar *buff, /* Temp area for key */ + *lastkey,*lastkey2; /* Last used search key */ + uchar *first_mbr_key; /* Searhed spatial key */ + byte *rec_buff; /* Tempbuff for recordpack */ + uchar *int_keypos, /* Save position for next/previous */ *int_maxpos; /* -""- */ uint int_nod_flag; /* -""- */ uint32 int_keytree_version; /* -""- */ @@ -325,6 +326,7 @@ typedef struct st_mi_sort_param uchar **sort_keys; byte *rec_buff; void *wordlist, *wordptr; + MEM_ROOT wordroot; char *record; MY_TMPDIR *tmpdir; int (*key_cmp)(struct st_mi_sort_param *, const void *, const void *); diff --git a/storage/myisam/sort.c b/storage/myisam/sort.c index c9562461f56633d7e3039b29a3467649977ceb09..44e7e8b464e23f9ba4773063a52b18d6eff894b2 100644 --- a/storage/myisam/sort.c +++ b/storage/myisam/sort.c @@ -447,6 +447,7 @@ err: close_cached_file(&info->tempfile_for_exceptions); ok: + free_root(&info->wordroot, MYF(0)); remove_io_thread(&info->read_cache); pthread_mutex_lock(&info->sort_info->mutex); info->sort_info->threads_running--;