Skip to content

M
MariaDB
Commit d70dbb08 authored by unknown's avatar unknown
Browse files
Options

Merge bk-internal.mysql.com:/home/bk/mysql-5.1-new 

into serg.mylan:/usr/home/serg/Abk/mysql-5.1
parents b89c4e23 288f6223
Hide whitespace changes
Inline Side-by-side
Showing with 184 additions and 120 deletions
client/Makefile.am
View file @ d70dbb08
......@@ -83,13 +83,13 @@ link_sources:
for f in $(sql_src) ; do \
rm -f $$f; \
@LN_CP_F@ $(top_srcdir)/sql/$$f $$f; \
done; \
for f in $(strings_src) ; do \
rm -f $(srcdir)/$$f; \
@LN_CP_F@ $(top_srcdir)/strings/$$f $$f; \
done; \
rm -f $(srcdir)/my_user.c; \
@LN_CP_F@ $(top_srcdir)/sql-common/my_user.c my_user.c
done;
for f in $(strings_src) ; do \
rm -f $(srcdir)/$$f; \
@LN_CP_F@ $(top_srcdir)/strings/$$f $$f; \
done;
-rm -f $(srcdir)/my_user.c;
@LN_CP_F@ $(top_srcdir)/sql-common/my_user.c my_user.c
# Don't update the files from bitkeeper
......
include/mysql/plugin.h
View file @ d70dbb08
......@@ -98,9 +98,11 @@ struct st_mysql_plugin
API for Full-text [pre]parser plugin. (MYSQL_FTPARSER_PLUGIN)
*/
#define MYSQL_FTPARSER_INTERFACE_VERSION 0x0000
#define MYSQL_FTPARSER_INTERFACE_VERSION 0x0100
/* Parsing modes. Set in MYSQL_FTPARSER_PARAM::mode */
enum enum_ftparser_mode
{
/*
Fast and simple mode. This mode is used for indexing, and natural
language queries.
......@@ -109,7 +111,7 @@ struct st_mysql_plugin
index. Stopwords or too short/long words should not be returned. The
'boolean_info' argument of mysql_add_word() does not have to be set.
*/
#define MYSQL_FTPARSER_SIMPLE_MODE 0
MYSQL_FTPARSER_SIMPLE_MODE= 0,
/*
Parse with stopwords mode. This mode is used in boolean searches for
......@@ -120,7 +122,7 @@ struct st_mysql_plugin
or long. The 'boolean_info' argument of mysql_add_word() does not
have to be set.
*/
#define MYSQL_FTPARSER_WITH_STOPWORDS 1
MYSQL_FTPARSER_WITH_STOPWORDS= 1,
/*
Parse in boolean mode. This mode is used to parse a boolean query string.
......@@ -133,7 +135,8 @@ struct st_mysql_plugin
MYSQL_FTPARSER_WITH_STOPWORDS mode, no word should be ignored.
Instead, use FT_TOKEN_STOPWORD for the token type of such a word.
*/
#define MYSQL_FTPARSER_FULL_BOOLEAN_INFO 2
MYSQL_FTPARSER_FULL_BOOLEAN_INFO= 2
};
/*
Token types for boolean mode searching (used for the type member of
......@@ -198,6 +201,17 @@ typedef struct st_mysql_ftparser_boolean_info
char *quot;
} MYSQL_FTPARSER_BOOLEAN_INFO;
/*
The following flag means that buffer with a string (document, word)
may be overwritten by the caller before the end of the parsing (that is
before st_mysql_ftparser::deinit() call). If one needs the string
to survive between two successive calls of the parsing function, she
needs to save a copy of it. The flag may be set by MySQL before calling
st_mysql_ftparser::parse(), or it may be set by a plugin before calling
st_mysql_ftparser_param::mysql_parse() or
st_mysql_ftparser_param::mysql_add_word().
*/
#define MYSQL_FTFLAGS_NEED_COPY 1
/*
An argument of the full-text parser plugin. This structure is
......@@ -209,22 +223,20 @@ typedef struct st_mysql_ftparser_boolean_info
to invoke the MySQL default parser. If plugin's role is to extract
textual data from .doc, .pdf or .xml content, it might extract
plaintext from the content, and then pass the text to the default
MySQL parser to be parsed. When mysql_parser is called, its param
argument should be given as the mysql_ftparam value.
MySQL parser to be parsed.
mysql_add_word: A server callback to add a new word. When parsing
a document, the server sets this to point at a function that adds
the word to MySQL full-text index. When parsing a search query,
this function will add the new word to the list of words to search
for. When mysql_add_word is called, its param argument should be
given as the mysql_ftparam value. boolean_info can be NULL for all
cases except when mode is MYSQL_FTPARSER_FULL_BOOLEAN_INFO.
for. The boolean_info argument can be NULL for all cases except
when mode is MYSQL_FTPARSER_FULL_BOOLEAN_INFO.
ftparser_state: A generic pointer. The plugin can set it to point
to information to be used internally for its own purposes.
mysql_ftparam: This is set by the server. It is passed as the first
argument to the mysql_parse or mysql_add_word callback. The plugin
mysql_ftparam: This is set by the server. It is used by MySQL functions
called via mysql_parse() and mysql_add_word() callback. The plugin
should not modify it.
cs: Information about the character set of the document or query string.
......@@ -233,21 +245,26 @@ typedef struct st_mysql_ftparser_boolean_info
length: Length of the document or query string, in bytes.
flags: See MYSQL_FTFLAGS_* constants above.
mode: The parsing mode. With boolean operators, with stopwords, or
nothing. See MYSQL_FTPARSER_* constants above.
nothing. See enum_ftparser_mode above.
*/
typedef struct st_mysql_ftparser_param
{
int (*mysql_parse)(void *param, char *doc, int doc_len);
int (*mysql_add_word)(void *param, char *word, int word_len,
int (*mysql_parse)(struct st_mysql_ftparser_param *,
char *doc, int doc_len);
int (*mysql_add_word)(struct st_mysql_ftparser_param *,
char *word, int word_len,
MYSQL_FTPARSER_BOOLEAN_INFO *boolean_info);
void *ftparser_state;
void *mysql_ftparam;
struct charset_info_st *cs;
char *doc;
int length;
int mode;
int flags;
enum enum_ftparser_mode mode;
} MYSQL_FTPARSER_PARAM;
/*
......
plugin/fulltext/plugin_example.c
View file @ d70dbb08
......@@ -144,10 +144,7 @@ static void add_word(MYSQL_FTPARSER_PARAM *param, char *word, size_t len)
MYSQL_FTPARSER_BOOLEAN_INFO bool_info=
{ FT_TOKEN_WORD, 0, 0, 0, 0, ' ', 0 };
if (param->mode == MYSQL_FTPARSER_FULL_BOOLEAN_INFO)
param->mysql_add_word(param->mysql_ftparam, word, len, &bool_info);
else
param->mysql_add_word(param->mysql_ftparam, word, len, 0);
param->mysql_add_word(param, word, len, &bool_info);
}
/*
......
sql/sql_plugin.cc
View file @ d70dbb08
......@@ -41,8 +41,8 @@ static int min_plugin_interface_version= 0x0000;
static int min_plugin_info_interface_version[MYSQL_MAX_PLUGIN_TYPE_NUM]=
{
0x0000,
0x0000,
0x0000
MYSQL_HANDLERTON_INTERFACE_VERSION,
MYSQL_FTPARSER_INTERFACE_VERSION
};
static int cur_plugin_info_interface_version[MYSQL_MAX_PLUGIN_TYPE_NUM]=
{
......@@ -50,6 +50,7 @@ static int cur_plugin_info_interface_version[MYSQL_MAX_PLUGIN_TYPE_NUM]=
MYSQL_HANDLERTON_INTERFACE_VERSION,
MYSQL_FTPARSER_INTERFACE_VERSION
};
static DYNAMIC_ARRAY plugin_dl_array;
static DYNAMIC_ARRAY plugin_array;
static HASH plugin_hash[MYSQL_MAX_PLUGIN_TYPE_NUM];
......
sql/table.cc
View file @ d70dbb08
......@@ -2355,12 +2355,23 @@ table_check_intact(TABLE *table, uint table_f_count,
// previous MySQL version
error= TRUE;
if (MYSQL_VERSION_ID > table->s->mysql_version)
{
my_error(ER_COL_COUNT_DOESNT_MATCH_PLEASE_UPDATE, MYF(0), table->alias,
table_f_count, table->s->fields, table->s->mysql_version,
MYSQL_VERSION_ID);
sql_print_error(ER(ER_COL_COUNT_DOESNT_MATCH_PLEASE_UPDATE),
table->alias, table_f_count, table->s->fields,
table->s->mysql_version, MYSQL_VERSION_ID);
DBUG_RETURN(error);
}
else if (MYSQL_VERSION_ID == table->s->mysql_version)
{
my_error(ER_COL_COUNT_DOESNT_MATCH_CORRUPTED,MYF(0), table->alias,
table_f_count, table->s->fields);
sql_print_error(ER(ER_COL_COUNT_DOESNT_MATCH_CORRUPTED), table->alias,
table_f_count, table->s->fields);
}
else
/*
moving from newer mysql to older one -> let's say not an error but
......
storage/myisam/ft_boolean_search.c
View file @ d70dbb08
......@@ -160,7 +160,6 @@ static int FTB_WORD_cmp_list(CHARSET_INFO *cs, FTB_WORD **a, FTB_WORD **b)
typedef struct st_my_ftb_param
{
MYSQL_FTPARSER_PARAM *up;
FTB *ftb;
FTB_EXPR *ftbe;
byte *up_quot;
......@@ -168,10 +167,11 @@ typedef struct st_my_ftb_param
} MY_FTB_PARAM;
static int ftb_query_add_word(void *param, char *word, int word_len,
static int ftb_query_add_word(MYSQL_FTPARSER_PARAM *param,
char *word, int word_len,
MYSQL_FTPARSER_BOOLEAN_INFO *info)
{
MY_FTB_PARAM *ftb_param= (MY_FTB_PARAM *)param;
MY_FTB_PARAM *ftb_param= param->mysql_ftparam;
FTB_WORD *ftbw;
FTB_EXPR *ftbe, *tmp_expr;
FT_WORD *phrase_word;
......@@ -269,9 +269,10 @@ static int ftb_query_add_word(void *param, char *word, int word_len,
}
static int ftb_parse_query_internal(void *param, char *query, int len)
static int ftb_parse_query_internal(MYSQL_FTPARSER_PARAM *param,
char *query, int len)
{
MY_FTB_PARAM *ftb_param= (MY_FTB_PARAM *)param;
MY_FTB_PARAM *ftb_param= param->mysql_ftparam;
MYSQL_FTPARSER_BOOLEAN_INFO info;
CHARSET_INFO *cs= ftb_param->ftb->charset;
char **start= &query;
......@@ -281,7 +282,7 @@ static int ftb_parse_query_internal(void *param, char *query, int len)
info.prev= ' ';
info.quot= 0;
while (ft_get_word(cs, start, end, &w, &info))
ftb_param->up->mysql_add_word(param, w.pos, w.len, &info);
param->mysql_add_word(param, w.pos, w.len, &info);
return(0);
}
......@@ -299,7 +300,6 @@ static void _ftb_parse_query(FTB *ftb, byte *query, uint len,
if (! (param= ftparser_call_initializer(ftb->info, ftb->keynr, 0)))
DBUG_VOID_RETURN;
ftb_param.up= param;
ftb_param.ftb= ftb;
ftb_param.depth= 0;
ftb_param.ftbe= ftb->root;
......@@ -311,6 +311,7 @@ static void _ftb_parse_query(FTB *ftb, byte *query, uint len,
param->cs= ftb->charset;
param->doc= query;
param->length= len;
param->flags= 0;
param->mode= MYSQL_FTPARSER_FULL_BOOLEAN_INFO;
parser->parse(param);
DBUG_VOID_RETURN;
......@@ -571,7 +572,6 @@ FT_INFO * ft_init_boolean_search(MI_INFO *info, uint keynr, byte *query,
typedef struct st_my_ftb_phrase_param
{
MYSQL_FTPARSER_PARAM *up;
LIST *phrase;
LIST *document;
CHARSET_INFO *cs;
......@@ -581,10 +581,11 @@ typedef struct st_my_ftb_phrase_param
} MY_FTB_PHRASE_PARAM;
static int ftb_phrase_add_word(void *param, char *word, int word_len,
static int ftb_phrase_add_word(MYSQL_FTPARSER_PARAM *param,
char *word, int word_len,
MYSQL_FTPARSER_BOOLEAN_INFO *boolean_info __attribute__((unused)))
{
MY_FTB_PHRASE_PARAM *phrase_param= (MY_FTB_PHRASE_PARAM *)param;
MY_FTB_PHRASE_PARAM *phrase_param= param->mysql_ftparam;
FT_WORD *w= (FT_WORD *)phrase_param->document->data;
LIST *phrase, *document;
w->pos= word;
......@@ -611,14 +612,15 @@ static int ftb_phrase_add_word(void *param, char *word, int word_len,
}
static int ftb_check_phrase_internal(void *param, char *document, int len)
static int ftb_check_phrase_internal(MYSQL_FTPARSER_PARAM *param,
char *document, int len)
{
FT_WORD word;
MY_FTB_PHRASE_PARAM *phrase_param= (MY_FTB_PHRASE_PARAM *)param;
MY_FTB_PHRASE_PARAM *phrase_param= param->mysql_ftparam;
const char *docend= document + len;
while (ft_simple_get_word(phrase_param->cs, &document, docend, &word, FALSE))
{
phrase_param->up->mysql_add_word(param, word.pos, word.len, 0);
param->mysql_add_word(param, word.pos, word.len, 0);
if (phrase_param->match)
return 1;
}
......@@ -651,7 +653,6 @@ static int _ftb_check_phrase(FTB *ftb, const byte *document, uint len,
if (! (param= ftparser_call_initializer(ftb->info, ftb->keynr, 1)))
DBUG_RETURN(0);
ftb_param.up= param;
ftb_param.phrase= ftbe->phrase;
ftb_param.document= ftbe->document;
ftb_param.cs= ftb->charset;
......@@ -665,6 +666,7 @@ static int _ftb_check_phrase(FTB *ftb, const byte *document, uint len,
param->cs= ftb->charset;
param->doc= (byte *)document;
param->length= len;
param->flags= 0;
param->mode= MYSQL_FTPARSER_WITH_STOPWORDS;
parser->parse(param);
DBUG_RETURN(ftb_param.match ? 1 : 0);
......@@ -820,16 +822,16 @@ int ft_boolean_read_next(FT_INFO *ftb, char *record)
typedef struct st_my_ftb_find_param
{
MYSQL_FTPARSER_PARAM *up;
FT_INFO *ftb;
FT_SEG_ITERATOR *ftsi;
} MY_FTB_FIND_PARAM;
static int ftb_find_relevance_add_word(void *param, char *word, int len,
static int ftb_find_relevance_add_word(MYSQL_FTPARSER_PARAM *param,
char *word, int len,
MYSQL_FTPARSER_BOOLEAN_INFO *boolean_info __attribute__((unused)))
{
MY_FTB_FIND_PARAM *ftb_param= (MY_FTB_FIND_PARAM *)param;
MY_FTB_FIND_PARAM *ftb_param= param->mysql_ftparam;
FT_INFO *ftb= ftb_param->ftb;
FTB_WORD *ftbw;
int a, b, c;
......@@ -859,14 +861,15 @@ static int ftb_find_relevance_add_word(void *param, char *word, int len,
}
static int ftb_find_relevance_parse(void *param, char *doc, int len)
static int ftb_find_relevance_parse(MYSQL_FTPARSER_PARAM *param,
char *doc, int len)
{
MY_FTB_FIND_PARAM *ftb_param=(MY_FTB_FIND_PARAM *)param;
MY_FTB_FIND_PARAM *ftb_param= param->mysql_ftparam;
FT_INFO *ftb= ftb_param->ftb;
char *end= doc + len;
FT_WORD w;
while (ft_simple_get_word(ftb->charset, &doc, end, &w, TRUE))
ftb_param->up->mysql_add_word(param, w.pos, w.len, 0);
param->mysql_add_word(param, w.pos, w.len, 0);
return(0);
}
......@@ -910,12 +913,12 @@ float ft_boolean_find_relevance(FT_INFO *ftb, byte *record, uint length)
_mi_ft_segiterator_init(ftb->info, ftb->keynr, record, &ftsi);
memcpy(&ftsi2, &ftsi, sizeof(ftsi));
ftb_param.up= param;
ftb_param.ftb= ftb;
ftb_param.ftsi= &ftsi2;
param->mysql_parse= ftb_find_relevance_parse;
param->mysql_add_word= ftb_find_relevance_add_word;
param->mysql_ftparam= (void *)&ftb_param;
param->flags= 0;
param->cs= ftb->charset;
param->mode= MYSQL_FTPARSER_SIMPLE_MODE;
while (_mi_ft_segiterator(&ftsi))
......
storage/myisam/ft_nlq_search.c
View file @ d70dbb08
......@@ -235,7 +235,9 @@ FT_INFO *ft_init_nlq_search(MI_INFO *info, uint keynr, byte *query,
NULL, NULL);
ft_parse_init(&wtree, aio.charset);
if (ft_parse(&wtree, query, query_len, 0, parser, ftparser_param))
ftparser_param->flags= 0;
if (ft_parse(&wtree, query, query_len, parser, ftparser_param,
&wtree.mem_root))
goto err;
if (tree_walk(&wtree, (tree_walk_action)&walk_and_match, &aio,
......@@ -255,7 +257,9 @@ FT_INFO *ft_init_nlq_search(MI_INFO *info, uint keynr, byte *query,
if (!(*info->read_record)(info,docid,record))
{
info->update|= HA_STATE_AKTIV;
_mi_ft_parse(&wtree, info, keynr, record, 1, ftparser_param);
ftparser_param->flags= MYSQL_FTFLAGS_NEED_COPY;
_mi_ft_parse(&wtree, info, keynr, record, ftparser_param,
&wtree.mem_root);
}
}
delete_queue(&best);
......
storage/myisam/ft_parser.c
View file @ d70dbb08
......@@ -24,15 +24,12 @@ typedef struct st_ft_docstat {
double sum;
} FT_DOCSTAT;
typedef struct st_my_ft_parser_param
{
MYSQL_FTPARSER_PARAM *up;
TREE *wtree;
my_bool with_alloc;
TREE *wtree;
MEM_ROOT *mem_root;
} MY_FT_PARSER_PARAM;
static int FT_WORD_cmp(CHARSET_INFO* cs, FT_WORD *w1, FT_WORD *w2)
{
return mi_compare_text(cs, (uchar*) w1->pos, w1->len,
......@@ -49,14 +46,14 @@ static int walk_and_copy(FT_WORD *word,uint32 count,FT_DOCSTAT *docstat)
/* transforms tree of words into the array, applying normalization */
FT_WORD * ft_linearize(TREE *wtree)
FT_WORD * ft_linearize(TREE *wtree, MEM_ROOT *mem_root)
{
FT_WORD *wlist,*p;
FT_DOCSTAT docstat;
DBUG_ENTER("ft_linearize");
if ((wlist=(FT_WORD *) my_malloc(sizeof(FT_WORD)*
(1+wtree->elements_in_tree),MYF(0))))
if ((wlist=(FT_WORD *) alloc_root(mem_root, sizeof(FT_WORD)*
(1+wtree->elements_in_tree))))
{
docstat.list=wlist;
docstat.uniq=wtree->elements_in_tree;
......@@ -241,19 +238,20 @@ void ft_parse_init(TREE *wtree, CHARSET_INFO *cs)
}
static int ft_add_word(void *param, byte *word, uint word_len,
static int ft_add_word(MYSQL_FTPARSER_PARAM *param,
char *word, int word_len,
MYSQL_FTPARSER_BOOLEAN_INFO *boolean_info __attribute__((unused)))
{
TREE *wtree;
FT_WORD w;
MY_FT_PARSER_PARAM *ft_param=param->mysql_ftparam;
DBUG_ENTER("ft_add_word");
wtree= ((MY_FT_PARSER_PARAM *)param)->wtree;
if (((MY_FT_PARSER_PARAM *)param)->with_alloc)
wtree= ft_param->wtree;
if (param->flags & MYSQL_FTFLAGS_NEED_COPY)
{
byte *ptr;
/* allocating the data in the tree - to avoid mallocs and frees */
DBUG_ASSERT(wtree->with_delete == 0);
ptr= (byte *)alloc_root(&wtree->mem_root, word_len);
ptr= (byte *)alloc_root(ft_param->mem_root, word_len);
memcpy(ptr, word, word_len);
w.pos= ptr;
}
......@@ -269,32 +267,32 @@ static int ft_add_word(void *param, byte *word, uint word_len,
}
static int ft_parse_internal(void *param, byte *doc, int doc_len)
static int ft_parse_internal(MYSQL_FTPARSER_PARAM *param,
byte *doc, int doc_len)
{
byte *end=doc+doc_len;
MY_FT_PARSER_PARAM *ft_param=(MY_FT_PARSER_PARAM *)param;
MY_FT_PARSER_PARAM *ft_param=param->mysql_ftparam;
TREE *wtree= ft_param->wtree;
FT_WORD w;
DBUG_ENTER("ft_parse_internal");
while (ft_simple_get_word(wtree->custom_arg, &doc, end, &w, TRUE))
if (ft_param->up->mysql_add_word(param, w.pos, w.len, 0))
if (param->mysql_add_word(param, w.pos, w.len, 0))
DBUG_RETURN(1);
DBUG_RETURN(0);
}
int ft_parse(TREE *wtree, byte *doc, int doclen, my_bool with_alloc,
int ft_parse(TREE *wtree, byte *doc, int doclen,
struct st_mysql_ftparser *parser,
MYSQL_FTPARSER_PARAM *param)
MYSQL_FTPARSER_PARAM *param, MEM_ROOT *mem_root)
{
MY_FT_PARSER_PARAM my_param;
DBUG_ENTER("ft_parse");
DBUG_ASSERT(parser);
my_param.up= param;
my_param.wtree= wtree;
my_param.with_alloc= with_alloc;
my_param.mem_root= mem_root;
param->mysql_parse= ft_parse_internal;
param->mysql_add_word= ft_add_word;
......@@ -356,6 +354,7 @@ MYSQL_FTPARSER_PARAM *ftparser_call_initializer(MI_INFO *info,
info->ftparser_param= (MYSQL_FTPARSER_PARAM *)
my_malloc(MAX_PARAM_NR * sizeof(MYSQL_FTPARSER_PARAM) *
info->s->ftparsers, MYF(MY_WME|MY_ZEROFILL));
init_alloc_root(&info->ft_memroot, FTPARSER_MEMROOT_ALLOC_SIZE, 0);
if (! info->ftparser_param)
return 0;
}
......@@ -387,6 +386,7 @@ MYSQL_FTPARSER_PARAM *ftparser_call_initializer(MI_INFO *info,
void ftparser_call_deinitializer(MI_INFO *info)
{
uint i, j, keys= info->s->state.header.keys;
free_root(&info->ft_memroot, MYF(0));
if (! info->ftparser_param)
return;
for (i= 0; i < keys; i++)
......
storage/myisam/ft_static.c
View file @ d70dbb08
......@@ -629,7 +629,7 @@ const char *ft_precompiled_stopwords[] = {
static int ft_default_parser_parse(MYSQL_FTPARSER_PARAM *param)
{
return param->mysql_parse(param->mysql_ftparam, param->doc, param->length);
return param->mysql_parse(param, param->doc, param->length);
}
struct st_mysql_ftparser ft_default_parser=
......
storage/myisam/ft_update.c
View file @ d70dbb08
......@@ -77,7 +77,7 @@ uint _mi_ft_segiterator(register FT_SEG_ITERATOR *ftsi)
{
uint pack_length= (ftsi->seg->bit_start);
ftsi->len= (pack_length == 1 ? (uint) *(uchar*) ftsi->pos :
uint2korr(ftsi->pos));
uint2korr(ftsi->pos));
ftsi->pos+= pack_length; /* Skip VARCHAR length */
DBUG_RETURN(1);
}
......@@ -95,9 +95,8 @@ uint _mi_ft_segiterator(register FT_SEG_ITERATOR *ftsi)
/* parses a document i.e. calls ft_parse for every keyseg */
uint _mi_ft_parse(TREE *parsed, MI_INFO *info, uint keynr,
const byte *record, my_bool with_alloc,
MYSQL_FTPARSER_PARAM *param)
uint _mi_ft_parse(TREE *parsed, MI_INFO *info, uint keynr, const byte *record,
MYSQL_FTPARSER_PARAM *param, MEM_ROOT *mem_root)
{
FT_SEG_ITERATOR ftsi;
struct st_mysql_ftparser *parser;
......@@ -110,14 +109,14 @@ uint _mi_ft_parse(TREE *parsed, MI_INFO *info, uint keynr,
while (_mi_ft_segiterator(&ftsi))
{
if (ftsi.pos)
if (ft_parse(parsed, (byte *)ftsi.pos, ftsi.len, with_alloc, parser,
param))
if (ft_parse(parsed, (byte *)ftsi.pos, ftsi.len, parser, param, mem_root))
DBUG_RETURN(1);
}
DBUG_RETURN(0);
}
FT_WORD * _mi_ft_parserecord(MI_INFO *info, uint keynr, const byte *record)
FT_WORD *_mi_ft_parserecord(MI_INFO *info, uint keynr, const byte *record,
MEM_ROOT *mem_root)
{
TREE ptree;
MYSQL_FTPARSER_PARAM *param;
......@@ -125,10 +124,11 @@ FT_WORD * _mi_ft_parserecord(MI_INFO *info, uint keynr, const byte *record)
if (! (param= ftparser_call_initializer(info, keynr, 0)))
DBUG_RETURN(NULL);
bzero((char*) &ptree, sizeof(ptree));
if (_mi_ft_parse(&ptree, info, keynr, record, 0, param))
param->flags= 0;
if (_mi_ft_parse(&ptree, info, keynr, record, param, mem_root))
DBUG_RETURN(NULL);
DBUG_RETURN(ft_linearize(&ptree));
DBUG_RETURN(ft_linearize(&ptree, mem_root));
}
static int _mi_ft_store(MI_INFO *info, uint keynr, byte *keybuf,
......@@ -206,10 +206,11 @@ int _mi_ft_update(MI_INFO *info, uint keynr, byte *keybuf,
int cmp, cmp2;
DBUG_ENTER("_mi_ft_update");
if (!(old_word=oldlist=_mi_ft_parserecord(info, keynr, oldrec)))
goto err0;
if (!(new_word=newlist=_mi_ft_parserecord(info, keynr, newrec)))
goto err1;
if (!(old_word=oldlist=_mi_ft_parserecord(info, keynr, oldrec,
&info->ft_memroot)) ||
!(new_word=newlist=_mi_ft_parserecord(info, keynr, newrec,
&info->ft_memroot)))
goto err;
error=0;
while(old_word->pos && new_word->pos)
......@@ -222,13 +223,13 @@ int _mi_ft_update(MI_INFO *info, uint keynr, byte *keybuf,
{
key_length=_ft_make_key(info,keynr,keybuf,old_word,pos);
if ((error=_mi_ck_delete(info,keynr,(uchar*) keybuf,key_length)))
goto err2;
goto err;
}
if (cmp > 0 || cmp2)
{
key_length=_ft_make_key(info,keynr,keybuf,new_word,pos);
if ((error=_mi_ck_write(info,keynr,(uchar*) keybuf,key_length)))
goto err2;
goto err;
}
if (cmp<=0) old_word++;
if (cmp>=0) new_word++;
......@@ -238,11 +239,8 @@ int _mi_ft_update(MI_INFO *info, uint keynr, byte *keybuf,
else if (new_word->pos)
error=_mi_ft_store(info,keynr,keybuf,new_word,pos);
err2:
my_free((char*) newlist,MYF(0));
err1:
my_free((char*) oldlist,MYF(0));
err0:
err:
free_root(&info->ft_memroot, MYF(MY_MARK_BLOCKS_FREE));
DBUG_RETURN(error);
}
......@@ -255,12 +253,13 @@ int _mi_ft_add(MI_INFO *info, uint keynr, byte *keybuf, const byte *record,
int error= -1;
FT_WORD *wlist;
DBUG_ENTER("_mi_ft_add");
DBUG_PRINT("enter",("keynr: %d",keynr));
if ((wlist=_mi_ft_parserecord(info, keynr, record)))
{
if ((wlist=_mi_ft_parserecord(info, keynr, record, &info->ft_memroot)))
error=_mi_ft_store(info,keynr,keybuf,wlist,pos);
my_free((char*) wlist,MYF(0));
}
free_root(&info->ft_memroot, MYF(MY_MARK_BLOCKS_FREE));
DBUG_PRINT("exit",("Return: %d",error));
DBUG_RETURN(error);
}
......@@ -275,11 +274,10 @@ int _mi_ft_del(MI_INFO *info, uint keynr, byte *keybuf, const byte *record,
DBUG_ENTER("_mi_ft_del");
DBUG_PRINT("enter",("keynr: %d",keynr));
if ((wlist=_mi_ft_parserecord(info, keynr, record)))
{
if ((wlist=_mi_ft_parserecord(info, keynr, record, &info->ft_memroot)))
error=_mi_ft_erase(info,keynr,keybuf,wlist,pos);
my_free((char*) wlist,MYF(0));
}
free_root(&info->ft_memroot, MYF(MY_MARK_BLOCKS_FREE));
DBUG_PRINT("exit",("Return: %d",error));
DBUG_RETURN(error);
}
......
storage/myisam/ftdefs.h
View file @ d70dbb08
......@@ -30,6 +30,8 @@
#define FT_MAX_WORD_LEN_FOR_SORT 31
#define FTPARSER_MEMROOT_ALLOC_SIZE 65536
#define COMPILE_STOPWORDS_IN
/* Interested readers may consult SMART
......@@ -119,12 +121,12 @@ void _mi_ft_segiterator_dummy_init(const byte *, uint, FT_SEG_ITERATOR *);
uint _mi_ft_segiterator(FT_SEG_ITERATOR *);
void ft_parse_init(TREE *, CHARSET_INFO *);
int ft_parse(TREE *, byte *, int, my_bool, struct st_mysql_ftparser *parser,
MYSQL_FTPARSER_PARAM *param);
FT_WORD * ft_linearize(TREE *);
FT_WORD * _mi_ft_parserecord(MI_INFO *, uint, const byte *);
uint _mi_ft_parse(TREE *, MI_INFO *, uint, const byte *, my_bool,
MYSQL_FTPARSER_PARAM *param);
int ft_parse(TREE *, byte *, int, struct st_mysql_ftparser *parser,
MYSQL_FTPARSER_PARAM *, MEM_ROOT *);
FT_WORD * ft_linearize(TREE *, MEM_ROOT *);
FT_WORD * _mi_ft_parserecord(MI_INFO *, uint, const byte *, MEM_ROOT *);
uint _mi_ft_parse(TREE *, MI_INFO *, uint, const byte *,
MYSQL_FTPARSER_PARAM *, MEM_ROOT *);
FT_INFO *ft_init_nlq_search(MI_INFO *, uint, byte *, uint, uint, byte *);
FT_INFO *ft_init_boolean_search(MI_INFO *, uint, byte *, uint, CHARSET_INFO *);
......
storage/myisam/mi_check.c
View file @ d70dbb08
......@@ -2117,6 +2117,7 @@ int mi_repair_by_sort(MI_CHECK *param, register MI_INFO *info,
my_seek(param->read_cache.file,0L,MY_SEEK_END,MYF(0));
sort_param.wordlist=NULL;
init_alloc_root(&sort_param.wordroot, FTPARSER_MEMROOT_ALLOC_SIZE, 0);
if (share->data_file_type == DYNAMIC_RECORD)
length=max(share->base.min_pack_length+1,share->base.min_block_length);
......@@ -2179,12 +2180,36 @@ int mi_repair_by_sort(MI_CHECK *param, register MI_INFO *info,
{
uint ft_max_word_len_for_sort=FT_MAX_WORD_LEN_FOR_SORT*
sort_param.keyinfo->seg->charset->mbmaxlen;
sort_info.max_records=
(ha_rows) (sort_info.filelength/ft_min_word_len+1);
sort_param.key_length+=ft_max_word_len_for_sort-HA_FT_MAXBYTELEN;
/*
fulltext indexes may have much more entries than the
number of rows in the table. We estimate the number here.
Note, built-in parser is always nr. 0 - see ftparser_call_initializer()
*/
if (sort_param.keyinfo->ftparser_nr == 0)
{
/*
for built-in parser the number of generated index entries
cannot be larger than the size of the data file divided
by the minimal word's length
*/
sort_info.max_records=
(ha_rows) (sort_info.filelength/ft_min_word_len+1);
}
else
{
/*
for external plugin parser we cannot tell anything at all :(
so, we'll use all the sort memory and start from ~10 buffpeks.
(see _create_index_by_sort)
*/
sort_info.max_records=
10*param->sort_buffer_length/sort_param.key_length;
}
sort_param.key_read=sort_ft_key_read;
sort_param.key_write=sort_ft_key_write;
sort_param.key_length+=ft_max_word_len_for_sort-HA_FT_MAXBYTELEN;
}
else
{
......@@ -2200,6 +2225,7 @@ int mi_repair_by_sort(MI_CHECK *param, register MI_INFO *info,
goto err;
}
param->calc_checksum=0; /* No need to calc glob_crc */
free_root(&sort_param.wordroot, MYF(0));
/* Set for next loop */
sort_info.max_records= (ha_rows) info->state->records;
......@@ -2589,6 +2615,7 @@ int mi_repair_parallel(MI_CHECK *param, register MI_INFO *info,
uint ft_max_word_len_for_sort=FT_MAX_WORD_LEN_FOR_SORT*
sort_param[i].keyinfo->seg->charset->mbmaxlen;
sort_param[i].key_length+=ft_max_word_len_for_sort-HA_FT_MAXBYTELEN;
init_alloc_root(&sort_param[i].wordroot, FTPARSER_MEMROOT_ALLOC_SIZE, 0);
}
}
sort_info.total_keys=i;
......@@ -2810,10 +2837,11 @@ static int sort_ft_key_read(MI_SORT_PARAM *sort_param, void *key)
{
for (;;)
{
my_free((char*) wptr, MYF(MY_ALLOW_ZERO_PTR));
free_root(&sort_param->wordroot, MYF(MY_MARK_BLOCKS_FREE));
if ((error=sort_get_next_record(sort_param)))
DBUG_RETURN(error);
if (!(wptr=_mi_ft_parserecord(info,sort_param->key,sort_param->record)))
if (!(wptr=_mi_ft_parserecord(info,sort_param->key,sort_param->record,
&sort_param->wordroot)))
DBUG_RETURN(1);
if (wptr->pos)
break;
......@@ -2837,7 +2865,7 @@ static int sort_ft_key_read(MI_SORT_PARAM *sort_param, void *key)
#endif
if (!wptr->pos)
{
my_free((char*) sort_param->wordlist, MYF(0));
free_root(&sort_param->wordroot, MYF(MY_MARK_BLOCKS_FREE));
sort_param->wordlist=0;
error=sort_write_record(sort_param);
}
......
storage/myisam/myisamdef.h
View file @ d70dbb08
......@@ -235,13 +235,14 @@ struct st_myisam_info {
/* accumulate indexfile changes between write's */
TREE *bulk_insert;
DYNAMIC_ARRAY *ft1_to_ft2; /* used only in ft1->ft2 conversion */
MYSQL_FTPARSER_PARAM *ftparser_param; /* share info between init/deinit */
char *filename; /* parameter to open filename */
uchar *buff, /* Temp area for key */
*lastkey,*lastkey2; /* Last used search key */
uchar *first_mbr_key; /* Searhed spatial key */
byte *rec_buff; /* Tempbuff for recordpack */
uchar *int_keypos, /* Save position for next/previous */
MEM_ROOT ft_memroot; /* used by the parser */
MYSQL_FTPARSER_PARAM *ftparser_param; /* share info between init/deinit */
char *filename; /* parameter to open filename */
uchar *buff, /* Temp area for key */
*lastkey,*lastkey2; /* Last used search key */
uchar *first_mbr_key; /* Searhed spatial key */
byte *rec_buff; /* Tempbuff for recordpack */
uchar *int_keypos, /* Save position for next/previous */
*int_maxpos; /* -""- */
uint int_nod_flag; /* -""- */
uint32 int_keytree_version; /* -""- */
......@@ -325,6 +326,7 @@ typedef struct st_mi_sort_param
uchar **sort_keys;
byte *rec_buff;
void *wordlist, *wordptr;
MEM_ROOT wordroot;
char *record;
MY_TMPDIR *tmpdir;
int (*key_cmp)(struct st_mi_sort_param *, const void *, const void *);
......
storage/myisam/sort.c
View file @ d70dbb08
......@@ -447,6 +447,7 @@ pthread_handler_t thr_find_all_keys(void *arg)
close_cached_file(&info->tempfile_for_exceptions);
ok:
free_root(&info->wordroot, MYF(0));
remove_io_thread(&info->read_cache);
pthread_mutex_lock(&info->sort_info->mutex);
info->sort_info->threads_running--;
......
unittest/Makefile.am
View file @ d70dbb08
......@@ -23,5 +23,5 @@ examples:
cd examples && $(MAKE)
unit: $(srcdir)/unit.pl
cp $< $@
cp $(srcdir)/unit.pl $@
chmod +x $@
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7