WL#2575 - Fulltext: Parser plugin for FTS

WL#2763 - MySQL plugin interface: step 1
Manual merge from CNET tree.


include/ft_global.h:
  Default parser added.
include/my_global.h:
  dlopen related code moved from sql_udf.cc into my_global.h
include/myisam.h:
  Added fulltext parser to MI_KEYDEF
libmysqld/Makefile.am:
  Added LIBDIR macro.
mysql-test/r/connect.result:
  Test result fixed: added plugin table
mysql-test/r/information_schema.result:
  Test result fixed: added plugin table.
mysql-test/r/mysqlcheck.result:
  Test result fixed: added plugin table.
mysql-test/r/system_mysql_db.result:
   Test fixed: added plugin table
mysql-test/t/system_mysql_db_fix.test:
   Test fixed: added plugin table
scripts/mysql_create_system_tables.sh:
  Added mysql.plugin table.
scripts/mysql_fix_privilege_tables.sql:
  Added mysql.plugin table.
sql/Makefile.am:
  Added LIBDIR macro.
sql/ha_myisam.cc:
  Pass fulltext parser from sql to myisam layer.
sql/lex.h:
  Plugin related symbols.
sql/mysqld.cc:
  Initialize/deinitialize plugins, pass opt_plugin_dir.
  plugin-dir renamed to plugin_dir.
  plugin_dir is relative to mysql_home now.
sql/set_var.cc:
  plugin_dir added to SHOW VARIABLES.
sql/share/errmsg.txt:
  Plugin related error messages.
sql/sql_class.h:
  Added parser to Key class.
  Hold parser_name instead of plugin in Key class.
sql/sql_lex.h:
  INSTALL/UNINSTALL PLUGIN commands.
sql/sql_parse.cc:
  INSTALL/UNINSTALL PLUGIN commands.
sql/sql_show.cc:
  SHOW CREATE TABLE: output parser name if index was created WITH PARSER.
sql/sql_table.cc:
  Pass fulltext parser from yacc to sql layer.
sql/sql_udf.cc:
  dlopen related code moved into my_global.h.
  Implemented better check for UDF path.
  UDF loads libraries that are under plugin_dir now.
sql/sql_yacc.yy:
  INSTALL/UNINSTALL PLUGIN syntax.
  Added WITH PARSER syntax to CREATE/ALTER TABLE/INDEX.
  opt_fulltext_parser must allocate memory, since it will be used afterwards.
sql/table.cc:
  Save/restore fulltext parser in extra data segment.
  Added DBUG_PRINTs.
storage/myisam/ft_boolean_search.c:
  Split functions so they can be used by fulltext parser.
  Use fulltext parser if specified.
storage/myisam/ft_nlq_search.c:
  Use fulltext parser.
storage/myisam/ft_parser.c:
  Split functions so they can be used by fulltext parser.
  Use fulltext parser if specified.
storage/myisam/ft_static.c:
  Default fulltext parser added.
storage/myisam/ft_update.c:
  Use fulltext parser.
storage/myisam/ftdefs.h:
  FTB_PARAM moved into plugin.h and renamed to MYSQL_FTPARSER_BOOLEAN_INFO.
storage/myisam/mi_open.c:
  Set default parser.

include/ft_global.h

@@ -53,6 +53,7 @@ extern ulong ft_min_word_len;
 extern ulong ft_max_word_len;
 extern ulong ft_query_expansion_limit;
 extern char  ft_boolean_syntax[15];
+extern struct st_mysql_ftparser ft_default_parser;
 
 int ft_init_stopwords(void);
 void ft_free_stopwords(void);

include/my_global.h

@@ -1380,4 +1380,23 @@ do { doubleget_union _tmp; \
 #define dlerror() ""
 #endif
 
+#ifdef HAVE_DLOPEN
+#if defined(__WIN__)
+#define dlsym(lib, name) GetProcAddress((HMODULE)lib, name)
+#define dlopen(libname, unused) LoadLibraryEx(libname, NULL, 0)
+#define dlclose(lib) FreeLibrary((HMODULE)lib)
+#elif !defined(OS2)
+#include <dlfcn.h>
+#endif
+#endif
+
+/* FreeBSD 2.2.2 does not define RTLD_NOW) */
+#ifndef RTLD_NOW
+#define RTLD_NOW 1
+#endif
+
+#ifndef HAVE_DLERROR
+#define dlerror() ""
+#endif
+
 #endif /* my_global_h */

include/myisam.h

@@ -32,6 +32,7 @@ extern "C" {
 #include "keycache.h"
 #endif
 #include "my_handler.h"
+#include <plugin.h>
 
 	/* defines used by myisam-funktions */
 
@@ -196,6 +197,7 @@ typedef struct st_mi_keydef		/* Key definition with open & info */
   uint32 version;			/* For concurrent read/write */
 
   HA_KEYSEG *seg,*end;
+  struct st_mysql_ftparser *parser;     /* Fulltext [pre]parser */
   int (*bin_search)(struct st_myisam_info *info,struct st_mi_keydef *keyinfo,
 		    uchar *page,uchar *key,
 		    uint key_len,uint comp_flag,uchar * *ret_pos,

libmysqld/Makefile.am

@@ -20,11 +20,13 @@
 MYSQLDATAdir =		$(localstatedir)
 MYSQLSHAREdir =		$(pkgdatadir)
 MYSQLBASEdir=		$(prefix)
+MYSQLLIBdir=            $(libdir)
 
 DEFS =			-DEMBEDDED_LIBRARY -DMYSQL_SERVER \
 			-DDEFAULT_MYSQL_HOME="\"$(MYSQLBASEdir)\"" \
 			-DDATADIR="\"$(MYSQLDATAdir)\"" \
-			-DSHAREDIR="\"$(MYSQLSHAREdir)\""
+			-DSHAREDIR="\"$(MYSQLSHAREdir)\"" \
+			-DLIBDIR="\"$(MYSQLLIBdir)\""
 INCLUDES=		@bdb_includes@ \
 			-I$(top_builddir)/include -I$(top_srcdir)/include \
 			-I$(top_srcdir)/sql -I$(top_srcdir)/sql/examples \

mysql-test/r/connect.result

@@ -9,6 +9,7 @@ help_keyword
 help_relation
 help_topic
 host
+plugin
 proc
 procs_priv
 tables_priv
@@ -36,6 +37,7 @@ help_keyword
 help_relation
 help_topic
 host
+plugin
 proc
 procs_priv
 tables_priv
@@ -71,6 +73,7 @@ help_keyword
 help_relation
 help_topic
 host
+plugin
 proc
 procs_priv
 tables_priv

mysql-test/r/information_schema.result

@@ -60,6 +60,7 @@ help_keyword
 help_relation
 help_topic
 host
+plugin
 proc
 procs_priv
 tables_priv
@@ -709,7 +710,7 @@ CREATE TABLE t_crashme ( f1 BIGINT);
 CREATE VIEW a1 (t_CRASHME) AS SELECT f1 FROM t_crashme GROUP BY f1;
 CREATE VIEW a2 AS SELECT t_CRASHME FROM a1;
 count(*)
-101
+102
 drop view a2, a1;
 drop table t_crashme;
 select table_schema,table_name, column_name from
@@ -779,7 +780,7 @@ flush privileges;
 SELECT table_schema, count(*) FROM information_schema.TABLES GROUP BY TABLE_SCHEMA;
 table_schema	count(*)
 information_schema	16
-mysql	17
+mysql	18
 create table t1 (i int, j int);
 create trigger trg1 before insert on t1 for each row
 begin

mysql-test/r/mysqlcheck.result

@@ -6,6 +6,7 @@ mysql.help_keyword                                 OK
 mysql.help_relation                                OK
 mysql.help_topic                                   OK
 mysql.host                                         OK
+mysql.plugin                                       OK
 mysql.proc                                         OK
 mysql.procs_priv                                   OK
 mysql.tables_priv                                  OK
@@ -23,6 +24,7 @@ mysql.help_keyword                                 OK
 mysql.help_relation                                OK
 mysql.help_topic                                   OK
 mysql.host                                         OK
+mysql.plugin                                       OK
 mysql.proc                                         OK
 mysql.procs_priv                                   OK
 mysql.tables_priv                                  OK

mysql-test/r/system_mysql_db.result

@@ -9,6 +9,7 @@ help_keyword
 help_relation
 help_topic
 host
+plugin
 proc
 procs_priv
 tables_priv

mysql-test/t/system_mysql_db_fix.test

@@ -85,7 +85,7 @@ INSERT INTO user VALUES ('localhost','',    '','N','N','N','N','N','N','N','N','
 
 -- disable_query_log
 
-DROP TABLE db, host, user, func, tables_priv, columns_priv, procs_priv, help_category, help_keyword, help_relation, help_topic, proc, time_zone, time_zone_leap_second, time_zone_name, time_zone_transition, time_zone_transition_type;
+DROP TABLE db, host, user, func, plugin, tables_priv, columns_priv, procs_priv, help_category, help_keyword, help_relation, help_topic, proc, time_zone, time_zone_leap_second, time_zone_name, time_zone_transition, time_zone_transition_type;
 
 -- enable_query_log
 

scripts/mysql_create_system_tables.sh

@@ -39,8 +39,8 @@ c_hc=""
 c_hr="" 
 c_hk="" 
 i_ht=""
-c_tzn="" c_tz="" c_tzt="" c_tztt="" c_tzls=""
-i_tzn="" i_tz="" i_tzt="" i_tztt="" i_tzls=""
+c_tzn="" c_tz="" c_tzt="" c_tztt="" c_tzls="" c_pl=""
+i_tzn="" i_tz="" i_tzt="" i_tztt="" i_tzls="" i_pl=""
 c_p="" c_pp=""
 
 # Check for old tables
@@ -202,6 +202,21 @@ then
   c_f="$c_f   comment='User defined functions';"
 fi
 
+if test ! -f $mdata/plugin.frm
+then
+  if test "$1" = "verbose" ; then
+    echo "Preparing plugin table" 1>&2;
+  fi
+
+  c_pl="$c_pl CREATE TABLE plugin ("
+  c_pl="$c_pl   name char(64) binary DEFAULT '' NOT NULL,"
+  c_pl="$c_pl   dl char(128) DEFAULT '' NOT NULL,"
+  c_pl="$c_pl   PRIMARY KEY (name)"
+  c_pl="$c_pl ) engine=MyISAM"
+  c_pl="$c_pl CHARACTER SET utf8 COLLATE utf8_bin"
+  c_pl="$c_pl   comment='MySQL plugins';"
+fi
+
 if test ! -f $mdata/tables_priv.frm
 then
   if test "$1" = "verbose" ; then
@@ -741,6 +756,9 @@ $i_u
 $c_f
 $i_f
 
+$c_pl
+$i_pl
+
 $c_t
 $c_c
 

scripts/mysql_fix_privilege_tables.sql

@@ -19,6 +19,12 @@ CREATE TABLE IF NOT EXISTS func (
   PRIMARY KEY (name)
 ) CHARACTER SET utf8 COLLATE utf8_bin;
 
+CREATE TABLE IF NOT EXISTS plugin (
+  name char(64) binary DEFAULT '' NOT NULL,
+  dl char(128) DEFAULT '' NOT NULL,
+  PRIMARY KEY (name)
+) CHARACTER SET utf8 COLLATE utf8_bin;
+
 ALTER TABLE user add File_priv enum('N','Y') COLLATE utf8_general_ci NOT NULL;
 
 -- Detect whether or not we had the Grant_priv column

sql/Makefile.am

@@ -19,6 +19,7 @@
 MYSQLDATAdir =		$(localstatedir)
 MYSQLSHAREdir =		$(pkgdatadir)
 MYSQLBASEdir=		$(prefix)
+MYSQLLIBdir=            $(pkglibdir)
 INCLUDES =		@ZLIB_INCLUDES@ \
 			@bdb_includes@ @innodb_includes@ @ndbcluster_includes@ \
 			-I$(top_builddir)/include -I$(top_srcdir)/include \
@@ -116,6 +117,7 @@ DEFS =			-DMYSQL_SERVER \
 			-DDEFAULT_MYSQL_HOME="\"$(MYSQLBASEdir)\"" \
 			-DDATADIR="\"$(MYSQLDATAdir)\"" \
 			-DSHAREDIR="\"$(MYSQLSHAREdir)\"" \
+			-DLIBDIR="\"$(MYSQLLIBdir)\"" \
 			@DEFS@
 
 BUILT_SOURCES =		sql_yacc.cc sql_yacc.h lex_hash.h

sql/ha_myisam.cc

@@ -280,6 +280,7 @@ err:
 
 int ha_myisam::open(const char *name, int mode, uint test_if_locked)
 {
+  uint i;
   if (!(file=mi_open(name, mode, test_if_locked)))
     return (my_errno ? my_errno : -1);
   
@@ -292,6 +293,14 @@ int ha_myisam::open(const char *name, int mode, uint test_if_locked)
     int_table_flags|=HA_REC_NOT_IN_SEQ;
   if (file->s->options & (HA_OPTION_CHECKSUM | HA_OPTION_COMPRESS_RECORD))
     int_table_flags|=HA_HAS_CHECKSUM;
+
+  for (i= 0; i < table->s->keys; i++)
+  {
+    struct st_plugin_int *parser= table->key_info[i].parser;
+    if (table->key_info[i].flags & HA_USES_PARSER)
+      file->s->keyinfo[i].parser=
+        (struct st_mysql_ftparser *)parser->plugin->info;
+  }
   return (0);
 }
 

sql/lex.h

@@ -246,6 +246,7 @@ static SYMBOL symbols[] = {
   { "INSENSITIVE",      SYM(INSENSITIVE_SYM)},
   { "INSERT",		SYM(INSERT)},
   { "INSERT_METHOD",    SYM(INSERT_METHOD)},
+  { "INSTALL",          SYM(INSTALL_SYM)},
   { "INT",		SYM(INT_SYM)},
   { "INT1",		SYM(TINYINT)},
   { "INT2",		SYM(SMALLINT)},
@@ -370,6 +371,7 @@ static SYMBOL symbols[] = {
   { "OUTER",		SYM(OUTER)},
   { "OUTFILE",		SYM(OUTFILE)},
   { "PACK_KEYS",	SYM(PACK_KEYS_SYM)},
+  { "PARSER",           SYM(PARSER_SYM)},
   { "PARTIAL",		SYM(PARTIAL)},
 #ifdef HAVE_PARTITION_DB
   { "PARTITION",        SYM(PARTITION_SYM)},
@@ -377,6 +379,7 @@ static SYMBOL symbols[] = {
   { "PARTITIONS",       SYM(PARTITIONS_SYM)},
   { "PASSWORD",		SYM(PASSWORD)},
   { "PHASE",            SYM(PHASE_SYM)},
+  { "PLUGIN",           SYM(PLUGIN_SYM)},
   { "POINT",		SYM(POINT_SYM)},
   { "POLYGON",		SYM(POLYGON)},
   { "PRECISION",	SYM(PRECISION)},
@@ -454,7 +457,7 @@ static SYMBOL symbols[] = {
   { "SNAPSHOT",         SYM(SNAPSHOT_SYM)},
   { "SMALLINT",		SYM(SMALLINT)},
   { "SOME",             SYM(ANY_SYM)},
-  { "SONAME",		SYM(UDF_SONAME_SYM)},
+  { "SONAME",		SYM(SONAME_SYM)},
   { "SOUNDS",		SYM(SOUNDS_SYM)},
   { "SPATIAL",		SYM(SPATIAL_SYM)},
   { "SPECIFIC",         SYM(SPECIFIC_SYM)},
@@ -525,6 +528,7 @@ static SYMBOL symbols[] = {
   { "UNIQUE",		SYM(UNIQUE_SYM)},
   { "UNKNOWN",		SYM(UNKNOWN_SYM)},
   { "UNLOCK",		SYM(UNLOCK_SYM)},
+  { "UNINSTALL",        SYM(UNINSTALL_SYM)},
   { "UNSIGNED",		SYM(UNSIGNED)},
   { "UNTIL",		SYM(UNTIL_SYM)},
   { "UPDATE",		SYM(UPDATE_SYM)},

sql/mysqld.cc

@@ -1072,10 +1072,13 @@ void clean_up(bool print_message)
   lex_free();				/* Free some memory */
   set_var_free();
   free_charsets();
-#ifdef HAVE_DLOPEN
   if (!opt_noacl)
+  {
+#ifdef HAVE_DLOPEN
     udf_free();
 #endif
+    plugin_free();
+  }
   (void) ha_panic(HA_PANIC_CLOSE);	/* close all tables and logs */
   if (tc_log)
     tc_log->close();
@@ -3413,10 +3416,13 @@ we force server id to 2, but this MySQL server will not act as a slave.");
   if (!opt_noacl)
     (void) grant_init();
 
-#ifdef HAVE_DLOPEN
   if (!opt_noacl)
+  {
+    plugin_init();
+#ifdef HAVE_DLOPEN
     udf_init();
 #endif
+  }
   if (opt_bootstrap) /* If running with bootstrap, do not start replication. */
     opt_skip_slave_start= 1;
   /*
@@ -4574,7 +4580,8 @@ enum options_mysqld
   OPT_TIMED_MUTEXES,
   OPT_OLD_STYLE_USER_LIMITS,
   OPT_LOG_SLOW_ADMIN_STATEMENTS,
-  OPT_TABLE_LOCK_WAIT_TIMEOUT
+  OPT_TABLE_LOCK_WAIT_TIMEOUT,
+  OPT_PLUGIN_DIR
 };
 
 
@@ -5724,6 +5731,10 @@ The minimum value for this variable is 4096.",
    (gptr*) &global_system_variables.optimizer_search_depth,
    (gptr*) &max_system_variables.optimizer_search_depth,
    0, GET_ULONG, OPT_ARG, MAX_TABLES+1, 0, MAX_TABLES+2, 0, 1, 0},
+  {"plugin_dir", OPT_PLUGIN_DIR,
+   "Directory for plugins.",
+   (gptr*) &opt_plugin_dir_ptr, (gptr*) &opt_plugin_dir_ptr, 0,
+   GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
    {"preload_buffer_size", OPT_PRELOAD_BUFFER_SIZE,
     "The size of the buffer that is allocated when preloading indexes",
     (gptr*) &global_system_variables.preload_buff_size,
@@ -6292,6 +6303,9 @@ static void mysql_init_variables(void)
 	  sizeof(mysql_real_data_home)-1);
   mysql_data_home_buff[0]=FN_CURLIB;	// all paths are relative from here
   mysql_data_home_buff[1]=0;
+  strmake(opt_plugin_dir, get_relative_path(LIBDIR),
+          sizeof(opt_plugin_dir) - 1);
+  opt_plugin_dir_ptr= opt_plugin_dir;
 
   /* Replication parameters */
   master_user= (char*) "test";
@@ -7215,6 +7229,7 @@ static void fix_paths(void)
   (void) my_load_path(mysql_home,mysql_home,""); // Resolve current dir
   (void) my_load_path(mysql_real_data_home,mysql_real_data_home,mysql_home);
   (void) my_load_path(pidfile_name,pidfile_name,mysql_real_data_home);
+  (void) my_load_path(opt_plugin_dir, opt_plugin_dir_ptr, mysql_home);
 
   char *sharedir=get_relative_path(SHAREDIR);
   if (test_if_hard_path(sharedir))

sql/set_var.cc

@@ -781,6 +781,7 @@ struct show_var_st init_vars[]= {
   {sys_optimizer_search_depth.name,(char*) &sys_optimizer_search_depth,
    SHOW_SYS},
   {"pid_file",                (char*) pidfile_name,                 SHOW_CHAR},
+  {"plugin_dir",              (char*) opt_plugin_dir,               SHOW_CHAR},
   {"port",                    (char*) &mysqld_port,                  SHOW_INT},
   {sys_preload_buff_size.name, (char*) &sys_preload_buff_size,      SHOW_SYS},
   {"protocol_version",        (char*) &protocol_version,            SHOW_INT},

sql/share/errmsg.txt

@@ -3006,7 +3006,7 @@ ER_CANT_FIND_DL_ENTRY
 	cze "Nemohu naj-Bt funkci '%-.64s' v knihovn'"
 	dan "Kan ikke finde funktionen '%-.64s' i bibliotek'"
 	nla "Kan functie '%-.64s' niet in library vinden"
-	eng "Can't find function '%-.64s' in library'"
+	eng "Can't find symbol '%-.64s' in library'"
 	jps "function '%-.64s' Cu[Ɍt鎖ł܂",
 	est "Ei leia funktsiooni '%-.64s' antud teegis"
 	fre "Impossible de trouver la fonction '%-.64s' dans la bibliothque'"
@@ -3018,7 +3018,7 @@ ER_CANT_FIND_DL_ENTRY
 	kor "̹ '%-.64s' Լ ã  ϴ."
 	por "No pode encontrar a funo '%-.64s' na biblioteca"
 	rum "Nu pot gasi functia '%-.64s' in libraria"
-	rus "   '%-.64s'  "
+	rus "   '%-.64s'  "
 	serbian "Ne mogu da pronadjem funkciju '%-.64s' u biblioteci"
 	slo "Nemem njs funkciu '%-.64s' v kninici'"
 	spa "No puedo encontrar funcin '%-.64s' en libraria'"

sql/sql_class.h

@@ -413,11 +413,13 @@ public:
   List<key_part_spec> columns;
   const char *name;
   bool generated;
+  LEX_STRING *parser_name;
 
   Key(enum Keytype type_par, const char *name_arg, enum ha_key_alg alg_par,
-      bool generated_arg, List<key_part_spec> &cols)
+      bool generated_arg, List<key_part_spec> &cols,
+      LEX_STRING *parser_arg= 0)
     :type(type_par), algorithm(alg_par), columns(cols), name(name_arg),
-    generated(generated_arg)
+    generated(generated_arg), parser_name(parser_arg)
   {}
   ~Key() {}
   /* Equality comparison of keys (ignoring name) */

sql/sql_lex.h

@@ -91,6 +91,7 @@ enum enum_sql_command {
   SQLCOM_CREATE_TRIGGER, SQLCOM_DROP_TRIGGER,
   SQLCOM_XA_START, SQLCOM_XA_END, SQLCOM_XA_PREPARE,
   SQLCOM_XA_COMMIT, SQLCOM_XA_ROLLBACK, SQLCOM_XA_RECOVER,
+  SQLCOM_INSTALL_PLUGIN, SQLCOM_UNINSTALL_PLUGIN,
   /* This should be the last !!! */
 
   SQLCOM_END

sql/sql_parse.cc

@@ -4780,6 +4780,15 @@ end_with_restore_list:
   case SQLCOM_XA_RECOVER:
     res= mysql_xa_recover(thd);
     break;
+  case SQLCOM_INSTALL_PLUGIN:
+    if (! (res= mysql_install_plugin(thd, &thd->lex->comment,
+                                     &thd->lex->ident)))
+      send_ok(thd);
+    break;
+  case SQLCOM_UNINSTALL_PLUGIN:
+    if (! (res= mysql_uninstall_plugin(thd, &thd->lex->comment)))
+      send_ok(thd);
+    break;
   default:
     DBUG_ASSERT(0);                             /* Impossible */
     send_ok(thd);

sql/sql_show.cc

@@ -948,6 +948,12 @@ store_create_info(THD *thd, TABLE_LIST *table_list, String *packet)
       }
     }
     packet->append(')');
+    if (key_info->parser)
+    {
+      packet->append(" WITH PARSER ", 13);
+      append_identifier(thd, packet, key_info->parser->name.str,
+                        key_info->parser->name.length);
+    }
   }
 
   /*

sql/sql_table.cc

@@ -1087,6 +1087,8 @@ static int mysql_prepare_table(THD *thd, HA_CREATE_INFO *create_info,
 	break;
     case Key::FULLTEXT:
 	key_info->flags= HA_FULLTEXT;
+	if ((key_info->parser_name= key->parser_name))
+          key_info->flags|= HA_USES_PARSER;
 	break;
     case Key::SPATIAL:
 #ifdef HAVE_SPATIAL

sql/sql_udf.cc

@@ -38,36 +38,10 @@
 #ifdef HAVE_DLOPEN
 extern "C"
 {
-#if defined(__WIN__)
-  void* dlsym(void* lib,const char* name)
-  {
-    return GetProcAddress((HMODULE)lib,name);
-  }
-  void* dlopen(const char* libname,int unused)
-  {
-    return LoadLibraryEx(libname,NULL,0);
-  }
-  void dlclose(void* lib)
-  {
-    FreeLibrary((HMODULE)lib);
-  }
-
-#elif !defined(OS2)
-#include <dlfcn.h>
-#endif
-
 #include <stdarg.h>
 #include <hash.h>
 }
 
-#ifndef RTLD_NOW
-#define RTLD_NOW 1				// For FreeBSD 2.2.2
-#endif
-
-#ifndef HAVE_DLERROR
-#define dlerror() ""
-#endif
-
 static bool initialized = 0;
 static MEM_ROOT mem;
 static HASH udf_hash;
@@ -194,8 +168,10 @@ void udf_init()
       This is done to ensure that only approved dll from the system
       directories are used (to make this even remotely secure).
     */
-    if (strchr(dl_name, '/') ||
-        IF_WIN(strchr(dl_name, '\\'),0) ||
+    if (my_strchr(files_charset_info, dl_name,
+                  dl_name + strlen(dl_name), '/') ||
+        IF_WIN(my_strchr(files_charset_info, dl_name,
+                         dl_name + strlen(dl_name), '\\'),0) ||
         strlen(name.str) > NAME_LEN)
     {
       sql_print_error("Invalid row in mysql.func table for function '%.64s'",
@@ -214,10 +190,13 @@ void udf_init()
     void *dl = find_udf_dl(tmp->dl);
     if (dl == NULL)
     {
-      if (!(dl = dlopen(tmp->dl, RTLD_NOW)))
+      char dlpath[FN_REFLEN];
+      strxnmov(dlpath, sizeof(dlpath) - 1, opt_plugin_dir, "/", tmp->dl,
+               NullS);
+      if (!(dl= dlopen(dlpath, RTLD_NOW)))
       {
 	/* Print warning to log */
-	sql_print_error(ER(ER_CANT_OPEN_LIBRARY), tmp->dl,errno,dlerror());
+	sql_print_error(ER(ER_CANT_OPEN_LIBRARY), dlpath, errno, dlerror());
 	/* Keep the udf in the hash so that we can remove it later */
 	continue;
       }
@@ -412,7 +391,9 @@ int mysql_create_function(THD *thd,udf_func *udf)
     This is done to ensure that only approved dll from the system
     directories are used (to make this even remotely secure).
   */
-  if (strchr(udf->dl, '/') || IF_WIN(strchr(udf->dl, '\\'),0))
+  if (my_strchr(files_charset_info, udf->dl, udf->dl + strlen(udf->dl), '/') ||
+      IF_WIN(strchr(files_charset_info, udf->dl,
+                    udf->dl + strlen(udf->dl), '\\'),0))
   {
     my_message(ER_UDF_NO_PATHS, ER(ER_UDF_NO_PATHS), MYF(0));
     DBUG_RETURN(1);
@@ -431,12 +412,14 @@ int mysql_create_function(THD *thd,udf_func *udf)
   }
   if (!(dl = find_udf_dl(udf->dl)))
   {
-    if (!(dl = dlopen(udf->dl, RTLD_NOW)))
+    char dlpath[FN_REFLEN];
+    strxnmov(dlpath, sizeof(dlpath) - 1, opt_plugin_dir, "/", udf->dl, NullS);
+    if (!(dl = dlopen(dlpath, RTLD_NOW)))
     {
       DBUG_PRINT("error",("dlopen of %s failed, error: %d (%s)",
-			  udf->dl,errno,dlerror()));
+			  dlpath, errno, dlerror()));
       my_error(ER_CANT_OPEN_LIBRARY, MYF(0),
-                      udf->dl, errno, dlerror());
+                      dlpath, errno, dlerror());
       goto err;
     }
     new_dl=1;

sql/sql_yacc.yy

@@ -335,6 +335,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, ulong *yystacksize);
 %token  INSENSITIVE_SYM
 %token  INSERT
 %token  INSERT_METHOD
+%token  INSTALL_SYM
 %token  INTERVAL_SYM
 %token  INTO
 %token  INT_SYM
@@ -470,12 +471,14 @@ bool my_yyoverflow(short **a, YYSTYPE **b, ulong *yystacksize);
 %token  OUTFILE
 %token  OUT_SYM
 %token  PACK_KEYS_SYM
+%token  PARSER_SYM
 %token  PARTIAL
 %token  PARTITION_SYM
 %token  PARTITIONS_SYM
 %token  PASSWORD
 %token  PARAM_MARKER
 %token  PHASE_SYM
+%token  PLUGIN_SYM
 %token  POINTFROMTEXT
 %token  POINT_SYM
 %token  POLYFROMTEXT
@@ -560,6 +563,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, ulong *yystacksize);
 %token  SLAVE
 %token  SMALLINT
 %token  SNAPSHOT_SYM
+%token  SONAME_SYM
 %token  SOUNDS_SYM
 %token  SPATIAL_SYM
 %token  SPECIFIC_SYM
@@ -621,13 +625,13 @@ bool my_yyoverflow(short **a, YYSTYPE **b, ulong *yystacksize);
 %token  TYPES_SYM
 %token  TYPE_SYM
 %token  UDF_RETURNS_SYM
-%token  UDF_SONAME_SYM
 %token  ULONGLONG_NUM
 %token  UNCOMMITTED_SYM
 %token  UNDEFINED_SYM
 %token  UNDERSCORE_CHARSET
 %token  UNDO_SYM
 %token  UNICODE_SYM
+%token  UNINSTALL_SYM
 %token  UNION_SYM
 %token  UNIQUE_SYM
 %token  UNIQUE_USERS
@@ -696,7 +700,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, ulong *yystacksize);
         sp_opt_label BIN_NUM label_ident
 
 %type <lex_str_ptr>
-	opt_table_alias
+	opt_table_alias opt_fulltext_parser
 
 %type <table>
 	table_ident table_ident_nodb references xid
@@ -844,7 +848,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, ulong *yystacksize);
 	statement sp_suid opt_view_list view_list or_replace algorithm
 	sp_c_chistics sp_a_chistics sp_chistic sp_c_chistic xa
         load_data opt_field_or_var_spec fields_or_vars opt_load_data_set_spec
-        view_user view_suid
+        install uninstall view_user view_suid
         partition_entry
 END_OF_INPUT
 
@@ -906,6 +910,7 @@ statement:
 	| handler
 	| help
 	| insert
+        | install
 	| kill
 	| load
 	| lock
@@ -930,6 +935,7 @@ statement:
 	| slave
 	| start
 	| truncate
+        | uninstall
 	| unlock
 	| update
 	| use
@@ -1187,11 +1193,15 @@ create:
 	    lex->col_list.empty();
 	    lex->change=NullS;
 	  }
-	   '(' key_list ')'
+	   '(' key_list ')' opt_fulltext_parser
 	  {
 	    LEX *lex=Lex;
-
-	    lex->key_list.push_back(new Key($2,$4.str, $5, 0, lex->col_list));
+	    if ($2 != Key::FULLTEXT && $12)
+	    {
+	      yyerror(ER(ER_SYNTAX_ERROR));
+	      YYABORT;
+	    }
+	    lex->key_list.push_back(new Key($2,$4.str,$5,0,lex->col_list,$12));
 	    lex->col_list.empty();
 	  }
 	| CREATE DATABASE opt_if_not_exists ident
@@ -1382,7 +1392,7 @@ sp_name:
 	;
 
 create_function_tail:
-	  RETURNS_SYM udf_type UDF_SONAME_SYM TEXT_STRING_sys
+	  RETURNS_SYM udf_type SONAME_SYM TEXT_STRING_sys
 	  {
 	    LEX *lex=Lex;
 	    lex->sql_command = SQLCOM_CREATE_FUNCTION;
@@ -3302,10 +3312,15 @@ column_def:
 	;
 
 key_def:
-	key_type opt_ident key_alg '(' key_list ')'
+	key_type opt_ident key_alg '(' key_list ')' opt_fulltext_parser
 	  {
 	    LEX *lex=Lex;
-	    lex->key_list.push_back(new Key($1,$2, $3, 0, lex->col_list));
+	    if ($1 != Key::FULLTEXT && $7)
+	    {
+	      yyerror(ER(ER_SYNTAX_ERROR));
+	      YYABORT;
+	    }
+	    lex->key_list.push_back(new Key($1,$2, $3, 0, lex->col_list, $7));
 	    lex->col_list.empty();		/* Alloced by sql_alloc */
 	  }
 	| opt_constraint constraint_key_type opt_ident key_alg '(' key_list ')'
@@ -3340,6 +3355,20 @@ key_def:
 	  }
 	;
 
+opt_fulltext_parser:
+        /* empty */               { $$= (LEX_STRING *)0; }
+	| WITH PARSER_SYM IDENT_sys
+          {
+            if (plugin_is_ready(&$3, MYSQL_FTPARSER_PLUGIN))
+              $$= (LEX_STRING *)sql_memdup(&$3, sizeof(LEX_STRING));
+            else
+            {
+              my_error(ER_FUNCTION_NOT_DEFINED, MYF(0), $3.str);
+              YYABORT;
+            }
+          }
+        ;
+
 opt_check_constraint:
 	/* empty */
 	| check_constraint
@@ -8153,10 +8182,13 @@ keyword:
 	| FLUSH_SYM		{}
 	| HANDLER_SYM		{}
 	| HELP_SYM		{}
+        | INSTALL_SYM           {}
 	| LANGUAGE_SYM          {}
 	| NO_SYM		{}
 	| OPEN_SYM		{}
+        | PARSER_SYM            {}
 	| PARTITION_SYM		{}
+        | PLUGIN_SYM            {}
         | PREPARE_SYM           {}
 	| REPAIR		{}
 	| RESET_SYM		{}
@@ -8166,10 +8198,12 @@ keyword:
 	| SECURITY_SYM		{}
 	| SIGNED_SYM		{}
 	| SLAVE			{}
+        | SONAME_SYM            {}
 	| START_SYM		{}
 	| STOP_SYM		{}
 	| TRUNCATE_SYM		{}
 	| UNICODE_SYM		{}
+        | UNINSTALL_SYM         {}
         | XA_SYM                {}
 	;
 
@@ -9732,4 +9766,19 @@ opt_migrate:
     | FOR_SYM MIGRATE_SYM   { Lex->xa_opt=XA_FOR_MIGRATE; }
     ;
 
+install:
+  INSTALL_SYM PLUGIN_SYM IDENT_sys SONAME_SYM TEXT_STRING_sys
+  {
+    LEX *lex= Lex;
+    lex->sql_command= SQLCOM_INSTALL_PLUGIN;
+    lex->comment= $3;
+    lex->ident= $5;
+  };
 
+uninstall:
+  UNINSTALL_SYM PLUGIN_SYM IDENT_sys
+  {
+    LEX *lex= Lex;
+    lex->sql_command= SQLCOM_UNINSTALL_PLUGIN;
+    lex->comment= $3;
+  };

sql/table.cc

@@ -323,6 +323,7 @@ int openfrm(THD *thd, const char *name, const char *alias, uint db_stat,
     if (! (share->connect_string.str= strmake_root(&outparam->mem_root,
             next_chunk + 2, share->connect_string.length)))
     {
+      DBUG_PRINT("EDS", ("strmake_root failed for connect_string"));
       my_free(buff, MYF(0));
       goto err;
     }
@@ -1064,9 +1065,17 @@ int openfrm(THD *thd, const char *name, const char *alias, uint db_stat,
 int closefrm(register TABLE *table)
 {
   int error=0;
+  uint idx;
+  KEY *key_info;
   DBUG_ENTER("closefrm");
   if (table->db_stat)
     error=table->file->close();
+  key_info= table->key_info;
+  for (idx= table->s->keys; idx; idx--, key_info++)
+  {
+    if (key_info->flags & HA_USES_PARSER)
+      plugin_unlock(key_info->parser);
+  }
   my_free((char*) table->alias, MYF(MY_ALLOW_ZERO_PTR));
   table->alias= 0;
   if (table->field)

storage/myisam/ft_boolean_search.c

@@ -91,6 +91,7 @@ struct st_ftb_expr
   float     weight;
   float     cur_weight;
   LIST     *phrase;               /* phrase words */
+  LIST     *document;             /* for phrase search */
   uint      yesses;               /* number of "yes" words matched */
   uint      nos;                  /* number of "no"  words matched */
   uint      ythresh;              /* number of "yes" words in expr */
@@ -154,85 +155,160 @@ static int FTB_WORD_cmp_list(CHARSET_INFO *cs, FTB_WORD **a, FTB_WORD **b)
   return i;
 }
 
-static void _ftb_parse_query(FTB *ftb, byte **start, byte *end,
-                      FTB_EXPR *up, uint depth, byte *up_quot)
+
+typedef struct st_my_ftb_param
 {
-  byte        res;
-  FTB_PARAM   param;
-  FT_WORD     w;
-  FTB_WORD   *ftbw;
-  FTB_EXPR   *ftbe;
-  FTB_EXPR   *tmp_expr;
-  FT_WORD    *phrase_word;
-  LIST       *phrase_list;
-  uint  extra=HA_FT_WLEN+ftb->info->s->rec_reflength; /* just a shortcut */
+  FTB *ftb;
+  FTB_EXPR *ftbe;
+  byte *up_quot;
+  uint depth;
+} MY_FTB_PARAM;
+
+
+static int ftb_query_add_word(void *param, byte *word, uint word_len,
+                              MYSQL_FTPARSER_BOOLEAN_INFO *info)
+{
+  MY_FTB_PARAM *ftb_param= (MY_FTB_PARAM *)param;
+  FTB_WORD *ftbw;
+  FTB_EXPR *ftbe, *tmp_expr;
+  FT_WORD *phrase_word;
+  LIST *tmp_element;
+  int r= info->weight_adjust;
+  float weight= (float)
+        (info->wasign ? nwghts : wghts)[(r>5)?5:((r<-5)?-5:r)];
+
+  switch (info->type) {
+    case FT_TOKEN_WORD:
+      ftbw= (FTB_WORD *)alloc_root(&ftb_param->ftb->mem_root,
+                                   sizeof(FTB_WORD) +
+                                   (info->trunc ? MI_MAX_KEY_BUFF :
+                                    word_len * ftb_param->ftb->charset->mbmaxlen +
+                                    HA_FT_WLEN +
+                                    ftb_param->ftb->info->s->rec_reflength));
+      ftbw->len= word_len + 1;
+      ftbw->flags= 0;
+      ftbw->off= 0;
+      if (info->yesno > 0) ftbw->flags|= FTB_FLAG_YES;
+      if (info->yesno < 0) ftbw->flags|= FTB_FLAG_NO;
+      if (info->trunc) ftbw->flags|= FTB_FLAG_TRUNC;
+      ftbw->weight= weight;
+      ftbw->up= ftb_param->ftbe;
+      ftbw->docid[0]= ftbw->docid[1]= HA_OFFSET_ERROR;
+      ftbw->ndepth= (info->yesno < 0) + ftb_param->depth;
+      ftbw->key_root= HA_OFFSET_ERROR;
+      memcpy(ftbw->word + 1, word, word_len);
+      ftbw->word[0]= word_len;
+      if (info->yesno > 0) ftbw->up->ythresh++;
+      queue_insert(&ftb_param->ftb->queue, (byte *)ftbw);
+      ftb_param->ftb->with_scan|= (info->trunc & FTB_FLAG_TRUNC);
+      for (tmp_expr= ftb_param->ftbe; tmp_expr->up; tmp_expr= tmp_expr->up)
+        if (! (tmp_expr->flags & FTB_FLAG_YES))
+          break;
+      ftbw->max_docid= &tmp_expr->max_docid;
+      /* fall through */
+    case FT_TOKEN_STOPWORD:
+      if (! ftb_param->up_quot) break;
+      phrase_word= (FT_WORD *)alloc_root(&ftb_param->ftb->mem_root, sizeof(FT_WORD));
+      tmp_element= (LIST *)alloc_root(&ftb_param->ftb->mem_root, sizeof(LIST));
+      phrase_word->pos= word;
+      phrase_word->len= word_len;
+      tmp_element->data= (void *)phrase_word;
+      ftb_param->ftbe->phrase= list_add(ftb_param->ftbe->phrase, tmp_element);
+      /* Allocate document list at this point.
+         It allows to avoid huge amount of allocs/frees for each row.*/
+      tmp_element= (LIST *)alloc_root(&ftb_param->ftb->mem_root, sizeof(LIST));
+      tmp_element->data= alloc_root(&ftb_param->ftb->mem_root, sizeof(FT_WORD));
+      ftb_param->ftbe->document=
+        list_add(ftb_param->ftbe->document, tmp_element);
+      break;
+    case FT_TOKEN_LEFT_PAREN:
+      ftbe=(FTB_EXPR *)alloc_root(&ftb_param->ftb->mem_root, sizeof(FTB_EXPR));
+      ftbe->flags= 0;
+      if (info->yesno > 0) ftbe->flags|= FTB_FLAG_YES;
+      if (info->yesno < 0) ftbe->flags|= FTB_FLAG_NO;
+      ftbe->weight= weight;
+      ftbe->up= ftb_param->ftbe;
+      ftbe->max_docid= ftbe->ythresh= ftbe->yweaks= 0;
+      ftbe->docid[0]= ftbe->docid[1]= HA_OFFSET_ERROR;
+      ftbe->phrase= NULL;
+      ftbe->document= 0;
+      if (info->quot) ftb_param->ftb->with_scan|= 2;
+      if (info->yesno > 0) ftbe->up->ythresh++;
+      ftb_param->ftbe= ftbe;
+      ftb_param->depth++;
+      ftb_param->up_quot= info->quot;
+      break;
+    case FT_TOKEN_RIGHT_PAREN:
+      if (ftb_param->ftbe->document)
+      {
+        /* Circuit document list */
+        for (tmp_element= ftb_param->ftbe->document;
+             tmp_element->next; tmp_element= tmp_element->next) /* no-op */;
+        tmp_element->next= ftb_param->ftbe->document;
+        ftb_param->ftbe->document->prev= tmp_element;
+      }
+      info->quot= 0;
+      if (ftb_param->ftbe->up)
+      {
+        DBUG_ASSERT(ftb_param->depth);
+        ftb_param->ftbe= ftb_param->ftbe->up;
+        ftb_param->depth--;
+        ftb_param->up_quot= 0;
+      }
+      break;
+    case FT_TOKEN_EOF:
+    default:
+      break;
+  }
+  return(0);
+}
+
+
+static int ftb_parse_query_internal(void *param, byte *query, uint len)
+{
+  MY_FTB_PARAM *ftb_param= (MY_FTB_PARAM *)param;
+  MYSQL_FTPARSER_BOOLEAN_INFO info;
+  CHARSET_INFO *cs= ftb_param->ftb->charset;
+  byte **start= &query;
+  byte *end= query + len;
+  FT_WORD w;
+
+  info.prev= ' ';
+  info.quot= 0;
+  while (ft_get_word(cs, start, end, &w, &info))
+    ftb_query_add_word(param, w.pos, w.len, &info);
+  return(0);
+}
+
+
+static void _ftb_parse_query(FTB *ftb, byte *query, uint len,
+                             struct st_mysql_ftparser *parser)
+{
+  MYSQL_FTPARSER_PARAM param;
+  MY_FTB_PARAM ftb_param;
+  DBUG_ENTER("_ftb_parse_query");
+  DBUG_ASSERT(parser);
 
   if (ftb->state != UNINITIALIZED)
     return;
 
-  param.prev=' ';
-  param.quot= up_quot;
-  while ((res=ft_get_word(ftb->charset,start,end,&w,&param)))
-  {
-    int   r=param.plusminus;
-    float weight= (float) (param.pmsign ? nwghts : wghts)[(r>5)?5:((r<-5)?-5:r)];
-    switch (res) {
-      case 1: /* word found */
-        ftbw=(FTB_WORD *)alloc_root(&ftb->mem_root,
-                                    sizeof(FTB_WORD) +
-                                    (param.trunc ? MI_MAX_KEY_BUFF :
-                                     w.len*ftb->charset->mbmaxlen+extra));
-        ftbw->len=w.len+1;
-        ftbw->flags=0;
-        ftbw->off=0;
-        if (param.yesno>0) ftbw->flags|=FTB_FLAG_YES;
-        if (param.yesno<0) ftbw->flags|=FTB_FLAG_NO;
-        if (param.trunc)   ftbw->flags|=FTB_FLAG_TRUNC;
-        ftbw->weight=weight;
-        ftbw->up=up;
-        ftbw->docid[0]=ftbw->docid[1]=HA_OFFSET_ERROR;
-        ftbw->ndepth= (param.yesno<0) + depth;
-        ftbw->key_root=HA_OFFSET_ERROR;
-        memcpy(ftbw->word+1, w.pos, w.len);
-        ftbw->word[0]=w.len;
-        if (param.yesno > 0) up->ythresh++;
-        queue_insert(& ftb->queue, (byte *)ftbw);
-        ftb->with_scan|=(param.trunc & FTB_FLAG_TRUNC);
-        for (tmp_expr= up; tmp_expr->up; tmp_expr= tmp_expr->up)
-          if (! (tmp_expr->flags & FTB_FLAG_YES))
-            break;
-        ftbw->max_docid= &tmp_expr->max_docid;
-      case 4: /* not indexed word (stopword or too short/long) */
-        if (! up_quot) break;
-        phrase_word= (FT_WORD *)alloc_root(&ftb->mem_root, sizeof(FT_WORD));
-        phrase_list= (LIST *)alloc_root(&ftb->mem_root, sizeof(LIST));
-        phrase_word->pos= w.pos;
-        phrase_word->len= w.len;
-        phrase_list->data= (void *)phrase_word;
-        up->phrase= list_add(up->phrase, phrase_list);
-        break;
-      case 2: /* left bracket */
-        ftbe=(FTB_EXPR *)alloc_root(&ftb->mem_root, sizeof(FTB_EXPR));
-        ftbe->flags=0;
-        if (param.yesno>0) ftbe->flags|=FTB_FLAG_YES;
-        if (param.yesno<0) ftbe->flags|=FTB_FLAG_NO;
-        ftbe->weight=weight;
-        ftbe->up=up;
-        ftbe->max_docid= ftbe->ythresh= ftbe->yweaks= 0;
-        ftbe->docid[0]=ftbe->docid[1]=HA_OFFSET_ERROR;
-        ftbe->phrase= NULL;
-        if (param.quot) ftb->with_scan|=2;
-        if (param.yesno > 0) up->ythresh++;
-        _ftb_parse_query(ftb, start, end, ftbe, depth+1, param.quot);
-        param.quot=0;
-        break;
-      case 3: /* right bracket */
-        if (up_quot) up->phrase= list_reverse(up->phrase);
-        return;
-    }
-  }
-  return;
+  ftb_param.ftb= ftb;
+  ftb_param.depth= 0;
+  ftb_param.ftbe= ftb->root;
+  ftb_param.up_quot= 0;
+
+  param.mysql_parse= ftb_parse_query_internal;
+  param.mysql_add_word= ftb_query_add_word;
+  param.ftparser_state= 0;
+  param.mysql_ftparam= (void *)&ftb_param;
+  param.cs= ftb->charset;
+  param.doc= query;
+  param.length= len;
+  param.mode= MYSQL_FTPARSER_FULL_BOOLEAN_INFO;
+  parser->parse(&param);
+  DBUG_VOID_RETURN;
 }
+ 
 
 static int _ftb_no_dupes_cmp(void* not_used __attribute__((unused)),
                              const void *a,const void *b)
@@ -463,8 +539,11 @@ FT_INFO * ft_init_boolean_search(MI_INFO *info, uint keynr, byte *query,
   ftbe->max_docid= ftbe->ythresh= ftbe->yweaks= 0;
   ftbe->docid[0]=ftbe->docid[1]=HA_OFFSET_ERROR;
   ftbe->phrase= NULL;
+  ftbe->document= 0;
   ftb->root=ftbe;
-  _ftb_parse_query(ftb, &query, query+query_len, ftbe, 0, NULL);
+  _ftb_parse_query(ftb, query, query_len, keynr == NO_SUCH_KEY ?
+                                          &ft_default_parser :
+                                          info->s->keyinfo[keynr].parser);
   ftb->list=(FTB_WORD **)alloc_root(&ftb->mem_root,
                                      sizeof(FTB_WORD *)*ftb->queue.elements);
   memcpy(ftb->list, ftb->queue.root+1, sizeof(FTB_WORD *)*ftb->queue.elements);
@@ -480,6 +559,62 @@ err:
 }
 
 
+typedef struct st_my_ftb_phrase_param
+{
+  LIST *phrase;
+  LIST *document;
+  CHARSET_INFO *cs;
+  uint phrase_length;
+  uint document_length;
+  uint match;
+} MY_FTB_PHRASE_PARAM;
+
+
+static int ftb_phrase_add_word(void *param, byte *word, uint word_len,
+    MYSQL_FTPARSER_BOOLEAN_INFO *boolean_info __attribute__((unused)))
+{
+  MY_FTB_PHRASE_PARAM *phrase_param= (MY_FTB_PHRASE_PARAM *)param;
+  FT_WORD *w= (FT_WORD *)phrase_param->document->data;
+  LIST *phrase, *document;
+  w->pos= word;
+  w->len= word_len;
+  phrase_param->document= phrase_param->document->prev;
+  if (phrase_param->phrase_length > phrase_param->document_length)
+  {
+    phrase_param->document_length++;
+    return 0;
+  }
+  /* TODO: rewrite phrase search to avoid
+     comparing the same word twice. */
+  for (phrase= phrase_param->phrase, document= phrase_param->document->next;
+       phrase; phrase= phrase->next, document= document->next)
+  {
+    FT_WORD *phrase_word= (FT_WORD *)phrase->data;
+    FT_WORD *document_word= (FT_WORD *)document->data;
+    if (my_strnncoll(phrase_param->cs, phrase_word->pos, phrase_word->len,
+                     document_word->pos, document_word->len))
+      return 0;
+  }
+  phrase_param->match++;
+  return 0;
+}
+
+
+static int ftb_check_phrase_internal(void *param, byte *document, uint len)
+{
+  FT_WORD word;
+  MY_FTB_PHRASE_PARAM *phrase_param= (MY_FTB_PHRASE_PARAM *)param;
+  const byte *docend= document + len;
+  while (ft_simple_get_word(phrase_param->cs, &document, docend, &word, FALSE))
+  {
+    ftb_phrase_add_word(param, word.pos, word.len, 0);
+    if (phrase_param->match)
+      return 1;
+  }
+  return 0;
+}
+
+
 /*
   Checks if given buffer matches phrase list.
 
@@ -494,32 +629,31 @@ err:
     1 is returned if phrase found, 0 else.
 */
 
-static int _ftb_check_phrase(const byte *s0, const byte *e0,
-                LIST *phrase, CHARSET_INFO *cs)
+static int _ftb_check_phrase(const byte *document, uint len,
+                FTB_EXPR *ftbe, CHARSET_INFO *cs,
+                struct st_mysql_ftparser *parser)
 {
-  FT_WORD h_word;
-  const byte *h_start= s0;
-  DBUG_ENTER("_ftb_strstr");
-  DBUG_ASSERT(phrase);
-
-  while (ft_simple_get_word(cs, (byte **)&h_start, e0, &h_word, FALSE))
-  {
-    FT_WORD *n_word;
-    LIST *phrase_element= phrase;
-    const byte *h_start1= h_start;
-    for (;;)
-    {
-      n_word= (FT_WORD *)phrase_element->data;
-      if (my_strnncoll(cs, (const uchar *) h_word.pos, h_word.len,
-		       (const uchar *) n_word->pos, n_word->len))
-        break;
-      if (! (phrase_element= phrase_element->next))
-        DBUG_RETURN(1);
-      if (! ft_simple_get_word(cs, (byte **)&h_start1, e0, &h_word, FALSE))
-        DBUG_RETURN(0);
-    }
-  }
-  DBUG_RETURN(0);
+  MY_FTB_PHRASE_PARAM ftb_param;
+  MYSQL_FTPARSER_PARAM param;
+  DBUG_ENTER("_ftb_check_phrase");
+  DBUG_ASSERT(parser);
+  ftb_param.phrase= ftbe->phrase;
+  ftb_param.document= ftbe->document;
+  ftb_param.cs= cs;
+  ftb_param.phrase_length= list_length(ftbe->phrase);
+  ftb_param.document_length= 1;
+  ftb_param.match= 0;
+
+  param.mysql_parse= ftb_check_phrase_internal;
+  param.mysql_add_word= ftb_phrase_add_word;
+  param.ftparser_state= 0;
+  param.mysql_ftparam= (void *)&ftb_param;
+  param.cs= cs;
+  param.doc= (byte *)document;
+  param.length= len;
+  param.mode= MYSQL_FTPARSER_WITH_STOPWORDS;
+  parser->parse(&param);
+  DBUG_RETURN(ftb_param.match ? 1 : 0);
 }
 
 
@@ -530,6 +664,9 @@ static void _ftb_climb_the_tree(FTB *ftb, FTB_WORD *ftbw, FT_SEG_ITERATOR *ftsi_
   float weight=ftbw->weight;
   int  yn=ftbw->flags, ythresh, mode=(ftsi_orig != 0);
   my_off_t curdoc=ftbw->docid[mode];
+  struct st_mysql_ftparser *parser= ftb->keynr == NO_SUCH_KEY ?
+                                    &ft_default_parser :
+                                    ftb->info->s->keyinfo[ftb->keynr].parser;
 
   for (ftbe=ftbw->up; ftbe; ftbe=ftbe->up)
   {
@@ -559,8 +696,8 @@ static void _ftb_climb_the_tree(FTB *ftb, FTB_WORD *ftbw, FT_SEG_ITERATOR *ftsi_
           {
             if (!ftsi.pos)
               continue;
-            not_found = ! _ftb_check_phrase(ftsi.pos, ftsi.pos+ftsi.len,
-                                      ftbe->phrase, ftb->charset);
+            not_found = ! _ftb_check_phrase(ftsi.pos, ftsi.len,
+                                      ftbe, ftb->charset, parser);
           }
           if (not_found) break;
         } /* ftbe->quot */
@@ -667,14 +804,67 @@ err:
 }
 
 
-float ft_boolean_find_relevance(FT_INFO *ftb, byte *record, uint length)
+typedef struct st_my_ftb_find_param
 {
-  FT_WORD word;
+  FT_INFO *ftb;
+  FT_SEG_ITERATOR *ftsi;
+} MY_FTB_FIND_PARAM;
+
+
+static int ftb_find_relevance_add_word(void *param, byte *word, uint len,
+             MYSQL_FTPARSER_BOOLEAN_INFO *boolean_info __attribute__((unused)))
+{
+  MY_FTB_FIND_PARAM *ftb_param= (MY_FTB_FIND_PARAM *)param;
+  FT_INFO *ftb= ftb_param->ftb;
   FTB_WORD *ftbw;
+  int a, b, c;
+  for (a= 0, b= ftb->queue.elements, c= (a+b)/2; b-a>1; c= (a+b)/2)
+  {
+    ftbw= ftb->list[c];
+    if (mi_compare_text(ftb->charset, (uchar*)word, len,
+                        (uchar*)ftbw->word+1, ftbw->len-1,
+                        (my_bool)(ftbw->flags&FTB_FLAG_TRUNC), 0) > 0)
+      b= c;
+    else
+      a= c;
+  }
+  for (; c >= 0; c--)
+  {
+    ftbw= ftb->list[c];
+    if (mi_compare_text(ftb->charset, (uchar*)word, len,
+                        (uchar*)ftbw->word + 1,ftbw->len - 1,
+                        (my_bool)(ftbw->flags & FTB_FLAG_TRUNC), 0))
+      break;
+    if (ftbw->docid[1] == ftb->info->lastpos)
+      continue;
+    ftbw->docid[1]= ftb->info->lastpos;
+     _ftb_climb_the_tree(ftb, ftbw, ftb_param->ftsi);
+  }
+  return(0);
+}
+
+
+static int ftb_find_relevance_parse(void *param, byte *doc, uint len)
+{
+  FT_INFO *ftb= ((MY_FTB_FIND_PARAM *)param)->ftb;
+  byte *end= doc + len;
+  FT_WORD w;
+  while (ft_simple_get_word(ftb->charset, &doc, end, &w, TRUE))
+    ftb_find_relevance_add_word(param, w.pos, w.len, 0);
+  return(0);
+}
+
+
+float ft_boolean_find_relevance(FT_INFO *ftb, byte *record, uint length)
+{
   FTB_EXPR *ftbe;
   FT_SEG_ITERATOR ftsi, ftsi2;
-  const byte *end;
   my_off_t  docid=ftb->info->lastpos;
+  MY_FTB_FIND_PARAM ftb_param;
+  MYSQL_FTPARSER_PARAM param;
+  struct st_mysql_ftparser *parser= ftb->keynr == NO_SUCH_KEY ?
+                                    &ft_default_parser :
+                                    ftb->info->s->keyinfo[ftb->keynr].parser;
 
   if (docid == HA_OFFSET_ERROR)
     return -2.0;
@@ -702,41 +892,23 @@ float ft_boolean_find_relevance(FT_INFO *ftb, byte *record, uint length)
     _mi_ft_segiterator_init(ftb->info, ftb->keynr, record, &ftsi);
   memcpy(&ftsi2, &ftsi, sizeof(ftsi));
 
+  ftb_param.ftb= ftb;
+  ftb_param.ftsi= &ftsi2;
+  param.mysql_parse= ftb_find_relevance_parse;
+  param.mysql_add_word= ftb_find_relevance_add_word;
+  param.ftparser_state= 0;
+  param.mysql_ftparam= (void *)&ftb_param;
+  param.cs= ftb->charset;
+  param.mode= MYSQL_FTPARSER_SIMPLE_MODE;
   while (_mi_ft_segiterator(&ftsi))
   {
     if (!ftsi.pos)
       continue;
 
-    end=ftsi.pos+ftsi.len;
-    while (ft_simple_get_word(ftb->charset, (byte **) &ftsi.pos,
-                              (byte *) end, &word, TRUE))
-    {
-      int a, b, c;
-      for (a=0, b=ftb->queue.elements, c=(a+b)/2; b-a>1; c=(a+b)/2)
-      {
-        ftbw=ftb->list[c];
-        if (mi_compare_text(ftb->charset, (uchar*) word.pos, word.len,
-                            (uchar*) ftbw->word+1, ftbw->len-1,
-                            (my_bool) (ftbw->flags&FTB_FLAG_TRUNC),0) >0)
-          b=c;
-        else
-          a=c;
-      }
-      for (; c>=0; c--)
-      {
-        ftbw=ftb->list[c];
-        if (mi_compare_text(ftb->charset, (uchar*) word.pos, word.len,
-                            (uchar*) ftbw->word+1,ftbw->len-1,
-                            (my_bool) (ftbw->flags&FTB_FLAG_TRUNC),0))
-          break;
-        if (ftbw->docid[1] == docid)
-          continue;
-        ftbw->docid[1]=docid;
-        _ftb_climb_the_tree(ftb, ftbw, &ftsi2);
-      }
-    }
+    param.doc= (byte *)ftsi.pos;
+    param.length= ftsi.len;
+    parser->parse(&param);
   }
-
   ftbe=ftb->root;
   if (ftbe->docid[1]==docid && ftbe->cur_weight>0 &&
       ftbe->yesses>=ftbe->ythresh && !ftbe->nos)

storage/myisam/ft_nlq_search.c

@@ -230,7 +230,7 @@ FT_INFO *ft_init_nlq_search(MI_INFO *info, uint keynr, byte *query,
             NULL, NULL);
 
   ft_parse_init(&wtree, aio.charset);
-  if (ft_parse(&wtree,query,query_len,0))
+  if (ft_parse(&wtree, query, query_len, 0, info->s->keyinfo[keynr].parser))
     goto err;
 
   if (tree_walk(&wtree, (tree_walk_action)&walk_and_match, &aio,

storage/myisam/ft_parser.c

@@ -24,6 +24,14 @@ typedef struct st_ft_docstat {
   double sum;
 } FT_DOCSTAT;
 
+
+typedef struct st_my_ft_parser_param
+{
+  TREE *wtree;
+  my_bool with_alloc;
+} MY_FT_PARSER_PARAM;
+
+
 static int FT_WORD_cmp(CHARSET_INFO* cs, FT_WORD *w1, FT_WORD *w2)
 {
   return mi_compare_text(cs, (uchar*) w1->pos, w1->len,
@@ -102,13 +110,14 @@ my_bool ft_boolean_check_syntax_string(const byte *str)
   4 - stopword found
 */
 byte ft_get_word(CHARSET_INFO *cs, byte **start, byte *end,
-                 FT_WORD *word, FTB_PARAM *param)
+                 FT_WORD *word, MYSQL_FTPARSER_BOOLEAN_INFO *param)
 {
   byte *doc=*start;
   uint mwc, length, mbl;
 
   param->yesno=(FTB_YES==' ') ? 1 : (param->quot != 0);
-  param->plusminus=param->pmsign=0;
+  param->weight_adjust= param->wasign= 0;
+  param->type= FT_TOKEN_EOF;
 
   while (doc<end)
   {
@@ -119,7 +128,8 @@ byte ft_get_word(CHARSET_INFO *cs, byte **start, byte *end,
       {
         param->quot=doc;
         *start=doc+1;
-        return 3; /* FTB_RBR */
+        param->type= FT_TOKEN_RIGHT_PAREN;
+        goto ret;
       }
       if (!param->quot)
       {
@@ -128,21 +138,22 @@ byte ft_get_word(CHARSET_INFO *cs, byte **start, byte *end,
           /* param->prev=' '; */
           *start=doc+1;
           if (*doc == FTB_LQUOT) param->quot=*start;
-          return (*doc == FTB_RBR)+2;
+          param->type= (*doc == FTB_RBR ? FT_TOKEN_RIGHT_PAREN : FT_TOKEN_LEFT_PAREN);
+          goto ret;
         }
         if (param->prev == ' ')
         {
           if (*doc == FTB_YES ) { param->yesno=+1;    continue; } else
           if (*doc == FTB_EGAL) { param->yesno= 0;    continue; } else
           if (*doc == FTB_NO  ) { param->yesno=-1;    continue; } else
-          if (*doc == FTB_INC ) { param->plusminus++; continue; } else
-          if (*doc == FTB_DEC ) { param->plusminus--; continue; } else
-          if (*doc == FTB_NEG ) { param->pmsign=!param->pmsign; continue; }
+          if (*doc == FTB_INC ) { param->weight_adjust++; continue; } else
+          if (*doc == FTB_DEC ) { param->weight_adjust--; continue; } else
+          if (*doc == FTB_NEG ) { param->wasign= !param->wasign; continue; }
         }
       }
       param->prev=*doc;
       param->yesno=(FTB_YES==' ') ? 1 : (param->quot != 0);
-      param->plusminus=param->pmsign=0;
+      param->weight_adjust= param->wasign= 0;
     }
 
     mwc=length=0;
@@ -161,20 +172,24 @@ byte ft_get_word(CHARSET_INFO *cs, byte **start, byte *end,
          || param->trunc) && length < ft_max_word_len)
     {
       *start=doc;
-      return 1;
+      param->type= FT_TOKEN_WORD;
+      goto ret;
     }
     else if (length) /* make sure length > 0 (if start contains spaces only) */
     {
       *start= doc;
-      return 4;
+      param->type= FT_TOKEN_STOPWORD;
+      goto ret;
     }
   }
   if (param->quot)
   {
     param->quot=*start=doc;
-    return 3; /* FTB_RBR */
+    param->type= 3; /* FT_RBR */
+    goto ret;
   }
-  return 0;
+ret:
+  return param->type;
 }
 
 byte ft_simple_get_word(CHARSET_INFO *cs, byte **start, const byte *end,
@@ -220,30 +235,67 @@ void ft_parse_init(TREE *wtree, CHARSET_INFO *cs)
   DBUG_VOID_RETURN;
 }
 
-int ft_parse(TREE *wtree, byte *doc, int doclen, my_bool with_alloc)
+
+static int ft_add_word(void *param, byte *word, uint word_len,
+             MYSQL_FTPARSER_BOOLEAN_INFO *boolean_info __attribute__((unused)))
 {
-  byte   *end=doc+doclen;
+  TREE *wtree;
   FT_WORD w;
-  DBUG_ENTER("ft_parse");
-
-  while (ft_simple_get_word(wtree->custom_arg, &doc, end, &w, TRUE))
+  DBUG_ENTER("ft_add_word");
+  wtree= ((MY_FT_PARSER_PARAM *)param)->wtree;
+  if (((MY_FT_PARSER_PARAM *)param)->with_alloc)
   {
-    if (with_alloc)
-    {
-      byte *ptr;
-      /* allocating the data in the tree - to avoid mallocs and frees */
-      DBUG_ASSERT(wtree->with_delete==0);
-      ptr=(byte *)alloc_root(& wtree->mem_root,w.len);
-      memcpy(ptr, w.pos, w.len);
-      w.pos=ptr;
-    }
-    if (!tree_insert(wtree, &w, 0, wtree->custom_arg))
-      goto err;
+    byte *ptr;
+    /* allocating the data in the tree - to avoid mallocs and frees */
+    DBUG_ASSERT(wtree->with_delete == 0);
+    ptr= (byte *)alloc_root(&wtree->mem_root, word_len);
+    memcpy(ptr, word, word_len);
+    w.pos= ptr;
+  }
+  else
+    w.pos= word;
+  w.len= word_len;
+  if (!tree_insert(wtree, &w, 0, wtree->custom_arg))
+  {
+    delete_tree(wtree);
+    DBUG_RETURN(1);
   }
   DBUG_RETURN(0);
+}
 
-err:
-  delete_tree(wtree);
-  DBUG_RETURN(1);
+
+static int ft_parse_internal(void *param, byte *doc, uint doc_len)
+{
+  byte   *end=doc+doc_len;
+  FT_WORD w;
+  TREE *wtree;
+  DBUG_ENTER("ft_parse_internal");
+
+  wtree= ((MY_FT_PARSER_PARAM *)param)->wtree;
+  while (ft_simple_get_word(wtree->custom_arg, &doc, end, &w, TRUE))
+    if (ft_add_word(param, w.pos, w.len, 0))
+      DBUG_RETURN(1);
+  DBUG_RETURN(0);
 }
 
+
+int ft_parse(TREE *wtree, byte *doc, int doclen, my_bool with_alloc,
+                    struct st_mysql_ftparser *parser)
+{
+  MYSQL_FTPARSER_PARAM param;
+  MY_FT_PARSER_PARAM my_param;
+  DBUG_ENTER("ft_parse");
+  DBUG_ASSERT(parser);
+  my_param.wtree= wtree;
+  my_param.with_alloc= with_alloc;
+
+  param.mysql_parse= ft_parse_internal;
+  param.mysql_add_word= ft_add_word;
+  param.ftparser_state= 0;
+  param.mysql_ftparam= &my_param;
+  param.cs= wtree->custom_arg;
+  param.doc= doc;
+  param.length= doclen;
+  param.mode= MYSQL_FTPARSER_SIMPLE_MODE;
+  DBUG_RETURN(parser->parse(&param));
+}

storage/myisam/ft_static.c

@@ -626,3 +626,14 @@ const char *ft_precompiled_stopwords[] = {
 #endif
 
   NULL };
+
+static int ft_default_parser_parse(MYSQL_FTPARSER_PARAM *param)
+{
+  return param->mysql_parse(param->mysql_ftparam, param->doc, param->length);
+}
+
+struct st_mysql_ftparser ft_default_parser=
+{
+  MYSQL_FTPARSER_INTERFACE_VERSION, ft_default_parser_parse, 0, 0
+};
+

storage/myisam/ft_update.c

@@ -99,15 +99,17 @@ uint _mi_ft_parse(TREE *parsed, MI_INFO *info, uint keynr,
                   const byte *record, my_bool with_alloc)
 {
   FT_SEG_ITERATOR ftsi;
+  struct st_mysql_ftparser *parser;
   DBUG_ENTER("_mi_ft_parse");
 
   _mi_ft_segiterator_init(info, keynr, record, &ftsi);
 
   ft_parse_init(parsed, info->s->keyinfo[keynr].seg->charset);
+  parser= info->s->keyinfo[keynr].parser;
   while (_mi_ft_segiterator(&ftsi))
   {
     if (ftsi.pos)
-      if (ft_parse(parsed, (byte *)ftsi.pos, ftsi.len, with_alloc))
+      if (ft_parse(parsed, (byte *)ftsi.pos, ftsi.len, with_alloc, parser))
         DBUG_RETURN(1);
   }
   DBUG_RETURN(0);

storage/myisam/ftdefs.h

@@ -22,6 +22,7 @@
 #include <m_ctype.h>
 #include <my_tree.h>
 #include <queues.h>
+#include <plugin.h>
 
 #define true_word_char(s,X)	(my_isalnum(s,X) || (X)=='_')
 #define misc_word_char(X)	((X)=='\'')
@@ -98,20 +99,12 @@ typedef struct st_ft_word {
   double weight;
 } FT_WORD;
 
-typedef struct st_ftb_param {
-  byte prev;
-  int  yesno;
-  int  plusminus;
-  bool pmsign;
-  bool trunc;
-  byte *quot;
-} FTB_PARAM;
-
 int is_stopword(char *word, uint len);
 
 uint _ft_make_key(MI_INFO *, uint , byte *, FT_WORD *, my_off_t);
 
-byte ft_get_word(CHARSET_INFO *, byte **, byte *, FT_WORD *, FTB_PARAM *);
+byte ft_get_word(CHARSET_INFO *, byte **, byte *, FT_WORD *,
+                 MYSQL_FTPARSER_BOOLEAN_INFO *);
 byte ft_simple_get_word(CHARSET_INFO *, byte **, const byte *,
                         FT_WORD *, my_bool);
 
@@ -126,7 +119,7 @@ void _mi_ft_segiterator_dummy_init(const byte *, uint, FT_SEG_ITERATOR *);
 uint _mi_ft_segiterator(FT_SEG_ITERATOR *);
 
 void ft_parse_init(TREE *, CHARSET_INFO *);
-int ft_parse(TREE *, byte *, int, my_bool);
+int ft_parse(TREE *, byte *, int, my_bool, struct st_mysql_ftparser *parser);
 FT_WORD * ft_linearize(TREE *);
 FT_WORD * _mi_ft_parserecord(MI_INFO *, uint, const byte *);
 uint _mi_ft_parse(TREE *, MI_INFO *, uint, const byte *, my_bool);

storage/myisam/mi_open.c

@@ -1047,6 +1047,7 @@ char *mi_keydef_read(char *ptr, MI_KEYDEF *keydef)
    keydef->block_size	= keydef->block_length/MI_MIN_KEY_BLOCK_LENGTH-1;
    keydef->underflow_block_length=keydef->block_length/3;
    keydef->version	= 0;			/* Not saved */
+   keydef->parser       = &ft_default_parser;
    return ptr;
 }