MDEV-34057 Inconsistent FTS state in concurrent scenarios

Problem:
=======
- This commit is a merge of mysql commit 129ee47ef994652081a11ee9040c0488e5275b14.
InnoDB FTS can be in inconsistent state when sync operation
terminates the server before committing the operation. This
could lead to incorrect synced doc id and incorrect query results.

Solution:
========
- During sync commit operation, InnoDB should pass
the sync transaction to update the max doc id
in the config table.

fts_read_synced_doc_id() : This function is used
to read only synced doc id from the config table.

mysql-test/suite/innodb_fts/r/fts_sync_commit_resiliency.result

+CREATE TABLE opening_lines (
+id INT UNSIGNED AUTO_INCREMENT NOT NULL PRIMARY KEY,
+opening_line TEXT(500),
+author VARCHAR(200),
+title VARCHAR(200)
+) ENGINE=InnoDB;
+CREATE FULLTEXT INDEX idx ON opening_lines(opening_line);
+CREATE FULLTEXT INDEX ft_idx1 ON opening_lines(title);
+INSERT INTO opening_lines(opening_line,author,title) VALUES
+('Call me Ishmael.','Herman Melville','Moby Dick'),
+('A screaming comes across the sky.','Thomas Pynchon','Gravity\'s Rainbow'),
+       ('I am an invisible man.','Ralph Ellison','Invisible Man'),
+       ('Where now? Who now? When now?','Samuel Beckett','The Unnamable'),
+       ('It was love at first sight.','Joseph Heller','Catch-22'),
+       ('All this happened, more or less.','Kurt Vonnegut','Slaughterhouse-Five'),
+       ('Mrs. Dalloway said she would buy the flowers herself.','Virginia Woolf','Mrs. Dalloway'),
+       ('It was a pleasure to burn.','Ray Bradbury','Fahrenheit 451');
+SET GLOBAL innodb_ft_aux_table='test/opening_lines';
+SELECT * FROM information_schema.innodb_ft_config;
+KEY	VALUE
+optimize_checkpoint_limit	180
+synced_doc_id	0
+stopword_table_name	
+use_stopword	1
+SELECT * FROM opening_lines WHERE MATCH(opening_line) AGAINST('Ishmael');
+id	opening_line	author	title
+1	Call me Ishmael.	Herman Melville	Moby Dick
+SELECT * FROM opening_lines WHERE MATCH(opening_line) AGAINST('invisible');
+id	opening_line	author	title
+3	I am an invisible man.	Ralph Ellison	Invisible Man
+SELECT * FROM opening_lines;
+id	opening_line	author	title
+1	Call me Ishmael.	Herman Melville	Moby Dick
+2	A screaming comes across the sky.	Thomas Pynchon	Gravity's Rainbow
+3	I am an invisible man.	Ralph Ellison	Invisible Man
+4	Where now? Who now? When now?	Samuel Beckett	The Unnamable
+5	It was love at first sight.	Joseph Heller	Catch-22
+6	All this happened, more or less.	Kurt Vonnegut	Slaughterhouse-Five
+7	Mrs. Dalloway said she would buy the flowers herself.	Virginia Woolf	Mrs. Dalloway
+8	It was a pleasure to burn.	Ray Bradbury	Fahrenheit 451
+SET GLOBAL innodb_optimize_fulltext_only=ON;
+SET DEBUG_SYNC='fts_crash_before_commit_sync SIGNAL hung WAIT_FOR ever';
+OPTIMIZE TABLE opening_lines;
+connect con1,localhost,root,,;
+SET DEBUG_SYNC='now WAIT_FOR hung';
+# restart
+SELECT * FROM opening_lines WHERE MATCH(opening_line) AGAINST('Ishmael');
+id	opening_line	author	title
+1	Call me Ishmael.	Herman Melville	Moby Dick
+SELECT * FROM opening_lines WHERE MATCH(opening_line) AGAINST('invisible');
+id	opening_line	author	title
+3	I am an invisible man.	Ralph Ellison	Invisible Man
+SELECT * FROM opening_lines;
+id	opening_line	author	title
+1	Call me Ishmael.	Herman Melville	Moby Dick
+2	A screaming comes across the sky.	Thomas Pynchon	Gravity's Rainbow
+3	I am an invisible man.	Ralph Ellison	Invisible Man
+4	Where now? Who now? When now?	Samuel Beckett	The Unnamable
+5	It was love at first sight.	Joseph Heller	Catch-22
+6	All this happened, more or less.	Kurt Vonnegut	Slaughterhouse-Five
+7	Mrs. Dalloway said she would buy the flowers herself.	Virginia Woolf	Mrs. Dalloway
+8	It was a pleasure to burn.	Ray Bradbury	Fahrenheit 451
+DROP TABLE opening_lines;

mysql-test/suite/innodb_fts/t/fts_sync_commit_resiliency.opt

+--innodb_ft_config

mysql-test/suite/innodb_fts/t/fts_sync_commit_resiliency.test

+# Test database resiliency against scenario where the server crashes
+# right before fts_sync_commit commits its transaction
+source include/have_innodb.inc;
+source include/have_debug.inc;
+source include/not_embedded.inc;
+source include/have_debug_sync.inc;
+
+CREATE TABLE opening_lines (
+       id INT UNSIGNED AUTO_INCREMENT NOT NULL PRIMARY KEY,
+       opening_line TEXT(500),
+       author VARCHAR(200),
+       title VARCHAR(200)
+       ) ENGINE=InnoDB;
+
+CREATE FULLTEXT INDEX idx ON opening_lines(opening_line);
+CREATE FULLTEXT INDEX ft_idx1 ON opening_lines(title);
+
+INSERT INTO opening_lines(opening_line,author,title) VALUES
+       ('Call me Ishmael.','Herman Melville','Moby Dick'),
+       ('A screaming comes across the sky.','Thomas Pynchon','Gravity\'s Rainbow'),
+       ('I am an invisible man.','Ralph Ellison','Invisible Man'),
+       ('Where now? Who now? When now?','Samuel Beckett','The Unnamable'),
+       ('It was love at first sight.','Joseph Heller','Catch-22'),
+       ('All this happened, more or less.','Kurt Vonnegut','Slaughterhouse-Five'),
+       ('Mrs. Dalloway said she would buy the flowers herself.','Virginia Woolf','Mrs. Dalloway'),
+       ('It was a pleasure to burn.','Ray Bradbury','Fahrenheit 451');
+
+SET GLOBAL innodb_ft_aux_table='test/opening_lines';
+SELECT * FROM information_schema.innodb_ft_config;
+
+SELECT * FROM opening_lines WHERE MATCH(opening_line) AGAINST('Ishmael');
+SELECT * FROM opening_lines WHERE MATCH(opening_line) AGAINST('invisible');
+SELECT * FROM opening_lines;
+
+SET GLOBAL innodb_optimize_fulltext_only=ON;
+SET DEBUG_SYNC='fts_crash_before_commit_sync SIGNAL hung WAIT_FOR ever';
+send OPTIMIZE TABLE opening_lines;
+
+connect(con1,localhost,root,,);
+SET DEBUG_SYNC='now WAIT_FOR hung';
+let $shutdown_timeout=0;
+--source include/restart_mysqld.inc
+
+SELECT * FROM opening_lines WHERE MATCH(opening_line) AGAINST('Ishmael');
+SELECT * FROM opening_lines WHERE MATCH(opening_line) AGAINST('invisible');
+SELECT * FROM opening_lines;
+DROP TABLE opening_lines;

storage/innobase/fts/fts0fts.cc

@@ -2607,89 +2607,85 @@ fts_get_next_doc_id(
 	return(DB_SUCCESS);
 }
 
-/*********************************************************************//**
-This function fetch the Doc ID from CONFIG table, and compare with
+/** Read the synced document id from the fts configuration table
+@param table  fts table
+@param doc_id document id to be read
+@param trx    transaction to read from config table
+@return DB_SUCCESS in case of success */
+static
+dberr_t fts_read_synced_doc_id(const dict_table_t *table,
+                               doc_id_t *doc_id,
+                               trx_t *trx)
+{
+  dberr_t error;
+  que_t*  graph= NULL;
+  char    table_name[MAX_FULL_NAME_LEN];
+
+  fts_table_t fts_table;
+  fts_table.suffix= "CONFIG";
+  fts_table.table_id= table->id;
+  fts_table.type= FTS_COMMON_TABLE;
+  fts_table.table= table;
+  ut_a(table->fts->doc_col != ULINT_UNDEFINED);
+
+  trx->op_info = "update the next FTS document id";
+  pars_info_t *info= pars_info_create();
+  pars_info_bind_function(info, "my_func", fts_fetch_store_doc_id,
+                          doc_id);
+
+  fts_get_table_name(&fts_table, table_name);
+  pars_info_bind_id(info, "config_table", table_name);
+
+  graph= fts_parse_sql(
+           &fts_table, info,
+           "DECLARE FUNCTION my_func;\n"
+           "DECLARE CURSOR c IS SELECT value FROM $config_table"
+           " WHERE key = 'synced_doc_id' FOR UPDATE;\n"
+           "BEGIN\n"
+           ""
+           "OPEN c;\n"
+           "WHILE 1 = 1 LOOP\n"
+           "  FETCH c INTO my_func();\n"
+           "  IF c % NOTFOUND THEN\n"
+           "    EXIT;\n"
+           "  END IF;\n"
+           "END LOOP;\n"
+           "CLOSE c;");
+
+  *doc_id = 0;
+  error = fts_eval_sql(trx, graph);
+  fts_que_graph_free_check_lock(&fts_table, NULL, graph);
+  return error;
+}
+
+/** This function fetch the Doc ID from CONFIG table, and compare with
 the Doc ID supplied. And store the larger one to the CONFIG table.
+@param table fts  table
+@param cmp_doc_id Doc ID to compare
+@param doc_id     larger document id after comparing "cmp_doc_id" to
+	          the one stored in CONFIG table
+@param trx	  transaction
 @return DB_SUCCESS if OK */
-static MY_ATTRIBUTE((nonnull))
+static
 dberr_t
 fts_cmp_set_sync_doc_id(
-/*====================*/
-	const dict_table_t*	table,		/*!< in: table */
-	doc_id_t		cmp_doc_id,	/*!< in: Doc ID to compare */
-	ibool			read_only,	/*!< in: TRUE if read the
-						synced_doc_id only */
-	doc_id_t*		doc_id)		/*!< out: larger document id
-						after comparing "cmp_doc_id"
-						to the one stored in CONFIG
-						table */
+	const dict_table_t *table,
+	doc_id_t           cmp_doc_id,
+	doc_id_t           *doc_id,
+	trx_t	           *trx=nullptr)
 {
-	trx_t*		trx;
-	pars_info_t*	info;
-	dberr_t		error;
-	fts_table_t	fts_table;
-	que_t*		graph = NULL;
-	fts_cache_t*	cache = table->fts->cache;
-	char		table_name[MAX_FULL_NAME_LEN];
-retry:
-	ut_a(table->fts->doc_col != ULINT_UNDEFINED);
+	fts_cache_t*	cache= table->fts->cache;
+	dberr_t 	error = DB_SUCCESS;
+	const trx_t*	const caller_trx = trx;
 
-	fts_table.suffix = "CONFIG";
-	fts_table.table_id = table->id;
-	fts_table.type = FTS_COMMON_TABLE;
-	fts_table.table = table;
-
-	trx = trx_create();
-	if (srv_read_only_mode) {
+	if (trx == nullptr) {
+		trx = trx_create();
 		trx_start_internal_read_only(trx);
-	} else {
-		trx_start_internal(trx);
-	}
-
-	trx->op_info = "update the next FTS document id";
-
-	info = pars_info_create();
-
-	pars_info_bind_function(
-		info, "my_func", fts_fetch_store_doc_id, doc_id);
-
-	fts_get_table_name(&fts_table, table_name);
-	pars_info_bind_id(info, "config_table", table_name);
-
-	graph = fts_parse_sql(
-		&fts_table, info,
-		"DECLARE FUNCTION my_func;\n"
-		"DECLARE CURSOR c IS SELECT value FROM $config_table"
-		" WHERE key = 'synced_doc_id' FOR UPDATE;\n"
-		"BEGIN\n"
-		""
-		"OPEN c;\n"
-		"WHILE 1 = 1 LOOP\n"
-		"  FETCH c INTO my_func();\n"
-		"  IF c % NOTFOUND THEN\n"
-		"    EXIT;\n"
-		"  END IF;\n"
-		"END LOOP;\n"
-		"CLOSE c;");
-
-	*doc_id = 0;
-
-	error = fts_eval_sql(trx, graph);
-
-	fts_que_graph_free_check_lock(&fts_table, NULL, graph);
-
-	// FIXME: We need to retry deadlock errors
-	if (error != DB_SUCCESS) {
-		goto func_exit;
 	}
+retry:
+	error = fts_read_synced_doc_id(table, doc_id, trx);
 
-	if (read_only) {
-		/* InnoDB stores actual synced_doc_id value + 1 in
-		FTS_CONFIG table. Reduce the value by 1 while reading
-		after startup. */
-		if (*doc_id) *doc_id -= 1;
-		goto func_exit;
-	}
+	if (error != DB_SUCCESS) goto func_exit;
 
 	if (cmp_doc_id == 0 && *doc_id) {
 		cache->synced_doc_id = *doc_id - 1;
@@ -2714,6 +2710,10 @@ fts_cmp_set_sync_doc_id(
 
 func_exit:
 
+	if (caller_trx) {
+		return error;
+	}
+
 	if (UNIV_LIKELY(error == DB_SUCCESS)) {
 		fts_sql_commit(trx);
 	} else {
@@ -2721,6 +2721,7 @@ fts_cmp_set_sync_doc_id(
 
 		ib::error() << "(" << error << ") while getting next doc id "
 			"for table " << table->name;
+
 		fts_sql_rollback(trx);
 
 		if (error == DB_DEADLOCK) {
@@ -4201,8 +4202,8 @@ fts_sync_commit(
 
 	/* After each Sync, update the CONFIG table about the max doc id
 	we just sync-ed to index table */
-	error = fts_cmp_set_sync_doc_id(sync->table, sync->max_doc_id, FALSE,
-					&last_doc_id);
+	error = fts_cmp_set_sync_doc_id(sync->table, sync->max_doc_id,
+					&last_doc_id, trx);
 
 	/* Get the list of deleted documents that are either in the
 	cache or were headed there but were deleted before the add
@@ -4228,6 +4229,7 @@ fts_sync_commit(
 	rw_lock_x_unlock(&cache->lock);
 
 	if (UNIV_LIKELY(error == DB_SUCCESS)) {
+		DEBUG_SYNC_C("fts_crash_before_commit_sync");
 		fts_sql_commit(trx);
 	} else {
 		fts_sql_rollback(trx);
@@ -4901,7 +4903,7 @@ fts_init_doc_id(
 
 	/* Then compare this value with the ID value stored in the CONFIG
 	table. The larger one will be our new initial Doc ID */
-	fts_cmp_set_sync_doc_id(table, 0, FALSE, &max_doc_id);
+	fts_cmp_set_sync_doc_id(table, 0, &max_doc_id);
 
 	/* If DICT_TF2_FTS_ADD_DOC_ID is set, we are in the process of
 	creating index (and add doc id column. No need to recovery
@@ -6376,7 +6378,17 @@ fts_init_index(
 	start_doc = cache->synced_doc_id;
 
 	if (!start_doc) {
-		fts_cmp_set_sync_doc_id(table, 0, TRUE, &start_doc);
+		trx_t *trx = trx_create();
+		trx_start_internal_read_only(trx);
+		dberr_t err= fts_read_synced_doc_id(table, &start_doc, trx);
+		fts_sql_commit(trx);
+		trx->free();
+		if (err != DB_SUCCESS) {
+			goto func_exit;
+		}
+		if (start_doc) {
+			start_doc--;
+		}
 		cache->synced_doc_id = start_doc;
 	}