Merge 10.4 into 10.5 (MDEV-27913)

mysql-test/suite/innodb_fts/r/bug_32831765.result

+call mtr.add_suppression("InnoDB: Total InnoDB FTS size .* for the table .* exceeds the innodb_ft_cache_size .*");
 #
 # Bug#32831765 SERVER HITS OOM CONDITION WHEN LOADING TWO
 # INNODB TABLES WITH FTS INDEXES

mysql-test/suite/innodb_fts/t/bug_32831765.test

@@ -2,6 +2,7 @@
 --source include/have_debug.inc
 --source include/big_test.inc
 
+call mtr.add_suppression("InnoDB: Total InnoDB FTS size .* for the table .* exceeds the innodb_ft_cache_size .*");
 --echo #
 --echo # Bug#32831765 SERVER HITS OOM CONDITION WHEN LOADING TWO
 --echo # INNODB TABLES WITH FTS INDEXES

mysql-test/suite/sys_vars/r/innodb_ft_cache_size_basic.result

@@ -16,6 +16,15 @@ select * from information_schema.session_variables where variable_name='innodb_f
 VARIABLE_NAME	VARIABLE_VALUE
 INNODB_FT_CACHE_SIZE	8000000
 set global innodb_ft_cache_size=1;
-ERROR HY000: Variable 'innodb_ft_cache_size' is a read only variable
+Warnings:
+Warning	1292	Truncated incorrect innodb_ft_cache_size value: '1'
+SHOW VARIABLES like 'innodb_ft_cache_size';
+Variable_name	Value
+innodb_ft_cache_size	1600000
 set session innodb_ft_cache_size=1;
-ERROR HY000: Variable 'innodb_ft_cache_size' is a read only variable
+ERROR HY000: Variable 'innodb_ft_cache_size' is a GLOBAL variable and should be set with SET GLOBAL
+set global innodb_ft_cache_size=512*1024*1024;
+SHOW VARIABLES like 'innodb_ft_cache_size';
+Variable_name	Value
+innodb_ft_cache_size	536870912
+set global innodb_ft_cache_size=default;

mysql-test/suite/sys_vars/r/innodb_ft_total_cache_size_basic.result

@@ -16,6 +16,15 @@ select * from information_schema.session_variables where variable_name='innodb_f
 VARIABLE_NAME	VARIABLE_VALUE
 INNODB_FT_TOTAL_CACHE_SIZE	640000000
 set global innodb_ft_total_cache_size=1;
-ERROR HY000: Variable 'innodb_ft_total_cache_size' is a read only variable
+Warnings:
+Warning	1292	Truncated incorrect innodb_ft_total_cache_size value: '1'
 set session innodb_ft_total_cache_size=1;
-ERROR HY000: Variable 'innodb_ft_total_cache_size' is a read only variable
+ERROR HY000: Variable 'innodb_ft_total_cache_size' is a GLOBAL variable and should be set with SET GLOBAL
+SHOW VARIABLES like 'innodb_ft_total_cache_size';
+Variable_name	Value
+innodb_ft_total_cache_size	32000000
+set global innodb_ft_total_cache_size=512*1024*1024;
+show variables like 'innodb_ft_total_cache_size';
+Variable_name	Value
+innodb_ft_total_cache_size	536870912
+set global innodb_ft_total_cache_size=default;

mysql-test/suite/sys_vars/r/sysvars_innodb,32bit.rdiff

---- sysvars_innodb.result
-+++ sysvars_innodb,32bit.result
 @@ -49,7 +49,7 @@
  SESSION_VALUE	NULL
  DEFAULT_VALUE	8
@@ -121,7 +119,7 @@
  VARIABLE_COMMENT	Helps to save your data in case the disk image of the database becomes corrupt. Value 5 can return bogus data, and 6 can permanently corrupt data.
  NUMERIC_MIN_VALUE	0
  NUMERIC_MAX_VALUE	6
-@@ -925,7 +925,7 @@
+@@ -925,10 +925,10 @@
  SESSION_VALUE	NULL
  DEFAULT_VALUE	8000000
  VARIABLE_SCOPE	GLOBAL
@@ -129,7 +127,11 @@
 +VARIABLE_TYPE	INT UNSIGNED
  VARIABLE_COMMENT	InnoDB Fulltext search cache size in bytes
  NUMERIC_MIN_VALUE	1600000
- NUMERIC_MAX_VALUE	80000000
+-NUMERIC_MAX_VALUE	1099511627776
++NUMERIC_MAX_VALUE	536870912
+ NUMERIC_BLOCK_SIZE	0
+ ENUM_VALUE_LIST	NULL
+ READ_ONLY	NO
 @@ -961,7 +961,7 @@
  SESSION_VALUE	NULL
  DEFAULT_VALUE	84
@@ -179,7 +181,7 @@
  VARIABLE_COMMENT	InnoDB Fulltext search parallel sort degree, will round up to nearest power of 2 number
  NUMERIC_MIN_VALUE	1
  NUMERIC_MAX_VALUE	16
-@@ -1033,7 +1033,7 @@
+@@ -1033,10 +1033,10 @@
  SESSION_VALUE	NULL
  DEFAULT_VALUE	640000000
  VARIABLE_SCOPE	GLOBAL
@@ -187,8 +189,12 @@
 +VARIABLE_TYPE	INT UNSIGNED
  VARIABLE_COMMENT	Total memory allocated for InnoDB Fulltext Search cache
  NUMERIC_MIN_VALUE	32000000
- NUMERIC_MAX_VALUE	1600000000
-@@ -1093,22 +1093,22 @@
+-NUMERIC_MAX_VALUE	1099511627776
++NUMERIC_MAX_VALUE	1600000000
+ NUMERIC_BLOCK_SIZE	0
+ ENUM_VALUE_LIST	NULL
+ READ_ONLY	NO
+@@ -1081,22 +1081,22 @@
  SESSION_VALUE	NULL
  DEFAULT_VALUE	200
  VARIABLE_SCOPE	GLOBAL
@@ -216,7 +222,7 @@
  NUMERIC_BLOCK_SIZE	0
  ENUM_VALUE_LIST	NULL
  READ_ONLY	NO
-@@ -1153,7 +1153,7 @@
+@@ -1141,7 +1141,7 @@
  SESSION_VALUE	50
  DEFAULT_VALUE	50
  VARIABLE_SCOPE	SESSION
@@ -225,7 +231,7 @@
  VARIABLE_COMMENT	Timeout in seconds an InnoDB transaction may wait for a lock before being rolled back. Values above 100000000 disable the timeout.
  NUMERIC_MIN_VALUE	0
  NUMERIC_MAX_VALUE	1073741824
-@@ -1165,10 +1165,10 @@
+@@ -1153,10 +1153,10 @@
  SESSION_VALUE	NULL
  DEFAULT_VALUE	16777216
  VARIABLE_SCOPE	GLOBAL
@@ -238,7 +244,7 @@
  NUMERIC_BLOCK_SIZE	1024
  ENUM_VALUE_LIST	NULL
  READ_ONLY	YES
-@@ -1213,7 +1213,7 @@
+@@ -1201,7 +1201,7 @@
  SESSION_VALUE	NULL
  DEFAULT_VALUE	1
  VARIABLE_SCOPE	GLOBAL
@@ -247,7 +253,7 @@
  VARIABLE_COMMENT	Deprecated parameter with no effect.
  NUMERIC_MIN_VALUE	1
  NUMERIC_MAX_VALUE	100
-@@ -1261,7 +1261,7 @@
+@@ -1249,7 +1249,7 @@
  SESSION_VALUE	NULL
  DEFAULT_VALUE	8192
  VARIABLE_SCOPE	GLOBAL
@@ -256,9 +262,9 @@
  VARIABLE_COMMENT	Redo log write ahead unit size to avoid read-on-write, it should match the OS cache block IO size
  NUMERIC_MIN_VALUE	512
  NUMERIC_MAX_VALUE	16384
-@@ -1273,10 +1273,10 @@
+@@ -1261,10 +1261,10 @@
  SESSION_VALUE	NULL
- DEFAULT_VALUE	100
+ DEFAULT_VALUE	32
  VARIABLE_SCOPE	GLOBAL
 -VARIABLE_TYPE	BIGINT UNSIGNED
 +VARIABLE_TYPE	INT UNSIGNED
@@ -269,9 +275,9 @@
  NUMERIC_BLOCK_SIZE	0
  ENUM_VALUE_LIST	NULL
  READ_ONLY	NO
-@@ -1285,10 +1285,10 @@
+@@ -1273,10 +1273,10 @@
  SESSION_VALUE	NULL
- DEFAULT_VALUE	1024
+ DEFAULT_VALUE	1536
  VARIABLE_SCOPE	GLOBAL
 -VARIABLE_TYPE	BIGINT UNSIGNED
 +VARIABLE_TYPE	INT UNSIGNED
@@ -282,7 +288,7 @@
  NUMERIC_BLOCK_SIZE	0
  ENUM_VALUE_LIST	NULL
  READ_ONLY	NO
-@@ -1333,10 +1333,10 @@
+@@ -1321,10 +1321,10 @@
  SESSION_VALUE	NULL
  DEFAULT_VALUE	0
  VARIABLE_SCOPE	GLOBAL
@@ -295,7 +301,7 @@
  NUMERIC_BLOCK_SIZE	0
  ENUM_VALUE_LIST	NULL
  READ_ONLY	NO
-@@ -1345,7 +1345,7 @@
+@@ -1333,7 +1333,7 @@
  SESSION_VALUE	NULL
  DEFAULT_VALUE	0
  VARIABLE_SCOPE	GLOBAL
@@ -362,7 +368,7 @@
  VARIABLE_COMMENT	Number of pages that must be accessed sequentially for InnoDB to trigger a readahead.
  NUMERIC_MIN_VALUE	0
  NUMERIC_MAX_VALUE	64
-@@ -1705,7 +1705,7 @@
+@@ -1693,7 +1693,7 @@
  SESSION_VALUE	NULL
  DEFAULT_VALUE	1048576
  VARIABLE_SCOPE	GLOBAL
@@ -371,7 +377,7 @@
  VARIABLE_COMMENT	Memory buffer size for index creation
  NUMERIC_MIN_VALUE	65536
  NUMERIC_MAX_VALUE	67108864
-@@ -1873,7 +1873,7 @@
+@@ -1861,7 +1861,7 @@
  SESSION_VALUE	NULL
  DEFAULT_VALUE	1
  VARIABLE_SCOPE	GLOBAL
@@ -380,7 +386,7 @@
  VARIABLE_COMMENT	Size of the mutex/lock wait array.
  NUMERIC_MIN_VALUE	1
  NUMERIC_MAX_VALUE	1024
-@@ -1897,10 +1897,10 @@
+@@ -1885,10 +1885,10 @@
  SESSION_VALUE	NULL
  DEFAULT_VALUE	30
  VARIABLE_SCOPE	GLOBAL
@@ -393,7 +399,7 @@
  NUMERIC_BLOCK_SIZE	0
  ENUM_VALUE_LIST	NULL
  READ_ONLY	NO
-@@ -2005,7 +2005,7 @@
+@@ -1993,7 +1993,7 @@
  SESSION_VALUE	NULL
  DEFAULT_VALUE	128
  VARIABLE_SCOPE	GLOBAL
@@ -402,7 +408,7 @@
  VARIABLE_COMMENT	Deprecated parameter with no effect.
  NUMERIC_MIN_VALUE	0
  NUMERIC_MAX_VALUE	128
-@@ -2029,7 +2029,7 @@
+@@ -2017,7 +2017,7 @@
  SESSION_VALUE	NULL
  DEFAULT_VALUE	0
  VARIABLE_SCOPE	GLOBAL

mysql-test/suite/sys_vars/r/sysvars_innodb.result

@@ -928,10 +928,10 @@ VARIABLE_SCOPE	GLOBAL
 VARIABLE_TYPE	BIGINT UNSIGNED
 VARIABLE_COMMENT	InnoDB Fulltext search cache size in bytes
 NUMERIC_MIN_VALUE	1600000
-NUMERIC_MAX_VALUE	80000000
+NUMERIC_MAX_VALUE	1099511627776
 NUMERIC_BLOCK_SIZE	0
 ENUM_VALUE_LIST	NULL
-READ_ONLY	YES
+READ_ONLY	NO
 COMMAND_LINE_ARGUMENT	REQUIRED
 VARIABLE_NAME	INNODB_FT_ENABLE_DIAG_PRINT
 SESSION_VALUE	NULL
@@ -1036,10 +1036,10 @@ VARIABLE_SCOPE	GLOBAL
 VARIABLE_TYPE	BIGINT UNSIGNED
 VARIABLE_COMMENT	Total memory allocated for InnoDB Fulltext Search cache
 NUMERIC_MIN_VALUE	32000000
-NUMERIC_MAX_VALUE	1600000000
+NUMERIC_MAX_VALUE	1099511627776
 NUMERIC_BLOCK_SIZE	0
 ENUM_VALUE_LIST	NULL
-READ_ONLY	YES
+READ_ONLY	NO
 COMMAND_LINE_ARGUMENT	REQUIRED
 VARIABLE_NAME	INNODB_FT_USER_STOPWORD_TABLE
 SESSION_VALUE	

mysql-test/suite/sys_vars/t/innodb_ft_cache_size_basic.test

@@ -18,11 +18,12 @@ select * from information_schema.global_variables where variable_name='innodb_ft
 select * from information_schema.session_variables where variable_name='innodb_ft_cache_size';
 --enable_warnings
 
-#
-# show that it's read-only
-#
---error ER_INCORRECT_GLOBAL_LOCAL_VAR
 set global innodb_ft_cache_size=1;
---error ER_INCORRECT_GLOBAL_LOCAL_VAR
+SHOW VARIABLES like 'innodb_ft_cache_size';
+
+--error ER_GLOBAL_VARIABLE
 set session innodb_ft_cache_size=1;
 
+set global innodb_ft_cache_size=512*1024*1024;
+SHOW VARIABLES like 'innodb_ft_cache_size';
+set global innodb_ft_cache_size=default;

mysql-test/suite/sys_vars/t/innodb_ft_total_cache_size_basic.test

@@ -18,11 +18,11 @@ select * from information_schema.global_variables where variable_name='innodb_ft
 select * from information_schema.session_variables where variable_name='innodb_ft_total_cache_size';
 --enable_warnings
 
-#
-# show that it's read-only
-#
---error ER_INCORRECT_GLOBAL_LOCAL_VAR
 set global innodb_ft_total_cache_size=1;
---error ER_INCORRECT_GLOBAL_LOCAL_VAR
+--error ER_GLOBAL_VARIABLE
 set session innodb_ft_total_cache_size=1;
 
+SHOW VARIABLES like 'innodb_ft_total_cache_size';
+set global innodb_ft_total_cache_size=512*1024*1024;
+show variables like 'innodb_ft_total_cache_size';
+set global innodb_ft_total_cache_size=default;

storage/innobase/fts/fts0fts.cc

@@ -53,14 +53,14 @@ by looking up the key word in the obsolete table names */
 
 /** This is maximum FTS cache for each table and would be
 a configurable variable */
-ulong	fts_max_cache_size;
+Atomic_relaxed<size_t> fts_max_cache_size;
 
 /** Whether the total memory used for FTS cache is exhausted, and we will
 need a sync to free some memory */
 bool	fts_need_sync = false;
 
 /** Variable specifying the total memory allocated for FTS cache */
-ulong	fts_max_total_cache_size;
+Atomic_relaxed<size_t> fts_max_total_cache_size;
 
 /** This is FTS result cache limit for each query and would be
 a configurable variable */
@@ -4246,7 +4246,7 @@ fts_sync(
 	ulint		i;
 	dberr_t		error = DB_SUCCESS;
 	fts_cache_t*	cache = sync->table->fts->cache;
-
+	size_t		fts_cache_size= 0;
 	rw_lock_x_lock(&cache->lock);
 
 	/* Check if cache is being synced.
@@ -4271,11 +4271,17 @@ fts_sync(
 	fts_sync_begin(sync);
 
 begin_sync:
-	if (cache->total_size > fts_max_cache_size) {
+	fts_cache_size= fts_max_cache_size;
+	if (cache->total_size > fts_cache_size) {
 		/* Avoid the case: sync never finish when
 		insert/update keeps comming. */
 		ut_ad(sync->unlock_cache);
 		sync->unlock_cache = false;
+		ib::warn() << "Total InnoDB FTS size "
+			<< cache->total_size << " for the table "
+			<< cache->sync->table->name
+			<< " exceeds the innodb_ft_cache_size "
+			<< fts_cache_size;
 	}
 
 	for (i = 0; i < ib_vector_size(cache->indexes); ++i) {
@@ -4296,6 +4302,13 @@ fts_sync(
 		if (error != DB_SUCCESS) {
 			goto end_sync;
 		}
+
+		if (!sync->unlock_cache
+		    && cache->total_size < fts_max_cache_size) {
+			/* Reset the unlock cache if the value
+			is less than innodb_ft_cache_size */
+			sync->unlock_cache = true;
+		}
 	}
 
 	DBUG_EXECUTE_IF("fts_instrument_sync_interrupted",

storage/innobase/handler/ha_innodb.cc

@@ -662,6 +662,18 @@ innodb_stopword_table_validate(
 						for update function */
 	struct st_mysql_value*		value);	/*!< in: incoming string */
 
+static
+void innodb_ft_cache_size_update(THD*, st_mysql_sys_var*, void*, const void* save)
+{
+  fts_max_cache_size= *static_cast<const size_t*>(save);
+}
+
+static
+void innodb_ft_total_cache_size_update(THD*, st_mysql_sys_var*, void*, const void* save)
+{
+  fts_max_total_cache_size= *static_cast<const size_t*>(save);
+}
+
 static bool is_mysql_datadir_path(const char *path);
 
 /** Validate passed-in "value" is a valid directory name.
@@ -19439,15 +19451,35 @@ static MYSQL_SYSVAR_STR(ft_aux_table, innodb_ft_aux_table,
   "FTS internal auxiliary table to be checked",
   innodb_ft_aux_table_validate, NULL, NULL);
 
-static MYSQL_SYSVAR_ULONG(ft_cache_size, fts_max_cache_size,
-  PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY,
+#if UNIV_WORD_SIZE == 4
+
+static MYSQL_SYSVAR_SIZE_T(ft_cache_size,
+  *reinterpret_cast<size_t*>(&fts_max_cache_size),
+  PLUGIN_VAR_RQCMDARG,
   "InnoDB Fulltext search cache size in bytes",
-  NULL, NULL, 8000000, 1600000, 80000000, 0);
+  NULL, innodb_ft_cache_size_update, 8000000, 1600000, 1U << 29, 0);
 
-static MYSQL_SYSVAR_ULONG(ft_total_cache_size, fts_max_total_cache_size,
-  PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY,
+static MYSQL_SYSVAR_SIZE_T(ft_total_cache_size,
+  *reinterpret_cast<size_t*>(&fts_max_total_cache_size),
+  PLUGIN_VAR_RQCMDARG,
   "Total memory allocated for InnoDB Fulltext Search cache",
-  NULL, NULL, 640000000, 32000000, 1600000000, 0);
+  NULL, innodb_ft_total_cache_size_update, 640000000, 32000000, 1600000000, 0);
+
+#else
+
+static MYSQL_SYSVAR_SIZE_T(ft_cache_size,
+  *reinterpret_cast<size_t*>(&fts_max_cache_size),
+  PLUGIN_VAR_RQCMDARG,
+  "InnoDB Fulltext search cache size in bytes",
+  NULL, innodb_ft_cache_size_update, 8000000, 1600000, 1ULL << 40, 0);
+
+static MYSQL_SYSVAR_SIZE_T(ft_total_cache_size,
+  *reinterpret_cast<size_t*>(&fts_max_total_cache_size),
+  PLUGIN_VAR_RQCMDARG,
+  "Total memory allocated for InnoDB Fulltext Search cache",
+  NULL, innodb_ft_total_cache_size_update, 640000000, 32000000, 1ULL << 40, 0);
+
+#endif
 
 static MYSQL_SYSVAR_SIZE_T(ft_result_cache_limit, fts_result_cache_limit,
   PLUGIN_VAR_RQCMDARG,

storage/innobase/include/fts0fts.h

 /*****************************************************************************
 
 Copyright (c) 2011, 2018, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2016, 2021, MariaDB Corporation.
+Copyright (c) 2016, 2022, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -355,10 +355,10 @@ struct fts_stopword_t;
 extern const char*	fts_default_stopword[];
 
 /** Variable specifying the maximum FTS cache size for each table */
-extern ulong		fts_max_cache_size;
+extern Atomic_relaxed<size_t> fts_max_cache_size;
 
 /** Variable specifying the total memory allocated for FTS cache */
-extern ulong		fts_max_total_cache_size;
+extern Atomic_relaxed<size_t> fts_max_total_cache_size;
 
 /** Variable specifying the FTS result cache limit for each query */
 extern size_t		fts_result_cache_limit;