MDEV-8158 InnoDB: Failing assertion: new_state->key_version !=...

MDEV-8158 InnoDB: Failing assertion: new_state->key_version != ENCRYPTION_KEY_VERSION_INVALID on dynamic change of encryption variables

don't allow to enable srv_encrypt_tables if no
encryption plugin is loaded

mysql-test/suite/sys_vars/r/innodb_encrypt_tables_basic.result

@@ -19,17 +19,23 @@ where variable_name='innodb_encrypt_tables';
 VARIABLE_NAME	VARIABLE_VALUE
 INNODB_ENCRYPT_TABLES	OFF
 set global innodb_encrypt_tables=ON;
+ERROR 42000: Variable 'innodb_encrypt_tables' can't be set to the value of 'ON'
+show warnings;
+Level	Code	Message
+Warning	138	InnoDB: cannot enable encryption, encryption plugin is not available
+Error	1231	Variable 'innodb_encrypt_tables' can't be set to the value of 'ON'
 select @@global.innodb_encrypt_tables;
 @@global.innodb_encrypt_tables
-ON
+OFF
 set global innodb_encrypt_tables=OFF;
 select @@global.innodb_encrypt_tables;
 @@global.innodb_encrypt_tables
 OFF
 set global innodb_encrypt_tables=1;
+ERROR 42000: Variable 'innodb_encrypt_tables' can't be set to the value of '1'
 select @@global.innodb_encrypt_tables;
 @@global.innodb_encrypt_tables
-ON
+OFF
 set session innodb_encrypt_tables=1;
 ERROR HY000: Variable 'innodb_encrypt_tables' is a GLOBAL variable and should be set with SET GLOBAL
 set global innodb_encrypt_tables=1.1;

mysql-test/suite/sys_vars/t/innodb_encrypt_tables_basic.test

@@ -19,10 +19,13 @@ where variable_name='innodb_encrypt_tables';
 #
 # show that it's writable
 #
+--error ER_WRONG_VALUE_FOR_VAR
 set global innodb_encrypt_tables=ON;
+show warnings;
 select @@global.innodb_encrypt_tables;
 set global innodb_encrypt_tables=OFF;
 select @@global.innodb_encrypt_tables;
+--error ER_WRONG_VALUE_FOR_VAR
 set global innodb_encrypt_tables=1;
 select @@global.innodb_encrypt_tables;
 --error ER_GLOBAL_VARIABLE

storage/innobase/handler/ha_innodb.cc

@@ -547,6 +547,10 @@ ib_cb_t innodb_api_cb[] = {
 	(ib_cb_t) ib_cursor_stmt_begin
 };
 
+
+static void innodb_remember_check_sysvar_funcs();
+mysql_var_check_func check_sysvar_enum;
+
 static MYSQL_THDVAR_UINT(default_encryption_key_id, PLUGIN_VAR_RQCMDARG,
 			 "Default encryption key id used for table encryption.",
 			 NULL, NULL,
@@ -637,6 +641,17 @@ static ibool innodb_have_lzma=IF_LZMA(1, 0);
 static ibool innodb_have_bzip2=IF_BZIP2(1, 0);
 static ibool innodb_have_snappy=IF_SNAPPY(1, 0);
 
+static
+int
+innodb_encrypt_tables_validate(
+/*==================================*/
+	THD*				thd,	/*!< in: thread handle */
+	struct st_mysql_sys_var*	var,	/*!< in: pointer to system
+						variable */
+	void*				save,	/*!< out: immediate result
+						for update function */
+	struct st_mysql_value*		value);	/*!< in: incoming string */
+
 static const char innobase_hton_name[]= "InnoDB";
 
 static MYSQL_THDVAR_BOOL(support_xa, PLUGIN_VAR_OPCMDARG,
@@ -3157,6 +3172,8 @@ innobase_init(
 
 	innobase_hton->table_options = innodb_table_option_list;
 
+	innodb_remember_check_sysvar_funcs();
+
 	ut_a(DATA_MYSQL_TRUE_VARCHAR == (ulint)MYSQL_TYPE_VARCHAR);
 
 #ifndef DBUG_OFF
@@ -19165,8 +19182,9 @@ static TYPELIB srv_encrypt_tables_typelib = {
 static MYSQL_SYSVAR_ENUM(encrypt_tables, srv_encrypt_tables,
 			 PLUGIN_VAR_OPCMDARG,
 			 "Enable encryption for tables. "
-                         "Don't forget to enable --innodb-encrypt-log too",
-			 NULL, NULL, 0, &srv_encrypt_tables_typelib);
+			 "Don't forget to enable --innodb-encrypt-log too",
+			 innodb_encrypt_tables_validate, NULL, 0,
+			 &srv_encrypt_tables_typelib);
 
 static MYSQL_SYSVAR_UINT(encryption_threads, srv_n_fil_crypt_threads,
 			 PLUGIN_VAR_RQCMDARG,
@@ -20099,3 +20117,37 @@ innodb_compression_algorithm_validate(
 #endif
 	DBUG_RETURN(0);
 }
+
+static
+int
+innodb_encrypt_tables_validate(
+/*=================================*/
+	THD*				thd,	/*!< in: thread handle */
+	struct st_mysql_sys_var*	var,	/*!< in: pointer to system
+						variable */
+	void*				save,	/*!< out: immediate result
+						for update function */
+	struct st_mysql_value*		value)	/*!< in: incoming string */
+{
+	if (check_sysvar_enum(thd, var, save, value))
+		return 1;
+
+	long encrypt_tables = *(long*)save;
+
+	if (encrypt_tables
+	    && !encryption_key_id_exists(FIL_DEFAULT_ENCRYPTION_KEY)) {
+		push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN,
+				    HA_ERR_UNSUPPORTED,
+				    "InnoDB: cannot enable encryption, "
+		                    "encryption plugin is not available");
+		return 1;
+	}
+	return 0;
+}
+
+static void innodb_remember_check_sysvar_funcs()
+{
+	/* remember build-in sysvar check functions */
+	ut_ad((MYSQL_SYSVAR_NAME(checksum_algorithm).flags & 0x1FF) == PLUGIN_VAR_ENUM);
+	check_sysvar_enum = MYSQL_SYSVAR_NAME(checksum_algorithm).check;
+}

storage/xtradb/handler/ha_innodb.cc

@@ -612,6 +612,10 @@ ib_cb_t innodb_api_cb[] = {
 	(ib_cb_t) ib_cursor_stmt_begin
 };
 
+
+static void innodb_remember_check_sysvar_funcs();
+mysql_var_check_func check_sysvar_enum;
+
 static MYSQL_THDVAR_UINT(default_encryption_key_id, PLUGIN_VAR_RQCMDARG,
 			 "Default encryption key id used for table encryption.",
 			 NULL, NULL,
@@ -727,6 +731,17 @@ innodb_compression_algorithm_validate(
 						for update function */
 	struct st_mysql_value*		value);	/*!< in: incoming string */
 
+static
+int
+innodb_encrypt_tables_validate(
+/*==================================*/
+	THD*				thd,	/*!< in: thread handle */
+	struct st_mysql_sys_var*	var,	/*!< in: pointer to system
+						variable */
+	void*				save,	/*!< out: immediate result
+						for update function */
+	struct st_mysql_value*		value);	/*!< in: incoming string */
+
 static const char innobase_hton_name[]= "InnoDB";
 
 static MYSQL_THDVAR_BOOL(support_xa, PLUGIN_VAR_OPCMDARG,
@@ -3503,6 +3518,8 @@ innobase_init(
         innobase_hton->fake_trx_id=wsrep_fake_trx_id;
 #endif /* WITH_WSREP */
 
+	innodb_remember_check_sysvar_funcs();
+
 	ut_a(DATA_MYSQL_TRUE_VARCHAR == (ulint)MYSQL_TYPE_VARCHAR);
 
 #ifndef DBUG_OFF
@@ -20347,8 +20364,9 @@ static TYPELIB srv_encrypt_tables_typelib = {
 static MYSQL_SYSVAR_ENUM(encrypt_tables, srv_encrypt_tables,
 			 PLUGIN_VAR_OPCMDARG,
 			 "Enable encryption for tables. "
-                         "Don't forget to enable --innodb-encrypt-log too",
-			 NULL, NULL, 0, &srv_encrypt_tables_typelib);
+			 "Don't forget to enable --innodb-encrypt-log too",
+			 innodb_encrypt_tables_validate, NULL, 0,
+			 &srv_encrypt_tables_typelib);
 
 static MYSQL_SYSVAR_UINT(encryption_threads, srv_n_fil_crypt_threads,
 			 PLUGIN_VAR_RQCMDARG,
@@ -21327,3 +21345,37 @@ innodb_compression_algorithm_validate(
 #endif
 	DBUG_RETURN(0);
 }
+
+static
+int
+innodb_encrypt_tables_validate(
+/*=================================*/
+	THD*				thd,	/*!< in: thread handle */
+	struct st_mysql_sys_var*	var,	/*!< in: pointer to system
+						variable */
+	void*				save,	/*!< out: immediate result
+						for update function */
+	struct st_mysql_value*		value)	/*!< in: incoming string */
+{
+	if (check_sysvar_enum(thd, var, save, value))
+		return 1;
+
+	long encrypt_tables = *(long*)save;
+
+	if (encrypt_tables
+	    && !encryption_key_id_exists(FIL_DEFAULT_ENCRYPTION_KEY)) {
+		push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN,
+				    HA_ERR_UNSUPPORTED,
+				    "InnoDB: cannot enable encryption, "
+		                    "encryption plugin is not available");
+		return 1;
+	}
+	return 0;
+}
+
+static void innodb_remember_check_sysvar_funcs()
+{
+	/* remember build-in sysvar check functions */
+	ut_ad((MYSQL_SYSVAR_NAME(checksum_algorithm).flags & 0x1FF) == PLUGIN_VAR_ENUM);
+	check_sysvar_enum = MYSQL_SYSVAR_NAME(checksum_algorithm).check;
+}