MDEV-19281: Plugin implementation for the Hashicorp Vault KMS

- Authentication is done using the Hashicorp Vault's token
  authentication method;
- If additional client authentication is required, then the
  path to the CA authentication bundle file may be passed
  as a plugin parameter;
- The creation of the keys and their management is carried
  out using the Hashicorp Vault KMS and their tools;
- Key values stored as hexadecimal strings;
- Key values caching is supported.
- Implemented a time-invalidated cache for key values and
  for key version numbers received from the Hashicorp Valult
  server;
- The plugin uses libcurl (https) as an interface to
  the HashiCorp Vault server;
- JSON parsing is performed through the JSON service
  (through the include/mysql/service_json.h);
- HashiCorp Vault 1.2.4 was used for development and testing.

debian/control

@@ -983,6 +983,15 @@ Description: CrackLib Password Validation Plugin for MariaDB
  .
  Install and configure this to enforce stronger passwords for MariaDB users.
 
+Package: mariadb-plugin-hashicorp-key-management
+Architecture: any
+Depends: mariadb-server-10.9 (= ${binary:Version}),
+         ${misc:Depends},
+         ${shlibs:Depends}
+Description: Hashicorp Key Management plugin for MariaDB
+ This encryption plugin uses Hashicorp Vault for storing encryption
+ keys for MariaDB Data-at-Rest encryption.
+
 Package: mariadb-plugin-provider-bzip2
 Architecture: any
 Depends: mariadb-server,

debian/mariadb-plugin-hashicorp-key-management.install

+etc/mysql/mariadb.conf.d/hashicorp_key_management.cnf
+usr/lib/mysql/plugin/hashicorp_key_management.so

plugin/hashicorp_key_management/CMakeLists.txt

+INCLUDE(FindCURL)
+IF(NOT CURL_FOUND)
+  # Can't build plugin
+  RETURN()
+ENDIF()
+
+INCLUDE_DIRECTORIES(${CURL_INCLUDE_DIR})
+
+set(CPACK_RPM_hashicorp-key-management_PACKAGE_SUMMARY "Hashicorp Key Management plugin for MariaDB" PARENT_SCOPE)
+set(CPACK_RPM_hashicorp-key-management_PACKAGE_DESCRIPTION "This encryption plugin uses Hashicorp Vault for storing encryption
+ keys for MariaDB Data-at-Rest encryption." PARENT_SCOPE)
+
+MYSQL_ADD_PLUGIN(HASHICORP_KEY_MANAGEMENT
+  hashicorp_key_management_plugin.cc
+  LINK_LIBRARIES ${CURL_LIBRARIES}
+  CONFIG hashicorp_key_management.cnf
+  COMPONENT hashicorp-key-management)
+
+INSTALL_DOCUMENTATION(hashicorp_key_management.txt
+        COMPONENT hashicorp-key-management)

plugin/hashicorp_key_management/hashicorp_key_management.cnf

+# Copyright (C) 2019-2022 MariaDB Corporation
+#
+# This is a default configuration for the Hashicorp Vault plugin.
+# You can read more about the parameters of this plugin in the
+# hashicorp_key_management.txt file.
+#
+# NOTE THAT YOU MUST MANUALLY UNCOMMENT THE "plugin-load-add"
+# LINE AND ALL THE NECESSARY PARAMETERS BELOW, SETTING THEM
+# TO APPROPRIATE VALUES!
+#
+[mariadb]
+
+#
+# To use Hashicorp Vault KMS, the plugin must be preloaded and
+# activated on the server:
+#
+#plugin-load-add=hashicorp_key_management.so
+
+# Most of its parameters should not be changed during plugin
+# operation and therefore must be preconfigured as part of
+# the server configuration:
+
+#
+# HTTP[s] URL that is used to connect to the Hashicorp Vault server.
+# It must include the name of the scheme ("https://" for a secure
+# connection) and, according to the API rules for storages of the
+# key-value type in Hashicorp Vault, after the server address, the
+# path must begin with the "/v1/" string (as prefix), for example:
+# "https://127.0.0.1:8200/v1/my_secrets"
+#
+#hashicorp-key-management-vault-url="<url>"
+
+#
+# Authentication token that passed to the Hashicorp Vault
+# in the request header:
+#
+#hashicorp-key-management-token="<token>"
+
+#
+# Optional path to the Certificate Authority (CA) bundle
+# (is a file that contains root and intermediate certificates):
+#
+#hashicorp-key-management-vault-ca="<path>"
+
+#
+# Set the duration (in seconds) for the Hashicorp Vault server
+# connection timeout. The allowed range is from 1 to 86400 seconds.
+# The user can also specify a zero value, which means the default
+# timeout value set by the libcurl library (currently 300 seconds):
+#
+#hashicorp-key-management-timeout=15
+
+#
+# Number of server request retries in case of timeout:
+#
+#hashicorp-key-management-retries=3
+
+#
+# Enable key caching (storing key values received from
+# the Hashicorp Vault server in the local memory):
+#
+#hashicorp-key-management-caching-enabled="on"
+
+#
+# This parameter instructs the plugin to use the key values
+# or version numbers taken from the cache in the event of a
+# timeout when accessing the vault server. By default this
+# option is disabled.
+#
+# Please note that key values or version numbers will be read
+# from the cache when the timeout expires only after the number
+# of attempts to read them from the storage server that specified
+# by the hashicorp-key-management-retries parameter has been
+# exhausted:
+#
+#hashicorp-key-management-use-cache-on-timeout="off"
+
+#
+# The time (in milliseconds) after which the value of the key
+# stored in the cache becomes invalid and an attempt to read this
+# data causes a new request send to the vault server. By default,
+# cache entries become invalid after 60,000 milliseconds (after
+# one minute).
+#
+# If the value of this parameter is zero, then the keys will always
+# be considered invalid, but they still can be used if the vault
+# server is unavailable and the corresponding cache operating mode
+# (--[loose-]hashicorp-key-management-use-cache-on-timeout="on")
+# is enabled.
+#
+#hashicorp-key-management-cache-timeout=0
+
+#
+# The time (in milliseconds) after which the information about
+# latest version number of the key (which stored in the cache)
+# becomes invalid and an attempt to read this information causes
+# a new request send to the vault server.
+#
+# If the value of this parameter is zero, then information abount
+# latest key version numbers always considered invalid, unless
+# there is no communication with the vault server and use of the
+# cache is allowed when the server is unavailable.
+#
+# By default, this parameter is zero, that is, the latest version
+# numbers for the keys stored in the cache are considered always
+# invalid, except when the vault server is unavailable and use
+# of the cache is allowed on server failures.
+#
+#hashicorp-key-management-cache-version-timeout=0

plugin/hashicorp_key_management/hashicorp_key_management.txt

+This file describes a hasicorp_key_management plugin that is used to
+implement encryption using keys stored in the Hashicorp Vault KMS.
+
+The current version of this plugin implements the following features:
+
+- Authentication is done using the Hashicorp Vault's token
+  authentication method;
+- If additional client authentication is required, then the
+  path to the CA authentication bundle file may be passed
+  as a plugin parameter;
+- The creation of the keys and their management is carried
+  out using the Hashicorp Vault KMS and their tools;
+- The plugin uses libcurl (https) as an interface to
+  the HashiCorp Vault server;
+- JSON parsing is performed through the JSON service
+  (through the include/mysql/service_json.h);
+- HashiCorp Vault 1.2.4 was used for development and testing.
+
+Since we require support for key versioning, then the key-value
+storage must be configured in Hashicorp Vault as a key-value storage
+that uses the interface of the second version. For example, you can
+create it as follows:
+
+~$ vault secrets enable -path /test -version=2 kv
+
+Key names must correspond to their numerical identifiers.
+Key identifiers itself, their possible values and rules of use
+are described in more detail in the MariaDB main documentation.
+
+From the point of view of the key-value storage (in terms
+of Hashicorp Vault), the key is a secret containing one key-value
+pair with the name "data" and a value representing a binary string
+containing the key value, for example:
+
+~$ vault kv get /test/1
+
+====== Metadata ======
+Key              Value
+---              -----
+created_time     2019-12-14T14:19:19.42432951Z
+deletion_time    n/a
+destroyed        false
+version          1
+
+==== Data ====
+Key     Value
+---     -----
+data    0123456789ABCDEF0123456789ABCDEF
+
+Keys values are strings containing binary data. MariaDB currently
+uses the AES algorithm with 256-bit keys as the default encryption
+method. In this case, the keys that will be stored in the Hashicorp
+Vault should be 32-byte strings. Most likely you will use some utilities
+for creating and administering keys designed to work with Hashicorp
+Vault. But in the simplest case, keys can be created from the command
+line through the vault utility, for example, as follows:
+
+~$ vault kv put /test/1 data="0123456789ABCDEF0123456789ABCDEF"
+
+If you use default encryption (AES), you should ensure that the
+key length is 32 bytes, otherwise it may fail to use InnoDB as
+a data storage.
+
+The plugin currently does not unseal Hashicorp Vault on its own,
+you must do this in advance and on your own.
+
+To use Hashicorp Vault KMS, the plugin must be preloaded and
+activated on the server. Most of its parameters should not be
+changed during plugin operation and therefore must be preconfigured
+as part of the server configuration through configuration file or
+command line options:
+
+--plugin-load-add=hashicorp_key_management.so
+--loose-hashicorp-key-management
+--loose-hashicorp-key-management-vault-url="$VAULT_ADDR/v1/test"
+--loose-hashicorp-key-management-token="$VAULT_TOKEN"
+
+Currently, the plugin supports the following parameters, which
+must be set in advance and cannot be changed during server
+operation:
+
+--[loose-]hashicorp-key-management-vault-url="<url>"
+
+  HTTP[s] URL that is used to connect to the Hashicorp Vault
+  server. It must include the name of the scheme (https://
+  for a secure connection) and, according to the API rules
+  for storages of the key-value type in Hashicorp Vault,
+  after the server address, the path must begin with the
+  "/v1/" string (as prefix), for example:
+
+     https://127.0.0.1:8200/v1/my_secrets
+
+  By default, the path is not set, therefore you must
+  replace with the correct path to your secrets.
+
+--[loose-]hashicorp-key-management-token="<token>"
+
+  Authentication token that passed to the Hashicorp Vault
+  in the request header.
+
+  By default, this parameter contains an empty string,
+  so you must specify the correct value for it, otherwise
+  the Hashicorp Vault server will refuse authorization.
+
+--[loose-]hashicorp-key-management-vault-ca="<path>"
+
+  Path to the Certificate Authority (CA) bundle (is a file
+  that contains root and intermediate certificates).
+
+  By default, this parameter contains an empty string,
+  which means no CA bundle.
+
+--[loose-]hashicorp-key-management-timeout=<timeout>
+
+  Set the duration (in seconds) for the Hashicorp Vault server
+  connection timeout. The default value is 15 seconds. The allowed
+  range is from 1 to 86400 seconds. The user can also specify a zero
+  value, which means the default timeout value set by the libcurl
+  library (currently 300 seconds).
+
+--[loose-]hashicorp-key-management-retries=<retries>
+
+  Number of server request retries in case of timeout.
+  Default is three retries.
+
+--[loose-]hashicorp-key-management-caching-enabled="on"|"off"
+
+  Enable key caching (storing key values received from
+  the Hashicorp Vault server in the local memory). By default
+  caching is enabled.
+
+--[loose-]hashicorp-key-management-use-cache-on-timeout="on"|"off"
+
+  This parameter instructs the plugin to use the key values
+  or version numbers taken from the cache in the event of a
+  timeout when accessing the vault server. By default this
+  option is disabled.
+
+  Please note that key values or version numbers will be read
+  from the cache when the timeout expires only after the number
+  of attempts to read them from the storage server that specified
+  by the --[loose-]hashicorp-key-management-retries parameter
+  has been exhausted.
+
+--[loose-]hashicorp-key-management-cache-timeout=<timeout>
+
+  The time (in milliseconds) after which the value of the key
+  stored in the cache becomes invalid and an attempt to read this
+  data causes a new request send to the vault server. By default,
+  cache entries become invalid after 60,000 milliseconds (after
+  one minute).
+
+  If the value of this parameter is zero, then the keys will always
+  be considered invalid, but they still can be used if the vault
+  server is unavailable and the corresponding cache operating mode
+  (--[loose-]hashicorp-key-management-use-cache-on-timeout="on")
+  is enabled.
+
+--[loose-]hashicorp-key-management-cache-version-timeout=<timeout>
+
+  The time (in milliseconds) after which the information about
+  latest version number of the key (which stored in the cache)
+  becomes invalid and an attempt to read this information causes
+  a new request send to the vault server.
+
+  If the value of this parameter is zero, then information abount
+  latest key version numbers always considered invalid, unless
+  there is no communication with the vault server and use of the
+  cache is allowed when the server is unavailable.
+
+  By default, this parameter is zero, that is, the latest version
+  numbers for the keys stored in the cache are considered always
+  invalid, except when the vault server is unavailable and use
+  of the cache is allowed on server failures.

plugin/hashicorp_key_management/mysql-test/vault/include/have_hashicorp_key_management_plugin.inc

+if (`SELECT COUNT(*)=0 FROM INFORMATION_SCHEMA.PLUGINS WHERE PLUGIN_NAME = 'hashicorp_key_management' AND PLUGIN_STATUS='ACTIVE'`)
+{
+  --skip Test requires active hashicorp_key_management plugin
+}

plugin/hashicorp_key_management/mysql-test/vault/include/have_hashicorp_key_management_plugin.opt

+--plugin-load-add=$HASHICORP_KEY_MANAGEMENT_SO
+--loose-hashicorp-key-management
+--loose-hashicorp-key-management-vault-url="$VAULT_ADDR/v1/mariadbtest/"
+--loose-hashicorp-key-management-token="$VAULT_TOKEN"
+--loose-hashicorp-key-management-timeout=60

plugin/hashicorp_key_management/mysql-test/vault/include/have_mariabackup.inc

+if (!$XTRABACKUP) {
+  --skip Needs mariabackup;
+}

plugin/hashicorp_key_management/mysql-test/vault/r/hashicorp_encode.result

+SHOW GLOBAL variables LIKE "hashicorp%";
+Variable_name	Value
+hashicorp_key_management_cache_timeout	60000
+hashicorp_key_management_cache_version_timeout	0
+hashicorp_key_management_caching_enabled	ON
+hashicorp_key_management_max_retries	3
+hashicorp_key_management_timeout	60
+hashicorp_key_management_use_cache_on_timeout	OFF
+hashicorp_key_management_vault_ca	
+hashicorp_key_management_vault_url	VAULT_ADDR/v1/mariadbtest/
+create table t1(c1 bigint not null, b char(200)) engine=innodb encrypted=yes encryption_key_id=1;
+show create table t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `c1` bigint(20) NOT NULL,
+  `b` char(200) DEFAULT NULL
+) ENGINE=InnoDB DEFAULT CHARSET=latin1 `encrypted`=yes `encryption_key_id`=1
+insert t1 values (12345, repeat('1234567890', 20));
+alter table t1 encryption_key_id=2;
+show create table t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `c1` bigint(20) NOT NULL,
+  `b` char(200) DEFAULT NULL
+) ENGINE=InnoDB DEFAULT CHARSET=latin1 `encrypted`=yes `encryption_key_id`=2
+alter table t1 encryption_key_id=33;
+ERROR HY000: Table storage engine 'InnoDB' does not support the create option 'ENCRYPTION_KEY_ID'
+show create table t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `c1` bigint(20) NOT NULL,
+  `b` char(200) DEFAULT NULL
+) ENGINE=InnoDB DEFAULT CHARSET=latin1 `encrypted`=yes `encryption_key_id`=2
+alter table t1 encryption_key_id=3;
+show create table t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `c1` bigint(20) NOT NULL,
+  `b` char(200) DEFAULT NULL
+) ENGINE=InnoDB DEFAULT CHARSET=latin1 `encrypted`=yes `encryption_key_id`=3
+alter table t1 encryption_key_id=4;
+show create table t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `c1` bigint(20) NOT NULL,
+  `b` char(200) DEFAULT NULL
+) ENGINE=InnoDB DEFAULT CHARSET=latin1 `encrypted`=yes `encryption_key_id`=4
+drop table t1;

plugin/hashicorp_key_management/mysql-test/vault/r/hashicorp_key_rotation_age.result

+SHOW GLOBAL variables LIKE "hashicorp%";
+Variable_name	Value
+hashicorp_key_management_cache_timeout	60000
+hashicorp_key_management_cache_version_timeout	0
+hashicorp_key_management_caching_enabled	ON
+hashicorp_key_management_max_retries	3
+hashicorp_key_management_timeout	60
+hashicorp_key_management_use_cache_on_timeout	OFF
+hashicorp_key_management_vault_ca	
+hashicorp_key_management_vault_url	VAULT_ADDR/v1/mariadbtest/
+# Restart the server with encryption
+# restart: with restart_parameters
+CREATE TABLE t1 (f1 INT, f2 VARCHAR(256))engine=innodb;
+INSERT INTO t1 VALUES(1, 'MariaDB'), (2, 'Robot'), (3, 'Science');
+INSERT INTO t1 SELECT * FROM t1;
+CREATE TABLE t2(f1 INT, f2 VARCHAR(256))engine=innodb;
+INSERT INTO t2 SELECT * FROM t1;
+CREATE TABLE t3(f1 INT, f2 VARCHAR(256))engine=innodb encrypted=yes;
+INSERT INTO t3 SELECT * FROM t1;
+CREATE TABLE t33(f1 INT, f2 VARCHAR(256)) engine=innodb encrypted=yes encryption_key_id=2;
+INSERT INTO t33 VALUES (12345, '1234567890');
+# Restart the server with encryption and rotate key age
+# restart: with restart_parameters
+# Wait until encryption threads have encrypted all tablespaces
+SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 AND NAME LIKE "test/%";
+NAME
+SELECT NAME, CURRENT_KEY_VERSION FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0 AND NAME LIKE "test/%";
+NAME	CURRENT_KEY_VERSION
+test/t1	1
+test/t2	1
+test/t3	1
+test/t33	1
+# Restart the server with innodb_encryption_rotate_key_age= 0
+# restart: with restart_parameters
+create table t4 (f1 int not null)engine=innodb encrypted=NO;
+alter table t33 encryption_key_id=111;
+ERROR HY000: Table storage engine 'InnoDB' does not support the create option 'ENCRYPTION_KEY_ID'
+# Update key value to version 2
+alter table t33 encryption_key_id=222;
+ERROR HY000: Table storage engine 'InnoDB' does not support the create option 'ENCRYPTION_KEY_ID'
+# Wait until encryption threads have encrypted all tablespaces
+SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 AND NAME LIKE "test/%";
+NAME
+test/t4
+SELECT NAME, CURRENT_KEY_VERSION FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0 AND NAME LIKE "test/%";
+NAME	CURRENT_KEY_VERSION
+test/t1	2
+test/t2	2
+test/t3	2
+test/t33	1
+# Disable encryption when innodb_encryption_rotate_key_age is 0
+set global innodb_encrypt_tables = OFF;
+# Wait until encryption threads to decrypt all encrypted tablespaces
+SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 AND NAME LIKE "test/%";
+NAME
+test/t1
+test/t2
+test/t4
+# Display only encrypted create tables (t3)
+SELECT NAME, CURRENT_KEY_VERSION FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0 AND NAME LIKE "test/%";
+NAME	CURRENT_KEY_VERSION
+test/t3	2
+test/t33	1
+alter table t33 encryption_key_id=333;
+ERROR HY000: Table storage engine 'InnoDB' does not support the create option 'ENCRYPTION_KEY_ID'
+# Update key value to version 3
+alter table t33 encryption_key_id=444;
+ERROR HY000: Table storage engine 'InnoDB' does not support the create option 'ENCRYPTION_KEY_ID'
+# Enable encryption when innodb_encryption_rotate_key_age is 0
+set global innodb_encrypt_tables = ON;
+# Wait until encryption threads to encrypt all unencrypted tablespaces
+SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 AND NAME LIKE "test/%";
+NAME
+test/t4
+# Display only unencrypted create tables (t4)
+SELECT NAME, CURRENT_KEY_VERSION FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0 AND NAME LIKE "test/%";
+NAME	CURRENT_KEY_VERSION
+test/t1	3
+test/t2	3
+test/t3	3
+test/t33	1
+# restart: with restart_parameters
+alter table t33 encryption_key_id=555;
+ERROR HY000: Table storage engine 'InnoDB' does not support the create option 'ENCRYPTION_KEY_ID'
+SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 AND NAME LIKE "test/%";
+NAME
+test/t4
+SELECT NAME, CURRENT_KEY_VERSION FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0 AND NAME LIKE "test/%";
+NAME	CURRENT_KEY_VERSION
+test/t1	3
+test/t2	3
+test/t3	3
+test/t33	1
+DROP TABLE t4, t3, t2, t1;
+DROP TABLE t33;
+# restart

plugin/hashicorp_key_management/mysql-test/vault/r/hashicorp_mariabackup.result

+CREATE TABLE t(i INT) ENGINE INNODB encrypted=yes encryption_key_id=1;
+INSERT INTO t VALUES(1);
+# mariabackup backup
+INSERT INTO t VALUES(2);
+# mariabackup prepare
+# shutdown server
+# remove datadir
+# mariabackup move back
+# restart
+SELECT * FROM t;
+i
+1
+DROP TABLE t;

plugin/hashicorp_key_management/mysql-test/vault/suite.pm

+package My::Suite::Vault;
+
+@ISA = qw(My::Suite);
+
+use strict;
+
+return "Hashicorp Vault key management utility not found"
+  unless `sh -c "command -v vault"`;
+
+return "You need to set the value of the VAULT_ADDR variable"
+  unless $ENV{VAULT_ADDR};
+
+return "You need to set the value of the VAULT_TOKEN variable"
+  unless $ENV{VAULT_TOKEN};
+
+bless {};

plugin/hashicorp_key_management/mysql-test/vault/t/hashicorp_deinit.inc

+--exec vault secrets disable mariadbtest > /dev/null

plugin/hashicorp_key_management/mysql-test/vault/t/hashicorp_encode.test

+--source include/have_innodb.inc
+--source hashicorp_plugin.inc
+
+--source hashicorp_init.inc
+
+replace_result $VAULT_ADDR VAULT_ADDR;
+SHOW GLOBAL variables LIKE "hashicorp%";
+
+create table t1(c1 bigint not null, b char(200)) engine=innodb encrypted=yes encryption_key_id=1;
+show create table t1;
+insert t1 values (12345, repeat('1234567890', 20));
+
+alter table t1 encryption_key_id=2;
+show create table t1;
+--error ER_ILLEGAL_HA_CREATE_OPTION
+alter table t1 encryption_key_id=33;
+show create table t1;
+alter table t1 encryption_key_id=3;
+show create table t1;
+alter table t1 encryption_key_id=4;
+show create table t1;
+
+drop table t1;
+
+--source hashicorp_deinit.inc

plugin/hashicorp_key_management/mysql-test/vault/t/hashicorp_init.inc

+--exec vault secrets disable mariadbtest > /dev/null
+--exec vault secrets enable -path /mariadbtest -version=2 kv > /dev/null
+--exec vault kv put /mariadbtest/1 data="123456789ABCDEF0123456789ABCDEF0" > /dev/null
+--exec vault kv put /mariadbtest/2 data="23456789ABCDEF0123456789ABCDEF01" > /dev/null
+--exec vault kv put /mariadbtest/3 data="00000000000000000000000000000000" > /dev/null
+--exec vault kv put /mariadbtest/3 data="00000000000000000000000000000001" > /dev/null
+--exec vault kv put /mariadbtest/4 data="456789ABCDEF0123456789ABCDEF0123" > /dev/null

plugin/hashicorp_key_management/mysql-test/vault/t/hashicorp_key_rotation_age.opt

+--loose-hashicorp-key-management-cache-version-timeout=0

plugin/hashicorp_key_management/mysql-test/vault/t/hashicorp_key_rotation_age.test

+--source include/have_innodb.inc
+--source include/not_embedded.inc
+--source hashicorp_plugin.inc
+
+--source hashicorp_init.inc
+
+replace_result $VAULT_ADDR VAULT_ADDR;
+SHOW GLOBAL variables LIKE "hashicorp%";
+
+--echo # Restart the server with encryption
+let $default_parameters="--innodb-tablespaces-encryption --innodb_encrypt_tables=ON";
+let $restart_noprint=1;
+let $restart_parameters=$default_parameters;
+--source include/restart_mysqld.inc
+
+CREATE TABLE t1 (f1 INT, f2 VARCHAR(256))engine=innodb;
+INSERT INTO t1 VALUES(1, 'MariaDB'), (2, 'Robot'), (3, 'Science');
+INSERT INTO t1 SELECT * FROM t1;
+
+CREATE TABLE t2(f1 INT, f2 VARCHAR(256))engine=innodb;
+INSERT INTO t2 SELECT * FROM t1;
+
+CREATE TABLE t3(f1 INT, f2 VARCHAR(256))engine=innodb encrypted=yes;
+INSERT INTO t3 SELECT * FROM t1;
+
+CREATE TABLE t33(f1 INT, f2 VARCHAR(256)) engine=innodb encrypted=yes encryption_key_id=2;
+INSERT INTO t33 VALUES (12345, '1234567890');
+
+--echo # Restart the server with encryption and rotate key age
+
+let $restart_parameters=$default_parameters --innodb_encryption_threads=5 --innodb_encryption_rotate_key_age=16384;
+--source include/restart_mysqld.inc
+
+--echo # Wait until encryption threads have encrypted all tablespaces
+
+--let $tables_count= `select count(*) + 1 from information_schema.tables where engine = 'InnoDB'`
+--let $wait_timeout= 600
+--let $wait_condition=SELECT COUNT(*) >= $tables_count FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0;
+--source include/wait_condition.inc
+
+SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 AND NAME LIKE "test/%";
+--sorted_result
+SELECT NAME, CURRENT_KEY_VERSION FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0 AND NAME LIKE "test/%";
+
+--echo # Restart the server with innodb_encryption_rotate_key_age= 0
+
+let $restart_parameters=$default_parameters --innodb_encryption_threads=1 --innodb_encryption_rotate_key_age=0;
+--source include/restart_mysqld.inc
+
+create table t4 (f1 int not null)engine=innodb encrypted=NO;
+
+# artificial error useful for debugging a plugin
+--error ER_ILLEGAL_HA_CREATE_OPTION
+alter table t33 encryption_key_id=111;
+
+--echo # Update key value to version 2
+--exec vault kv put /mariadbtest/1 data="11112222333344445555666677778888" > /dev/null
+--sleep 2
+
+# artificial error useful for debugging a plugin
+--error ER_ILLEGAL_HA_CREATE_OPTION
+alter table t33 encryption_key_id=222;
+
+--echo # Wait until encryption threads have encrypted all tablespaces
+
+--let $tables_count= `select count(*) from information_schema.tables where engine = 'InnoDB'`
+--let $wait_timeout= 600
+--let $wait_condition=SELECT COUNT(*) >= $tables_count FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0;
+--source include/wait_condition.inc
+
+SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 AND NAME LIKE "test/%";
+--sorted_result
+SELECT NAME, CURRENT_KEY_VERSION FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0 AND NAME LIKE "test/%";
+
+--echo # Disable encryption when innodb_encryption_rotate_key_age is 0
+set global innodb_encrypt_tables = OFF;
+
+--echo # Wait until encryption threads to decrypt all encrypted tablespaces
+
+--let $tables_count= `select count(*) - 1 from information_schema.tables where engine = 'InnoDB'`
+--let $wait_timeout= 600
+--let $wait_condition=SELECT COUNT(*) >= $tables_count FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 AND ROTATING_OR_FLUSHING = 0;
+--source include/wait_condition.inc
+
+--sorted_result
+SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 AND NAME LIKE "test/%";
+--echo # Display only encrypted create tables (t3)
+--sorted_result
+SELECT NAME, CURRENT_KEY_VERSION FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0 AND NAME LIKE "test/%";
+
+# artificial error useful for debugging a plugin
+--error ER_ILLEGAL_HA_CREATE_OPTION
+alter table t33 encryption_key_id=333;
+
+--echo # Update key value to version 3
+--exec vault kv put /mariadbtest/1 data="5555222233334444555566667777AAAA" > /dev/null
+--sleep 2
+
+# artificial error useful for debugging a plugin
+--error ER_ILLEGAL_HA_CREATE_OPTION
+alter table t33 encryption_key_id=444;
+
+--echo # Enable encryption when innodb_encryption_rotate_key_age is 0
+set global innodb_encrypt_tables = ON;
+
+--echo # Wait until encryption threads to encrypt all unencrypted tablespaces
+
+--let $tables_count= `select count(*) from information_schema.tables where engine = 'InnoDB'`
+--let $wait_timeout= 600
+--let $wait_condition=SELECT COUNT(*) >= $tables_count FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0;
+--source include/wait_condition.inc
+
+SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 AND NAME LIKE "test/%";
+--echo # Display only unencrypted create tables (t4)
+--sorted_result
+SELECT NAME, CURRENT_KEY_VERSION FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0 AND NAME LIKE "test/%";
+
+--let $restart_parameters=$default_parameters
+--source include/restart_mysqld.inc
+
+# artificial error useful for debugging a plugin
+--error ER_ILLEGAL_HA_CREATE_OPTION
+alter table t33 encryption_key_id=555;
+
+SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 AND NAME LIKE "test/%";
+--sorted_result
+SELECT NAME, CURRENT_KEY_VERSION FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0 AND NAME LIKE "test/%";
+
+DROP TABLE t4, t3, t2, t1;
+DROP TABLE t33;
+
+--let $restart_parameters=
+--source include/restart_mysqld.inc
+
+--source hashicorp_deinit.inc

plugin/hashicorp_key_management/mysql-test/vault/t/hashicorp_mariabackup.test

+--source include/big_test.inc
+--source include/have_innodb.inc
+--source include/have_mariabackup.inc
+--source hashicorp_plugin.inc
+
+--source hashicorp_init.inc
+
+CREATE TABLE t(i INT) ENGINE INNODB encrypted=yes encryption_key_id=1;
+INSERT INTO t VALUES(1);
+
+echo # mariabackup backup;
+let $targetdir=$MYSQLTEST_VARDIR/tmp/backup;
+
+--disable_result_log
+exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir;
+--enable_result_log
+
+INSERT INTO t VALUES(2);
+
+echo # mariabackup prepare;
+--disable_result_log
+exec $XTRABACKUP --prepare --target-dir=$targetdir;
+
+let $_datadir= `SELECT @@datadir`;
+echo # shutdown server;
+--source include/shutdown_mysqld.inc
+echo # remove datadir;
+rmdir $_datadir;
+echo # mariabackup move back;
+exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --copy-back --datadir=$_datadir --target-dir=$targetdir --parallel=2 --throttle=1;
+--source include/start_mysqld.inc
+--enable_result_log
+
+SELECT * FROM t;
+DROP TABLE t;
+
+rmdir $targetdir;
+
+--source hashicorp_deinit.inc

plugin/hashicorp_key_management/mysql-test/vault/t/hashicorp_plugin.inc

+--source include/have_hashicorp_key_management_plugin.inc
+--source include/not_windows.inc