Commit c51f615b authored by Monty's avatar Monty

Added options --s3-protocol-version and --s3-host-name

parent 3e83034e
set @save_s3_protocol_version=@@global.s3_protocol_version;
set @@global.s3_protocol_version="Original";
create table t1 (pk int primary key, a int);
insert into t1 values (1,1),(2,2),(3,3),(4,4);
alter table t1 engine=S3;
ERROR HY000: Got error from put_object(database/t1/frm): 8 Access Denied
drop table t1;
set @@global.s3_protocol_version=@save_s3_protocol_version;
--source include/have_s3.inc
if (`SELECT @@s3_host_name <> "s3.amazonaws.com"`)
{
skip Not connected to AWS;
}
--source create_database.inc
#
# Check options against amazon
#
set @save_s3_protocol_version=@@global.s3_protocol_version;
set @@global.s3_protocol_version="Original";
create table t1 (pk int primary key, a int);
insert into t1 values (1,1),(2,2),(3,3),(4,4);
--replace_result $database database
--error 3
alter table t1 engine=S3;
drop table t1;
#
# clean up
#
set @@global.s3_protocol_version=@save_s3_protocol_version;
--source drop_database.inc
...@@ -88,10 +88,12 @@ Variable_name Value ...@@ -88,10 +88,12 @@ Variable_name Value
s3_access_key X s3_access_key X
s3_block_size X s3_block_size X
s3_bucket X s3_bucket X
s3_host_name X
s3_pagecache_age_threshold X s3_pagecache_age_threshold X
s3_pagecache_buffer_size X s3_pagecache_buffer_size X
s3_pagecache_division_limit X s3_pagecache_division_limit X
s3_pagecache_file_hash_size X s3_pagecache_file_hash_size X
s3_protocol_version X
s3_region X s3_region X
s3_secret_key X s3_secret_key X
show status like "s3%"; show status like "s3%";
......
...@@ -3,3 +3,9 @@ ...@@ -3,3 +3,9 @@
[mysqld.1] [mysqld.1]
s3=ON s3=ON
#s3-host-name=s3.amazonaws.com
#s3-protocol-version=Amazon
#s3-bucket=MariaDB
#s3-access-key=...
#s3-secret-key=...
#s3-region=eu-north-1
...@@ -27,19 +27,23 @@ ...@@ -27,19 +27,23 @@
#include <libmarias3/marias3.h> #include <libmarias3/marias3.h>
#include "s3_func.h" #include "s3_func.h"
static const char *op_types[]= {"to_s3", "from_s3", "delete_from_s3", NullS};
static TYPELIB op_typelib= {array_elements(op_types)-1,"", op_types, NULL};
#define OP_IMPOSSIBLE array_elements(op_types)
static const char *load_default_groups[]= { "aria_s3_copy", 0 }; static const char *load_default_groups[]= { "aria_s3_copy", 0 };
static const char *opt_s3_access_key, *opt_s3_secret_key; static const char *opt_s3_access_key, *opt_s3_secret_key;
static const char *opt_s3_region="eu-north-1"; static const char *opt_s3_region="eu-north-1";
static const char *opt_s3_host_name= DEFAULT_AWS_HOST_NAME;
static const char *opt_database; static const char *opt_database;
static const char *opt_s3_bucket="MariaDB"; static const char *opt_s3_bucket="MariaDB";
static my_bool opt_compression, opt_verbose, opt_force, opt_s3_debug; static my_bool opt_compression, opt_verbose, opt_force, opt_s3_debug;
static int opt_operation= -1; static ulong opt_operation= OP_IMPOSSIBLE, opt_protocol_version= 1;
static ulong opt_block_size; static ulong opt_block_size;
static char **default_argv=0; static char **default_argv=0;
static const char *op_types[]= {"to_s3", "from_s3", "delete_from_s3", NullS};
static TYPELIB op_typelib= {array_elements(op_types)-1,"", op_types, NULL};
static ms3_st *global_s3_client= 0; static ms3_st *global_s3_client= 0;
static struct my_option my_long_options[] = static struct my_option my_long_options[] =
{ {
{"help", '?', "Display this help and exit.", 0, 0, 0, GET_NO_ARG, NO_ARG, 0, {"help", '?', "Display this help and exit.", 0, 0, 0, GET_NO_ARG, NO_ARG, 0,
...@@ -56,12 +60,15 @@ static struct my_option my_long_options[] = ...@@ -56,12 +60,15 @@ static struct my_option my_long_options[] =
{"s3_bucket", 'b', "AWS prefix for tables", {"s3_bucket", 'b', "AWS prefix for tables",
(char**) &opt_s3_bucket, (char**) &opt_s3_bucket, 0, (char**) &opt_s3_bucket, (char**) &opt_s3_bucket, 0,
GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
{"s3_host_name", 'h', "Host name to S3 provider",
(char**) &opt_s3_host_name, (char**) &opt_s3_host_name, 0,
GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
{"compress", 'c', "Use compression", &opt_compression, &opt_compression, {"compress", 'c', "Use compression", &opt_compression, &opt_compression,
0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
{"op", 'o', "Operation to excecute. One of 'from_s3', 'to_s3' or " {"op", 'o', "Operation to excecute. One of 'from_s3', 'to_s3' or "
"'delete_from_s3'", "'delete_from_s3'",
&opt_operation, &opt_operation, &op_typelib, &opt_operation, &opt_operation, &op_typelib,
GET_ENUM, REQUIRED_ARG, -1, 0, 0, 0, 0, 0}, GET_ENUM, REQUIRED_ARG, OP_IMPOSSIBLE, 0, 0, 0, 0, 0},
{"database", 'd', {"database", 'd',
"Database for copied table (second prefix). " "Database for copied table (second prefix). "
"If not given, the directory of the table file is used", "If not given, the directory of the table file is used",
...@@ -69,6 +76,10 @@ static struct my_option my_long_options[] = ...@@ -69,6 +76,10 @@ static struct my_option my_long_options[] =
{"s3_block_size", 'B', "Block size for data/index blocks in s3", {"s3_block_size", 'B', "Block size for data/index blocks in s3",
&opt_block_size, &opt_block_size, 0, GET_ULONG, REQUIRED_ARG, &opt_block_size, &opt_block_size, 0, GET_ULONG, REQUIRED_ARG,
4*1024*1024, 64*1024, 16*1024*1024, MALLOC_OVERHEAD, 1024, 0 }, 4*1024*1024, 64*1024, 16*1024*1024, MALLOC_OVERHEAD, 1024, 0 },
{"s3_protocol_version", 'L',
"Protocol used to communication with S3. One of \"Amazon\" or \"Original\".",
&opt_protocol_version, &opt_protocol_version, &s3_protocol_typelib,
GET_ENUM, REQUIRED_ARG, 1, 0, 0, 0, 0, 0},
{"force", 'f', "Force copy even if target exists", {"force", 'f', "Force copy even if target exists",
&opt_force, &opt_force, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, &opt_force, &opt_force, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
{"verbose", 'v', "Write more information", &opt_verbose, &opt_verbose, {"verbose", 'v', "Write more information", &opt_verbose, &opt_verbose,
...@@ -101,7 +112,7 @@ static void usage(void) ...@@ -101,7 +112,7 @@ static void usage(void)
" see the PUBLIC for details.\n"); " see the PUBLIC for details.\n");
puts("Copy an Aria table to and from s3"); puts("Copy an Aria table to and from s3");
printf("Usage: %s --aws-access-key=# --aws-secret-access-key=# --aws-region # " printf("Usage: %s --aws-access-key=# --aws-secret-access-key=# --aws-region # "
"--op=(from|to) [OPTIONS] tables[.MAI]\n", "--op=(from_s3 | to_s3 | delete_from_s3) [OPTIONS] tables[.MAI]\n",
my_progname_short); my_progname_short);
print_defaults("my", load_default_groups); print_defaults("my", load_default_groups);
puts(""); puts("");
...@@ -170,9 +181,9 @@ static void get_options(register int *argc,register char ***argv) ...@@ -170,9 +181,9 @@ static void get_options(register int *argc,register char ***argv)
fprintf(stderr, "--aws-secret-access-key was not given\n"); fprintf(stderr, "--aws-secret-access-key was not given\n");
my_exit(-1); my_exit(-1);
} }
if ((int) opt_operation == -1) if (opt_operation == OP_IMPOSSIBLE)
{ {
fprintf(stderr, "You must specify an operation with --op=[from|to]\n"); fprintf(stderr, "You must specify an operation with --op=[from_s3|to_s3|delete_from_s3]\n");
my_exit(-1); my_exit(-1);
} }
if (opt_s3_debug) if (opt_s3_debug)
...@@ -189,7 +200,7 @@ int main(int argc, char** argv) ...@@ -189,7 +200,7 @@ int main(int argc, char** argv)
s3_init_library(); s3_init_library();
if (!(global_s3_client= ms3_init(opt_s3_access_key, if (!(global_s3_client= ms3_init(opt_s3_access_key,
opt_s3_secret_key, opt_s3_secret_key,
opt_s3_region, NULL))) opt_s3_region, opt_s3_host_name)))
{ {
fprintf(stderr, "Can't open connection to S3, error: %d %s", errno, fprintf(stderr, "Can't open connection to S3, error: %d %s", errno,
ms3_error(errno)); ms3_error(errno));
...@@ -198,7 +209,10 @@ int main(int argc, char** argv) ...@@ -198,7 +209,10 @@ int main(int argc, char** argv)
{ {
size_t block_size= opt_block_size; size_t block_size= opt_block_size;
uint8_t protocol_version= (uint8_t) opt_protocol_version+1;
ms3_set_option(global_s3_client, MS3_OPT_BUFFER_CHUNK_SIZE, &block_size); ms3_set_option(global_s3_client, MS3_OPT_BUFFER_CHUNK_SIZE, &block_size);
ms3_set_option(global_s3_client, MS3_OPT_FORCE_PROTOCOL_VERSION,
&protocol_version);
} }
for (; *argv ; argv++) for (; *argv ; argv++)
......
...@@ -68,16 +68,18 @@ ...@@ -68,16 +68,18 @@
#include "s3_func.h" #include "s3_func.h"
#include "aria_backup.h" #include "aria_backup.h"
#define DEFAULT_AWS_HOST_NAME "s3.amazonaws.com"
static PAGECACHE s3_pagecache; static PAGECACHE s3_pagecache;
static ulong s3_block_size; static ulong s3_block_size, s3_protocol_version;
static ulong s3_pagecache_division_limit, s3_pagecache_age_threshold; static ulong s3_pagecache_division_limit, s3_pagecache_age_threshold;
static ulong s3_pagecache_file_hash_size; static ulong s3_pagecache_file_hash_size;
static ulonglong s3_pagecache_buffer_size; static ulonglong s3_pagecache_buffer_size;
static char *s3_bucket, *s3_access_key=0, *s3_secret_key=0, *s3_region; static char *s3_bucket, *s3_access_key=0, *s3_secret_key=0, *s3_region;
static char *s3_host_name;
static char *s3_tmp_access_key=0, *s3_tmp_secret_key=0; static char *s3_tmp_access_key=0, *s3_tmp_secret_key=0;
handlerton *s3_hton= 0; handlerton *s3_hton= 0;
/* Don't show access or secret keys to users if they exists */ /* Don't show access or secret keys to users if they exists */
static void update_access_key(MYSQL_THD thd, static void update_access_key(MYSQL_THD thd,
...@@ -115,6 +117,12 @@ static MYSQL_SYSVAR_ULONG(block_size, s3_block_size, ...@@ -115,6 +117,12 @@ static MYSQL_SYSVAR_ULONG(block_size, s3_block_size,
"Block size for S3", 0, 0, "Block size for S3", 0, 0,
4*1024*1024, 65536, 16*1024*1024, 8192); 4*1024*1024, 65536, 16*1024*1024, 8192);
static MYSQL_SYSVAR_ENUM(protocol_version, s3_protocol_version,
PLUGIN_VAR_RQCMDARG,
"Protocol used to communication with S3. One of "
"\"Amazon\" or \"Original\".",
NULL, NULL, 1, &s3_protocol_typelib);
static MYSQL_SYSVAR_ULONG(pagecache_age_threshold, static MYSQL_SYSVAR_ULONG(pagecache_age_threshold,
s3_pagecache_age_threshold, PLUGIN_VAR_RQCMDARG, s3_pagecache_age_threshold, PLUGIN_VAR_RQCMDARG,
"This characterizes the number of hits a hot block has to be untouched " "This characterizes the number of hits a hot block has to be untouched "
...@@ -148,6 +156,10 @@ static MYSQL_SYSVAR_STR(bucket, s3_bucket, ...@@ -148,6 +156,10 @@ static MYSQL_SYSVAR_STR(bucket, s3_bucket,
PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY,
"AWS bucket", "AWS bucket",
0, 0, "MariaDB"); 0, 0, "MariaDB");
static MYSQL_SYSVAR_STR(host_name, s3_host_name,
PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY,
"AWS bucket",
0, 0, DEFAULT_AWS_HOST_NAME);
static MYSQL_SYSVAR_STR(access_key, s3_tmp_access_key, static MYSQL_SYSVAR_STR(access_key, s3_tmp_access_key,
PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY | PLUGIN_VAR_MEMALLOC, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY | PLUGIN_VAR_MEMALLOC,
"AWS access key", "AWS access key",
...@@ -241,6 +253,8 @@ static my_bool s3_info_init(S3_INFO *info) ...@@ -241,6 +253,8 @@ static my_bool s3_info_init(S3_INFO *info)
{ {
if (!s3_usable()) if (!s3_usable())
return 1; return 1;
info->protocol_version= (uint8_t) s3_protocol_version+1;
lex_string_set(&info->host_name, s3_host_name);
lex_string_set(&info->access_key, s3_access_key); lex_string_set(&info->access_key, s3_access_key);
lex_string_set(&info->secret_key, s3_secret_key); lex_string_set(&info->secret_key, s3_secret_key);
lex_string_set(&info->region, s3_region); lex_string_set(&info->region, s3_region);
...@@ -702,10 +716,12 @@ static SHOW_VAR status_variables[]= { ...@@ -702,10 +716,12 @@ static SHOW_VAR status_variables[]= {
static struct st_mysql_sys_var* system_variables[]= { static struct st_mysql_sys_var* system_variables[]= {
MYSQL_SYSVAR(block_size), MYSQL_SYSVAR(block_size),
MYSQL_SYSVAR(protocol_version),
MYSQL_SYSVAR(pagecache_age_threshold), MYSQL_SYSVAR(pagecache_age_threshold),
MYSQL_SYSVAR(pagecache_buffer_size), MYSQL_SYSVAR(pagecache_buffer_size),
MYSQL_SYSVAR(pagecache_division_limit), MYSQL_SYSVAR(pagecache_division_limit),
MYSQL_SYSVAR(pagecache_file_hash_size), MYSQL_SYSVAR(pagecache_file_hash_size),
MYSQL_SYSVAR(host_name),
MYSQL_SYSVAR(bucket), MYSQL_SYSVAR(bucket),
MYSQL_SYSVAR(access_key), MYSQL_SYSVAR(access_key),
MYSQL_SYSVAR(secret_key), MYSQL_SYSVAR(secret_key),
......
...@@ -37,6 +37,11 @@ static void convert_frm_to_disk_format(uchar *header); ...@@ -37,6 +37,11 @@ static void convert_frm_to_disk_format(uchar *header);
static int s3_read_frm_from_disk(const char *filename, uchar **to, static int s3_read_frm_from_disk(const char *filename, uchar **to,
size_t *to_size); size_t *to_size);
/* Used by ha_s3.cc and tools to define different protocol options */
static const char *protocol_types[]= {"Original", "Amazon", NullS};
TYPELIB s3_protocol_typelib= {array_elements(protocol_types)-1,"",
protocol_types, NULL};
/****************************************************************************** /******************************************************************************
Allocations handler for libmarias3 Allocations handler for libmarias3
...@@ -137,13 +142,15 @@ ms3_st *s3_open_connection(S3_INFO *s3) ...@@ -137,13 +142,15 @@ ms3_st *s3_open_connection(S3_INFO *s3)
if (!(s3_client= ms3_init(s3->access_key.str, if (!(s3_client= ms3_init(s3->access_key.str,
s3->secret_key.str, s3->secret_key.str,
s3->region.str, s3->region.str,
NULL))) s3->host_name.str)))
{ {
my_printf_error(HA_ERR_NO_SUCH_TABLE, my_printf_error(HA_ERR_NO_SUCH_TABLE,
"Can't open connection to S3, error: %d %s", MYF(0), "Can't open connection to S3, error: %d %s", MYF(0),
errno, ms3_error(errno)); errno, ms3_error(errno));
my_errno= HA_ERR_NO_SUCH_TABLE; my_errno= HA_ERR_NO_SUCH_TABLE;
} }
ms3_set_option(s3_client, MS3_OPT_FORCE_PROTOCOL_VERSION,
&s3->protocol_version);
return s3_client; return s3_client;
} }
...@@ -282,11 +289,12 @@ int aria_copy_to_s3(ms3_st *s3_client, const char *aws_bucket, ...@@ -282,11 +289,12 @@ int aria_copy_to_s3(ms3_st *s3_client, const char *aws_bucket,
char filename[FN_REFLEN]; char filename[FN_REFLEN];
char *aws_path_end, *end; char *aws_path_end, *end;
uchar *alloc_block= 0, *block; uchar *alloc_block= 0, *block;
ms3_status_st status;
File file= -1; File file= -1;
my_off_t file_size; my_off_t file_size;
size_t frm_length; size_t frm_length;
int error; int error;
ms3_status_st status; my_bool frm_created= 0;
DBUG_ENTER("aria_copy_to_s3"); DBUG_ENTER("aria_copy_to_s3");
aws_path_end= strxmov(aws_path, database, "/", table_name, NullS); aws_path_end= strxmov(aws_path, database, "/", table_name, NullS);
...@@ -324,6 +332,7 @@ int aria_copy_to_s3(ms3_st *s3_client, const char *aws_bucket, ...@@ -324,6 +332,7 @@ int aria_copy_to_s3(ms3_st *s3_client, const char *aws_bucket,
0)) 0))
goto err; goto err;
frm_created= 1;
my_free(alloc_block); my_free(alloc_block);
alloc_block= 0; alloc_block= 0;
} }
...@@ -347,7 +356,7 @@ int aria_copy_to_s3(ms3_st *s3_client, const char *aws_bucket, ...@@ -347,7 +356,7 @@ int aria_copy_to_s3(ms3_st *s3_client, const char *aws_bucket,
{ {
fprintf(stderr, fprintf(stderr,
"Aria table %s doesn't match criteria to be copied to S3.\n" "Aria table %s doesn't match criteria to be copied to S3.\n"
"It should be non-transactional and should have row_format page", "It should be non-transactional and should have row_format page\n",
path); path);
goto err; goto err;
} }
...@@ -432,6 +441,11 @@ int aria_copy_to_s3(ms3_st *s3_client, const char *aws_bucket, ...@@ -432,6 +441,11 @@ int aria_copy_to_s3(ms3_st *s3_client, const char *aws_bucket,
DBUG_RETURN(0); DBUG_RETURN(0);
err: err:
if (frm_created)
{
end= strmov(aws_path_end,"/frm");
(void) s3_delete_object(s3_client, aws_bucket, aws_path, 0);
}
if (file >= 0) if (file >= 0)
my_close(file, MYF(0)); my_close(file, MYF(0));
my_free(alloc_block); my_free(alloc_block);
......
...@@ -23,17 +23,24 @@ ...@@ -23,17 +23,24 @@
C_MODE_START C_MODE_START
#include <libmarias3/marias3.h> #include <libmarias3/marias3.h>
#define DEFAULT_AWS_HOST_NAME "s3.amazonaws.com"
extern TYPELIB s3_protocol_typelib;
/* Store information about a s3 connection */ /* Store information about a s3 connection */
typedef struct s3_info typedef struct s3_info
{ {
LEX_CSTRING access_key, secret_key, region, bucket; LEX_CSTRING access_key, secret_key, region, bucket, host_name;
/* The following will be filled in by maria_open() */ /* The following will be filled in by maria_open() */
LEX_CSTRING database, table; LEX_CSTRING database, table;
/* Sent to open to verify version */ /* Sent to open to verify version */
LEX_CUSTRING tabledef_version; LEX_CUSTRING tabledef_version;
/* Protocol for the list bucket API call. 1 for Amazon, 2 for some others */
uint8_t protocol_version;
} S3_INFO; } S3_INFO;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment