Skip to content

M
mariadb
Commit f7002c05 authored by Jan Lindström's avatar Jan Lindström
Browse files
Options

MDEV-8250: InnoDB: Page compressed tables are not compressed and... 

MDEV-8250: InnoDB: Page compressed tables are not compressed and compressed+encrypted tables cause crash

Analysis: Problem is that both encrypted tables and compressed tables use
FIL header offset FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION to store
required metadata. Furhermore, for only compressed tables currently
code skips compression.

Fixes:
- Only encrypted pages store key_version to FIL header offset FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION,
  no need to fix
- Only compressed pages store compression algorithm to FIL header offset FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION,
  no need to fix as they have different page type FIL_PAGE_PAGE_COMPRESSED
- Compressed and encrypted pages now use a new page type FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED and
  key_version is stored on FIL header offset FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION and compression
  method is stored after FIL header similar way as compressed size, so that first
  FIL_PAGE_COMPRESSED_SIZE is stored followed by FIL_PAGE_COMPRESSION_METHOD
- Fix buf_page_encrypt_before_write function to really compress pages if compression is enabled
- Fix buf_page_decrypt_after_read function to really decompress pages if compression is used
- Small style fixes
parent cd70bed0
Hide whitespace changes
Inline Side-by-side
Showing with 569 additions and 409 deletions
mysql-test/suite/encryption/t/innodb-page_encryption_compression.test
View file @ f7002c05
...@@ -6,6 +6,7 @@ ...@@ -6,6 +6,7 @@
let $innodb_compression_algorithm_orig=`SELECT @@innodb_compression_algorithm`; let $innodb_compression_algorithm_orig=`SELECT @@innodb_compression_algorithm`;
let $innodb_file_format_orig = `SELECT @@innodb_file_format`; let $innodb_file_format_orig = `SELECT @@innodb_file_format`;
let $innodb_file_per_table_orig = `SELECT @@innodb_file_per_table`; let $innodb_file_per_table_orig = `SELECT @@innodb_file_per_table`;
let $encryption = `SELECT @@innodb_encrypt_tables`;
--enable_query_log --enable_query_log
SET GLOBAL innodb_file_format = `Barracuda`; SET GLOBAL innodb_file_format = `Barracuda`;
...@@ -124,4 +125,5 @@ drop table innodb_dynamic; ...@@ -124,4 +125,5 @@ drop table innodb_dynamic;
EVAL SET GLOBAL innodb_compression_algorithm = $innodb_compression_algorithm_orig; EVAL SET GLOBAL innodb_compression_algorithm = $innodb_compression_algorithm_orig;
EVAL SET GLOBAL innodb_file_per_table = $innodb_file_per_table_orig; EVAL SET GLOBAL innodb_file_per_table = $innodb_file_per_table_orig;
EVAL SET GLOBAL innodb_file_format = $innodb_file_format_orig; EVAL SET GLOBAL innodb_file_format = $innodb_file_format_orig;
EVAL SET GLOBAL innodb_encrypt_tables = $encryption;
--enable_query_log --enable_query_log
storage/innobase/buf/buf0buf.cc
View file @ f7002c05
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
Copyright (c) 1995, 2014, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 1995, 2014, Oracle and/or its affiliates. All Rights Reserved.
Copyright (c) 2008, Google Inc. Copyright (c) 2008, Google Inc.
Copyright (c) 2013, SkySQL Ab. All Rights Reserved. Copyright (c) 2013, 2015, MariaDB Corporation. All Rights Reserved.
Portions of this file contain modifications contributed and copyrighted by Portions of this file contain modifications contributed and copyrighted by
Google, Inc. Those modifications are gratefully acknowledged and are described Google, Inc. Those modifications are gratefully acknowledged and are described
...@@ -2453,13 +2453,26 @@ buf_block_align_instance( ...@@ -2453,13 +2453,26 @@ buf_block_align_instance(
ut_ad(page_get_page_no(page_align(ptr)) ut_ad(page_get_page_no(page_align(ptr))
== 0xffffffff); == 0xffffffff);
break; break;
case BUF_BLOCK_FILE_PAGE: case BUF_BLOCK_FILE_PAGE: {
ulint space = page_get_space_id(page_align(ptr));
ulint offset = page_get_page_no(page_align(ptr));
if (block->page.space != space ||
block->page.offset != offset) {
ib_logf(IB_LOG_LEVEL_ERROR,
"Corruption: Block space_id %lu != page space_id %lu or "
"Block offset %lu != page offset %lu",
(ulint)block->page.space, space,
(ulint)block->page.offset, offset);
}
ut_ad(block->page.space ut_ad(block->page.space
== page_get_space_id(page_align(ptr))); == page_get_space_id(page_align(ptr)));
ut_ad(block->page.offset ut_ad(block->page.offset
== page_get_page_no(page_align(ptr))); == page_get_page_no(page_align(ptr)));
break; break;
} }
}
mutex_exit(&block->mutex); mutex_exit(&block->mutex);
#endif /* UNIV_DEBUG */ #endif /* UNIV_DEBUG */
...@@ -5706,49 +5719,54 @@ buf_page_encrypt_before_write( ...@@ -5706,49 +5719,54 @@ buf_page_encrypt_before_write(
ulint space_id) /*!< in: space id */ ulint space_id) /*!< in: space id */
{ {
fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space_id); fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space_id);
ulint zip_size = buf_page_get_zip_size(bpage);
ulint page_size = (zip_size) ? zip_size : UNIV_PAGE_SIZE;
buf_pool_t* buf_pool = buf_pool_from_bpage(bpage);
bool page_compressed = fil_space_is_page_compressed(bpage->space);
bpage->real_size = UNIV_PAGE_SIZE; bpage->real_size = UNIV_PAGE_SIZE;
bool encrypted = true;
fil_page_type_validate(src_frame); fil_page_type_validate(src_frame);
if (crypt_data != NULL && crypt_data->encryption == FIL_SPACE_ENCRYPTION_OFF) { if (bpage->offset == 0) {
/* Encryption is disabled */ /* Page 0 of a tablespace is not encrypted/compressed */
ut_ad(bpage->key_version == 0);
return const_cast<byte*>(src_frame); return const_cast<byte*>(src_frame);
} }
if (!srv_encrypt_tables && (crypt_data == NULL || crypt_data->encryption == FIL_SPACE_ENCRYPTION_DEFAULT)) { if (bpage->space == TRX_SYS_SPACE && bpage->offset == TRX_SYS_PAGE_NO) {
/* Encryption is disabled */ /* don't encrypt/compress page as it contains address to dblwr buffer */
bpage->key_version = 0;
return const_cast<byte*>(src_frame); return const_cast<byte*>(src_frame);
} }
if (bpage->offset == 0) { if (crypt_data != NULL && crypt_data->encryption == FIL_SPACE_ENCRYPTION_OFF) {
/* Page 0 of a tablespace is not encrypted */ /* Encryption is disabled */
ut_ad(bpage->key_version == 0); encrypted = false;
return const_cast<byte*>(src_frame); }
if (!srv_encrypt_tables && (crypt_data == NULL || crypt_data->encryption == FIL_SPACE_ENCRYPTION_DEFAULT)) {
/* Encryption is disabled */
encrypted = false;
} }
/* Is encryption needed? */ /* Is encryption needed? */
if (crypt_data->type == CRYPT_SCHEME_UNENCRYPTED) { if (crypt_data == NULL || crypt_data->type == CRYPT_SCHEME_UNENCRYPTED) {
/* An unencrypted table */ /* An unencrypted table */
bpage->key_version = 0; bpage->key_version = 0;
return const_cast<byte*>(src_frame); encrypted = false;
} }
if (bpage->space == TRX_SYS_SPACE && bpage->offset == TRX_SYS_PAGE_NO) { if (!encrypted && !page_compressed) {
/* don't encrypt page as it contains address to dblwr buffer */ /* No need to encrypt or page compress the page */
bpage->key_version = 0;
return const_cast<byte*>(src_frame); return const_cast<byte*>(src_frame);
} }
ulint zip_size = buf_page_get_zip_size(bpage);
ulint page_size = (zip_size) ? zip_size : UNIV_PAGE_SIZE;
buf_pool_t* buf_pool = buf_pool_from_bpage(bpage);
bool page_compressed = fil_space_is_page_compressed(bpage->space);
/* Find free slot from temporary memory array */ /* Find free slot from temporary memory array */
buf_tmp_buffer_t* slot = buf_pool_reserve_tmp_slot(buf_pool, page_compressed); buf_tmp_buffer_t* slot = buf_pool_reserve_tmp_slot(buf_pool, page_compressed);
bpage->slot = slot; bpage->slot = slot;
byte *dst_frame = slot->crypt_buf; byte *dst_frame = bpage->slot->out_buf = slot->crypt_buf;
if (!page_compressed) { if (!page_compressed) {
/* Encrypt page content */ /* Encrypt page content */
...@@ -5778,6 +5796,7 @@ buf_page_encrypt_before_write( ...@@ -5778,6 +5796,7 @@ buf_page_encrypt_before_write(
page_size, page_size,
fil_space_get_page_compression_level(bpage->space), fil_space_get_page_compression_level(bpage->space),
block_size, block_size,
encrypted,
&out_len, &out_len,
IF_LZO(slot->lzo_mem, NULL) IF_LZO(slot->lzo_mem, NULL)
); );
...@@ -5785,14 +5804,18 @@ buf_page_encrypt_before_write( ...@@ -5785,14 +5804,18 @@ buf_page_encrypt_before_write(
bpage->real_size = out_len; bpage->real_size = out_len;
fil_page_type_validate(tmp); fil_page_type_validate(tmp);
if(encrypted) {
/* And then we encrypt the page content */ /* And then we encrypt the page content */
fil_space_encrypt(bpage->space, fil_space_encrypt(bpage->space,
bpage->offset, bpage->offset,
bpage->newest_modification, bpage->newest_modification,
tmp, tmp,
zip_size, zip_size,
dst_frame); dst_frame);
} else {
bpage->slot->out_buf = dst_frame = tmp;
}
} }
fil_page_type_validate(dst_frame); fil_page_type_validate(dst_frame);
...@@ -5809,13 +5832,6 @@ buf_page_decrypt_after_read( ...@@ -5809,13 +5832,6 @@ buf_page_decrypt_after_read(
/*========================*/ /*========================*/
buf_page_t* bpage) /*!< in/out: buffer page read from disk */ buf_page_t* bpage) /*!< in/out: buffer page read from disk */
{ {
ut_ad(bpage->key_version == 0);
if (bpage->offset == 0) {
/* File header pages are not encrypted */
return (TRUE);
}
ulint zip_size = buf_page_get_zip_size(bpage); ulint zip_size = buf_page_get_zip_size(bpage);
ulint size = (zip_size) ? zip_size : UNIV_PAGE_SIZE; ulint size = (zip_size) ? zip_size : UNIV_PAGE_SIZE;
...@@ -5823,52 +5839,61 @@ buf_page_decrypt_after_read( ...@@ -5823,52 +5839,61 @@ buf_page_decrypt_after_read(
((buf_block_t*) bpage)->frame; ((buf_block_t*) bpage)->frame;
unsigned key_version = unsigned key_version =
mach_read_from_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); mach_read_from_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
bool page_compressed = fil_page_is_compressed(dst_frame); bool page_compressed = fil_page_is_compressed(dst_frame);
bool page_compressed_encrypted = fil_page_is_compressed_encrypted(dst_frame);
buf_pool_t* buf_pool = buf_pool_from_bpage(bpage); buf_pool_t* buf_pool = buf_pool_from_bpage(bpage);
if (key_version == 0) { ut_ad(bpage->key_version == 0);
/* the page we read is unencrypted */
if (page_compressed) { if (bpage->offset == 0) {
/* Find free slot from temporary memory array */ /* File header pages are not encrypted/compressed */
buf_tmp_buffer_t* slot = buf_pool_reserve_tmp_slot(buf_pool, page_compressed); return (TRUE);
}
fil_page_type_validate(dst_frame); if (page_compressed) {
/* the page we read is unencrypted */
/* Find free slot from temporary memory array */
buf_tmp_buffer_t* slot = buf_pool_reserve_tmp_slot(buf_pool, page_compressed);
fil_decompress_page(slot->comp_buf, fil_page_type_validate(dst_frame);
dst_frame,
size,
&bpage->write_size);
/* Mark this slot as free */ fil_decompress_page(slot->comp_buf,
slot->reserved = false; dst_frame,
} size,
&bpage->write_size);
/* Mark this slot as free */
slot->reserved = false;
key_version = 0;
fil_page_type_validate(dst_frame); fil_page_type_validate(dst_frame);
} else { } else {
/* the page we read is encrypted */ buf_tmp_buffer_t* slot = NULL;
/* Find free slot from temporary memory array */
buf_tmp_buffer_t* slot = buf_pool_reserve_tmp_slot(buf_pool, page_compressed);
memcpy(slot->crypt_buf, dst_frame, size); if (key_version) {
/* Find free slot from temporary memory array */
fil_page_type_validate(dst_frame); slot = buf_pool_reserve_tmp_slot(buf_pool, page_compressed);
fil_page_type_validate(slot->crypt_buf); memcpy(slot->crypt_buf, dst_frame, size);
/* decrypt from crypt_buf to dst_frame */ fil_page_type_validate(dst_frame);
fil_space_decrypt(bpage->space, fil_page_type_validate(slot->crypt_buf);
slot->crypt_buf, /* decrypt from crypt_buf to dst_frame */
size, fil_space_decrypt(bpage->space,
dst_frame); slot->crypt_buf,
size,
dst_frame);
fil_page_type_validate(dst_frame);
fil_page_type_validate(slot->crypt_buf);
}
fil_page_type_validate(dst_frame); if (page_compressed_encrypted) {
fil_page_type_validate(slot->crypt_buf); if (!slot) {
slot = buf_pool_reserve_tmp_slot(buf_pool, page_compressed);
}
/* decompress from dst_frame to comp_buf and then copy to #ifdef UNIV_DEBUG
buffer pool */ fil_page_type_validate(dst_frame);
if (page_compressed) { #endif
fil_decompress_page(slot->comp_buf, fil_decompress_page(slot->comp_buf,
dst_frame, dst_frame,
size, size,
...@@ -5878,7 +5903,9 @@ buf_page_decrypt_after_read( ...@@ -5878,7 +5903,9 @@ buf_page_decrypt_after_read(
fil_page_type_validate(dst_frame); fil_page_type_validate(dst_frame);
/* Mark this slot as free */ /* Mark this slot as free */
slot->reserved = false; if (slot) {
slot->reserved = false;
}
} }
bpage->key_version = key_version; bpage->key_version = key_version;
......
storage/innobase/fil/fil0crypt.cc
View file @ f7002c05
...@@ -591,8 +591,7 @@ fil_space_encrypt( ...@@ -591,8 +591,7 @@ fil_space_encrypt(
ut_error; ut_error;
} }
ibool page_compressed = (mach_read_from_2(src_frame+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED); ibool page_compressed = (orig_page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED);
ulint page_comp_method = mach_read_from_8(src_frame+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
/* FIL page header is not encrypted */ /* FIL page header is not encrypted */
memcpy(dst_frame, src_frame, FIL_PAGE_DATA); memcpy(dst_frame, src_frame, FIL_PAGE_DATA);
...@@ -607,17 +606,6 @@ fil_space_encrypt( ...@@ -607,17 +606,6 @@ fil_space_encrypt(
byte* dst = dst_frame + FIL_PAGE_DATA; byte* dst = dst_frame + FIL_PAGE_DATA;
uint32 dstlen = 0; uint32 dstlen = 0;
/* For page compressed tables we encrypt only the actual compressed
payload. Note that first two bytes of page data is actual payload
size and that should not be encrypted. */
if (page_compressed) {
ulint payload = mach_read_from_2(src_frame + FIL_PAGE_DATA);
mach_write_to_2(dst_frame + FIL_PAGE_DATA, payload);
srclen = payload;
src+=2;
dst+=2;
}
int rc = encryption_scheme_encrypt(src, srclen, dst, &dstlen, int rc = encryption_scheme_encrypt(src, srclen, dst, &dstlen,
crypt_data, key_version, crypt_data, key_version,
space, offset, lsn); space, offset, lsn);
...@@ -641,43 +629,39 @@ fil_space_encrypt( ...@@ -641,43 +629,39 @@ fil_space_encrypt(
memcpy(dst_frame + page_size - FIL_PAGE_DATA_END, memcpy(dst_frame + page_size - FIL_PAGE_DATA_END,
src_frame + page_size - FIL_PAGE_DATA_END, src_frame + page_size - FIL_PAGE_DATA_END,
FIL_PAGE_DATA_END); FIL_PAGE_DATA_END);
}
/* handle post encryption checksum */ /* handle post encryption checksum */
ib_uint32_t checksum = 0; ib_uint32_t checksum = 0;
srv_checksum_algorithm_t algorithm = srv_checksum_algorithm_t algorithm =
static_cast<srv_checksum_algorithm_t>(srv_checksum_algorithm); static_cast<srv_checksum_algorithm_t>(srv_checksum_algorithm);
if (zip_size == 0) { if (zip_size == 0) {
switch (algorithm) { switch (algorithm) {
case SRV_CHECKSUM_ALGORITHM_CRC32: case SRV_CHECKSUM_ALGORITHM_CRC32:
case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32: case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32:
checksum = buf_calc_page_crc32(dst_frame); checksum = buf_calc_page_crc32(dst_frame);
break; break;
case SRV_CHECKSUM_ALGORITHM_INNODB: case SRV_CHECKSUM_ALGORITHM_INNODB:
case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB: case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB:
checksum = (ib_uint32_t) buf_calc_page_new_checksum( checksum = (ib_uint32_t) buf_calc_page_new_checksum(
dst_frame); dst_frame);
break; break;
case SRV_CHECKSUM_ALGORITHM_NONE: case SRV_CHECKSUM_ALGORITHM_NONE:
case SRV_CHECKSUM_ALGORITHM_STRICT_NONE: case SRV_CHECKSUM_ALGORITHM_STRICT_NONE:
checksum = BUF_NO_CHECKSUM_MAGIC; checksum = BUF_NO_CHECKSUM_MAGIC;
break; break;
/* no default so the compiler will emit a warning /* no default so the compiler will emit a warning
* if new enum is added and not handled here */ * if new enum is added and not handled here */
}
} else {
checksum = page_zip_calc_checksum(dst_frame, zip_size,
algorithm);
} }
// store the post-encryption checksum after the key-version
mach_write_to_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4,
checksum);
} else { } else {
mach_write_to_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4, checksum = page_zip_calc_checksum(dst_frame, zip_size,
page_comp_method); algorithm);
} }
// store the post-encryption checksum after the key-version
mach_write_to_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4, checksum);
srv_stats.pages_encrypted.inc(); srv_stats.pages_encrypted.inc();
} }
...@@ -721,8 +705,7 @@ fil_space_decrypt( ...@@ -721,8 +705,7 @@ fil_space_decrypt(
{ {
ulint page_type = mach_read_from_2(src_frame+FIL_PAGE_TYPE); ulint page_type = mach_read_from_2(src_frame+FIL_PAGE_TYPE);
uint key_version = mach_read_from_4(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); uint key_version = mach_read_from_4(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
bool page_compressed = (page_type == FIL_PAGE_PAGE_COMPRESSED); bool page_compressed = (page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED);
ulint page_comp_method = mach_read_from_4(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4);
if (key_version == ENCRYPTION_KEY_NOT_ENCRYPTED) { if (key_version == ENCRYPTION_KEY_NOT_ENCRYPTED) {
//TODO: is this really needed ? //TODO: is this really needed ?
...@@ -748,17 +731,6 @@ fil_space_decrypt( ...@@ -748,17 +731,6 @@ fil_space_decrypt(
uint32 dstlen = 0; uint32 dstlen = 0;
ulint srclen = page_size - (FIL_PAGE_DATA + FIL_PAGE_DATA_END); ulint srclen = page_size - (FIL_PAGE_DATA + FIL_PAGE_DATA_END);
/* For page compressed tables we decrypt only the actual compressed
payload. Note that first two bytes of page data is actual payload
size and that should not be decrypted. */
if (page_compressed) {
ulint compressed_len = mach_read_from_2(src_frame + FIL_PAGE_DATA);
src+=2;
dst+=2;
mach_write_to_2(dst_frame + FIL_PAGE_DATA, compressed_len);
srclen = compressed_len;
}
int rc = encryption_scheme_decrypt(src, srclen, dst, &dstlen, int rc = encryption_scheme_decrypt(src, srclen, dst, &dstlen,
crypt_data, key_version, crypt_data, key_version,
space, offset, lsn); space, offset, lsn);
...@@ -785,8 +757,6 @@ fil_space_decrypt( ...@@ -785,8 +757,6 @@ fil_space_decrypt(
// clear key-version & crypt-checksum from dst // clear key-version & crypt-checksum from dst
memset(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, 0, 8); memset(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, 0, 8);
} else {
mach_write_to_8(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, page_comp_method);
} }
srv_stats.pages_decrypted.inc(); srv_stats.pages_decrypted.inc();
......
storage/innobase/fil/fil0pagecompress.cc
View file @ f7002c05
...@@ -87,29 +87,35 @@ static ulint srv_data_read, srv_data_written; ...@@ -87,29 +87,35 @@ static ulint srv_data_read, srv_data_written;
For page compressed pages compress the page before actual write For page compressed pages compress the page before actual write
operation. operation.
@return compressed page to be written*/ @return compressed page to be written*/
UNIV_INTERN
byte* byte*
fil_compress_page( fil_compress_page(
/*==============*/ /*==============*/
ulint space_id, /*!< in: tablespace id of the ulint space_id, /*!< in: tablespace id of the
table. */ table. */
byte* buf, /*!< in: buffer from which to write; in aio byte* buf, /*!< in: buffer from which to write; in aio
this must be appropriately aligned */ this must be appropriately aligned */
byte* out_buf, /*!< out: compressed buffer */ byte* out_buf, /*!< out: compressed buffer */
ulint len, /*!< in: length of input buffer.*/ ulint len, /*!< in: length of input buffer.*/
ulint compression_level, /* in: compression level */ ulint level, /* in: compression level */
ulint block_size, /*!< in: block size */ ulint block_size, /*!< in: block size */
ulint* out_len, /*!< out: actual length of compressed bool encrypted, /*!< in: is page also encrypted */
page */ ulint* out_len, /*!< out: actual length of compressed
byte* lzo_mem) /*!< in: temporal memory used by LZO */ page */
byte* lzo_mem) /*!< in: temporal memory used by LZO */
{ {
int err = Z_OK; int err = Z_OK;
int level = 0; int comp_level = level;
ulint header_len = FIL_PAGE_DATA + FIL_PAGE_COMPRESSED_SIZE; ulint header_len = FIL_PAGE_DATA + FIL_PAGE_COMPRESSED_SIZE;
ulint write_size=0; ulint write_size=0;
/* Cache to avoid change during function execution */ /* Cache to avoid change during function execution */
ulint comp_method = innodb_compression_algorithm; ulint comp_method = innodb_compression_algorithm;
ulint orig_page_type; ulint orig_page_type;
if (encrypted) {
header_len += FIL_PAGE_COMPRESSION_METHOD_SIZE;
}
ut_ad(buf); ut_ad(buf);
ut_ad(out_buf); ut_ad(out_buf);
ut_ad(len); ut_ad(len);
...@@ -128,17 +134,14 @@ fil_compress_page( ...@@ -128,17 +134,14 @@ fil_compress_page(
return (buf); return (buf);
} }
level = compression_level;
ut_ad(fil_space_is_page_compressed(space_id));
fil_system_enter(); fil_system_enter();
fil_space_t* space = fil_space_get_by_id(space_id); fil_space_t* space = fil_space_get_by_id(space_id);
fil_system_exit(); fil_system_exit();
/* If no compression level was provided to this table, use system /* If no compression level was provided to this table, use system
default level */ default level */
if (level == 0) { if (comp_level == 0) {
level = page_zip_level; comp_level = page_zip_level;
} }
#ifdef UNIV_PAGECOMPRESS_DEBUG #ifdef UNIV_PAGECOMPRESS_DEBUG
...@@ -159,13 +162,16 @@ fil_compress_page( ...@@ -159,13 +162,16 @@ fil_compress_page(
if (err == 0) { if (err == 0) {
/* If error we leave the actual page as it was */ /* If error we leave the actual page as it was */
#ifndef UNIV_PAGECOMPRESS_DEBUG
if (space->printed_compression_failure == false) { if (space->printed_compression_failure == false) {
#endif
ib_logf(IB_LOG_LEVEL_WARN, ib_logf(IB_LOG_LEVEL_WARN,
"Compression failed for space %lu name %s len %lu rt %d write %lu.", "Compression failed for space %lu name %s len %lu rt %d write %lu.",
space_id, fil_space_name(space), len, err, write_size); space_id, fil_space_name(space), len, err, write_size);
space->printed_compression_failure = true; space->printed_compression_failure = true;
#ifndef UNIV_PAGECOMPRESS_DEBUG
} }
#endif
srv_stats.pages_page_compression_error.inc(); srv_stats.pages_page_compression_error.inc();
*out_len = len; *out_len = len;
return (buf); return (buf);
...@@ -196,7 +202,7 @@ fil_compress_page( ...@@ -196,7 +202,7 @@ fil_compress_page(
size_t out_pos=0; size_t out_pos=0;
err = lzma_easy_buffer_encode( err = lzma_easy_buffer_encode(
compression_level, comp_level,
LZMA_CHECK_NONE, LZMA_CHECK_NONE,
NULL, /* No custom allocator, use malloc/free */ NULL, /* No custom allocator, use malloc/free */
reinterpret_cast<uint8_t*>(buf), reinterpret_cast<uint8_t*>(buf),
...@@ -212,7 +218,6 @@ fil_compress_page( ...@@ -212,7 +218,6 @@ fil_compress_page(
space_id, fil_space_name(space), len, err, out_pos); space_id, fil_space_name(space), len, err, out_pos);
space->printed_compression_failure = true; space->printed_compression_failure = true;
} }
srv_stats.pages_page_compression_error.inc(); srv_stats.pages_page_compression_error.inc();
*out_len = len; *out_len = len;
return (buf); return (buf);
...@@ -274,7 +279,7 @@ fil_compress_page( ...@@ -274,7 +279,7 @@ fil_compress_page(
#endif /* HAVE_SNAPPY */ #endif /* HAVE_SNAPPY */
case PAGE_ZLIB_ALGORITHM: case PAGE_ZLIB_ALGORITHM:
err = compress2(out_buf+header_len, (ulong*)&write_size, buf, len, level); err = compress2(out_buf+header_len, (ulong*)&write_size, buf, len, comp_level);
if (err != Z_OK) { if (err != Z_OK) {
/* If error we leave the actual page as it was */ /* If error we leave the actual page as it was */
...@@ -296,7 +301,6 @@ fil_compress_page( ...@@ -296,7 +301,6 @@ fil_compress_page(
*out_len = len; *out_len = len;
return (buf); return (buf);
break; break;
default: default:
ut_error; ut_error;
break; break;
...@@ -306,19 +310,29 @@ fil_compress_page( ...@@ -306,19 +310,29 @@ fil_compress_page(
memcpy(out_buf, buf, FIL_PAGE_DATA); memcpy(out_buf, buf, FIL_PAGE_DATA);
/* Set up the checksum */ /* Set up the checksum */
mach_write_to_4(out_buf+FIL_PAGE_SPACE_OR_CHKSUM, BUF_NO_CHECKSUM_MAGIC); mach_write_to_4(out_buf+FIL_PAGE_SPACE_OR_CHKSUM, BUF_NO_CHECKSUM_MAGIC);
/* Set up the correct page type */
mach_write_to_2(out_buf+FIL_PAGE_TYPE, FIL_PAGE_PAGE_COMPRESSED); /* Set up the compression algorithm */
/* Set up the flush lsn to be compression algorithm */
mach_write_to_8(out_buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, comp_method); mach_write_to_8(out_buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, comp_method);
if (encrypted) {
/* Set up the correct page type */
mach_write_to_2(out_buf+FIL_PAGE_TYPE, FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED);
mach_write_to_2(out_buf+FIL_PAGE_DATA+FIL_PAGE_COMPRESSED_SIZE, comp_method);
} else {
/* Set up the correct page type */
mach_write_to_2(out_buf+FIL_PAGE_TYPE, FIL_PAGE_PAGE_COMPRESSED);
}
/* Set up the actual payload lenght */ /* Set up the actual payload lenght */
mach_write_to_2(out_buf+FIL_PAGE_DATA, write_size); mach_write_to_2(out_buf+FIL_PAGE_DATA, write_size);
#ifdef UNIV_DEBUG #ifdef UNIV_DEBUG
/* Verify */ /* Verify */
ut_ad(fil_page_is_compressed(out_buf)); ut_ad(fil_page_is_compressed(out_buf) || fil_page_is_compressed_encrypted(out_buf));
ut_ad(mach_read_from_4(out_buf+FIL_PAGE_SPACE_OR_CHKSUM) == BUF_NO_CHECKSUM_MAGIC); ut_ad(mach_read_from_4(out_buf+FIL_PAGE_SPACE_OR_CHKSUM) == BUF_NO_CHECKSUM_MAGIC);
ut_ad(mach_read_from_2(out_buf+FIL_PAGE_DATA) == write_size); ut_ad(mach_read_from_2(out_buf+FIL_PAGE_DATA) == write_size);
ut_ad(mach_read_from_8(out_buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION) == (ulint)comp_method); ut_ad(mach_read_from_8(out_buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION) == (ulint)comp_method ||
mach_read_from_2(out_buf+FIL_PAGE_DATA+FIL_PAGE_COMPRESSED_SIZE) == (ulint)comp_method);
/* Verify that page can be decompressed */ /* Verify that page can be decompressed */
{ {
...@@ -360,7 +374,6 @@ fil_compress_page( ...@@ -360,7 +374,6 @@ fil_compress_page(
space_id, fil_space_name(space), len, write_size); space_id, fil_space_name(space), len, write_size);
#endif /* UNIV_PAGECOMPRESS_DEBUG */ #endif /* UNIV_PAGECOMPRESS_DEBUG */
srv_stats.page_compression_saved.add((len - write_size)); srv_stats.page_compression_saved.add((len - write_size));
srv_stats.pages_page_compressed.inc(); srv_stats.pages_page_compressed.inc();
...@@ -379,29 +392,37 @@ fil_compress_page( ...@@ -379,29 +392,37 @@ fil_compress_page(
/****************************************************************//** /****************************************************************//**
For page compressed pages decompress the page after actual read For page compressed pages decompress the page after actual read
operation. */ operation. */
UNIV_INTERN
void void
fil_decompress_page( fil_decompress_page(
/*================*/ /*================*/
byte* page_buf, /*!< in: preallocated buffer or NULL */ byte* page_buf, /*!< in: preallocated buffer or NULL */
byte* buf, /*!< out: buffer from which to read; in aio byte* buf, /*!< out: buffer from which to read; in aio
this must be appropriately aligned */ this must be appropriately aligned */
ulong len, /*!< in: length of output buffer.*/ ulong len, /*!< in: length of output buffer.*/
ulint* write_size) /*!< in/out: Actual payload size of ulint* write_size) /*!< in/out: Actual payload size of
the compressed data. */ the compressed data. */
{ {
int err = 0; int err = 0;
ulint actual_size = 0; ulint actual_size = 0;
ulint compression_alg = 0; ulint compression_alg = 0;
byte *in_buf; byte *in_buf;
ulint ptype; ulint ptype;
ulint header_len = FIL_PAGE_DATA + FIL_PAGE_COMPRESSED_SIZE;
ut_ad(buf); ut_ad(buf);
ut_ad(len); ut_ad(len);
ptype = mach_read_from_2(buf+FIL_PAGE_TYPE); ptype = mach_read_from_2(buf+FIL_PAGE_TYPE);
if (ptype == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) {
header_len += FIL_PAGE_COMPRESSION_METHOD_SIZE;
}
/* Do not try to uncompressed pages that are not compressed */ /* Do not try to uncompressed pages that are not compressed */
if (ptype != FIL_PAGE_PAGE_COMPRESSED && ptype != FIL_PAGE_TYPE_COMPRESSED) { if (ptype != FIL_PAGE_PAGE_COMPRESSED &&
ptype != FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED &&
ptype != FIL_PAGE_TYPE_COMPRESSED) {
return; return;
} }
...@@ -415,7 +436,8 @@ fil_decompress_page( ...@@ -415,7 +436,8 @@ fil_decompress_page(
/* Before actual decompress, make sure that page type is correct */ /* Before actual decompress, make sure that page type is correct */
if (mach_read_from_4(buf+FIL_PAGE_SPACE_OR_CHKSUM) != BUF_NO_CHECKSUM_MAGIC || if (mach_read_from_4(buf+FIL_PAGE_SPACE_OR_CHKSUM) != BUF_NO_CHECKSUM_MAGIC ||
mach_read_from_2(buf+FIL_PAGE_TYPE) != FIL_PAGE_PAGE_COMPRESSED) { (ptype != FIL_PAGE_PAGE_COMPRESSED &&
ptype != FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED)) {
ib_logf(IB_LOG_LEVEL_ERROR, ib_logf(IB_LOG_LEVEL_ERROR,
"Corruption: We try to uncompress corrupted page" "Corruption: We try to uncompress corrupted page"
" CRC %lu type %lu len %lu.", " CRC %lu type %lu len %lu.",
...@@ -427,7 +449,11 @@ fil_decompress_page( ...@@ -427,7 +449,11 @@ fil_decompress_page(
} }
/* Get compression algorithm */ /* Get compression algorithm */
compression_alg = mach_read_from_8(buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); if (ptype == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) {
compression_alg = mach_read_from_2(buf+FIL_PAGE_DATA+FIL_PAGE_COMPRESSED_SIZE);
} else {
compression_alg = mach_read_from_8(buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
}
/* Get the actual size of compressed page */ /* Get the actual size of compressed page */
actual_size = mach_read_from_2(buf+FIL_PAGE_DATA); actual_size = mach_read_from_2(buf+FIL_PAGE_DATA);
...@@ -456,7 +482,7 @@ fil_decompress_page( ...@@ -456,7 +482,7 @@ fil_decompress_page(
switch(compression_alg) { switch(compression_alg) {
case PAGE_ZLIB_ALGORITHM: case PAGE_ZLIB_ALGORITHM:
err= uncompress(in_buf, &len, buf+FIL_PAGE_DATA+FIL_PAGE_COMPRESSED_SIZE, (unsigned long)actual_size); err= uncompress(in_buf, &len, buf+header_len, (unsigned long)actual_size);
/* If uncompress fails it means that page is corrupted */ /* If uncompress fails it means that page is corrupted */
if (err != Z_OK) { if (err != Z_OK) {
...@@ -475,7 +501,7 @@ fil_decompress_page( ...@@ -475,7 +501,7 @@ fil_decompress_page(
#ifdef HAVE_LZ4 #ifdef HAVE_LZ4
case PAGE_LZ4_ALGORITHM: case PAGE_LZ4_ALGORITHM:
err = LZ4_decompress_fast((const char *)buf+FIL_PAGE_DATA+FIL_PAGE_COMPRESSED_SIZE, (char *)in_buf, len); err = LZ4_decompress_fast((const char *)buf+header_len, (char *)in_buf, len);
if (err != (int)actual_size) { if (err != (int)actual_size) {
ib_logf(IB_LOG_LEVEL_ERROR, ib_logf(IB_LOG_LEVEL_ERROR,
...@@ -490,10 +516,9 @@ fil_decompress_page( ...@@ -490,10 +516,9 @@ fil_decompress_page(
break; break;
#endif /* HAVE_LZ4 */ #endif /* HAVE_LZ4 */
#ifdef HAVE_LZO #ifdef HAVE_LZO
case PAGE_LZO_ALGORITHM: case PAGE_LZO_ALGORITHM: {
{ ulint olen=0;
ulint olen=0; err = lzo1x_decompress((const unsigned char *)buf+header_len,
err = lzo1x_decompress((const unsigned char *)buf+FIL_PAGE_DATA+FIL_PAGE_COMPRESSED_SIZE,
actual_size,(unsigned char *)in_buf, &olen, NULL); actual_size,(unsigned char *)in_buf, &olen, NULL);
if (err != LZO_E_OK || (olen == 0 || olen > UNIV_PAGE_SIZE)) { if (err != LZO_E_OK || (olen == 0 || olen > UNIV_PAGE_SIZE)) {
...@@ -521,7 +546,7 @@ fil_decompress_page( ...@@ -521,7 +546,7 @@ fil_decompress_page(
&memlimit, &memlimit,
0, 0,
NULL, NULL,
buf+FIL_PAGE_DATA+FIL_PAGE_COMPRESSED_SIZE, buf+header_len,
&src_pos, &src_pos,
actual_size, actual_size,
in_buf, in_buf,
...@@ -550,7 +575,7 @@ fil_decompress_page( ...@@ -550,7 +575,7 @@ fil_decompress_page(
err = BZ2_bzBuffToBuffDecompress( err = BZ2_bzBuffToBuffDecompress(
(char *)in_buf, (char *)in_buf,
&dst_pos, &dst_pos,
(char *)(buf+FIL_PAGE_DATA+FIL_PAGE_COMPRESSED_SIZE), (char *)(buf+header_len),
actual_size, actual_size,
1, 1,
0); 0);
...@@ -575,7 +600,7 @@ fil_decompress_page( ...@@ -575,7 +600,7 @@ fil_decompress_page(
ulint olen = 0; ulint olen = 0;
cstatus = snappy_uncompress( cstatus = snappy_uncompress(
(const char *)(buf+FIL_PAGE_DATA+FIL_PAGE_COMPRESSED_SIZE), (const char *)(buf+header_len),
actual_size, actual_size,
(char *)in_buf, (char *)in_buf,
&olen); &olen);
......
storage/innobase/handler/ha_innodb.cc
View file @ f7002c05
...@@ -20042,6 +20042,7 @@ innodb_compression_algorithm_validate( ...@@ -20042,6 +20042,7 @@ innodb_compression_algorithm_validate(
} }
compression_algorithm = *reinterpret_cast<ulong*>(save); compression_algorithm = *reinterpret_cast<ulong*>(save);
(void)compression_algorithm;
#ifndef HAVE_LZ4 #ifndef HAVE_LZ4
if (compression_algorithm == PAGE_LZ4_ALGORITHM) { if (compression_algorithm == PAGE_LZ4_ALGORITHM) {
......
storage/innobase/include/buf0buf.h
View file @ f7002c05
...@@ -1506,6 +1506,9 @@ typedef struct { ...@@ -1506,6 +1506,9 @@ typedef struct {
can be read while it's being flushed */ can be read while it's being flushed */
byte* comp_buf_free; /*!< for compression, allocated byte* comp_buf_free; /*!< for compression, allocated
buffer that is then alligned */ buffer that is then alligned */
byte* out_buf; /*!< resulting buffer after
encryption/compression. This is a
pointer and not allocated. */
} buf_tmp_buffer_t; } buf_tmp_buffer_t;
/** The common buffer control block structure /** The common buffer control block structure
......
storage/innobase/include/buf0buf.ic
View file @ f7002c05
...@@ -2,6 +2,7 @@ ...@@ -2,6 +2,7 @@
Copyright (c) 1995, 2014, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 1995, 2014, Oracle and/or its affiliates. All Rights Reserved.
Copyright (c) 2008, Google Inc. Copyright (c) 2008, Google Inc.
Copyright (c) 2014, 2015, MariaDB Corporation.
Portions of this file contain modifications contributed and copyrighted by Portions of this file contain modifications contributed and copyrighted by
Google, Inc. Those modifications are gratefully acknowledged and are described Google, Inc. Those modifications are gratefully acknowledged and are described
...@@ -1465,8 +1466,10 @@ buf_page_get_frame( ...@@ -1465,8 +1466,10 @@ buf_page_get_frame(
/*===============*/ /*===============*/
const buf_page_t* bpage) /*!< in: buffer pool page */ const buf_page_t* bpage) /*!< in: buffer pool page */
{ {
if (bpage->slot && bpage->slot->crypt_buf) { /* In encryption/compression buffer pool page may contain extra
return bpage->slot->crypt_buf; buffer where result is stored. */
if (bpage->slot && bpage->slot->out_buf) {
return bpage->slot->out_buf;
} else if (bpage->zip.data) { } else if (bpage->zip.data) {
return bpage->zip.data; return bpage->zip.data;
} else { } else {
......
storage/innobase/include/fil0fil.h
View file @ f7002c05
...@@ -142,8 +142,11 @@ extern fil_addr_t fil_addr_null; ...@@ -142,8 +142,11 @@ extern fil_addr_t fil_addr_null;
#define FIL_PAGE_DATA 38 /*!< start of the data on the page */ #define FIL_PAGE_DATA 38 /*!< start of the data on the page */
/* Following are used when page compression is used */ /* Following are used when page compression is used */
#define FIL_PAGE_COMPRESSED_SIZE 2 /*!< Number of bytes used to store #define FIL_PAGE_COMPRESSED_SIZE 2 /*!< Number of bytes used to store
actual payload data size on actual payload data size on
compressed pages. */ compressed pages. */
#define FIL_PAGE_COMPRESSION_METHOD_SIZE 2
/*!< Number of bytes used to store
actual compression method. */
/* @} */ /* @} */
/** File page trailer @{ */ /** File page trailer @{ */
#define FIL_PAGE_END_LSN_OLD_CHKSUM 8 /*!< the low 4 bytes of this are used #define FIL_PAGE_END_LSN_OLD_CHKSUM 8 /*!< the low 4 bytes of this are used
...@@ -154,6 +157,8 @@ extern fil_addr_t fil_addr_null; ...@@ -154,6 +157,8 @@ extern fil_addr_t fil_addr_null;
/* @} */ /* @} */
/** File page types (values of FIL_PAGE_TYPE) @{ */ /** File page types (values of FIL_PAGE_TYPE) @{ */
#define FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED 37401 /*!< Page is compressed and
then encrypted */
#define FIL_PAGE_PAGE_COMPRESSED 34354 /*!< page compressed page */ #define FIL_PAGE_PAGE_COMPRESSED 34354 /*!< page compressed page */
#define FIL_PAGE_INDEX 17855 /*!< B-tree node */ #define FIL_PAGE_INDEX 17855 /*!< B-tree node */
#define FIL_PAGE_UNDO_LOG 2 /*!< Undo log page */ #define FIL_PAGE_UNDO_LOG 2 /*!< Undo log page */
......
storage/innobase/include/fil0fil.ic
View file @ f7002c05
...@@ -118,6 +118,7 @@ fil_page_type_validate( ...@@ -118,6 +118,7 @@ fil_page_type_validate(
#ifdef UNIV_ENCRYPTION_EXTRA_DEBUG #ifdef UNIV_ENCRYPTION_EXTRA_DEBUG
uint key_version = mach_read_from_4(page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); uint key_version = mach_read_from_4(page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
bool page_compressed = (page_type == FIL_PAGE_PAGE_COMPRESSED); bool page_compressed = (page_type == FIL_PAGE_PAGE_COMPRESSED);
bool page_compressed_encrypted = (page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED);
ulint space = mach_read_from_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); ulint space = mach_read_from_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID);
ulint offset = mach_read_from_4(page + FIL_PAGE_OFFSET); ulint offset = mach_read_from_4(page + FIL_PAGE_OFFSET);
ib_uint64_t lsn = mach_read_from_8(page + FIL_PAGE_LSN); ib_uint64_t lsn = mach_read_from_8(page + FIL_PAGE_LSN);
...@@ -136,6 +137,7 @@ fil_page_type_validate( ...@@ -136,6 +137,7 @@ fil_page_type_validate(
/* Validate page type */ /* Validate page type */
if (!((page_type == FIL_PAGE_PAGE_COMPRESSED || if (!((page_type == FIL_PAGE_PAGE_COMPRESSED ||
page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED ||
page_type == FIL_PAGE_INDEX || page_type == FIL_PAGE_INDEX ||
page_type == FIL_PAGE_UNDO_LOG || page_type == FIL_PAGE_UNDO_LOG ||
page_type == FIL_PAGE_INODE || page_type == FIL_PAGE_INODE ||
...@@ -151,6 +153,7 @@ fil_page_type_validate( ...@@ -151,6 +153,7 @@ fil_page_type_validate(
page_type == FIL_PAGE_TYPE_COMPRESSED))) { page_type == FIL_PAGE_TYPE_COMPRESSED))) {
ut_ad(page_type == FIL_PAGE_PAGE_COMPRESSED || ut_ad(page_type == FIL_PAGE_PAGE_COMPRESSED ||
page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED ||
page_type == FIL_PAGE_INDEX || page_type == FIL_PAGE_INDEX ||
page_type == FIL_PAGE_UNDO_LOG || page_type == FIL_PAGE_UNDO_LOG ||
page_type == FIL_PAGE_INODE || page_type == FIL_PAGE_INODE ||
......
storage/innobase/include/fil0pagecompress.h
View file @ f7002c05
/***************************************************************************** /*****************************************************************************
Copyright (C) 2013, 2014 SkySQL Ab. All Rights Reserved. Copyright (C) 2013, 2015 MariaDB Corporation. All Rights Reserved.
This program is free software; you can redistribute it and/or modify it under This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software the terms of the GNU General Public License as published by the Free Software
...@@ -34,6 +34,7 @@ Created 11/12/2013 Jan Lindström jan.lindstrom@skysql.com ...@@ -34,6 +34,7 @@ Created 11/12/2013 Jan Lindström jan.lindstrom@skysql.com
Returns the page compression level flag of the space, or 0 if the space Returns the page compression level flag of the space, or 0 if the space
is not compressed. The tablespace must be cached in the memory cache. is not compressed. The tablespace must be cached in the memory cache.
@return page compression level if page compressed, ULINT_UNDEFINED if space not found */ @return page compression level if page compressed, ULINT_UNDEFINED if space not found */
UNIV_INTERN
ulint ulint
fil_space_get_page_compression_level( fil_space_get_page_compression_level(
/*=================================*/ /*=================================*/
...@@ -42,14 +43,16 @@ fil_space_get_page_compression_level( ...@@ -42,14 +43,16 @@ fil_space_get_page_compression_level(
Returns the page compression flag of the space, or false if the space Returns the page compression flag of the space, or false if the space
is not compressed. The tablespace must be cached in the memory cache. is not compressed. The tablespace must be cached in the memory cache.
@return true if page compressed, false if not or space not found */ @return true if page compressed, false if not or space not found */
UNIV_INTERN
ibool ibool
fil_space_is_page_compressed( fil_space_is_page_compressed(
/*=========================*/ /*=========================*/
ulint id); /*!< in: space id */ ulint id); /*!< in: space id */
/*******************************************************************//** /*******************************************************************//**
Returns the page compression flag of the space, or false if the space Returns the page compression flag of the space, or false if the space
is not compressed. The tablespace must be cached in the memory cache. is not compressed. The tablespace must be cached in the memory cache.
@return true if page compressed, false if not or space not found */ @return true if page compressed, false if not or space not found */
UNIV_INTERN
ibool ibool
fil_space_get_page_compressed( fil_space_get_page_compressed(
/*=========================*/ /*=========================*/
...@@ -58,88 +61,106 @@ fil_space_get_page_compressed( ...@@ -58,88 +61,106 @@ fil_space_get_page_compressed(
Returns the atomic writes flag of the space, or false if the space Returns the atomic writes flag of the space, or false if the space
is not using atomic writes. The tablespace must be cached in the memory cache. is not using atomic writes. The tablespace must be cached in the memory cache.
@return atomic write table option value */ @return atomic write table option value */
UNIV_INTERN
atomic_writes_t atomic_writes_t
fil_space_get_atomic_writes( fil_space_get_atomic_writes(
/*=========================*/ /*=========================*/
ulint id); /*!< in: space id */ ulint id); /*!< in: space id */
/*******************************************************************//** /*******************************************************************//**
Find out wheather the page is index page or not Find out wheather the page is index page or not
@return true if page type index page, false if not */ @return true if page type index page, false if not */
UNIV_INTERN
ibool ibool
fil_page_is_index_page( fil_page_is_index_page(
/*===================*/ /*===================*/
byte *buf); /*!< in: page */ byte *buf); /*!< in: page */
/****************************************************************//** /****************************************************************//**
Get the name of the compression algorithm used for page Get the name of the compression algorithm used for page
compression. compression.
@return compression algorithm name or "UNKNOWN" if not known*/ @return compression algorithm name or "UNKNOWN" if not known*/
UNIV_INTERN
const char* const char*
fil_get_compression_alg_name( fil_get_compression_alg_name(
/*=========================*/ /*=========================*/
ulint comp_alg); /*!<in: compression algorithm number */ ulint comp_alg); /*!<in: compression algorithm number */
/****************************************************************//** /****************************************************************//**
For page compressed pages compress the page before actual write For page compressed pages compress the page before actual write
operation. operation.
@return compressed page to be written*/ @return compressed page to be written*/
UNIV_INTERN
byte* byte*
fil_compress_page( fil_compress_page(
/*==============*/ /*==============*/
ulint space_id, /*!< in: tablespace id of the ulint space_id, /*!< in: tablespace id of the
table. */ table. */
byte* buf, /*!< in: buffer from which to write; in aio byte* buf, /*!< in: buffer from which to write; in aio
this must be appropriately aligned */ this must be appropriately aligned */
byte* out_buf, /*!< out: compressed buffer */ byte* out_buf, /*!< out: compressed buffer */
ulint len, /*!< in: length of input buffer.*/ ulint len, /*!< in: length of input buffer.*/
ulint compression_level, /*!< in: compression level */ ulint level, /* in: compression level */
ulint block_size, /*!< in: block size */ ulint block_size, /*!< in: block size */
ulint* out_len, /*!< out: actual length of compressed bool encrypted, /*!< in: is page also encrypted */
page */ ulint* out_len, /*!< out: actual length of compressed
byte* lzo_mem); /*!< in: temporal memory used by LZO */ page */
byte* lzo_mem); /*!< in: temporal memory used by LZO */
/****************************************************************//** /****************************************************************//**
For page compressed pages decompress the page after actual read For page compressed pages decompress the page after actual read
operation. operation. */
@return uncompressed page */ UNIV_INTERN
void void
fil_decompress_page( fil_decompress_page(
/*================*/ /*================*/
byte* page_buf, /*!< in: preallocated buffer or NULL */ byte* page_buf, /*!< in: preallocated buffer or NULL */
byte* buf, /*!< out: buffer from which to read; in aio byte* buf, /*!< out: buffer from which to read; in aio
this must be appropriately aligned */ this must be appropriately aligned */
ulong len, /*!< in: length of output buffer.*/ ulong len, /*!< in: length of output buffer.*/
ulint* write_size); /*!< in/out: Actual payload size of ulint* write_size); /*!< in/out: Actual payload size of
the compressed data. */ the compressed data. */
/****************************************************************//** /****************************************************************//**
Get space id from fil node Get space id from fil node
@return space id*/ @return space id*/
UNIV_INTERN
ulint ulint
fil_node_get_space_id( fil_node_get_space_id(
/*==================*/ /*==================*/
fil_node_t* node); /*!< in: Node where to get space id*/ fil_node_t* node); /*!< in: Node where to get space id*/
/****************************************************************//** /****************************************************************//**
Get block size from fil node Get block size from fil node
@return block size*/ @return block size*/
UNIV_INTERN
ulint ulint
fil_node_get_block_size( fil_node_get_block_size(
fil_node_t* node); /*!< in: Node where to get block fil_node_t* node); /*!< in: Node where to get block
size */ size */
/*******************************************************************//** /*******************************************************************//**
Find out wheather the page is page compressed Find out wheather the page is page compressed
@return true if page is page compressed*/ @return true if page is page compressed*/
UNIV_INTERN
ibool ibool
fil_page_is_compressed( fil_page_is_compressed(
/*===================*/ /*===================*/
byte *buf); /*!< in: page */ byte* buf); /*!< in: page */
/*******************************************************************//**
Find out wheather the page is page compressed
@return true if page is page compressed*/
UNIV_INTERN
ibool
fil_page_is_compressed_encrypted(
/*=============================*/
byte* buf); /*!< in: page */
/*******************************************************************//** /*******************************************************************//**
Find out wheather the page is page compressed with lzo method Find out wheather the page is page compressed with lzo method
@return true if page is page compressed with lzo method*/ @return true if page is page compressed with lzo method*/
UNIV_INTERN
ibool ibool
fil_page_is_lzo_compressed( fil_page_is_lzo_compressed(
/*=======================*/ /*=======================*/
byte *buf); /*!< in: page */ byte* buf); /*!< in: page */
#endif #endif
storage/innobase/include/fsp0pagecompress.ic
View file @ f7002c05
/***************************************************************************** /*****************************************************************************
Copyright (C) 2013,2014 SkySQL Ab. All Rights Reserved. Copyright (C) 2013, 2015, MariaDB Corporation. All Rights Reserved.
This program is free software; you can redistribute it and/or modify it under This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software the terms of the GNU General Public License as published by the Free Software
...@@ -67,7 +67,7 @@ UNIV_INLINE ...@@ -67,7 +67,7 @@ UNIV_INLINE
ibool ibool
fil_page_is_index_page( fil_page_is_index_page(
/*===================*/ /*===================*/
byte *buf) /*!< in: page */ byte* buf) /*!< in: page */
{ {
return(mach_read_from_2(buf+FIL_PAGE_TYPE) == FIL_PAGE_INDEX); return(mach_read_from_2(buf+FIL_PAGE_TYPE) == FIL_PAGE_INDEX);
} }
...@@ -79,11 +79,23 @@ UNIV_INLINE ...@@ -79,11 +79,23 @@ UNIV_INLINE
ibool ibool
fil_page_is_compressed( fil_page_is_compressed(
/*===================*/ /*===================*/
byte *buf) /*!< in: page */ byte* buf) /*!< in: page */
{ {
return(mach_read_from_2(buf+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED); return(mach_read_from_2(buf+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED);
} }
/*******************************************************************//**
Find out wheather the page is page compressed
@return true if page is page compressed, false if not */
UNIV_INLINE
ibool
fil_page_is_compressed_encrypted(
/*=============================*/
byte* buf) /*!< in: page */
{
return(mach_read_from_2(buf+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED);
}
/*******************************************************************//** /*******************************************************************//**
Returns the page compression level of the space, or 0 if the space Returns the page compression level of the space, or 0 if the space
is not compressed. The tablespace must be cached in the memory cache. is not compressed. The tablespace must be cached in the memory cache.
...@@ -136,7 +148,7 @@ UNIV_INLINE ...@@ -136,7 +148,7 @@ UNIV_INLINE
const char* const char*
fil_get_compression_alg_name( fil_get_compression_alg_name(
/*=========================*/ /*=========================*/
ulint comp_alg) /*!<in: compression algorithm number */ ulint comp_alg) /*!<in: compression algorithm number */
{ {
switch(comp_alg) { switch(comp_alg) {
case PAGE_UNCOMPRESSED: case PAGE_UNCOMPRESSED:
...@@ -190,8 +202,10 @@ UNIV_INLINE ...@@ -190,8 +202,10 @@ UNIV_INLINE
ibool ibool
fil_page_is_lzo_compressed( fil_page_is_lzo_compressed(
/*=======================*/ /*=======================*/
byte *buf) /*!< in: page */ byte* buf) /*!< in: page */
{ {
return(mach_read_from_2(buf+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED && return((mach_read_from_2(buf+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED &&
mach_read_from_8(buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION) == PAGE_LZO_ALGORITHM); mach_read_from_8(buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION) == PAGE_LZO_ALGORITHM) ||
(mach_read_from_2(buf+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED &&
mach_read_from_2(buf+FIL_PAGE_DATA+FIL_PAGE_COMPRESSED_SIZE) == PAGE_LZO_ALGORITHM));
} }
storage/xtradb/buf/buf0buf.cc
View file @ f7002c05
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
Copyright (c) 1995, 2014, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 1995, 2014, Oracle and/or its affiliates. All Rights Reserved.
Copyright (c) 2008, Google Inc. Copyright (c) 2008, Google Inc.
Copyright (c) 2013, SkySQL Ab. All Rights Reserved. Copyright (c) 2013, 2015, MariaDB Corporation. All Rights Reserved.
Portions of this file contain modifications contributed and copyrighted by Portions of this file contain modifications contributed and copyrighted by
Google, Inc. Those modifications are gratefully acknowledged and are described Google, Inc. Those modifications are gratefully acknowledged and are described
...@@ -2449,13 +2449,26 @@ buf_block_align_instance( ...@@ -2449,13 +2449,26 @@ buf_block_align_instance(
ut_ad(page_get_page_no(page_align(ptr)) ut_ad(page_get_page_no(page_align(ptr))
== 0xffffffff); == 0xffffffff);
break; break;
case BUF_BLOCK_FILE_PAGE: case BUF_BLOCK_FILE_PAGE: {
ulint space = page_get_space_id(page_align(ptr));
ulint offset = page_get_page_no(page_align(ptr));
if (block->page.space != space ||
block->page.offset != offset) {
ib_logf(IB_LOG_LEVEL_ERROR,
"Corruption: Block space_id %lu != page space_id %lu or "
"Block offset %lu != page offset %lu",
(ulint)block->page.space, space,
(ulint)block->page.offset, offset);
}
ut_ad(block->page.space ut_ad(block->page.space
== page_get_space_id(page_align(ptr))); == page_get_space_id(page_align(ptr)));
ut_ad(block->page.offset ut_ad(block->page.offset
== page_get_page_no(page_align(ptr))); == page_get_page_no(page_align(ptr)));
break; break;
} }
}
mutex_exit(&block->mutex); mutex_exit(&block->mutex);
#endif /* UNIV_DEBUG */ #endif /* UNIV_DEBUG */
...@@ -5865,49 +5878,54 @@ buf_page_encrypt_before_write( ...@@ -5865,49 +5878,54 @@ buf_page_encrypt_before_write(
ulint space_id) /*!< in: space id */ ulint space_id) /*!< in: space id */
{ {
fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space_id); fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space_id);
ulint zip_size = buf_page_get_zip_size(bpage);
ulint page_size = (zip_size) ? zip_size : UNIV_PAGE_SIZE;
buf_pool_t* buf_pool = buf_pool_from_bpage(bpage);
bool page_compressed = fil_space_is_page_compressed(bpage->space);
bpage->real_size = UNIV_PAGE_SIZE; bpage->real_size = UNIV_PAGE_SIZE;
bool encrypted = true;
fil_page_type_validate(src_frame); fil_page_type_validate(src_frame);
if (crypt_data != NULL && crypt_data->encryption == FIL_SPACE_ENCRYPTION_OFF) { if (bpage->offset == 0) {
/* Encryption is disabled */ /* Page 0 of a tablespace is not encrypted/compressed */
ut_ad(bpage->key_version == 0);
return const_cast<byte*>(src_frame); return const_cast<byte*>(src_frame);
} }
if (!srv_encrypt_tables && (crypt_data == NULL || crypt_data->encryption == FIL_SPACE_ENCRYPTION_DEFAULT)) { if (bpage->space == TRX_SYS_SPACE && bpage->offset == TRX_SYS_PAGE_NO) {
/* Encryption is disabled */ /* don't encrypt/compress page as it contains address to dblwr buffer */
bpage->key_version = 0;
return const_cast<byte*>(src_frame); return const_cast<byte*>(src_frame);
} }
if (bpage->offset == 0) { if (crypt_data != NULL && crypt_data->encryption == FIL_SPACE_ENCRYPTION_OFF) {
/* Page 0 of a tablespace is not encrypted */ /* Encryption is disabled */
ut_ad(bpage->key_version == 0); encrypted = false;
return const_cast<byte*>(src_frame); }
if (!srv_encrypt_tables && (crypt_data == NULL || crypt_data->encryption == FIL_SPACE_ENCRYPTION_DEFAULT)) {
/* Encryption is disabled */
encrypted = false;
} }
/* Is encryption needed? */ /* Is encryption needed? */
if (crypt_data->type == CRYPT_SCHEME_UNENCRYPTED) { if (crypt_data == NULL || crypt_data->type == CRYPT_SCHEME_UNENCRYPTED) {
/* An unencrypted table */ /* An unencrypted table */
bpage->key_version = 0; bpage->key_version = 0;
return const_cast<byte*>(src_frame); encrypted = false;
} }
if (bpage->space == TRX_SYS_SPACE && bpage->offset == TRX_SYS_PAGE_NO) { if (!encrypted && !page_compressed) {
/* don't encrypt page as it contains address to dblwr buffer */ /* No need to encrypt or page compress the page */
bpage->key_version = 0;
return const_cast<byte*>(src_frame); return const_cast<byte*>(src_frame);
} }
ulint zip_size = buf_page_get_zip_size(bpage);
ulint page_size = (zip_size) ? zip_size : UNIV_PAGE_SIZE;
buf_pool_t* buf_pool = buf_pool_from_bpage(bpage);
bool page_compressed = fil_space_is_page_compressed(bpage->space);
/* Find free slot from temporary memory array */ /* Find free slot from temporary memory array */
buf_tmp_buffer_t* slot = buf_pool_reserve_tmp_slot(buf_pool, page_compressed); buf_tmp_buffer_t* slot = buf_pool_reserve_tmp_slot(buf_pool, page_compressed);
bpage->slot = slot; bpage->slot = slot;
byte *dst_frame = slot->crypt_buf; byte *dst_frame = bpage->slot->out_buf = slot->crypt_buf;
if (!page_compressed) { if (!page_compressed) {
/* Encrypt page content */ /* Encrypt page content */
...@@ -5937,6 +5955,7 @@ buf_page_encrypt_before_write( ...@@ -5937,6 +5955,7 @@ buf_page_encrypt_before_write(
page_size, page_size,
fil_space_get_page_compression_level(bpage->space), fil_space_get_page_compression_level(bpage->space),
block_size, block_size,
encrypted,
&out_len, &out_len,
IF_LZO(slot->lzo_mem, NULL) IF_LZO(slot->lzo_mem, NULL)
); );
...@@ -5944,14 +5963,18 @@ buf_page_encrypt_before_write( ...@@ -5944,14 +5963,18 @@ buf_page_encrypt_before_write(
bpage->real_size = out_len; bpage->real_size = out_len;
fil_page_type_validate(tmp); fil_page_type_validate(tmp);
if(encrypted) {
/* And then we encrypt the page content */ /* And then we encrypt the page content */
fil_space_encrypt(bpage->space, fil_space_encrypt(bpage->space,
bpage->offset, bpage->offset,
bpage->newest_modification, bpage->newest_modification,
tmp, tmp,
zip_size, zip_size,
dst_frame); dst_frame);
} else {
bpage->slot->out_buf = dst_frame = tmp;
}
} }
fil_page_type_validate(dst_frame); fil_page_type_validate(dst_frame);
...@@ -5968,13 +5991,6 @@ buf_page_decrypt_after_read( ...@@ -5968,13 +5991,6 @@ buf_page_decrypt_after_read(
/*========================*/ /*========================*/
buf_page_t* bpage) /*!< in/out: buffer page read from disk */ buf_page_t* bpage) /*!< in/out: buffer page read from disk */
{ {
ut_ad(bpage->key_version == 0);
if (bpage->offset == 0) {
/* File header pages are not encrypted */
return (TRUE);
}
ulint zip_size = buf_page_get_zip_size(bpage); ulint zip_size = buf_page_get_zip_size(bpage);
ulint size = (zip_size) ? zip_size : UNIV_PAGE_SIZE; ulint size = (zip_size) ? zip_size : UNIV_PAGE_SIZE;
...@@ -5983,54 +5999,72 @@ buf_page_decrypt_after_read( ...@@ -5983,54 +5999,72 @@ buf_page_decrypt_after_read(
unsigned key_version = unsigned key_version =
mach_read_from_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); mach_read_from_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
bool page_compressed = fil_page_is_compressed(dst_frame); bool page_compressed = fil_page_is_compressed(dst_frame);
bool page_compressed_encrypted = fil_page_is_compressed_encrypted(dst_frame);
buf_pool_t* buf_pool = buf_pool_from_bpage(bpage); buf_pool_t* buf_pool = buf_pool_from_bpage(bpage);
if (key_version == 0) { ut_ad(bpage->key_version == 0);
if (bpage->offset == 0) {
/* File header pages are not encrypted/compressed */
return (TRUE);
}
if (page_compressed) {
/* the page we read is unencrypted */ /* the page we read is unencrypted */
if (page_compressed) { /* Find free slot from temporary memory array */
/* Find free slot from temporary memory array */ buf_tmp_buffer_t* slot = buf_pool_reserve_tmp_slot(buf_pool, page_compressed);
buf_tmp_buffer_t* slot = buf_pool_reserve_tmp_slot(buf_pool, page_compressed);
fil_page_type_validate(dst_frame); fil_page_type_validate(dst_frame);
fil_decompress_page(slot->comp_buf, fil_decompress_page(slot->comp_buf,
dst_frame, dst_frame,
size, size,
&bpage->write_size); &bpage->write_size);
/* Mark this slot as free */ /* Mark this slot as free */
slot->reserved = false; slot->reserved = false;
} key_version = 0;
fil_page_type_validate(dst_frame); fil_page_type_validate(dst_frame);
} else { } else {
/* Find free slot from temporary memory array */ buf_tmp_buffer_t* slot = NULL;
buf_tmp_buffer_t* slot = buf_pool_reserve_tmp_slot(buf_pool, page_compressed);
memcpy(slot->crypt_buf, dst_frame, size);
fil_page_type_validate(dst_frame); if (key_version) {
fil_page_type_validate(slot->crypt_buf); /* Find free slot from temporary memory array */
slot = buf_pool_reserve_tmp_slot(buf_pool, page_compressed);
memcpy(slot->crypt_buf, dst_frame, size);
/* decrypt from crypt_buf to dst_frame */ fil_page_type_validate(dst_frame);
fil_space_decrypt(bpage->space, fil_page_type_validate(slot->crypt_buf);
slot->crypt_buf, /* decrypt from crypt_buf to dst_frame */
size, fil_space_decrypt(bpage->space,
dst_frame); slot->crypt_buf,
size,
dst_frame);
fil_page_type_validate(dst_frame);
fil_page_type_validate(slot->crypt_buf);
}
fil_page_type_validate(dst_frame); if (page_compressed_encrypted) {
fil_page_type_validate(slot->crypt_buf); if (!slot) {
slot = buf_pool_reserve_tmp_slot(buf_pool, page_compressed);
}
if (page_compressed) { #ifdef UNIV_DEBUG
fil_page_type_validate(dst_frame);
#endif
fil_decompress_page(slot->comp_buf, fil_decompress_page(slot->comp_buf,
dst_frame, dst_frame,
size, size,
&bpage->write_size); &bpage->write_size);
}
fil_page_type_validate(dst_frame); fil_page_type_validate(dst_frame);
}
/* Mark this slot as free */ /* Mark this slot as free */
slot->reserved = false; if (slot) {
slot->reserved = false;
}
} }
bpage->key_version = key_version; bpage->key_version = key_version;
......
storage/xtradb/fil/fil0crypt.cc
View file @ f7002c05
...@@ -591,8 +591,7 @@ fil_space_encrypt( ...@@ -591,8 +591,7 @@ fil_space_encrypt(
ut_error; ut_error;
} }
ibool page_compressed = (mach_read_from_2(src_frame+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED); ibool page_compressed = (orig_page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED);
ulint page_comp_method = mach_read_from_8(src_frame+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
/* FIL page header is not encrypted */ /* FIL page header is not encrypted */
memcpy(dst_frame, src_frame, FIL_PAGE_DATA); memcpy(dst_frame, src_frame, FIL_PAGE_DATA);
...@@ -607,17 +606,6 @@ fil_space_encrypt( ...@@ -607,17 +606,6 @@ fil_space_encrypt(
byte* dst = dst_frame + FIL_PAGE_DATA; byte* dst = dst_frame + FIL_PAGE_DATA;
uint32 dstlen = 0; uint32 dstlen = 0;
/* For page compressed tables we encrypt only the actual compressed
payload. Note that first two bytes of page data is actual payload
size and that should not be encrypted. */
if (page_compressed) {
ulint payload = mach_read_from_2(src_frame + FIL_PAGE_DATA);
mach_write_to_2(dst_frame + FIL_PAGE_DATA, payload);
srclen = payload;
src+=2;
dst+=2;
}
int rc = encryption_scheme_encrypt(src, srclen, dst, &dstlen, int rc = encryption_scheme_encrypt(src, srclen, dst, &dstlen,
crypt_data, key_version, crypt_data, key_version,
space, offset, lsn); space, offset, lsn);
...@@ -641,43 +629,39 @@ fil_space_encrypt( ...@@ -641,43 +629,39 @@ fil_space_encrypt(
memcpy(dst_frame + page_size - FIL_PAGE_DATA_END, memcpy(dst_frame + page_size - FIL_PAGE_DATA_END,
src_frame + page_size - FIL_PAGE_DATA_END, src_frame + page_size - FIL_PAGE_DATA_END,
FIL_PAGE_DATA_END); FIL_PAGE_DATA_END);
}
/* handle post encryption checksum */ /* handle post encryption checksum */
ib_uint32_t checksum = 0; ib_uint32_t checksum = 0;
srv_checksum_algorithm_t algorithm = srv_checksum_algorithm_t algorithm =
static_cast<srv_checksum_algorithm_t>(srv_checksum_algorithm); static_cast<srv_checksum_algorithm_t>(srv_checksum_algorithm);
if (zip_size == 0) { if (zip_size == 0) {
switch (algorithm) { switch (algorithm) {
case SRV_CHECKSUM_ALGORITHM_CRC32: case SRV_CHECKSUM_ALGORITHM_CRC32:
case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32: case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32:
checksum = buf_calc_page_crc32(dst_frame); checksum = buf_calc_page_crc32(dst_frame);
break; break;
case SRV_CHECKSUM_ALGORITHM_INNODB: case SRV_CHECKSUM_ALGORITHM_INNODB:
case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB: case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB:
checksum = (ib_uint32_t) buf_calc_page_new_checksum( checksum = (ib_uint32_t) buf_calc_page_new_checksum(
dst_frame); dst_frame);
break; break;
case SRV_CHECKSUM_ALGORITHM_NONE: case SRV_CHECKSUM_ALGORITHM_NONE:
case SRV_CHECKSUM_ALGORITHM_STRICT_NONE: case SRV_CHECKSUM_ALGORITHM_STRICT_NONE:
checksum = BUF_NO_CHECKSUM_MAGIC; checksum = BUF_NO_CHECKSUM_MAGIC;
break; break;
/* no default so the compiler will emit a warning /* no default so the compiler will emit a warning
* if new enum is added and not handled here */ * if new enum is added and not handled here */
}
} else {
checksum = page_zip_calc_checksum(dst_frame, zip_size,
algorithm);
} }
// store the post-encryption checksum after the key-version
mach_write_to_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4,
checksum);
} else { } else {
mach_write_to_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4, checksum = page_zip_calc_checksum(dst_frame, zip_size,
page_comp_method); algorithm);
} }
// store the post-encryption checksum after the key-version
mach_write_to_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4, checksum);
srv_stats.pages_encrypted.inc(); srv_stats.pages_encrypted.inc();
} }
...@@ -721,8 +705,7 @@ fil_space_decrypt( ...@@ -721,8 +705,7 @@ fil_space_decrypt(
{ {
ulint page_type = mach_read_from_2(src_frame+FIL_PAGE_TYPE); ulint page_type = mach_read_from_2(src_frame+FIL_PAGE_TYPE);
uint key_version = mach_read_from_4(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); uint key_version = mach_read_from_4(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
bool page_compressed = (page_type == FIL_PAGE_PAGE_COMPRESSED); bool page_compressed = (page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED);
ulint page_comp_method = mach_read_from_4(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4);
if (key_version == ENCRYPTION_KEY_NOT_ENCRYPTED) { if (key_version == ENCRYPTION_KEY_NOT_ENCRYPTED) {
//TODO: is this really needed ? //TODO: is this really needed ?
...@@ -748,17 +731,6 @@ fil_space_decrypt( ...@@ -748,17 +731,6 @@ fil_space_decrypt(
uint32 dstlen = 0; uint32 dstlen = 0;
ulint srclen = page_size - (FIL_PAGE_DATA + FIL_PAGE_DATA_END); ulint srclen = page_size - (FIL_PAGE_DATA + FIL_PAGE_DATA_END);
/* For page compressed tables we decrypt only the actual compressed
payload. Note that first two bytes of page data is actual payload
size and that should not be decrypted. */
if (page_compressed) {
ulint compressed_len = mach_read_from_2(src_frame + FIL_PAGE_DATA);
src+=2;
dst+=2;
mach_write_to_2(dst_frame + FIL_PAGE_DATA, compressed_len);
srclen = compressed_len;
}
int rc = encryption_scheme_decrypt(src, srclen, dst, &dstlen, int rc = encryption_scheme_decrypt(src, srclen, dst, &dstlen,
crypt_data, key_version, crypt_data, key_version,
space, offset, lsn); space, offset, lsn);
...@@ -785,8 +757,6 @@ fil_space_decrypt( ...@@ -785,8 +757,6 @@ fil_space_decrypt(
// clear key-version & crypt-checksum from dst // clear key-version & crypt-checksum from dst
memset(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, 0, 8); memset(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, 0, 8);
} else {
mach_write_to_8(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, page_comp_method);
} }
srv_stats.pages_decrypted.inc(); srv_stats.pages_decrypted.inc();
......
storage/xtradb/fil/fil0pagecompress.cc
View file @ f7002c05
...@@ -87,29 +87,35 @@ static ulint srv_data_read, srv_data_written; ...@@ -87,29 +87,35 @@ static ulint srv_data_read, srv_data_written;
For page compressed pages compress the page before actual write For page compressed pages compress the page before actual write
operation. operation.
@return compressed page to be written*/ @return compressed page to be written*/
UNIV_INTERN
byte* byte*
fil_compress_page( fil_compress_page(
/*==============*/ /*==============*/
ulint space_id, /*!< in: tablespace id of the ulint space_id, /*!< in: tablespace id of the
table. */ table. */
byte* buf, /*!< in: buffer from which to write; in aio byte* buf, /*!< in: buffer from which to write; in aio
this must be appropriately aligned */ this must be appropriately aligned */
byte* out_buf, /*!< out: compressed buffer */ byte* out_buf, /*!< out: compressed buffer */
ulint len, /*!< in: length of input buffer.*/ ulint len, /*!< in: length of input buffer.*/
ulint compression_level, /* in: compression level */ ulint level, /* in: compression level */
ulint block_size, /*!< in: block size */ ulint block_size, /*!< in: block size */
ulint* out_len, /*!< out: actual length of compressed bool encrypted, /*!< in: is page also encrypted */
page */ ulint* out_len, /*!< out: actual length of compressed
byte* lzo_mem) /*!< in: temporal memory used by LZO */ page */
byte* lzo_mem) /*!< in: temporal memory used by LZO */
{ {
int err = Z_OK; int err = Z_OK;
int level = compression_level; int comp_level = level;
ulint header_len = FIL_PAGE_DATA + FIL_PAGE_COMPRESSED_SIZE; ulint header_len = FIL_PAGE_DATA + FIL_PAGE_COMPRESSED_SIZE;
ulint write_size=0; ulint write_size=0;
/* Cache to avoid change during function execution */ /* Cache to avoid change during function execution */
ulint comp_method = innodb_compression_algorithm; ulint comp_method = innodb_compression_algorithm;
ulint orig_page_type; ulint orig_page_type;
if (encrypted) {
header_len += FIL_PAGE_COMPRESSION_METHOD_SIZE;
}
ut_ad(buf); ut_ad(buf);
ut_ad(out_buf); ut_ad(out_buf);
ut_ad(len); ut_ad(len);
...@@ -134,8 +140,8 @@ fil_compress_page( ...@@ -134,8 +140,8 @@ fil_compress_page(
/* If no compression level was provided to this table, use system /* If no compression level was provided to this table, use system
default level */ default level */
if (level == 0) { if (comp_level == 0) {
level = page_zip_level; comp_level = page_zip_level;
} }
#ifdef UNIV_PAGECOMPRESS_DEBUG #ifdef UNIV_PAGECOMPRESS_DEBUG
...@@ -156,13 +162,16 @@ fil_compress_page( ...@@ -156,13 +162,16 @@ fil_compress_page(
if (err == 0) { if (err == 0) {
/* If error we leave the actual page as it was */ /* If error we leave the actual page as it was */
#ifndef UNIV_PAGECOMPRESS_DEBUG
if (space->printed_compression_failure == false) { if (space->printed_compression_failure == false) {
#endif
ib_logf(IB_LOG_LEVEL_WARN, ib_logf(IB_LOG_LEVEL_WARN,
"Compression failed for space %lu name %s len %lu rt %d write %lu.", "Compression failed for space %lu name %s len %lu rt %d write %lu.",
space_id, fil_space_name(space), len, err, write_size); space_id, fil_space_name(space), len, err, write_size);
space->printed_compression_failure = true; space->printed_compression_failure = true;
#ifndef UNIV_PAGECOMPRESS_DEBUG
} }
#endif
srv_stats.pages_page_compression_error.inc(); srv_stats.pages_page_compression_error.inc();
*out_len = len; *out_len = len;
return (buf); return (buf);
...@@ -193,7 +202,7 @@ fil_compress_page( ...@@ -193,7 +202,7 @@ fil_compress_page(
size_t out_pos=0; size_t out_pos=0;
err = lzma_easy_buffer_encode( err = lzma_easy_buffer_encode(
compression_level, comp_level,
LZMA_CHECK_NONE, LZMA_CHECK_NONE,
NULL, /* No custom allocator, use malloc/free */ NULL, /* No custom allocator, use malloc/free */
reinterpret_cast<uint8_t*>(buf), reinterpret_cast<uint8_t*>(buf),
...@@ -270,7 +279,7 @@ fil_compress_page( ...@@ -270,7 +279,7 @@ fil_compress_page(
#endif /* HAVE_SNAPPY */ #endif /* HAVE_SNAPPY */
case PAGE_ZLIB_ALGORITHM: case PAGE_ZLIB_ALGORITHM:
err = compress2(out_buf+header_len, (ulong*)&write_size, buf, len, level); err = compress2(out_buf+header_len, (ulong*)&write_size, buf, len, comp_level);
if (err != Z_OK) { if (err != Z_OK) {
/* If error we leave the actual page as it was */ /* If error we leave the actual page as it was */
...@@ -301,19 +310,29 @@ fil_compress_page( ...@@ -301,19 +310,29 @@ fil_compress_page(
memcpy(out_buf, buf, FIL_PAGE_DATA); memcpy(out_buf, buf, FIL_PAGE_DATA);
/* Set up the checksum */ /* Set up the checksum */
mach_write_to_4(out_buf+FIL_PAGE_SPACE_OR_CHKSUM, BUF_NO_CHECKSUM_MAGIC); mach_write_to_4(out_buf+FIL_PAGE_SPACE_OR_CHKSUM, BUF_NO_CHECKSUM_MAGIC);
/* Set up the correct page type */
mach_write_to_2(out_buf+FIL_PAGE_TYPE, FIL_PAGE_PAGE_COMPRESSED); /* Set up the compression algorithm */
/* Set up the flush lsn to be compression algorithm */
mach_write_to_8(out_buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, comp_method); mach_write_to_8(out_buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, comp_method);
if (encrypted) {
/* Set up the correct page type */
mach_write_to_2(out_buf+FIL_PAGE_TYPE, FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED);
mach_write_to_2(out_buf+FIL_PAGE_DATA+FIL_PAGE_COMPRESSED_SIZE, comp_method);
} else {
/* Set up the correct page type */
mach_write_to_2(out_buf+FIL_PAGE_TYPE, FIL_PAGE_PAGE_COMPRESSED);
}
/* Set up the actual payload lenght */ /* Set up the actual payload lenght */
mach_write_to_2(out_buf+FIL_PAGE_DATA, write_size); mach_write_to_2(out_buf+FIL_PAGE_DATA, write_size);
#ifdef UNIV_DEBUG #ifdef UNIV_DEBUG
/* Verify */ /* Verify */
ut_ad(fil_page_is_compressed(out_buf)); ut_ad(fil_page_is_compressed(out_buf) || fil_page_is_compressed_encrypted(out_buf));
ut_ad(mach_read_from_4(out_buf+FIL_PAGE_SPACE_OR_CHKSUM) == BUF_NO_CHECKSUM_MAGIC); ut_ad(mach_read_from_4(out_buf+FIL_PAGE_SPACE_OR_CHKSUM) == BUF_NO_CHECKSUM_MAGIC);
ut_ad(mach_read_from_2(out_buf+FIL_PAGE_DATA) == write_size); ut_ad(mach_read_from_2(out_buf+FIL_PAGE_DATA) == write_size);
ut_ad(mach_read_from_8(out_buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION) == (ulint)comp_method); ut_ad(mach_read_from_8(out_buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION) == (ulint)comp_method ||
mach_read_from_2(out_buf+FIL_PAGE_DATA+FIL_PAGE_COMPRESSED_SIZE) == (ulint)comp_method);
/* Verify that page can be decompressed */ /* Verify that page can be decompressed */
{ {
...@@ -373,29 +392,37 @@ fil_compress_page( ...@@ -373,29 +392,37 @@ fil_compress_page(
/****************************************************************//** /****************************************************************//**
For page compressed pages decompress the page after actual read For page compressed pages decompress the page after actual read
operation. */ operation. */
UNIV_INTERN
void void
fil_decompress_page( fil_decompress_page(
/*================*/ /*================*/
byte* page_buf, /*!< in: preallocated buffer or NULL */ byte* page_buf, /*!< in: preallocated buffer or NULL */
byte* buf, /*!< out: buffer from which to read; in aio byte* buf, /*!< out: buffer from which to read; in aio
this must be appropriately aligned */ this must be appropriately aligned */
ulong len, /*!< in: length of output buffer.*/ ulong len, /*!< in: length of output buffer.*/
ulint* write_size) /*!< in/out: Actual payload size of ulint* write_size) /*!< in/out: Actual payload size of
the compressed data. */ the compressed data. */
{ {
int err = 0; int err = 0;
ulint actual_size = 0; ulint actual_size = 0;
ulint compression_alg = 0; ulint compression_alg = 0;
byte *in_buf; byte *in_buf;
ulint ptype; ulint ptype;
ulint header_len = FIL_PAGE_DATA + FIL_PAGE_COMPRESSED_SIZE;
ut_ad(buf); ut_ad(buf);
ut_ad(len); ut_ad(len);
ptype = mach_read_from_2(buf+FIL_PAGE_TYPE); ptype = mach_read_from_2(buf+FIL_PAGE_TYPE);
if (ptype == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) {
header_len += FIL_PAGE_COMPRESSION_METHOD_SIZE;
}
/* Do not try to uncompressed pages that are not compressed */ /* Do not try to uncompressed pages that are not compressed */
if (ptype != FIL_PAGE_PAGE_COMPRESSED && ptype != FIL_PAGE_TYPE_COMPRESSED) { if (ptype != FIL_PAGE_PAGE_COMPRESSED &&
ptype != FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED &&
ptype != FIL_PAGE_TYPE_COMPRESSED) {
return; return;
} }
...@@ -409,7 +436,8 @@ fil_decompress_page( ...@@ -409,7 +436,8 @@ fil_decompress_page(
/* Before actual decompress, make sure that page type is correct */ /* Before actual decompress, make sure that page type is correct */
if (mach_read_from_4(buf+FIL_PAGE_SPACE_OR_CHKSUM) != BUF_NO_CHECKSUM_MAGIC || if (mach_read_from_4(buf+FIL_PAGE_SPACE_OR_CHKSUM) != BUF_NO_CHECKSUM_MAGIC ||
mach_read_from_2(buf+FIL_PAGE_TYPE) != FIL_PAGE_PAGE_COMPRESSED) { (ptype != FIL_PAGE_PAGE_COMPRESSED &&
ptype != FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED)) {
ib_logf(IB_LOG_LEVEL_ERROR, ib_logf(IB_LOG_LEVEL_ERROR,
"Corruption: We try to uncompress corrupted page" "Corruption: We try to uncompress corrupted page"
" CRC %lu type %lu len %lu.", " CRC %lu type %lu len %lu.",
...@@ -421,7 +449,11 @@ fil_decompress_page( ...@@ -421,7 +449,11 @@ fil_decompress_page(
} }
/* Get compression algorithm */ /* Get compression algorithm */
compression_alg = mach_read_from_8(buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); if (ptype == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) {
compression_alg = mach_read_from_2(buf+FIL_PAGE_DATA+FIL_PAGE_COMPRESSED_SIZE);
} else {
compression_alg = mach_read_from_8(buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
}
/* Get the actual size of compressed page */ /* Get the actual size of compressed page */
actual_size = mach_read_from_2(buf+FIL_PAGE_DATA); actual_size = mach_read_from_2(buf+FIL_PAGE_DATA);
...@@ -450,7 +482,7 @@ fil_decompress_page( ...@@ -450,7 +482,7 @@ fil_decompress_page(
switch(compression_alg) { switch(compression_alg) {
case PAGE_ZLIB_ALGORITHM: case PAGE_ZLIB_ALGORITHM:
err= uncompress(in_buf, &len, buf+FIL_PAGE_DATA+FIL_PAGE_COMPRESSED_SIZE, (unsigned long)actual_size); err= uncompress(in_buf, &len, buf+header_len, (unsigned long)actual_size);
/* If uncompress fails it means that page is corrupted */ /* If uncompress fails it means that page is corrupted */
if (err != Z_OK) { if (err != Z_OK) {
...@@ -469,7 +501,7 @@ fil_decompress_page( ...@@ -469,7 +501,7 @@ fil_decompress_page(
#ifdef HAVE_LZ4 #ifdef HAVE_LZ4
case PAGE_LZ4_ALGORITHM: case PAGE_LZ4_ALGORITHM:
err = LZ4_decompress_fast((const char *)buf+FIL_PAGE_DATA+FIL_PAGE_COMPRESSED_SIZE, (char *)in_buf, len); err = LZ4_decompress_fast((const char *)buf+header_len, (char *)in_buf, len);
if (err != (int)actual_size) { if (err != (int)actual_size) {
ib_logf(IB_LOG_LEVEL_ERROR, ib_logf(IB_LOG_LEVEL_ERROR,
...@@ -486,7 +518,7 @@ fil_decompress_page( ...@@ -486,7 +518,7 @@ fil_decompress_page(
#ifdef HAVE_LZO #ifdef HAVE_LZO
case PAGE_LZO_ALGORITHM: { case PAGE_LZO_ALGORITHM: {
ulint olen=0; ulint olen=0;
err = lzo1x_decompress((const unsigned char *)buf+FIL_PAGE_DATA+FIL_PAGE_COMPRESSED_SIZE, err = lzo1x_decompress((const unsigned char *)buf+header_len,
actual_size,(unsigned char *)in_buf, &olen, NULL); actual_size,(unsigned char *)in_buf, &olen, NULL);
if (err != LZO_E_OK || (olen == 0 || olen > UNIV_PAGE_SIZE)) { if (err != LZO_E_OK || (olen == 0 || olen > UNIV_PAGE_SIZE)) {
...@@ -514,7 +546,7 @@ fil_decompress_page( ...@@ -514,7 +546,7 @@ fil_decompress_page(
&memlimit, &memlimit,
0, 0,
NULL, NULL,
buf+FIL_PAGE_DATA+FIL_PAGE_COMPRESSED_SIZE, buf+header_len,
&src_pos, &src_pos,
actual_size, actual_size,
in_buf, in_buf,
...@@ -543,7 +575,7 @@ fil_decompress_page( ...@@ -543,7 +575,7 @@ fil_decompress_page(
err = BZ2_bzBuffToBuffDecompress( err = BZ2_bzBuffToBuffDecompress(
(char *)in_buf, (char *)in_buf,
&dst_pos, &dst_pos,
(char *)(buf+FIL_PAGE_DATA+FIL_PAGE_COMPRESSED_SIZE), (char *)(buf+header_len),
actual_size, actual_size,
1, 1,
0); 0);
...@@ -568,7 +600,7 @@ fil_decompress_page( ...@@ -568,7 +600,7 @@ fil_decompress_page(
ulint olen = 0; ulint olen = 0;
cstatus = snappy_uncompress( cstatus = snappy_uncompress(
(const char *)(buf+FIL_PAGE_DATA+FIL_PAGE_COMPRESSED_SIZE), (const char *)(buf+header_len),
actual_size, actual_size,
(char *)in_buf, (char *)in_buf,
&olen); &olen);
......
storage/xtradb/handler/ha_innodb.cc
View file @ f7002c05
...@@ -21293,6 +21293,7 @@ innodb_compression_algorithm_validate( ...@@ -21293,6 +21293,7 @@ innodb_compression_algorithm_validate(
} }
compression_algorithm = *reinterpret_cast<ulong*>(save); compression_algorithm = *reinterpret_cast<ulong*>(save);
(void)compression_algorithm;
#ifndef HAVE_LZ4 #ifndef HAVE_LZ4
if (compression_algorithm == PAGE_LZ4_ALGORITHM) { if (compression_algorithm == PAGE_LZ4_ALGORITHM) {
......
storage/xtradb/include/buf0buf.h
View file @ f7002c05
...@@ -1534,6 +1534,9 @@ typedef struct { ...@@ -1534,6 +1534,9 @@ typedef struct {
can be read while it's being flushed */ can be read while it's being flushed */
byte* comp_buf_free; /*!< for compression, allocated byte* comp_buf_free; /*!< for compression, allocated
buffer that is then alligned */ buffer that is then alligned */
byte* out_buf; /*!< resulting buffer after
encryption/compression. This is a
pointer and not allocated. */
} buf_tmp_buffer_t; } buf_tmp_buffer_t;
/** The common buffer control block structure /** The common buffer control block structure
......
storage/xtradb/include/buf0buf.ic
View file @ f7002c05
...@@ -2,6 +2,7 @@ ...@@ -2,6 +2,7 @@
Copyright (c) 1995, 2014, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 1995, 2014, Oracle and/or its affiliates. All Rights Reserved.
Copyright (c) 2008, Google Inc. Copyright (c) 2008, Google Inc.
Copyright (c) 2014, 2015, MariaDB Corporation.
Portions of this file contain modifications contributed and copyrighted by Portions of this file contain modifications contributed and copyrighted by
Google, Inc. Those modifications are gratefully acknowledged and are described Google, Inc. Those modifications are gratefully acknowledged and are described
...@@ -926,8 +927,10 @@ buf_page_get_frame( ...@@ -926,8 +927,10 @@ buf_page_get_frame(
/*===============*/ /*===============*/
const buf_page_t* bpage) /*!< in: buffer pool page */ const buf_page_t* bpage) /*!< in: buffer pool page */
{ {
if (bpage->slot && bpage->slot->crypt_buf) { /* In encryption/compression buffer pool page may contain extra
return bpage->slot->crypt_buf; buffer where result is stored. */
if (bpage->slot && bpage->slot->out_buf) {
return bpage->slot->out_buf;
} else if (bpage->zip.data) { } else if (bpage->zip.data) {
return bpage->zip.data; return bpage->zip.data;
} else { } else {
......
storage/xtradb/include/fil0fil.h
View file @ f7002c05
...@@ -138,8 +138,11 @@ extern fil_addr_t fil_addr_null; ...@@ -138,8 +138,11 @@ extern fil_addr_t fil_addr_null;
#define FIL_PAGE_DATA 38 /*!< start of the data on the page */ #define FIL_PAGE_DATA 38 /*!< start of the data on the page */
/* Following are used when page compression is used */ /* Following are used when page compression is used */
#define FIL_PAGE_COMPRESSED_SIZE 2 /*!< Number of bytes used to store #define FIL_PAGE_COMPRESSED_SIZE 2 /*!< Number of bytes used to store
actual payload data size on actual payload data size on
compressed pages. */ compressed pages. */
#define FIL_PAGE_COMPRESSION_METHOD_SIZE 2
/*!< Number of bytes used to store
actual compression method. */
/* @} */ /* @} */
/** File page trailer @{ */ /** File page trailer @{ */
#define FIL_PAGE_END_LSN_OLD_CHKSUM 8 /*!< the low 4 bytes of this are used #define FIL_PAGE_END_LSN_OLD_CHKSUM 8 /*!< the low 4 bytes of this are used
...@@ -150,6 +153,8 @@ extern fil_addr_t fil_addr_null; ...@@ -150,6 +153,8 @@ extern fil_addr_t fil_addr_null;
/* @} */ /* @} */
/** File page types (values of FIL_PAGE_TYPE) @{ */ /** File page types (values of FIL_PAGE_TYPE) @{ */
#define FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED 37401 /*!< Page is compressed and
then encrypted */
#define FIL_PAGE_PAGE_COMPRESSED 34354 /*!< Page compressed page */ #define FIL_PAGE_PAGE_COMPRESSED 34354 /*!< Page compressed page */
#define FIL_PAGE_INDEX 17855 /*!< B-tree node */ #define FIL_PAGE_INDEX 17855 /*!< B-tree node */
#define FIL_PAGE_UNDO_LOG 2 /*!< Undo log page */ #define FIL_PAGE_UNDO_LOG 2 /*!< Undo log page */
......
storage/xtradb/include/fil0fil.ic
View file @ f7002c05
...@@ -118,6 +118,7 @@ fil_page_type_validate( ...@@ -118,6 +118,7 @@ fil_page_type_validate(
#ifdef UNIV_ENCRYPTION_EXTRA_DEBUG #ifdef UNIV_ENCRYPTION_EXTRA_DEBUG
uint key_version = mach_read_from_4(page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); uint key_version = mach_read_from_4(page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
bool page_compressed = (page_type == FIL_PAGE_PAGE_COMPRESSED); bool page_compressed = (page_type == FIL_PAGE_PAGE_COMPRESSED);
bool page_compressed_encrypted = (page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED);
ulint space = mach_read_from_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); ulint space = mach_read_from_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID);
ulint offset = mach_read_from_4(page + FIL_PAGE_OFFSET); ulint offset = mach_read_from_4(page + FIL_PAGE_OFFSET);
ib_uint64_t lsn = mach_read_from_8(page + FIL_PAGE_LSN); ib_uint64_t lsn = mach_read_from_8(page + FIL_PAGE_LSN);
...@@ -137,6 +138,7 @@ fil_page_type_validate( ...@@ -137,6 +138,7 @@ fil_page_type_validate(
/* Validate page type */ /* Validate page type */
if (!((page_type == FIL_PAGE_PAGE_COMPRESSED || if (!((page_type == FIL_PAGE_PAGE_COMPRESSED ||
page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED ||
page_type == FIL_PAGE_INDEX || page_type == FIL_PAGE_INDEX ||
page_type == FIL_PAGE_UNDO_LOG || page_type == FIL_PAGE_UNDO_LOG ||
page_type == FIL_PAGE_INODE || page_type == FIL_PAGE_INODE ||
...@@ -152,6 +154,7 @@ fil_page_type_validate( ...@@ -152,6 +154,7 @@ fil_page_type_validate(
page_type == FIL_PAGE_TYPE_COMPRESSED))) { page_type == FIL_PAGE_TYPE_COMPRESSED))) {
ut_ad(page_type == FIL_PAGE_PAGE_COMPRESSED || ut_ad(page_type == FIL_PAGE_PAGE_COMPRESSED ||
page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED ||
page_type == FIL_PAGE_INDEX || page_type == FIL_PAGE_INDEX ||
page_type == FIL_PAGE_UNDO_LOG || page_type == FIL_PAGE_UNDO_LOG ||
page_type == FIL_PAGE_INODE || page_type == FIL_PAGE_INODE ||
......
storage/xtradb/include/fil0pagecompress.h
View file @ f7002c05
/***************************************************************************** /*****************************************************************************
Copyright (C) 2013, 2014 SkySQL Ab. All Rights Reserved. Copyright (C) 2013, 2015 MariaDB Corporation. All Rights Reserved.
This program is free software; you can redistribute it and/or modify it under This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software the terms of the GNU General Public License as published by the Free Software
...@@ -34,6 +34,7 @@ Created 11/12/2013 Jan Lindström jan.lindstrom@skysql.com ...@@ -34,6 +34,7 @@ Created 11/12/2013 Jan Lindström jan.lindstrom@skysql.com
Returns the page compression level flag of the space, or 0 if the space Returns the page compression level flag of the space, or 0 if the space
is not compressed. The tablespace must be cached in the memory cache. is not compressed. The tablespace must be cached in the memory cache.
@return page compression level if page compressed, ULINT_UNDEFINED if space not found */ @return page compression level if page compressed, ULINT_UNDEFINED if space not found */
UNIV_INTERN
ulint ulint
fil_space_get_page_compression_level( fil_space_get_page_compression_level(
/*=================================*/ /*=================================*/
...@@ -42,14 +43,16 @@ fil_space_get_page_compression_level( ...@@ -42,14 +43,16 @@ fil_space_get_page_compression_level(
Returns the page compression flag of the space, or false if the space Returns the page compression flag of the space, or false if the space
is not compressed. The tablespace must be cached in the memory cache. is not compressed. The tablespace must be cached in the memory cache.
@return true if page compressed, false if not or space not found */ @return true if page compressed, false if not or space not found */
UNIV_INTERN
ibool ibool
fil_space_is_page_compressed( fil_space_is_page_compressed(
/*=========================*/ /*=========================*/
ulint id); /*!< in: space id */ ulint id); /*!< in: space id */
/*******************************************************************//** /*******************************************************************//**
Returns the page compression flag of the space, or false if the space Returns the page compression flag of the space, or false if the space
is not compressed. The tablespace must be cached in the memory cache. is not compressed. The tablespace must be cached in the memory cache.
@return true if page compressed, false if not or space not found */ @return true if page compressed, false if not or space not found */
UNIV_INTERN
ibool ibool
fil_space_get_page_compressed( fil_space_get_page_compressed(
/*=========================*/ /*=========================*/
...@@ -58,88 +61,106 @@ fil_space_get_page_compressed( ...@@ -58,88 +61,106 @@ fil_space_get_page_compressed(
Returns the atomic writes flag of the space, or false if the space Returns the atomic writes flag of the space, or false if the space
is not using atomic writes. The tablespace must be cached in the memory cache. is not using atomic writes. The tablespace must be cached in the memory cache.
@return atomic write table option value */ @return atomic write table option value */
UNIV_INTERN
atomic_writes_t atomic_writes_t
fil_space_get_atomic_writes( fil_space_get_atomic_writes(
/*=========================*/ /*=========================*/
ulint id); /*!< in: space id */ ulint id); /*!< in: space id */
/*******************************************************************//** /*******************************************************************//**
Find out wheather the page is index page or not Find out wheather the page is index page or not
@return true if page type index page, false if not */ @return true if page type index page, false if not */
UNIV_INTERN
ibool ibool
fil_page_is_index_page( fil_page_is_index_page(
/*===================*/ /*===================*/
byte *buf); /*!< in: page */ byte *buf); /*!< in: page */
/****************************************************************//** /****************************************************************//**
Get the name of the compression algorithm used for page Get the name of the compression algorithm used for page
compression. compression.
@return compression algorithm name or "UNKNOWN" if not known*/ @return compression algorithm name or "UNKNOWN" if not known*/
UNIV_INTERN
const char* const char*
fil_get_compression_alg_name( fil_get_compression_alg_name(
/*=========================*/ /*=========================*/
ulint comp_alg); /*!<in: compression algorithm number */ ulint comp_alg); /*!<in: compression algorithm number */
/****************************************************************//** /****************************************************************//**
For page compressed pages compress the page before actual write For page compressed pages compress the page before actual write
operation. operation.
@return compressed page to be written*/ @return compressed page to be written*/
UNIV_INTERN
byte* byte*
fil_compress_page( fil_compress_page(
/*==============*/ /*==============*/
ulint space_id, /*!< in: tablespace id of the ulint space_id, /*!< in: tablespace id of the
table. */ table. */
byte* buf, /*!< in: buffer from which to write; in aio byte* buf, /*!< in: buffer from which to write; in aio
this must be appropriately aligned */ this must be appropriately aligned */
byte* out_buf, /*!< out: compressed buffer */ byte* out_buf, /*!< out: compressed buffer */
ulint len, /*!< in: length of input buffer.*/ ulint len, /*!< in: length of input buffer.*/
ulint compression_level, /*!< in: compression level */ ulint level, /* in: compression level */
ulint block_size, /*!< in: block size */ ulint block_size, /*!< in: block size */
ulint* out_len, /*!< out: actual length of compressed bool encrypted, /*!< in: is page also encrypted */
page */ ulint* out_len, /*!< out: actual length of compressed
byte* lzo_mem); /*!< in: temporal memory used by LZO */ page */
byte* lzo_mem); /*!< in: temporal memory used by LZO */
/****************************************************************//** /****************************************************************//**
For page compressed pages decompress the page after actual read For page compressed pages decompress the page after actual read
operation. operation. */
@return uncompressed page */ UNIV_INTERN
void void
fil_decompress_page( fil_decompress_page(
/*================*/ /*================*/
byte* page_buf, /*!< in: preallocated buffer or NULL */ byte* page_buf, /*!< in: preallocated buffer or NULL */
byte* buf, /*!< out: buffer from which to read; in aio byte* buf, /*!< out: buffer from which to read; in aio
this must be appropriately aligned */ this must be appropriately aligned */
ulong len, /*!< in: length of output buffer.*/ ulong len, /*!< in: length of output buffer.*/
ulint* write_size); /*!< in/out: Actual payload size of ulint* write_size); /*!< in/out: Actual payload size of
the compressed data. */ the compressed data. */
/****************************************************************//** /****************************************************************//**
Get space id from fil node Get space id from fil node
@return space id*/ @return space id*/
UNIV_INTERN
ulint ulint
fil_node_get_space_id( fil_node_get_space_id(
/*==================*/ /*==================*/
fil_node_t* node); /*!< in: Node where to get space id*/ fil_node_t* node); /*!< in: Node where to get space id*/
/****************************************************************//** /****************************************************************//**
Get block size from fil node Get block size from fil node
@return block size*/ @return block size*/
UNIV_INTERN
ulint ulint
fil_node_get_block_size( fil_node_get_block_size(
fil_node_t* node); /*!< in: Node where to get block fil_node_t* node); /*!< in: Node where to get block
size */ size */
/*******************************************************************//** /*******************************************************************//**
Find out wheather the page is page compressed Find out wheather the page is page compressed
@return true if page is page compressed*/ @return true if page is page compressed*/
UNIV_INTERN
ibool ibool
fil_page_is_compressed( fil_page_is_compressed(
/*===================*/ /*===================*/
byte *buf); /*!< in: page */ byte* buf); /*!< in: page */
/*******************************************************************//**
Find out wheather the page is page compressed
@return true if page is page compressed*/
UNIV_INTERN
ibool
fil_page_is_compressed_encrypted(
/*=============================*/
byte* buf); /*!< in: page */
/*******************************************************************//** /*******************************************************************//**
Find out wheather the page is page compressed with lzo method Find out wheather the page is page compressed with lzo method
@return true if page is page compressed with lzo method*/ @return true if page is page compressed with lzo method*/
UNIV_INTERN
ibool ibool
fil_page_is_lzo_compressed( fil_page_is_lzo_compressed(
/*=======================*/ /*=======================*/
byte *buf); /*!< in: page */ byte* buf); /*!< in: page */
#endif #endif
storage/xtradb/include/fsp0pagecompress.ic
View file @ f7002c05
/***************************************************************************** /*****************************************************************************
Copyright (C) 2013, 2014, SkySQL Ab. All Rights Reserved. Copyright (C) 2013, 2015, MariaDB Corporation. All Rights Reserved.
This program is free software; you can redistribute it and/or modify it under This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software the terms of the GNU General Public License as published by the Free Software
...@@ -67,7 +67,7 @@ UNIV_INLINE ...@@ -67,7 +67,7 @@ UNIV_INLINE
ibool ibool
fil_page_is_index_page( fil_page_is_index_page(
/*===================*/ /*===================*/
byte *buf) /*!< in: page */ byte* buf) /*!< in: page */
{ {
return(mach_read_from_2(buf+FIL_PAGE_TYPE) == FIL_PAGE_INDEX); return(mach_read_from_2(buf+FIL_PAGE_TYPE) == FIL_PAGE_INDEX);
} }
...@@ -79,11 +79,23 @@ UNIV_INLINE ...@@ -79,11 +79,23 @@ UNIV_INLINE
ibool ibool
fil_page_is_compressed( fil_page_is_compressed(
/*===================*/ /*===================*/
byte *buf) /*!< in: page */ byte* buf) /*!< in: page */
{ {
return(mach_read_from_2(buf+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED); return(mach_read_from_2(buf+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED);
} }
/*******************************************************************//**
Find out wheather the page is page compressed
@return true if page is page compressed, false if not */
UNIV_INLINE
ibool
fil_page_is_compressed_encrypted(
/*=============================*/
byte* buf) /*!< in: page */
{
return(mach_read_from_2(buf+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED);
}
/*******************************************************************//** /*******************************************************************//**
Returns the page compression level of the space, or 0 if the space Returns the page compression level of the space, or 0 if the space
is not compressed. The tablespace must be cached in the memory cache. is not compressed. The tablespace must be cached in the memory cache.
...@@ -136,7 +148,7 @@ UNIV_INLINE ...@@ -136,7 +148,7 @@ UNIV_INLINE
const char* const char*
fil_get_compression_alg_name( fil_get_compression_alg_name(
/*=========================*/ /*=========================*/
ulint comp_alg) /*!<in: compression algorithm number */ ulint comp_alg) /*!<in: compression algorithm number */
{ {
switch(comp_alg) { switch(comp_alg) {
case PAGE_UNCOMPRESSED: case PAGE_UNCOMPRESSED:
...@@ -190,8 +202,10 @@ UNIV_INLINE ...@@ -190,8 +202,10 @@ UNIV_INLINE
ibool ibool
fil_page_is_lzo_compressed( fil_page_is_lzo_compressed(
/*=======================*/ /*=======================*/
byte *buf) /*!< in: page */ byte* buf) /*!< in: page */
{ {
return(mach_read_from_2(buf+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED && return((mach_read_from_2(buf+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED &&
mach_read_from_8(buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION) == PAGE_LZO_ALGORITHM); mach_read_from_8(buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION) == PAGE_LZO_ALGORITHM) ||
(mach_read_from_2(buf+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED &&
mach_read_from_2(buf+FIL_PAGE_DATA+FIL_PAGE_COMPRESSED_SIZE) == PAGE_LZO_ALGORITHM));
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7