MDEV-9793: getting mysqld crypto key from key version failed

Make sure that we read all possible encryption keys from checkpoint
and if log block checksum does not match, print all found
checkpoint encryption keys.

storage/innobase/include/log0crypt.h

 /*****************************************************************************
 
 Copyright (C) 2013, 2015, Google Inc. All Rights Reserved.
-Copyright (C) 2014, 2015, MariaDB Corporation. All Rights Reserved.
+Copyright (C) 2014, 2016, MariaDB Corporation. All Rights Reserved.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -117,4 +117,12 @@ log_crypt_print_error(
 /*==================*/
 	log_crypt_err_t 	err_info); 	/*!< out: error info */
 
+/*********************************************************************//**
+Print checkpoint no from log block and all encryption keys from
+checkpoints if they are present. Used for problem analysis. */
+void
+log_crypt_print_checkpoint_keys(
+/*============================*/
+	const byte* log_block);
+
 #endif  // log0crypt.h

storage/innobase/log/log0crypt.cc

@@ -127,11 +127,34 @@ static
 const crypt_info_t*
 get_crypt_info(
 /*===========*/
-	const byte* log_block) {
+	const byte* log_block)
+{
 	ib_uint64_t checkpoint_no = log_block_get_checkpoint_no(log_block);
 	return get_crypt_info(checkpoint_no);
 }
 
+/*********************************************************************//**
+Print checkpoint no from log block and all encryption keys from
+checkpoints if they are present. Used for problem analysis. */
+void
+log_crypt_print_checkpoint_keys(
+/*============================*/
+	const byte* log_block)
+{
+	ib_uint64_t checkpoint_no = log_block_get_checkpoint_no(log_block);
+
+	if (crypt_info.size()) {
+		fprintf(stderr, "InnoDB: redo log checkpoint: %lu [ chk key ]: ", checkpoint_no);
+		for (size_t i = 0; i < crypt_info.size(); i++) {
+			struct crypt_info_t* it = &crypt_info[i];
+			fprintf(stderr, "[ %lu %u ] ",
+				it->checkpoint_no,
+				it->key_version);
+		}
+		fprintf(stderr, "\n");
+	}
+}
+
 /*********************************************************************//**
 Call AES CTR to encrypt/decrypt log blocks. */
 static
@@ -280,10 +303,13 @@ static
 bool
 add_crypt_info(crypt_info_t* info)
 {
+	const crypt_info_t* found=NULL;
 	/* so that no one is searching array while we modify it */
 	ut_ad(mutex_own(&(log_sys->mutex)));
 
-	if (get_crypt_info(info->checkpoint_no) != NULL) {
+	found = get_crypt_info(info->checkpoint_no);
+
+	if (found != NULL && found->checkpoint_no == info->checkpoint_no) {
 		// already present...
 		return true;
 	}

storage/innobase/log/log0recv.cc

@@ -2714,6 +2714,8 @@ recv_scan_log_recs(
 
 			/* Garbage or an incompletely written log block */
 
+			/* Print checkpoint encryption keys if present */
+			log_crypt_print_checkpoint_keys(log_block);
 			finished = TRUE;
 
 			if (maybe_encrypted) {

storage/xtradb/include/log0crypt.h

 /*****************************************************************************
 
 Copyright (C) 2013, 2015, Google Inc. All Rights Reserved.
-Copyright (C) 2014, 2015, MariaDB Corporation. All Rights Reserved.
+Copyright (C) 2014, 2016, MariaDB Corporation. All Rights Reserved.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -117,4 +117,12 @@ log_crypt_print_error(
 /*==================*/
 	log_crypt_err_t 	err_info); 	/*!< out: error info */
 
+/*********************************************************************//**
+Print checkpoint no from log block and all encryption keys from
+checkpoints if they are present. Used for problem analysis. */
+void
+log_crypt_print_checkpoint_keys(
+/*============================*/
+	const byte* log_block);
+
 #endif  // log0crypt.h

storage/xtradb/log/log0crypt.cc

@@ -127,11 +127,34 @@ static
 const crypt_info_t*
 get_crypt_info(
 /*===========*/
-	const byte* log_block) {
+	const byte* log_block)
+{
 	ib_uint64_t checkpoint_no = log_block_get_checkpoint_no(log_block);
 	return get_crypt_info(checkpoint_no);
 }
 
+/*********************************************************************//**
+Print checkpoint no from log block and all encryption keys from
+checkpoints if they are present. Used for problem analysis. */
+void
+log_crypt_print_checkpoint_keys(
+/*============================*/
+	const byte* log_block)
+{
+	ib_uint64_t checkpoint_no = log_block_get_checkpoint_no(log_block);
+
+	if (crypt_info.size()) {
+		fprintf(stderr, "InnoDB: redo log checkpoint: %lu [ chk key ]: ", checkpoint_no);
+		for (size_t i = 0; i < crypt_info.size(); i++) {
+			struct crypt_info_t* it = &crypt_info[i];
+			fprintf(stderr, "[ %lu %u ] ",
+				it->checkpoint_no,
+				it->key_version);
+		}
+		fprintf(stderr, "\n");
+	}
+}
+
 /*********************************************************************//**
 Call AES CTR to encrypt/decrypt log blocks. */
 static
@@ -280,10 +303,13 @@ static
 bool
 add_crypt_info(crypt_info_t* info)
 {
+	const crypt_info_t* found=NULL;
 	/* so that no one is searching array while we modify it */
 	ut_ad(mutex_own(&(log_sys->mutex)));
 
-	if (get_crypt_info(info->checkpoint_no) != NULL) {
+	found = get_crypt_info(info->checkpoint_no);
+
+	if (found != NULL && found->checkpoint_no == info->checkpoint_no) {
 		// already present...
 		return true;
 	}

storage/xtradb/log/log0recv.cc

@@ -2786,6 +2786,8 @@ recv_scan_log_recs(
 
 			/* Garbage or an incompletely written log block */
 
+			/* Print checkpoint encryption keys if present */
+			log_crypt_print_checkpoint_keys(log_block);
 			finished = TRUE;
 
 			if (maybe_encrypted) {