MDEV-33274 The test encryption.innodb-redo-nokeys often fails

If we fail to open a tablespace while looking for FILE_CHECKPOINT, we
set the corruption flag. Specifically, if encryption key is missing, we
would not be able to open an encrypted tablespace and the flag could be
set. We miss checking for this flag and report "Missing FILE_CHECKPOINT"

Address review comment to improve the test. Flush pages before starting
no-checkpoint block. It should improve the number of cases where the
test is skipped because some intermediate checkpoint is triggered.

mysql-test/suite/encryption/r/innodb-redo-nokeys.result

@@ -17,6 +17,15 @@ insert into t2 select * from t1;
 insert into t3 select * from t1;
 insert into t4 select * from t1;
 commit;
+
+# Flush all dirty pages from buffer pool
+SET @no_checkpoint_save_pct= @@GLOBAL.innodb_max_dirty_pages_pct;
+SET @no_checkpoint_save_pct_lwm= @@GLOBAL.innodb_max_dirty_pages_pct_lwm;
+SET GLOBAL innodb_max_dirty_pages_pct_lwm=0.0;
+SET GLOBAL innodb_max_dirty_pages_pct=0.0;
+SET GLOBAL innodb_max_dirty_pages_pct= @no_checkpoint_save_pct;
+SET GLOBAL innodb_max_dirty_pages_pct_lwm= @no_checkpoint_save_pct_lwm;
+
 CREATE TABLE t5 (a VARCHAR(8)) ENGINE=InnoDB ENCRYPTED=YES;
 SET GLOBAL innodb_flush_log_at_trx_commit=1;
 begin;

mysql-test/suite/encryption/t/innodb-redo-nokeys.test

@@ -38,7 +38,9 @@ insert into t3 select * from t1;
 insert into t4 select * from t1;
 commit;
 
+let $no_checkpoint_flush= 1;
 --source ../../suite/innodb/include/no_checkpoint_start.inc
+
 #
 # We test redo log page read at recv_read_page using
 # keys that are not in std_data/keys.txt. If checkpoint

mysql-test/suite/innodb/include/no_checkpoint_start.inc

 # Preparation for using no_checkpoint_end.inc
 
+# no_checkpoint_flush: Set to trigger flushing the dirty pages from buffer pool
+# and checkpoint before the "no checkpoint" block.
+
+if ($no_checkpoint_flush) {
+  --echo
+  --echo # Flush all dirty pages from buffer pool
+  SET @no_checkpoint_save_pct= @@GLOBAL.innodb_max_dirty_pages_pct;
+  SET @no_checkpoint_save_pct_lwm= @@GLOBAL.innodb_max_dirty_pages_pct_lwm;
+
+  SET GLOBAL innodb_max_dirty_pages_pct_lwm=0.0;
+  SET GLOBAL innodb_max_dirty_pages_pct=0.0;
+
+  let $wait_condition =
+  SELECT variable_value = 0
+  FROM information_schema.global_status
+  WHERE variable_name = 'INNODB_BUFFER_POOL_PAGES_DIRTY';
+  --source include/wait_condition.inc
+
+  SET GLOBAL innodb_max_dirty_pages_pct= @no_checkpoint_save_pct;
+  SET GLOBAL innodb_max_dirty_pages_pct_lwm= @no_checkpoint_save_pct_lwm;
+  --echo
+}
+
 let MYSQLD_DATADIR= `select @@datadir`;
 --replace_regex /.*Last checkpoint at[ 	]*([0-9]+).*/\1/
 let CHECKPOINT_LSN=`SHOW ENGINE INNODB STATUS`;

storage/innobase/log/log0recv.cc

@@ -3538,6 +3538,15 @@ recv_recovery_from_checkpoint_start(lsn_t flush_lsn)
 		return(DB_ERROR);
 	}
 
+	/* If we fail to open a tablespace while looking for FILE_CHECKPOINT, we
+	set the corruption flag. Specifically, if encryption key is missing, we
+	would not be able to open an encrypted tablespace and the flag could be
+	set. */
+	if (recv_sys.found_corrupt_fs) {
+		mysql_mutex_unlock(&log_sys.mutex);
+		return DB_ERROR;
+	}
+
 	if (recv_sys.mlog_checkpoint_lsn == 0) {
 		lsn_t scan_lsn = log_sys.log.scanned_lsn;
 		if (!srv_read_only_mode && scan_lsn != checkpoint_lsn) {