MDEV-21216 InnoDB does dirty read of TRX_SYS page before recovery

InnoDB startup was discovering undo tablespaces in a dirty way.
It was reading a possibly stale copy of the TRX_SYS page before
processing any redo log records.

srv_start(): Do not call buf_pool_invalidate(). Invoke
trx_rseg_get_n_undo_tablespaces() after the recovery has been initiated.

recv_recovery_from_checkpoint_start(): Assert that the buffer pool is
empty. This used to be guaranteed by the buf_pool_invalidate() call.

trx_rseg_get_n_undo_tablespaces(): Move to the calling compilation unit,
and reimplement in a simpler way.

srv_undo_tablespace_create(): Remove the constant parameter
size=SRV_UNDO_TABLESPACE_SIZE_IN_PAGES.

srv_undo_tablespace_open(): Reimplement in a cleaner way, with
more robust error handling.

srv_all_undo_tablespaces_open(): Split from srv_undo_tablespaces_init().

srv_undo_tablespaces_init(): Read all "undo001","undo002" tablespace
files directly, without consulting the TRX_SYS page via calling
trx_rseg_get_n_undo_tablespaces().

This is joint work with Thirunarayanan Balathandayuthapani.

mysql-test/suite/innodb/r/log_file.result

@@ -217,7 +217,7 @@ SELECT * FROM INFORMATION_SCHEMA.ENGINES
 WHERE engine = 'innodb'
 AND support IN ('YES', 'DEFAULT', 'ENABLED');
 ENGINE	SUPPORT	COMMENT	TRANSACTIONS	XA	SAVEPOINTS
-FOUND 1 /InnoDB: Unable to open undo tablespace.*undo002/ in mysqld.1.err
+FOUND 1 /InnoDB: Expected to open innodb_undo_tablespaces=3 but was able to find only 1/ in mysqld.1.err
 bak_ib_logfile0
 bak_ib_logfile1
 bak_ib_logfile2
@@ -255,7 +255,7 @@ SELECT * FROM INFORMATION_SCHEMA.ENGINES
 WHERE engine = 'innodb'
 AND support IN ('YES', 'DEFAULT', 'ENABLED');
 ENGINE	SUPPORT	COMMENT	TRANSACTIONS	XA	SAVEPOINTS
-FOUND 1 /InnoDB: Unable to open undo tablespace.*undo001/ in mysqld.1.err
+FOUND 1 /InnoDB: Expected to open innodb_undo_tablespaces=3 but was able to find only 0/ in mysqld.1.err
 bak_ib_logfile0
 bak_ib_logfile1
 bak_ib_logfile2

mysql-test/suite/innodb/t/log_file.test

@@ -171,7 +171,7 @@ let SEARCH_PATTERN=undo tablespace .*undo003.* exists\. Creating system tablespa
 --source include/start_mysqld.inc
 eval $check_no_innodb;
 --source include/shutdown_mysqld.inc
-let SEARCH_PATTERN=InnoDB: Unable to open undo tablespace.*undo002;
+let SEARCH_PATTERN=InnoDB: Expected to open innodb_undo_tablespaces=3 but was able to find only 1;
 --source include/search_pattern_in_file.inc
 # clean up & Restore
 --source ../include/log_file_cleanup.inc
@@ -183,7 +183,7 @@ let SEARCH_PATTERN=InnoDB: Unable to open undo tablespace.*undo002;
 --source include/start_mysqld.inc
 eval $check_no_innodb;
 --source include/shutdown_mysqld.inc
-let SEARCH_PATTERN=InnoDB: Unable to open undo tablespace.*undo001;
+let SEARCH_PATTERN=InnoDB: Expected to open innodb_undo_tablespaces=3 but was able to find only 0;
 --source include/search_pattern_in_file.inc
 
 # clean up & Restore

mysql-test/suite/mariabackup/undo_space_id.result

-# Create 2 UNDO TABLESPACE(UNDO003, UNDO004)
+# Create 2 UNDO TABLESPACE(UNDO001(space_id =3), UNDO002(space_id =4))
 CREATE TABLE t1(a varchar(60)) ENGINE INNODB;
 start transaction;
 INSERT INTO t1 VALUES(1);
 # xtrabackup backup
 # Display undo log files from target directory
-undo003
-undo004
+undo001
+undo002
 # xtrabackup prepare
 # Display undo log files from targer directory
-undo003
-undo004
+undo001
+undo002
 DROP TABLE t1;

mysql-test/suite/mariabackup/undo_space_id.test

 --source include/have_innodb.inc
 --source include/have_debug.inc
 
---echo # Create 2 UNDO TABLESPACE(UNDO003, UNDO004)
+--echo # Create 2 UNDO TABLESPACE(UNDO001(space_id =3), UNDO002(space_id =4))
 
 let $basedir=$MYSQLTEST_VARDIR/tmp/backup;
 

storage/innobase/include/trx0rseg.h

@@ -85,17 +85,6 @@ trx_rseg_create(ulint space_id)
 void
 trx_temp_rseg_create();
 
-/********************************************************************
-Get the number of unique rollback tablespaces in use except space id 0.
-The last space id will be the sentinel value ULINT_UNDEFINED. The array
-will be sorted on space id. Note: space_ids should have have space for
-TRX_SYS_N_RSEGS + 1 elements.
-@return number of unique rollback tablespaces in use. */
-ulint
-trx_rseg_get_n_undo_tablespaces(
-/*============================*/
-	ulint*		space_ids);	/*!< out: array of space ids of
-					UNDO tablespaces */
 /* Number of undo log slots in a rollback segment file copy */
 #define TRX_RSEG_N_SLOTS	(srv_page_size / 16)
 

storage/innobase/log/log0recv.cc

@@ -3414,6 +3414,15 @@ recv_recovery_from_checkpoint_start(lsn_t flush_lsn)
 	ut_ad(srv_operation == SRV_OPERATION_NORMAL
 	      || srv_operation == SRV_OPERATION_RESTORE
 	      || srv_operation == SRV_OPERATION_RESTORE_EXPORT);
+#ifdef UNIV_DEBUG
+	for (ulint i= 0; i < srv_buf_pool_instances; i++) {
+		buf_pool_t* buf_pool = buf_pool_from_array(i);
+		buf_flush_list_mutex_enter(buf_pool);
+		ut_ad(UT_LIST_GET_LEN(buf_pool->LRU) == 0);
+		ut_ad(UT_LIST_GET_LEN(buf_pool->unzip_LRU) == 0);
+		buf_flush_list_mutex_exit(buf_pool);
+	}
+#endif
 
 	/* Initialize red-black tree for fast insertions into the
 	flush_list during recovery process. */

storage/innobase/trx/trx0rseg.cc

@@ -31,8 +31,6 @@ Created 3/26/1996 Heikki Tuuri
 #include "trx0purge.h"
 #include "srv0mon.h"
 
-#include <algorithm>
-
 #ifdef WITH_WSREP
 #include <mysql/service_wsrep.h>
 
@@ -712,55 +710,6 @@ trx_temp_rseg_create()
 	}
 }
 
-/********************************************************************
-Get the number of unique rollback tablespaces in use except space id 0.
-The last space id will be the sentinel value ULINT_UNDEFINED. The array
-will be sorted on space id. Note: space_ids should have have space for
-TRX_SYS_N_RSEGS + 1 elements.
-@return number of unique rollback tablespaces in use. */
-ulint
-trx_rseg_get_n_undo_tablespaces(
-/*============================*/
-	ulint*		space_ids)	/*!< out: array of space ids of
-					UNDO tablespaces */
-{
-	mtr_t mtr;
-	mtr.start();
-
-	buf_block_t* sys_header = trx_sysf_get(&mtr, false);
-	if (!sys_header) {
-		mtr.commit();
-		return 0;
-	}
-
-	ulint* end = space_ids;
-
-	for (ulint rseg_id = 0; rseg_id < TRX_SYS_N_RSEGS; rseg_id++) {
-		uint32_t page_no = trx_sysf_rseg_get_page_no(sys_header,
-							     rseg_id);
-
-		if (page_no == FIL_NULL) {
-			continue;
-		}
-
-		if (ulint space = trx_sysf_rseg_get_space(sys_header,
-							  rseg_id)) {
-			if (std::find(space_ids, end, space) == end) {
-				*end++ = space;
-			}
-		}
-	}
-
-	mtr.commit();
-
-	ut_a(end - space_ids <= TRX_SYS_N_RSEGS);
-	*end = ULINT_UNDEFINED;
-
-	std::sort(space_ids, end);
-
-	return ulint(end - space_ids);
-}
-
 /** Update the offset information about the end of the binlog entry
 which corresponds to the transaction just being committed.
 In a replication slave, this updates the master binlog position