MDEV-33980 mariadb-backup --backup is missing retry logic for undo tablespaces

Problem:
========
- Currently mariabackup have to reread the pages in case they are
modified by server concurrently. But while reading the undo
tablespace, mariabackup failed to do reread the page in case of
error.

Fix:
===
Mariabackup --backup functionality should have retry logic
while reading the undo tablespaces.

mysql-test/suite/mariabackup/undo_space_id.result

@@ -11,3 +11,10 @@ undo004
 undo003
 undo004
 DROP TABLE t1;
+#
+#  MDEV-33980  mariadb-backup --backup is missing
+#       retry logic for undo tablespaces
+#
+# xtrabackup backup
+# Display undo log files from target directory
+FOUND 5 /Retrying to read undo tablespace*/ in backup.log

mysql-test/suite/mariabackup/undo_space_id.test

@@ -23,3 +23,22 @@ list_files $basedir undo*;
 
 DROP TABLE t1;
 rmdir $basedir;
+
+--echo #
+--echo #  MDEV-33980  mariadb-backup --backup is missing
+--echo #       retry logic for undo tablespaces
+--echo #
+let $backuplog= $MYSQLTEST_VARDIR/tmp/backup.log;
+--echo # xtrabackup backup
+--disable_result_log
+--error 1
+exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf  --backup --target-dir=$basedir --dbug=+d,undo_space_read_fail > $backuplog;
+--enable_result_log
+--echo # Display undo log files from target directory
+list_files $basedir undo*;
+
+--let SEARCH_PATTERN=Retrying to read undo tablespace*
+--let SEARCH_FILE=$backuplog
+--source include/search_pattern_in_file.inc
+rmdir $basedir;
+remove_file $backuplog;

storage/innobase/os/os0file.cc

@@ -7560,7 +7560,15 @@ bool fil_node_t::read_page0(bool first)
 	}
 	ut_free(buf2);
 
+	DBUG_EXECUTE_IF("undo_space_read_fail",
+			if (space_id == srv_undo_space_id_start) {
+				goto wrong_space_id;
+			});
+
 	if (UNIV_UNLIKELY(space_id != space->id)) {
+#ifndef DBUG_OFF
+wrong_space_id:
+#endif
 		ib::error() << "Expected tablespace id " << space->id
 			<< " but found " << space_id
 			<< " in the file " << name;

storage/innobase/srv/srv0start.cc

@@ -100,6 +100,7 @@ Created 2/16/1996 Heikki Tuuri
 #include "zlib.h"
 #include "ut0crc32.h"
 #include "btr0scrub.h"
+#include "log.h"
 
 /** Log sequence number immediately after startup */
 lsn_t	srv_start_lsn;
@@ -656,10 +657,11 @@ static bool srv_undo_tablespace_open(const char* name, ulint space_id,
 	pfs_os_file_t	fh;
 	bool		success;
 	char		undo_name[sizeof "innodb_undo000"];
+	ulint		n_retries = 5;
 
 	snprintf(undo_name, sizeof(undo_name),
 		 "innodb_undo%03u", static_cast<unsigned>(space_id));
-
+undo_retry:
 	fh = os_file_create(
 		innodb_data_file_key, name, OS_FILE_OPEN
 		| OS_FILE_ON_ERROR_NO_EXIT | OS_FILE_ON_ERROR_SILENT,
@@ -716,6 +718,15 @@ static bool srv_undo_tablespace_open(const char* name, ulint space_id,
 
 	mutex_exit(&fil_system.mutex);
 
+	if (!success && n_retries &&
+	    srv_operation == SRV_OPERATION_BACKUP) {
+		sql_print_information("InnoDB: Retrying to read undo "
+				      "tablespace %s", undo_name);
+		fil_space_free(space_id, false);
+		n_retries--;
+		goto undo_retry;
+	}
+
 	return success;
 }