MDEV-32939 If tables are frequently created, renamed, dropped, a backup cannot be restored

During mariadb-backup --backup, a table could be renamed, created and
dropped. We could have both oldname.ibd and oldname.new, and one of
the files would be deleted before the InnoDB recovery starts. The desired
end result would be that we will recover both oldname.ibd and newname.ibd.

During normal crash recovery, at most one file operation (create, rename,
delete) may require to be replayed from the write-ahead log before the
DDL recovery starts.

deferred_spaces.create(): In mariadb-backup --prepare, try to create the
file in case it does not exist.

fil_name_process(): Display a message about not found files not only
if innodb_force_recovery is set, but also in mariadb-backup --prepare.
If we are processing a FILE_RENAME for a tablespace whose recovery is
deferred, suppress the message and adjust the file name in case
fil_ibd_load() returns FIL_LOAD_NOT_FOUND or FIL_LOAD_DEFER.

fil_ibd_load(): Remove a redundant file name comparison.
The caller already compared that the file names are different.
We used to wrongly return FIL_LOAD_OK instead of FIL_LOAD_ID_CHANGED
if only the schema name differed, such as a/t1.ibd and b/t1.ibd.

Tested by: Matthias Leich
Reviewed by: Thirunarayanan Balathandayuthapani

storage/innobase/fil/fil0fil.cc

@@ -2486,21 +2486,15 @@ fil_ibd_load(
 	mysql_mutex_unlock(&fil_system.mutex);
 
 	if (space) {
-		/* Compare the filename we are trying to open with the
-		filename from the first node of the tablespace we opened
-		previously. Fail if it is different. */
-		fil_node_t* node = UT_LIST_GET_FIRST(space->chain);
-		if (0 != strcmp(innobase_basename(filename),
-				innobase_basename(node->name))) {
-			ib::info()
-				<< "Ignoring data file '" << filename
-				<< "' with space ID " << space->id
-				<< ". Another data file called " << node->name
-				<< " exists with the same space ID.";
+		sql_print_information("InnoDB: Ignoring data file '%s'"
+				      " with space ID " ULINTPF
+				      ". Another data file called %s"
+				      " exists"
+				      " with the same space ID.",
+				      filename, space->id,
+				      UT_LIST_GET_FIRST(space->chain)->name);
 		space = NULL;
-			return(FIL_LOAD_ID_CHANGED);
-		}
-		return(FIL_LOAD_OK);
+		return FIL_LOAD_ID_CHANGED;
 	}
 
 	if (srv_operation == SRV_OPERATION_RESTORE) {

storage/innobase/log/log0recv.cc

@@ -855,7 +855,22 @@ static struct
           filename= tbl_name + 1;
       }
     }
-    space->add(filename, OS_FILE_CLOSED, size, false, false);
+    pfs_os_file_t handle= OS_FILE_CLOSED;
+    if (srv_operation == SRV_OPERATION_RESTORE)
+    {
+      /* During mariadb-backup --backup, a table could be renamed,
+      created and dropped, and we may be missing the file at this
+      point of --prepare. Try to create the file if it does not exist
+      already. If the file exists, we'll pass handle=OS_FILE_CLOSED
+      and the file will be opened normally in fil_space_t::acquire()
+      inside recv_sys_t::recover_deferred(). */
+      bool success;
+      handle= os_file_create(innodb_data_file_key, filename,
+                             OS_FILE_CREATE | OS_FILE_ON_ERROR_NO_EXIT |
+                             OS_FILE_ON_ERROR_SILENT,
+                             OS_FILE_AIO, OS_DATA_FILE, false, &success);
+    }
+    space->add(filename, handle, size, false, false);
     space->recv_size= it->second.size;
     space->size_in_header= size;
     return space;
@@ -1278,7 +1293,8 @@ static void fil_name_process(const char *name, ulint len, uint32_t space_id,
 
 	file_name_t&	f = p.first->second;
 
-	if (auto d = deferred_spaces.find(space_id)) {
+	auto d = deferred_spaces.find(space_id);
+	if (d) {
 		if (deleted) {
 			d->deleted = true;
 			goto got_deleted;
@@ -1345,23 +1361,37 @@ static void fil_name_process(const char *name, ulint len, uint32_t space_id,
 			FILE_* record. */
 			ut_ad(space == NULL);
 
-			if (srv_force_recovery) {
+			if (srv_operation == SRV_OPERATION_RESTORE && d
+			    && ftype == FILE_RENAME) {
+rename:
+				d->file_name = fname.name;
+				f.name = fname.name;
+				break;
+			}
+
+			if (srv_force_recovery
+			    || srv_operation == SRV_OPERATION_RESTORE) {
 				/* Without innodb_force_recovery,
 				missing tablespaces will only be
 				reported in
 				recv_init_crash_recovery_spaces().
 				Enable some more diagnostics when
 				forcing recovery. */
-
-				ib::info()
-					<< "At LSN: " << recv_sys.recovered_lsn
-					<< ": unable to open file "
-					<< fname.name
-					<< " for tablespace " << space_id;
+				sql_print_information("InnoDB: At LSN: " LSN_PF
+						      ": unable to open file "
+						      "%s for tablespace "
+						      UINT32PF,
+						      recv_sys.recovered_lsn,
+						      fname.name.c_str(),
+						      space_id);
 			}
 			break;
 
 		case FIL_LOAD_DEFER:
+			if (d && ftype == FILE_RENAME
+			    && srv_operation == SRV_OPERATION_RESTORE) {
+				goto rename;
+			}
 			/* Skip the deferred spaces
 			when lsn is already processed */
 			if (store != store_t::STORE_IF_EXISTS) {

storage/innobase/os/os0file.cc

@@ -4146,7 +4146,6 @@ bool fil_node_t::read_page0()
         != DB_SUCCESS)
     {
       sql_print_error("InnoDB: Unable to read first page of file %s", name);
-corrupted:
       aligned_free(page);
       return false;
     }
@@ -4163,25 +4162,35 @@ bool fil_node_t::read_page0()
     if (!fil_space_t::is_valid_flags(flags, space->id))
     {
       ulint cflags= fsp_flags_convert_from_101(flags);
-      if (cflags == ULINT_UNDEFINED)
+      if (cflags != ULINT_UNDEFINED)
       {
-invalid:
-        ib::error() << "Expected tablespace flags "
-          << ib::hex(space->flags)
-          << " but found " << ib::hex(flags)
-          << " in the file " << name;
-        goto corrupted;
-      }
-
         ulint cf= cflags & ~FSP_FLAGS_MEM_MASK;
         ulint sf= space->flags & ~FSP_FLAGS_MEM_MASK;
 
-      if (!fil_space_t::is_flags_equal(cf, sf) &&
-          !fil_space_t::is_flags_equal(sf, cf))
-        goto invalid;
+        if (fil_space_t::is_flags_equal(cf, sf) ||
+            fil_space_t::is_flags_equal(sf, cf))
+        {
           flags= cflags;
+          goto flags_ok;
+        }
+      }
+
+      aligned_free(page);
+      goto invalid;
+    }
+
+    if (!fil_space_t::is_flags_equal((flags & ~FSP_FLAGS_MEM_MASK),
+                                     (space->flags & ~FSP_FLAGS_MEM_MASK)) &&
+        !fil_space_t::is_flags_equal((space->flags & ~FSP_FLAGS_MEM_MASK),
+                                     (flags & ~FSP_FLAGS_MEM_MASK)))
+    {
+invalid:
+      sql_print_error("InnoDB: Expected tablespace flags 0x%zx but found 0x%zx"
+                      " in the file %s", space->flags, flags, name);
+      return false;
     }
 
+  flags_ok:
     ut_ad(!(flags & FSP_FLAGS_MEM_MASK));
 
     /* Try to read crypt_data from page 0 if it is not yet read. */