MDEV-18200 MariaBackup full backup failed with InnoDB: Failing assertion: success

There are many filesystem related errors that can occur with
MariaBackup. These already outputed to stderr with a good description of
the error. Many of these are permission or resource (file descriptor)
limits where the assertion and resulting core crash doesn't offer
developers anything more than the log message. To the user, assertions
and core crashes come across as poor error handling.

As such we return an error and handle this all the way up the stack.

mysql-test/suite/mariabackup/data_directory.result

@@ -11,3 +11,9 @@ SELECT * FROM t;
 a
 1
 DROP TABLE t;
+#
+# MDEV-18200 MariaBackup full backup failed with InnoDB: Failing assertion: success
+#
+#
+# End of 10.4 tests
+#

mysql-test/suite/mariabackup/data_directory.test

@@ -21,4 +21,19 @@ rmdir $table_data_dir;
 SELECT * FROM t;
 DROP TABLE t;
 rmdir $targetdir;
+
+--echo #
+--echo # MDEV-18200 MariaBackup full backup failed with InnoDB: Failing assertion: success
+--echo #
+let $DATADIR= `select @@datadir`;
+chmod 0000 $DATADIR/ibdata1;
+--disable_result_log
+--error 1
+exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir;
+--enable_result_log
+chmod 0755 $DATADIR/ibdata1;
 rmdir $table_data_dir;
+rmdir $targetdir;
+--echo #
+--echo # End of 10.4 tests
+--echo #

storage/innobase/fil/fil0fil.cc

@@ -334,6 +334,7 @@ fil_node_t* fil_space_t::add(const char* name, pfs_os_file_t handle,
 	return node;
 }
 
+__attribute__((warn_unused_result, nonnull))
 /** Open a tablespace file.
 @param node  data file
 @return whether the file was successfully opened */
@@ -362,9 +363,9 @@ static bool fil_node_open_file_low(fil_node_t *node)
                                  : OS_FILE_OPEN | OS_FILE_ON_ERROR_NO_EXIT,
                                  OS_FILE_AIO, type,
                                  srv_read_only_mode, &success);
-    if (node->is_open())
+
+    if (success && node->is_open())
     {
-      ut_ad(success);
 #ifndef _WIN32
       if (!node->space->id && !srv_read_only_mode && my_disable_locking &&
           os_file_lock(node->handle, node->name))

storage/innobase/include/log0recv.h

@@ -254,6 +254,7 @@ struct recv_sys_t
   /** The contents of the doublewrite buffer */
   recv_dblwr_t dblwr;
 
+  __attribute__((warn_unused_result)) 
   inline dberr_t read(os_offset_t offset, span<byte> buf);
   inline size_t files_size();
   void close_files();

storage/innobase/log/log0log.cc

@@ -169,6 +169,7 @@ dberr_t log_file_t::close() noexcept
   return DB_SUCCESS;
 }
 
+__attribute__((warn_unused_result))
 dberr_t log_file_t::read(os_offset_t offset, span<byte> buf) noexcept
 {
   ut_ad(is_opened());

storage/innobase/log/log0recv.cc

@@ -1611,7 +1611,8 @@ ATTRIBUTE_COLD static dberr_t recv_log_recover_pre_10_2()
   if (source_offset < (log_sys.is_pmem() ? log_sys.file_size : 4096))
     memcpy_aligned<512>(buf, &log_sys.buf[source_offset & ~511], 512);
   else
-    recv_sys.read(source_offset & ~511, {buf, 512});
+    if (dberr_t err= recv_sys.read(source_offset & ~511, {buf, 512}))
+      return err;
 
   if (log_block_calc_checksum_format_0(buf) !=
       mach_read_from_4(my_assume_aligned<4>(buf + 508)) &&