MDEV-32364 Server crashes when starting server with high innodb_log_buffer_size

log_t::create(): Return whether the initialisation succeeded.
It may fail if too large an innodb_log_buffer_size is specified.

extra/mariabackup/xtrabackup.cc

@@ -4613,7 +4613,10 @@ static bool xtrabackup_backup_func()
 		goto fail;
 	}
 
-	log_sys.create();
+	if (!log_sys.create()) {
+		msg("Error: cannot initialize log subsystem");
+		goto fail;
+	}
 	log_sys.log.create();
 	log_sys.log.open_file(get_log_file_path());
 
@@ -6053,8 +6056,10 @@ static bool xtrabackup_prepare_func(char** argv)
 		}
 
 		sync_check_init();
+		if (!log_sys.create()) {
+			goto error_cleanup;
+		}
 		recv_sys.create();
-		log_sys.create();
 		recv_sys.recovery_on = true;
 
 		xb_fil_io_init();

mysql-test/suite/innodb/r/log_buffer_size.result

+# restart: --innodb-log-buffer-size=1125899906842624
+SELECT * FROM INFORMATION_SCHEMA.ENGINES WHERE engine = 'innodb'
+AND support IN ('YES', 'DEFAULT', 'ENABLED');
+ENGINE	SUPPORT	COMMENT	TRANSACTIONS	XA	SAVEPOINTS
+FOUND 1 /InnoDB: Cannot allocate memory; too large innodb_log_buffer_size\?/ in mysqld.1.err
+# restart

mysql-test/suite/innodb/t/log_buffer_size.test

+--source include/have_innodb.inc
+--source include/no_valgrind_without_big.inc
+# the parameter is only 32-bit on Windows
+--source include/not_windows.inc
+
+--disable_query_log
+call mtr.add_suppression("\\[Warning\\] option 'innodb-log-buffer-size': unsigned value 1125899906842624 adjusted");
+call mtr.add_suppression("InnoDB: Cannot allocate memory; too large innodb_log_buffer_size\\?");
+call mtr.add_suppression("\\[ERROR\\] InnoDB: Plugin initialization aborted");
+call mtr.add_suppression("\\[ERROR\\] Plugin 'InnoDB' registration as a STORAGE ENGINE failed");
+--enable_query_log
+
+--let $restart_parameters= --innodb-log-buffer-size=1125899906842624
+--source include/restart_mysqld.inc
+SELECT * FROM INFORMATION_SCHEMA.ENGINES WHERE engine = 'innodb'
+AND support IN ('YES', 'DEFAULT', 'ENABLED');
+
+--let SEARCH_FILE = $MYSQLTEST_VARDIR/log/mysqld.1.err
+--let SEARCH_PATTERN=InnoDB: Cannot allocate memory; too large innodb_log_buffer_size\\?
+--source include/search_pattern_in_file.inc
+
+--let $restart_parameters=
+--source include/restart_mysqld.inc

storage/innobase/include/log0log.h

@@ -681,8 +681,9 @@ struct log_t{
     return flushes.load(std::memory_order_relaxed);
   }
 
-  /** Initialise the redo log subsystem. */
-  void create();
+  /** Initialise the redo log subsystem.
+  @return whether the initialisation succeeded */
+  bool create();
 
   /** Shut down the redo log subsystem. */
   void close();

storage/innobase/log/log0log.cc

@@ -171,11 +171,10 @@ log_set_capacity(ulonglong file_size)
 }
 
 /** Initialize the redo log subsystem. */
-void log_t::create()
+bool log_t::create()
 {
   ut_ad(this == &log_sys);
   ut_ad(!is_initialised());
-  m_initialised= true;
 
   mysql_mutex_init(log_sys_mutex_key, &mutex, nullptr);
   mysql_mutex_init(log_flush_order_mutex_key, &flush_order_mutex, nullptr);
@@ -191,9 +190,18 @@ void log_t::create()
 
   buf= static_cast<byte*>(ut_malloc_dontdump(srv_log_buffer_size,
                                              PSI_INSTRUMENT_ME));
-  TRASH_ALLOC(buf, srv_log_buffer_size);
+  if (!buf)
+    return false;
   flush_buf= static_cast<byte*>(ut_malloc_dontdump(srv_log_buffer_size,
                                                    PSI_INSTRUMENT_ME));
+  if (!flush_buf)
+  {
+    ut_free_dodump(buf, srv_log_buffer_size);
+    buf= nullptr;
+    return false;
+  }
+
+  TRASH_ALLOC(buf, srv_log_buffer_size);
   TRASH_ALLOC(flush_buf, srv_log_buffer_size);
 
   max_buf_free= srv_log_buffer_size / LOG_BUF_FLUSH_RATIO -
@@ -220,6 +228,8 @@ void log_t::create()
   buf_free= LOG_BLOCK_HDR_SIZE;
   checkpoint_buf= static_cast<byte*>
     (aligned_malloc(OS_FILE_LOG_BLOCK_SIZE, OS_FILE_LOG_BLOCK_SIZE));
+  m_initialised= true;
+  return true;
 }
 
 file_os_io::file_os_io(file_os_io &&rhs) : m_fd(rhs.m_fd)

storage/innobase/srv/srv0start.cc

@@ -99,6 +99,7 @@ Created 2/16/1996 Heikki Tuuri
 #include "os0event.h"
 #include "zlib.h"
 #include "ut0crc32.h"
+#include "log.h"
 
 /** We are prepared for a situation that we have this many threads waiting for
 a semaphore inside InnoDB. srv_start() sets the value. */
@@ -1248,7 +1249,12 @@ dberr_t srv_start(bool create_new_db)
 	}
 #endif /* UNIV_DEBUG */
 
-	log_sys.create();
+	if (!log_sys.create()) {
+		sql_print_error("InnoDB: Cannot allocate memory;"
+				" too large innodb_log_buffer_size?");
+		return srv_init_abort(DB_ERROR);
+	}
+
 	recv_sys.create();
 	lock_sys.create(srv_lock_table_size);