MDEV-30551 InnoDB recovery hangs when buffer pool ran out of memory

- During non-last batch of multi-batch recovery, InnoDB holds
log_sys.mutex and preallocates the block which may intiate
page flush, which may initiate log flush, which requires
log_sys.mutex to acquire again. This leads to assert failure.
So InnoDB recovery should release log_sys.mutex before
preallocating the block.

mysql-test/suite/innodb/r/alter_copy.result

@@ -51,7 +51,7 @@ ADD INDEX(a,b,d), ADD INDEX(a,d,b), ADD INDEX(b,c,d), ADD INDEX(b,d,c),
 ALGORITHM=COPY;
 connection default;
 SET DEBUG_SYNC='now WAIT_FOR hung';
-# restart: --innodb-force-recovery=3
+# restart: --innodb-force-recovery=3 --debug_dbug=+d,recv_ran_out_of_buffer
 disconnect hang;
 #sql-alter.frm
 #sql-alter.ibd

mysql-test/suite/innodb/t/alter_copy.test

@@ -57,7 +57,7 @@ ALTER TABLE t ADD INDEX(b,c,d,a),ADD INDEX(b,c,a,d),ADD INDEX(b,a,c,d),ADD INDEX
 connection default;
 SET DEBUG_SYNC='now WAIT_FOR hung';
 let $shutdown_timeout=0;
---let $restart_parameters= --innodb-force-recovery=3
+--let $restart_parameters= --innodb-force-recovery=3 --debug_dbug="+d,recv_ran_out_of_buffer"
 --source include/restart_mysqld.inc
 disconnect hang;
 let $shutdown_timeout=;

storage/innobase/buf/buf0lru.cc

@@ -408,6 +408,11 @@ buf_block_t *buf_LRU_get_free_block(bool have_mutex)
 		mysql_mutex_assert_owner(&buf_pool.mutex);
 		goto got_mutex;
 	}
+	DBUG_EXECUTE_IF("recv_ran_out_of_buffer",
+			if (recv_recovery_is_on()
+			    && recv_sys.apply_log_recs) {
+				goto flush_lru;
+			});
 	mysql_mutex_lock(&buf_pool.mutex);
 got_mutex:
 	buf_LRU_check_size_of_non_data_objects();
@@ -493,7 +498,9 @@ buf_block_t *buf_LRU_get_free_block(bool have_mutex)
 	removing the block from buf_pool.page_hash and buf_pool.LRU is fairly
 	involved (particularly in case of ROW_FORMAT=COMPRESSED pages). We
 	can do that in a separate patch sometime in future. */
-
+#ifndef DBUG_OFF
+flush_lru:
+#endif
 	if (!buf_flush_LRU(innodb_lru_flush_size)) {
 		MONITOR_INC(MONITOR_LRU_SINGLE_FLUSH_FAILURE_COUNT);
 		++flush_failures;

storage/innobase/log/log0recv.cc

@@ -2692,8 +2692,23 @@ void recv_sys_t::apply(bool last_batch)
 
     fil_system.extend_to_recv_size();
 
+    /* Release the log_sys mutex in non-last batches of multi-batch
+    recovery mode and recv_sys.mutex before preallocating the
+    block because while preallocating the block which may initiate
+    log flush which requires log_sys mutex to acquire again, which
+    should be acquired before recv_sys.mutex in order to avoid
+    deadlocks. */
+    mutex_exit(&mutex);
+    if (!last_batch)
+      mysql_mutex_unlock(&log_sys.mutex);
+
+    mysql_mutex_assert_not_owner(&log_sys.mutex);
     buf_block_t *free_block= buf_LRU_get_free_block(false);
 
+    if (!last_batch)
+      mysql_mutex_lock(&log_sys.mutex);
+    mutex_enter(&mutex);
+
     for (map::iterator p= pages.begin(); p != pages.end(); )
     {
       const page_id_t page_id= p->first;
@@ -2708,7 +2723,10 @@ void recv_sys_t::apply(bool last_batch)
         if (UNIV_LIKELY(!!recover_low(page_id, p, mtr, free_block)))
         {
           mutex_exit(&mutex);
+          if (!last_batch) mysql_mutex_unlock(&log_sys.mutex);
+          mysql_mutex_assert_not_owner(&log_sys.mutex);
           free_block= buf_LRU_get_free_block(false);
+          if (!last_batch) mysql_mutex_lock(&log_sys.mutex);
           mutex_enter(&mutex);
           break;
         }