MDEV-29664 Assertion `!n_mysql_tables_in_use' failed in innobase_close_connection

When ha_end_bulk_insert() fails F_UNLCK was done twice: in
select_insert::prepare_eof() and in select_create::abort_result_set().

Now we avoid making F_UNLCK in prepare_eof() if error is non-zero.

mysql-test/main/create_or_replace2.result → mysql-test/main/create_replace_debug.result

@@ -91,3 +91,13 @@ ERROR HY000: Table 't2' was not locked with LOCK TABLES
 unlock tables;
 drop tables t1;
 set @@debug_dbug= @saved_debug_dbug;
+#
+# MDEV-29664 Assertion `!n_mysql_tables_in_use' failed in innobase_close_connection
+#
+create table t1 (x int);
+set @old_dbug= @@debug_dbug;
+set @@debug_dbug= '+d,ha_end_bulk_insert_fail';
+create or replace table t2 (y int) engine innodb select * from t1;
+ERROR HY000: Out of memory.
+set @@debug_dbug= @old_dbug;
+drop table t1;

mysql-test/main/create_or_replace2.test → mysql-test/main/create_replace_debug.test

@@ -86,3 +86,14 @@ unlock tables;
 
 drop tables t1;
 set @@debug_dbug= @saved_debug_dbug;
+
+--echo #
+--echo # MDEV-29664 Assertion `!n_mysql_tables_in_use' failed in innobase_close_connection
+--echo #
+create table t1 (x int);
+set @old_dbug= @@debug_dbug;
+set @@debug_dbug= '+d,ha_end_bulk_insert_fail';
+--error ER_OUT_OF_RESOURCES
+create or replace table t2 (y int) engine innodb select * from t1;
+set @@debug_dbug= @old_dbug;
+drop table t1;

sql/handler.cc

@@ -5030,6 +5030,11 @@ int handler::ha_end_bulk_insert()
   DBUG_ENTER("handler::ha_end_bulk_insert");
   DBUG_EXECUTE_IF("crash_end_bulk_insert",
                   { extra(HA_EXTRA_FLUSH) ; DBUG_SUICIDE();});
+  if (DBUG_IF("ha_end_bulk_insert_fail"))
+  {
+    my_error(ER_OUT_OF_RESOURCES, MYF(0));
+    DBUG_RETURN(HA_ERR_OUT_OF_MEM);
+  }
   estimation_rows_to_insert= 0;
   DBUG_RETURN(end_bulk_insert());
 }

sql/sql_insert.cc

@@ -4285,7 +4285,7 @@ bool select_insert::prepare_eof()
   table->file->extra(HA_EXTRA_NO_IGNORE_DUP_KEY);
   table->file->extra(HA_EXTRA_WRITE_CANNOT_REPLACE);
 
-  if (atomic_replace)
+  if (atomic_replace && !error)
   {
     DBUG_ASSERT(table->s->tmp_table);
 
@@ -5490,7 +5490,10 @@ void select_create::abort_result_set()
 
     if (atomic_replace)
     {
+      ulonglong save_options_bits= thd->variables.option_bits;
+      thd->variables.option_bits|= OPTION_NOT_AUTOCOMMIT;
       (void) table->file->ha_external_lock(thd, F_UNLCK);
+      thd->variables.option_bits= save_options_bits;
       (void) thd->drop_temporary_table(table, NULL, true);
     }
     else

storage/innobase/handler/ha_innodb.cc

@@ -16172,6 +16172,7 @@ ha_innobase::external_lock(
 
 	/* MySQL is releasing a table lock */
 
+	ut_ad(trx->n_mysql_tables_in_use);
 	trx->n_mysql_tables_in_use--;
 	m_mysql_has_locked = false;