MDEV-18322 Assertion "wrong page type" on instant ALTER TABLE

row_ins_clust_index_entry_low(): Invoke btr_set_instant() in the same
mini-transaction that has successfully inserted the metadata record.
In this way, if inserting the metadata record fails before any
undo log record was written for it, the index root page will remain
consistent.

innobase_instant_try(): Remove the btr_set_instant() call.

Reviewed by: Thirunarayanan Balathandayuthapani
Tested by: Matthias Leich

mysql-test/suite/innodb/r/instant_alter_bugs.result

@@ -495,4 +495,28 @@ SET GLOBAL innodb_purge_rseg_truncate_frequency=@save_frequency;
 CREATE TABLE t1 (i int AS (0) STORED, j INT) ENGINE=InnoDB;
 ALTER TABLE t1 ADD COLUMN i INT GENERATED ALWAYS AS (1), DROP COLUMN i;
 DROP TABLE t1;
+#
+# MDEV-18322 Assertion "wrong_page_type" on instant ALTER
+#
+BEGIN NOT ATOMIC
+DECLARE c TEXT
+DEFAULT(SELECT CONCAT('CREATE TABLE t1 (c',
+GROUP_CONCAT(seq SEPARATOR ' CHAR(200), c'),
+' CHAR(211)) ENGINE=InnoDB ROW_FORMAT=REDUNDANT')
+FROM seq_1_to_40);
+EXECUTE IMMEDIATE c;
+END;
+$$
+INSERT INTO t1 SET c1=NULL;
+ALTER TABLE t1 ADD c41 INT FIRST;
+ERROR 42000: Row size too large. The maximum row size for the used table type, not counting BLOBs, is 8123. This includes storage overhead, check the manual. You have to change some columns to TEXT or BLOBs
+ALTER TABLE t1 ADD c41 INT FIRST;
+ERROR 42000: Row size too large. The maximum row size for the used table type, not counting BLOBs, is 8123. This includes storage overhead, check the manual. You have to change some columns to TEXT or BLOBs
+CHECK TABLE t1;
+Table	Op	Msg_type	Msg_text
+test.t1	check	status	OK
+SELECT COUNT(*) FROM t1;
+COUNT(*)
+1
+DROP TABLE t1;
 # End of 10.4 tests

mysql-test/suite/innodb/t/instant_alter_bugs.test

 --source include/have_innodb.inc
+--source include/have_sequence.inc
 
 SET @save_frequency= @@GLOBAL.innodb_purge_rseg_truncate_frequency;
 SET GLOBAL innodb_purge_rseg_truncate_frequency=1;
@@ -526,4 +527,28 @@ CREATE TABLE t1 (i int AS (0) STORED, j INT) ENGINE=InnoDB;
 ALTER TABLE t1 ADD COLUMN i INT GENERATED ALWAYS AS (1), DROP COLUMN i;
 DROP TABLE t1;
 
+--echo #
+--echo # MDEV-18322 Assertion "wrong_page_type" on instant ALTER
+--echo #
+
+DELIMITER $$;
+BEGIN NOT ATOMIC
+  DECLARE c TEXT
+  DEFAULT(SELECT CONCAT('CREATE TABLE t1 (c',
+                        GROUP_CONCAT(seq SEPARATOR ' CHAR(200), c'),
+                        ' CHAR(211)) ENGINE=InnoDB ROW_FORMAT=REDUNDANT')
+          FROM seq_1_to_40);
+  EXECUTE IMMEDIATE c;
+END;
+$$
+DELIMITER ;$$
+INSERT INTO t1 SET c1=NULL;
+--error ER_TOO_BIG_ROWSIZE
+ALTER TABLE t1 ADD c41 INT FIRST;
+--error ER_TOO_BIG_ROWSIZE
+ALTER TABLE t1 ADD c41 INT FIRST;
+CHECK TABLE t1;
+SELECT COUNT(*) FROM t1;
+DROP TABLE t1;
+
 --echo # End of 10.4 tests

storage/innobase/handler/handler0alter.cc

@@ -6130,10 +6130,6 @@ static bool innobase_instant_try(
 			goto err_exit;
 		}
 
-		btr_set_instant(root, *index, &mtr);
-		mtr.commit();
-		mtr.start();
-		index->set_modified(mtr);
 		err = row_ins_clust_index_entry_low(
 			BTR_NO_LOCKING_FLAG, BTR_MODIFY_TREE, index,
 			index->n_uniq, entry, 0, thr);

storage/innobase/row/row0ins.cc

@@ -2657,14 +2657,17 @@ row_ins_clust_index_entry_low(
 		ut_ad(!dict_index_is_online_ddl(index));
 		ut_ad(!index->table->persistent_autoinc);
 		ut_ad(!index->is_instant());
+		ut_ad(!entry->info_bits);
 		mtr.set_log_mode(MTR_LOG_NO_REDO);
 	} else {
 		index->set_modified(mtr);
 
-		if (UNIV_UNLIKELY(entry->is_metadata())) {
+		if (UNIV_UNLIKELY(entry->info_bits != 0)) {
+			ut_ad(entry->is_metadata());
 			ut_ad(index->is_instant());
 			ut_ad(!dict_index_is_online_ddl(index));
 			ut_ad(mode == BTR_MODIFY_TREE);
+			ut_ad(flags == BTR_NO_LOCKING_FLAG);
 		} else {
 			if (mode == BTR_MODIFY_LEAF
 			    && dict_index_is_online_ddl(index)) {
@@ -2716,11 +2719,6 @@ row_ins_clust_index_entry_low(
 #endif /* UNIV_DEBUG */
 
 	if (UNIV_UNLIKELY(entry->info_bits != 0)) {
-		ut_ad(entry->is_metadata());
-		ut_ad(flags == BTR_NO_LOCKING_FLAG);
-		ut_ad(index->is_instant());
-		ut_ad(!dict_index_is_online_ddl(index));
-
 		const rec_t* rec = btr_cur_get_rec(cursor);
 
 		if (rec_get_info_bits(rec, page_rec_is_comp(rec))
@@ -2830,7 +2828,17 @@ row_ins_clust_index_entry_low(
 			}
 		}
 
+		if (err == DB_SUCCESS && entry->info_bits) {
+			if (buf_block_t* root
+			    = btr_root_block_get(index, RW_X_LATCH, &mtr)) {
+				btr_set_instant(root, *index, &mtr);
+			} else {
+				ut_ad("cannot find root page" == 0);
+			}
+		}
+
 		if (big_rec != NULL) {
+			ut_ad(err == DB_SUCCESS);
 			mtr_commit(&mtr);
 
 			/* Online table rebuild could read (and