MDEV-18649 Assertion supremum[7] == index.n_core_null_bytes failed

This is follow-up to MDEV-18048: Relax a too strict debug assertion

This assertion should have been relaxed when implementing the first part of
MDEV-15563: instant removal of NOT NULL attribute for ROW_FORMAT=REDUNDANT
tables.

For ROW_FORMAT=REDUNDANT, there is no bitmap of null columns;
the null flags are encoded in the end offset of each field.
We do not really care about the number of fields that can be NULL.

mysql-test/suite/innodb/r/instant_alter.result

@@ -702,6 +702,12 @@ CREATE TABLE t1 (a INT, b INT, c INT NOT NULL, d INT,
 e INT, f INT, g INT, h INT, j INT) ENGINE=InnoDB ROW_FORMAT=REDUNDANT;
 ALTER TABLE t1 MODIFY COLUMN c INT, MODIFY COLUMN a INT AFTER b;
 DROP TABLE t1;
+CREATE TABLE t1 (a INT NOT NULL, b INT NOT NULL) ENGINE=InnoDB ROW_FORMAT=REDUNDANT;
+INSERT INTO t1 VALUES (0,0);
+ALTER TABLE t1 MODIFY a INT AFTER b;
+ALTER TABLE t1 ADD COLUMN v INT AS (a) VIRTUAL;
+ALTER TABLE t1 MODIFY b INT NOT NULL AFTER a;
+DROP TABLE t1;
 CREATE TABLE t1 (a INT NOT NULL) ENGINE=InnoDB ROW_FORMAT=REDUNDANT;
 INSERT INTO t1 VALUES (1);
 ALTER TABLE t1 ADD COLUMN b INT;
@@ -1511,6 +1517,12 @@ CREATE TABLE t1 (a INT, b INT, c INT NOT NULL, d INT,
 e INT, f INT, g INT, h INT, j INT) ENGINE=InnoDB ROW_FORMAT=COMPACT;
 ALTER TABLE t1 MODIFY COLUMN c INT, MODIFY COLUMN a INT AFTER b;
 DROP TABLE t1;
+CREATE TABLE t1 (a INT NOT NULL, b INT NOT NULL) ENGINE=InnoDB ROW_FORMAT=COMPACT;
+INSERT INTO t1 VALUES (0,0);
+ALTER TABLE t1 MODIFY a INT AFTER b;
+ALTER TABLE t1 ADD COLUMN v INT AS (a) VIRTUAL;
+ALTER TABLE t1 MODIFY b INT NOT NULL AFTER a;
+DROP TABLE t1;
 CREATE TABLE t1 (a INT NOT NULL) ENGINE=InnoDB ROW_FORMAT=COMPACT;
 INSERT INTO t1 VALUES (1);
 ALTER TABLE t1 ADD COLUMN b INT;
@@ -2320,6 +2332,12 @@ CREATE TABLE t1 (a INT, b INT, c INT NOT NULL, d INT,
 e INT, f INT, g INT, h INT, j INT) ENGINE=InnoDB ROW_FORMAT=DYNAMIC;
 ALTER TABLE t1 MODIFY COLUMN c INT, MODIFY COLUMN a INT AFTER b;
 DROP TABLE t1;
+CREATE TABLE t1 (a INT NOT NULL, b INT NOT NULL) ENGINE=InnoDB ROW_FORMAT=DYNAMIC;
+INSERT INTO t1 VALUES (0,0);
+ALTER TABLE t1 MODIFY a INT AFTER b;
+ALTER TABLE t1 ADD COLUMN v INT AS (a) VIRTUAL;
+ALTER TABLE t1 MODIFY b INT NOT NULL AFTER a;
+DROP TABLE t1;
 CREATE TABLE t1 (a INT NOT NULL) ENGINE=InnoDB ROW_FORMAT=DYNAMIC;
 INSERT INTO t1 VALUES (1);
 ALTER TABLE t1 ADD COLUMN b INT;
@@ -2486,5 +2504,5 @@ SELECT variable_value-@old_instant instants
 FROM information_schema.global_status
 WHERE variable_name = 'innodb_instant_alter_column';
 instants
-174
+178
 SET GLOBAL innodb_purge_rseg_truncate_frequency= @saved_frequency;

mysql-test/suite/innodb/t/instant_alter.test

@@ -587,6 +587,15 @@ e INT, f INT, g INT, h INT, j INT) $engine;
 ALTER TABLE t1 MODIFY COLUMN c INT, MODIFY COLUMN a INT AFTER b;
 DROP TABLE t1;
 
+# MDEV-18649 Failing assertion on ALTER for ROW_FORMAT=REDUNDANT
+eval CREATE TABLE t1 (a INT NOT NULL, b INT NOT NULL) $engine;
+INSERT INTO t1 VALUES (0,0);
+ALTER TABLE t1 MODIFY a INT AFTER b;
+# Exploit MDEV-17468 to force the table definition to be reloaded
+ALTER TABLE t1 ADD COLUMN v INT AS (a) VIRTUAL;
+ALTER TABLE t1 MODIFY b INT NOT NULL AFTER a;
+DROP TABLE t1;
+
 # MDEV-18033/MDEV-18034 Failing assertion on ALTER
 eval CREATE TABLE t1 (a INT NOT NULL) $engine;
 INSERT INTO t1 VALUES (1);

storage/innobase/btr/btr0btr.cc

@@ -1933,7 +1933,10 @@ void btr_set_instant(buf_block_t* root, const dict_index_t& index, mtr_t* mtr)
 			ut_ad(index.table->instant);
 			ut_ad(!memcmp(infimum, field_ref_zero, 8));
 			ut_ad(!memcmp(supremum, field_ref_zero, 7));
-			ut_ad(supremum[7] == index.n_core_null_bytes);
+			/* The n_core_null_bytes only matters for
+			ROW_FORMAT=COMPACT and ROW_FORMAT=DYNAMIC tables. */
+			ut_ad(supremum[7] == index.n_core_null_bytes
+			      || !index.table->not_redundant());
 			return;
 		}
 		break;