MDEV-28122 Optimize table crash while applying online log

- InnoDB fails to check the overflow buffer while applying
the operation to the table that was rebuilt. This is caused
by commit 3cef4f8f (MDEV-515).

mysql-test/suite/innodb/r/innodb-table-online.result

@@ -470,6 +470,29 @@ SET DEBUG_SYNC="now SIGNAL con1_signal";
 connection con1;
 DROP TABLE t1;
 connection default;
+#
+#  MDEV-28122 Optimize table crash while applying online log
+#
+CREATE TABLE t1(f1 INT NOT NULL, f2 INT NOT NULL,
+f3 CHAR(200), f4 CHAR(200),
+f5 VARCHAR(87), PRIMARY KEY(f1))ENGINE=InnoDB;
+INSERT INTO t1 VALUES(6000, 6000, "InnoDB",
+"MariaDB", repeat('a', 87));
+SET DEBUG_SYNC="inplace_after_index_build SIGNAL dml_start WAIT_FOR dml_commit";
+ALTER TABLE t1 FORCE;
+connection con1;
+SET DEBUG_SYNC="now WAIT_FOR dml_start";
+BEGIN;
+INSERT INTO t1 SELECT seq, seq, "IDB", "MDB", repeat('a', 87) FROM seq_1_to_127;
+INSERT INTO t1 VALUES(128, 128, "IDB", "MDB", repeat('a', 86));
+INSERT INTO t1 VALUES(129, 129, "idb", "mdb", repeat('a', 2));
+COMMIT;
+SET DEBUG_SYNC="now SIGNAL dml_commit";
+connection default;
+CHECK TABLE t1;
+Table	Op	Msg_type	Msg_text
+test.t1	check	status	OK
+DROP TABLE t1;
 SET DEBUG_SYNC=RESET;
 disconnect con1;
 SET GLOBAL innodb_file_per_table = @global_innodb_file_per_table_orig;

mysql-test/suite/innodb/t/innodb-table-online.test

@@ -2,6 +2,7 @@
 --source include/innodb_encrypt_log.inc
 --source include/have_debug.inc
 --source include/have_debug_sync.inc
+--source include/have_sequence.inc
 
 let $innodb_metrics_select=
 SELECT name, count FROM INFORMATION_SCHEMA.INNODB_METRICS WHERE subsystem = 'ddl';
@@ -440,6 +441,32 @@ connection con1;
 reap;
 DROP TABLE t1;
 connection default;
+
+--echo #
+--echo #  MDEV-28122 Optimize table crash while applying online log
+--echo #
+CREATE TABLE t1(f1 INT NOT NULL, f2 INT NOT NULL,
+                f3 CHAR(200), f4 CHAR(200),
+                f5 VARCHAR(87), PRIMARY KEY(f1))ENGINE=InnoDB;
+INSERT INTO t1 VALUES(6000, 6000, "InnoDB",
+                      "MariaDB", repeat('a', 87));
+
+SET DEBUG_SYNC="inplace_after_index_build SIGNAL dml_start WAIT_FOR dml_commit";
+SEND ALTER TABLE t1 FORCE;
+
+connection con1;
+SET DEBUG_SYNC="now WAIT_FOR dml_start";
+BEGIN;
+INSERT INTO t1 SELECT seq, seq, "IDB", "MDB", repeat('a', 87) FROM seq_1_to_127;
+INSERT INTO t1 VALUES(128, 128, "IDB", "MDB", repeat('a', 86));
+INSERT INTO t1 VALUES(129, 129, "idb", "mdb", repeat('a', 2));
+COMMIT;
+SET DEBUG_SYNC="now SIGNAL dml_commit";
+
+connection default;
+reap;
+CHECK TABLE t1;
+DROP TABLE t1;
 SET DEBUG_SYNC=RESET;
 disconnect con1;
 

storage/innobase/row/row0log.cc

@@ -2167,6 +2167,11 @@ row_log_table_apply_op(
 
 	*error = DB_SUCCESS;
 
+	/* 3 = 1 (op type) + 1 (extra_size) + at least 1 byte payload */
+	if (mrec + 3 >= mrec_end) {
+		return(NULL);
+	}
+
 	const bool is_instant = log->is_instant(dup->index);
 	const mrec_t* const mrec_start = mrec;
 
@@ -2214,11 +2219,6 @@ row_log_table_apply_op(
 		break;
 
 	case ROW_T_DELETE:
-		/* 1 (extra_size) + at least 1 (payload) */
-		if (mrec + 2 >= mrec_end) {
-			return(NULL);
-		}
-
 		extra_size = *mrec++;
 		ut_ad(mrec < mrec_end);