MDEV-23244 ALTER TABLE…ADD PRIMARY KEY fails to flag duplicates

The fix of MDEV-13654 (commit ff81faf6)
wrongly caused ADD PRIMARY KEY to ignore duplicate PRIMARY KEY values
caused by concurrent DML transactions that had been started before the
ALTER TABLE operation (but did not access the table before the ALTER TABLE
started).

row_ins_duplicate_online(): Always report a duplicate key error
if DB_TRX_ID had been reset (it belongs to a transaction that had
started before the ALTER TABLE operation).

mysql-test/suite/innodb/r/alter_primary_key.result

+#
+# MDEV-23244 ALTER TABLE…ADD PRIMARY KEY fails to flag
+# duplicate key error from concurrent DML
+#
+CREATE TABLE t0 (pk INT PRIMARY KEY) ENGINE=InnoDB;
+CREATE TABLE t1 (c CHAR(2) NOT NULL) ENGINE=InnoDB;
+connect  con1,localhost,root,,;
+BEGIN;
+INSERT INTO t0 VALUES(1);
+connection default;
+SET DEBUG_SYNC='row_log_table_apply1_before SIGNAL dml WAIT_FOR dml_done';
+ALTER TABLE t1 ADD PRIMARY KEY(c(1));
+connection con1;
+SET DEBUG_SYNC='now WAIT_FOR dml';
+INSERT INTO t1 VALUES ('ab'),('ac');
+COMMIT;
+SET DEBUG_SYNC='now SIGNAL dml_done';
+disconnect con1;
+connection default;
+ERROR 23000: Duplicate entry 'a' for key 'PRIMARY'
+SET DEBUG_SYNC='RESET';
+SELECT * FROM t1;
+c
+ab
+ac
+DROP TABLE t0,t1;

mysql-test/suite/innodb/t/alter_primary_key.test

+--source innodb_default_row_format.inc
+--source include/have_debug.inc
+--source include/have_debug_sync.inc
+
+--echo #
+--echo # MDEV-23244 ALTER TABLE…ADD PRIMARY KEY fails to flag
+--echo # duplicate key error from concurrent DML
+--echo #
+
+CREATE TABLE t0 (pk INT PRIMARY KEY) ENGINE=InnoDB;
+CREATE TABLE t1 (c CHAR(2) NOT NULL) ENGINE=InnoDB;
+
+connect (con1,localhost,root,,);
+BEGIN;
+INSERT INTO t0 VALUES(1);
+
+connection default;
+SET DEBUG_SYNC='row_log_table_apply1_before SIGNAL dml WAIT_FOR dml_done';
+send ALTER TABLE t1 ADD PRIMARY KEY(c(1));
+
+connection con1;
+SET DEBUG_SYNC='now WAIT_FOR dml';
+INSERT INTO t1 VALUES ('ab'),('ac');
+COMMIT;
+SET DEBUG_SYNC='now SIGNAL dml_done';
+disconnect con1;
+
+connection default;
+--error ER_DUP_ENTRY
+reap;
+SET DEBUG_SYNC='RESET';
+
+SELECT * FROM t1;
+DROP TABLE t0,t1;

storage/innobase/row/row0ins.cc

@@ -2241,8 +2241,14 @@ row_ins_duplicate_online(
 		return(DB_SUCCESS);
 	}
 
-	if (fields == n_uniq + 2) {
-		/* rec is an exact match of entry. */
+	ulint trx_id_len;
+
+	if (fields == n_uniq + 2
+	    && memcmp(rec_get_nth_field(rec, offsets, n_uniq, &trx_id_len),
+		      reset_trx_id, DATA_TRX_ID_LEN + DATA_ROLL_PTR_LEN)) {
+		ut_ad(trx_id_len == DATA_TRX_ID_LEN);
+		/* rec is an exact match of entry, and DB_TRX_ID belongs
+		to a transaction that started after our ALTER TABLE. */
 		return(DB_SUCCESS_LOCKED_REC);
 	}