MDEV-26294 Duplicate entries in unique index not detected when changing collation

ha_innobase::check_if_supported_inplace_alter(): Refuse to change the
collation of a column that would become or remain indexed as part of
the ALTER TABLE operation.

In MariaDB Server 10.6, we will allow this type of operation;
that fix depends on MDEV-15250.

mysql-test/suite/innodb/r/change_column_collation.result

+#
+# MDEV-26294 Duplicate entries in unique index not detected when
+#	changing collation with INPLACE algorithm
+#
+SET NAMES utf8;
+CREATE TABLE t1 (
+id INT PRIMARY KEY,
+msg VARCHAR(100) CHARACTER SET utf8 COLLATE utf8_swedish_ci UNIQUE
+) ENGINE=INNODB;
+INSERT INTO t1 VALUES (1, 'aaa');
+INSERT INTO t1 VALUES (2, 'ååå');
+ALTER TABLE t1
+MODIFY msg VARCHAR(100) CHARACTER SET utf8 COLLATE utf8_unicode_ci,
+ALGORITHM=NOCOPY;
+ERROR 0A000: ALGORITHM=NOCOPY is not supported. Reason: Collation change on an indexed column. Try ALGORITHM=COPY
+ALTER TABLE t1 DROP INDEX msg,
+MODIFY msg VARCHAR(100) CHARACTER SET utf8 COLLATE utf8_unicode_ci,
+ALGORITHM=NOCOPY;
+DROP TABLE t1;

mysql-test/suite/innodb/r/instant_alter_charset,redundant.rdiff

---- instant_alter_charset.result	2019-06-17 14:36:02.311515062 +0300
-+++ instant_alter_charset,redundant.result	2019-06-17 14:50:11.888705725 +0300
-@@ -279,7 +279,6 @@
+@@ -288,7 +288,6 @@
  alter table boundary_255
  modify a varchar(70) charset utf8mb4,
  algorithm=instant;
@@ -8,7 +6,7 @@
  drop table boundary_255;
  create table t (
  a char(10) collate utf8mb3_general_ci,
-@@ -297,32 +296,21 @@
+@@ -306,35 +305,21 @@
  repeat('a', 10), repeat('a', 10)
  );
  alter table t modify a char(10) collate utf8mb4_general_ci, algorithm=instant;
@@ -27,16 +25,19 @@
 -test.t	check	status	OK
 +ERROR 0A000: ALGORITHM=INSTANT is not supported. Reason: Cannot change column type. Try ALGORITHM=COPY
  alter table t modify aa char(10) collate utf8mb4_general_ci, algorithm=instant;
+-ERROR 0A000: ALGORITHM=INSTANT is not supported. Reason: Collation change on an indexed column. Try ALGORITHM=COPY
 -check table t;
 -Table	Op	Msg_type	Msg_text
 -test.t	check	status	OK
 +ERROR 0A000: ALGORITHM=INSTANT is not supported. Reason: Cannot change column type. Try ALGORITHM=COPY
  alter table t modify bb char(70) collate utf8mb4_general_ci, algorithm=instant;
+-ERROR 0A000: ALGORITHM=INSTANT is not supported. Reason: Collation change on an indexed column. Try ALGORITHM=COPY
 -check table t;
 -Table	Op	Msg_type	Msg_text
 -test.t	check	status	OK
 +ERROR 0A000: ALGORITHM=INSTANT is not supported. Reason: Cannot change column type. Try ALGORITHM=COPY
  alter table t modify cc char(100) collate utf8mb4_general_ci, algorithm=instant;
+-ERROR 0A000: ALGORITHM=INSTANT is not supported. Reason: Collation change on an indexed column. Try ALGORITHM=COPY
 -check table t;
 -Table	Op	Msg_type	Msg_text
 -test.t	check	status	OK
@@ -44,8 +45,10 @@
  alter table t modify d char(10) collate utf8mb4_spanish_ci, algorithm=instant;
 +ERROR 0A000: ALGORITHM=INSTANT is not supported. Reason: Cannot change column type. Try ALGORITHM=COPY
  alter table t modify dd char(10) collate utf8mb4_spanish_ci, algorithm=instant;
--ERROR 0A000: ALGORITHM=INSTANT is not supported. Reason: ADD INDEX. Try ALGORITHM=NOCOPY
+-ERROR 0A000: ALGORITHM=INSTANT is not supported. Reason: Collation change on an indexed column. Try ALGORITHM=COPY
 +ERROR 0A000: ALGORITHM=INSTANT is not supported. Reason: Cannot change column type. Try ALGORITHM=COPY
  select * from t;
  a	b	c	aa	bb	cc	d	dd
  aaaaaaaaaa	aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa	aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa	aaaaaaaaaa	aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa	aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa	aaaaaaaaaa	aaaaaaaaaa
+
+mysqltest: Result length mismatch

mysql-test/suite/innodb/t/change_column_collation.test

+--source include/have_innodb.inc
+
+--echo #
+--echo # MDEV-26294 Duplicate entries in unique index not detected when
+--echo #	changing collation with INPLACE algorithm
+--echo #
+
+# Detect the duplicate entry after collation change of column
+
+SET NAMES utf8;
+CREATE TABLE t1 (
+  id INT PRIMARY KEY,
+  msg VARCHAR(100) CHARACTER SET utf8 COLLATE utf8_swedish_ci UNIQUE
+) ENGINE=INNODB;
+INSERT INTO t1 VALUES (1, 'aaa');
+INSERT INTO t1 VALUES (2, 'ååå');
+--error ER_ALTER_OPERATION_NOT_SUPPORTED_REASON
+ALTER TABLE t1
+MODIFY msg VARCHAR(100) CHARACTER SET utf8 COLLATE utf8_unicode_ci,
+ALGORITHM=NOCOPY;
+ALTER TABLE t1 DROP INDEX msg,
+MODIFY msg VARCHAR(100) CHARACTER SET utf8 COLLATE utf8_unicode_ci,
+ALGORITHM=NOCOPY;
+DROP TABLE t1;

mysql-test/suite/innodb/t/instant_alter_charset.test

@@ -227,9 +227,18 @@ create table key_part_bug (
   a varchar(150) charset utf8mb3 unique key
 ) engine=innodb;
 
+--error ER_ALTER_OPERATION_NOT_SUPPORTED_REASON
 alter table key_part_bug
   modify a varchar(150) charset utf8mb4,
   algorithm=instant;
+--error ER_ALTER_OPERATION_NOT_SUPPORTED_REASON
+alter table key_part_bug
+  modify a varchar(150) charset utf8mb4,
+  algorithm=nocopy;
+alter table key_part_bug
+  drop index a,
+  modify a varchar(150) charset utf8mb4,
+  algorithm=nocopy;
 
 drop table key_part_bug;
 
@@ -347,10 +356,13 @@ check table t;
 alter table t modify c char(100) collate utf8mb4_general_ci, algorithm=instant;
 check table t;
 
+--error ER_ALTER_OPERATION_NOT_SUPPORTED_REASON
 alter table t modify aa char(10) collate utf8mb4_general_ci, algorithm=instant;
 check table t;
+--error ER_ALTER_OPERATION_NOT_SUPPORTED_REASON
 alter table t modify bb char(70) collate utf8mb4_general_ci, algorithm=instant;
 check table t;
+--error ER_ALTER_OPERATION_NOT_SUPPORTED_REASON
 alter table t modify cc char(100) collate utf8mb4_general_ci, algorithm=instant;
 check table t;
 
@@ -442,11 +454,16 @@ while ($counter <= $data_size) {
 
   insert into tmp values ('AAA', 'AAA'), ('bbb', 'bbb');
 
+--error ER_ALTER_OPERATION_NOT_SUPPORTED_REASON
   eval alter table tmp
     change a a varchar(50) charset $to_charset collate $to_collate,
     modify b varchar(50) charset $to_charset collate $to_collate,
     algorithm=instant;
 
+  eval alter table tmp
+    change a a varchar(50) charset $to_charset collate $to_collate,
+    algorithm=instant;
+
   check table tmp;
 
   drop table tmp;
@@ -685,7 +702,10 @@ alter table t modify a char(10) collate latin1_general_cs, algorithm=inplace;
 
 --error ER_ALTER_OPERATION_NOT_SUPPORTED_REASON
 alter table t modify b char(10) collate latin1_general_cs, algorithm=instant;
+--error ER_ALTER_OPERATION_NOT_SUPPORTED_REASON
 alter table t modify b char(10) collate latin1_general_cs, algorithm=nocopy;
+alter table t modify b char(10) collate latin1_general_cs,
+drop index b_key, algorithm=nocopy;
 check table t;
 
 alter table t modify c char(10) collate latin1_general_cs, algorithm=instant;
@@ -708,7 +728,10 @@ alter table t modify a varchar(10) collate latin1_general_cs, algorithm=inplace;
 
 --error ER_ALTER_OPERATION_NOT_SUPPORTED_REASON
 alter table t modify b varchar(10) collate latin1_general_cs, algorithm=instant;
+--error ER_ALTER_OPERATION_NOT_SUPPORTED_REASON
 alter table t modify b varchar(10) collate latin1_general_cs, algorithm=nocopy;
+alter table t modify b varchar(10) collate latin1_general_cs,
+drop index b_key, algorithm=nocopy;
 check table t;
 
 alter table t modify c varchar(10) collate latin1_general_cs, algorithm=instant;

storage/innobase/handler/handler0alter.cc

 /*****************************************************************************
 
 Copyright (c) 2005, 2019, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2013, 2021, MariaDB Corporation.
+Copyright (c) 2013, 2022, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -2310,7 +2310,16 @@ ha_innobase::check_if_supported_inplace_alter(
 
 			if (new_field->field) {
 				/* This is an existing column. */
-				continue;
+
+				if (new_field->field->charset()
+				    == key_part->field->charset()) {
+					continue;
+				}
+
+				ha_alter_info->unsupported_reason =
+					"Collation change on"
+					" an indexed column";
+				DBUG_RETURN(HA_ALTER_INPLACE_NOT_SUPPORTED);
 			}
 
 			/* This is an added column. */