Bug 4531: unique key prefix interacts poorly with utf8

Check HEAP+HASH, binary collation

mysql-test/r/ctype_utf8.result

@@ -470,6 +470,42 @@ ERROR 23000: Duplicate entry 'ꪪꪪ' for key 1
 drop table t1;
 create table t1 (
 c char(10) character set utf8 collate utf8_bin,
+unique key a using hash (c(1))
+) engine=heap;
+show create table t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `c` char(10) character set utf8 collate utf8_bin default NULL,
+  UNIQUE KEY `a` (`c`(1))
+) ENGINE=HEAP DEFAULT CHARSET=latin1
+insert into t1 values ('a'),('b'),('c'),('d'),('e'),('f');
+insert into t1 values ('aa');
+ERROR 23000: Duplicate entry 'aa' for key 1
+insert into t1 values ('aaa');
+ERROR 23000: Duplicate entry 'aaa' for key 1
+insert into t1 values ('б');
+insert into t1 values ('бб');
+ERROR 23000: Duplicate entry 'б' for key 1
+insert into t1 values ('ббб');
+ERROR 23000: Duplicate entry 'б' for key 1
+select c as c_all from t1 order by c;
+c_all
+a
+b
+c
+d
+e
+f
+б
+select c as c_a from t1 where c='a';
+c_a
+a
+select c as c_a from t1 where c='б';
+c_a
+б
+drop table t1;
+create table t1 (
+c char(10) character set utf8 collate utf8_bin,
 unique key a using btree (c(1))
 ) engine=heap;
 show create table t1;

mysql-test/t/ctype_utf8.test

@@ -349,27 +349,25 @@ drop table t1;
 # Bug 4531: unique key prefix interacts poorly with utf8
 # Check HEAP+HASH, binary collation
 #
-# This doesn't work correctly yet.
-#
-#create table t1 (
-#c char(10) character set utf8 collate utf8_bin,
-#unique key a using hash (c(1))
-#) engine=heap;
-#show create table t1;
-#insert into t1 values ('a'),('b'),('c'),('d'),('e'),('f');
-#--error 1062
-#insert into t1 values ('aa');
-#--error 1062
-#insert into t1 values ('aaa');
-#insert into t1 values ('б');
-#--error 1062
-#insert into t1 values ('бб');
-#--error 1062
-#insert into t1 values ('ббб');
-#select c as c_all from t1 order by c;
-#select c as c_a from t1 where c='a';
-#select c as c_a from t1 where c='б';
-#drop table t1;
+create table t1 (
+c char(10) character set utf8 collate utf8_bin,
+unique key a using hash (c(1))
+) engine=heap;
+show create table t1;
+insert into t1 values ('a'),('b'),('c'),('d'),('e'),('f');
+--error 1062
+insert into t1 values ('aa');
+--error 1062
+insert into t1 values ('aaa');
+insert into t1 values ('б');
+--error 1062
+insert into t1 values ('бб');
+--error 1062
+insert into t1 values ('ббб');
+select c as c_all from t1 order by c;
+select c as c_a from t1 where c='a';
+select c as c_a from t1 where c='б';
+drop table t1;
 
 #
 # Bug 4531: unique key prefix interacts poorly with utf8

sql/ha_heap.cc

@@ -430,7 +430,7 @@ int ha_heap::create(const char *name, TABLE *table_arg,
       {
 	if (!f_is_packed(flag) &&
 	    f_packtype(flag) == (int) FIELD_TYPE_DECIMAL &&
-	    !(flag & FIELDFLAG_BINARY))
+	    !(field->charset() == &my_charset_bin))
 	  seg->type= (int) HA_KEYTYPE_TEXT;
 	else
 	  seg->type= (int) HA_KEYTYPE_BINARY;