ctype_utf8.test:

  Added a test case for bug #11484.
hp_hash.c:
  Fixed bug #11484.
  This bug in the function hp_rec_key_cmp resulted in wrong
  comparison of varchar multibyte keys if the bytes after 
  string values happened to be different. This caused wrong
  results for queries returning DISTINCT varchar fields in
  multibyte charsets (e.g. in utf8).


heap/hp_hash.c:
  Fixed bug #11484.
  This bug in the function hp_rec_key_cmp resulted in wrong
  comparison of varchar multibyte keys if the bytes after 
  string values happened to be different. This caused wrong
  results for queries returning DISTINCT varchar fields in
  multibyte charsets (e.g. in utf8).
mysql-test/t/ctype_utf8.test:
  Added a test case for bug #11484.

heap/hp_hash.c

@@ -552,9 +552,9 @@ int hp_rec_key_cmp(HP_KEYDEF *keydef, const byte *rec1, const byte *rec2,
       if (cs->mbmaxlen > 1)
       {
         uint char_length= seg->length / cs->mbmaxlen;
-        char_length1= my_charpos(cs, pos1, pos1 + char_length1, char_length);
+        char_length1= my_charpos(cs, pos1, pos1 + char_length1, char_length1);
         set_if_smaller(char_length1, seg->length);
-        char_length2= my_charpos(cs, pos2, pos2 + char_length2, char_length);
+        char_length2= my_charpos(cs, pos2, pos2 + char_length2, char_length2);
         set_if_smaller(char_length2, seg->length);
       }
 

mysql-test/r/ctype_utf8.result

@@ -950,3 +950,22 @@ hex(a)
 5B
 E880BD
 drop table t1;
+CREATE TABLE t1(id varchar(20) NOT NULL) DEFAULT CHARSET=utf8;
+INSERT INTO t1 VALUES ('xxx'), ('aa'), ('yyy'), ('aa');
+SELECT id FROM t1;
+id
+xxx
+aa
+yyy
+aa
+SELECT DISTINCT id FROM t1;
+id
+xxx
+aa
+yyy
+SELECT DISTINCT id FROM t1 ORDER BY id;
+id
+aa
+xxx
+yyy
+DROP TABLE t1;

mysql-test/t/ctype_utf8.test

@@ -800,3 +800,16 @@ insert into t1 values (_utf8 0xe880bd);
 insert into t1 values (_utf8 0x5b);
 select hex(a) from t1;
 drop table t1;
+
+#
+# Test for bug #11484: wrong results for a DISTINCT varchar column in uft8. 
+#
+
+CREATE TABLE t1(id varchar(20) NOT NULL) DEFAULT CHARSET=utf8;
+INSERT INTO t1 VALUES ('xxx'), ('aa'), ('yyy'), ('aa');
+
+SELECT id FROM t1;
+SELECT DISTINCT id FROM t1;
+SELECT DISTINCT id FROM t1 ORDER BY id;
+
+DROP TABLE t1;