MDEV-12592 Illegal mix of collations with the HEX function

Fixing repertoire for HEX() from MY_REPERTOIRE_EXTENDED to MY_REPERTOIRE_ASCII.

mysql-test/r/func_str.result

@@ -4661,3 +4661,20 @@ DROP TABLE t1;
 #
 # End of 10.1 tests
 #
+#
+# Start of 10.3 tests
+#
+#
+# MDEV-12592 Illegal mix of collations with the HEX function
+#
+SET NAMES utf8;
+CREATE TABLE t1 (a VARCHAR(10) CHARACTER SET latin1);
+INSERT INTO t1 VALUES (0x09),('a');
+SELECT IF(a<' ',HEX(a),a) FROM t1 ORDER BY a;
+IF(a<' ',HEX(a),a)
+09
+a
+DROP TABLE  t1;
+#
+# End of 10.3 tests
+#

mysql-test/t/func_str.test

@@ -1806,3 +1806,23 @@ DROP TABLE t1;
 --echo #
 --echo # End of 10.1 tests
 --echo #
+
+
+--echo #
+--echo # Start of 10.3 tests
+--echo #
+
+--echo #
+--echo # MDEV-12592 Illegal mix of collations with the HEX function
+--echo #
+
+SET NAMES utf8;
+CREATE TABLE t1 (a VARCHAR(10) CHARACTER SET latin1);
+INSERT INTO t1 VALUES (0x09),('a');
+SELECT IF(a<' ',HEX(a),a) FROM t1 ORDER BY a;
+DROP TABLE  t1;
+
+
+--echo #
+--echo # End of 10.3 tests
+--echo #

sql/item_strfunc.h

@@ -1074,7 +1074,7 @@ class Item_func_hex :public Item_str_ascii_checksum_func
   }
   void fix_length_and_dec()
   {
-    collation.set(default_charset());
+    collation.set(default_charset(), DERIVATION_COERCIBLE, MY_REPERTOIRE_ASCII);
     decimals=0;
     fix_char_length(args[0]->max_length * 2);
     m_arg0_type_handler= args[0]->type_handler();