MDEV-11685: sql_mode can't be set with non-ascii connection charset

The supplied sql_mode(s) should be converted to ASCII first,
before comparing it with the sql_mode set.

mysql-test/r/ctype_ucs.result

@@ -4576,6 +4576,16 @@ SELECT CHAR_LENGTH(TRIM(BOTH 0x00 FROM _ucs2 0x0061));
 CHAR_LENGTH(TRIM(BOTH 0x00 FROM _ucs2 0x0061))
 1
 #
+# MDEV-11685: sql_mode can't be set with non-ascii connection charset
+#
+SET character_set_connection=ucs2;
+SET sql_mode='NO_ENGINE_SUBSTITUTION';
+SELECT @@sql_mode;
+@@sql_mode
+NO_ENGINE_SUBSTITUTION
+SET sql_mode=DEFAULT;
+SET NAMES utf8;
+#
 # End of 5.5 tests
 #
 #

mysql-test/r/ctype_utf16.result

@@ -1580,6 +1580,16 @@ ERROR HY000: Invalid utf16 character string: 'DE9899'
 DO LPAD(_utf16 0x0061 COLLATE utf16_unicode_ci, 10000, 0x0061DE989999);
 ERROR HY000: Invalid utf16 character string: 'DE9899'
 #
+# MDEV-11685: sql_mode can't be set with non-ascii connection charset
+#
+SET character_set_connection=utf16;
+SET sql_mode='NO_ENGINE_SUBSTITUTION';
+SELECT @@sql_mode;
+@@sql_mode
+NO_ENGINE_SUBSTITUTION
+SET sql_mode=DEFAULT;
+SET NAMES utf8;
+#
 # End of 5.5 tests
 #
 #

mysql-test/r/ctype_utf32.result

@@ -1666,6 +1666,16 @@ c
 Warnings:
 Warning	1300	Invalid utf32 character string: '\xFF\xFF\x00\x00'
 #
+# MDEV-11685: sql_mode can't be set with non-ascii connection charset
+#
+SET character_set_connection=utf32;
+SET sql_mode='NO_ENGINE_SUBSTITUTION';
+SELECT @@sql_mode;
+@@sql_mode
+NO_ENGINE_SUBSTITUTION
+SET sql_mode=DEFAULT;
+SET NAMES utf8;
+#
 # End of 5.5 tests
 #
 #

mysql-test/t/ctype_ucs.test

@@ -816,6 +816,14 @@ SELECT CHAR_LENGTH(TRIM(BOTH 0x0001 FROM _ucs2 0x0061));
 SELECT CHAR_LENGTH(TRIM(BOTH 0x61 FROM _ucs2 0x0061));
 SELECT CHAR_LENGTH(TRIM(BOTH 0x00 FROM _ucs2 0x0061));
 
+--echo #
+--echo # MDEV-11685: sql_mode can't be set with non-ascii connection charset
+--echo #
+SET character_set_connection=ucs2;
+SET sql_mode='NO_ENGINE_SUBSTITUTION';
+SELECT @@sql_mode;
+SET sql_mode=DEFAULT;
+SET NAMES utf8;
 
 --echo #
 --echo # End of 5.5 tests

mysql-test/t/ctype_utf16.test

@@ -795,6 +795,15 @@ DO RPAD(_utf16 0x0061 COLLATE utf16_unicode_ci, 10000, 0x0061DE989999);
 --error ER_INVALID_CHARACTER_STRING
 DO LPAD(_utf16 0x0061 COLLATE utf16_unicode_ci, 10000, 0x0061DE989999);
 
+--echo #
+--echo # MDEV-11685: sql_mode can't be set with non-ascii connection charset
+--echo #
+SET character_set_connection=utf16;
+SET sql_mode='NO_ENGINE_SUBSTITUTION';
+SELECT @@sql_mode;
+SET sql_mode=DEFAULT;
+SET NAMES utf8;
+
 --echo #
 --echo # End of 5.5 tests
 --echo #

mysql-test/t/ctype_utf32.test

@@ -894,6 +894,15 @@ SELECT CHAR_LENGTH(TRIM(BOTH 0x00 FROM _utf32 0x00000061));
 #
 select hex(lower(cast(0xffff0000 as char character set utf32))) as c;
 
+--echo #
+--echo # MDEV-11685: sql_mode can't be set with non-ascii connection charset
+--echo #
+SET character_set_connection=utf32;
+SET sql_mode='NO_ENGINE_SUBSTITUTION';
+SELECT @@sql_mode;
+SET sql_mode=DEFAULT;
+SET NAMES utf8;
+
 --echo #
 --echo # End of 5.5 tests
 --echo #

sql/sys_vars.ic

@@ -1347,7 +1347,7 @@ public:
 
     if (var->value->result_type() == STRING_RESULT)
     {
-      if (!(res=var->value->val_str(&str)))
+      if (!(res=var->value->val_str_ascii(&str)))
         return true;
       else
       {