MDEV-23330 Server crash or ASAN negative-size-param in my_strnncollsp_binary /...

MDEV-23330 Server crash or ASAN negative-size-param in my_strnncollsp_binary / SORT_FIELD_ATTR::compare_packed_varstrings

In case of skipping decodding fix charset to be binary.

mysql-test/main/func_des_encrypt.result

@@ -35,3 +35,12 @@ DES_DECRYPT(DES_ENCRYPT('1234'))	DES_DECRYPT(DES_ENCRYPT('12345'))	DES_DECRYPT(D
 1234	12345	123456	1234567
 DROP TABLE t1;
 End of 5.0 tests
+#
+# MDEV-23330: Server crash or ASAN negative-size-param in
+# my_strnncollsp_binary / SORT_FIELD_ATTR::compare_packed_varstrings
+#
+CREATE TABLE t1 (a CHAR(240), b BIT(48));
+INSERT INTO t1 VALUES ('a',b'0001'),('b',b'0010'),('c',b'0011'),('d',b'0100'),('e',b'0001'),('f',b'0101'),('g',b'0110'),('h',b'0111'),('i',b'1000'),('j',b'1001');
+SELECT DES_DECRYPT(a, 'x'), BINARY b FROM t1 GROUP BY 1, 2 WITH ROLLUP;
+DROP TABLE t1;
+# End of 10.5 tests

mysql-test/main/func_des_encrypt.test

@@ -37,3 +37,19 @@ SELECT
 DROP TABLE t1;
 
 --Echo End of 5.0 tests
+
+--echo #
+--echo # MDEV-23330: Server crash or ASAN negative-size-param in
+--echo # my_strnncollsp_binary / SORT_FIELD_ATTR::compare_packed_varstrings
+--echo #
+
+CREATE TABLE t1 (a CHAR(240), b BIT(48));
+INSERT INTO t1 VALUES ('a',b'0001'),('b',b'0010'),('c',b'0011'),('d',b'0100'),('e',b'0001'),('f',b'0101'),('g',b'0110'),('h',b'0111'),('i',b'1000'),('j',b'1001');
+--disable_result_log
+SELECT DES_DECRYPT(a, 'x'), BINARY b FROM t1 GROUP BY 1, 2 WITH ROLLUP;
+--enable_result_log
+
+# Cleanup
+DROP TABLE t1;
+
+--echo # End of 10.5 tests

sql/item_strfunc.cc

@@ -831,7 +831,10 @@ String *Item_func_des_decrypt::val_str(String *str)
     return 0;
   length= res->length();
   if (length < 9 || (length % 8) != 1 || !((*res)[0] & 128))
+  {
+    res->set_charset(&my_charset_bin);
     return res;				// Skip decryption if not encrypted
+  }
 
   if (arg_count == 1)			// If automatic uncompression
   {