Bug#54661 sha2() returns BINARY result

Problem: sha2() reported its result as BINARY

Fix:
- Inheriting Item_func_sha2 from Item_str_ascii_func
- Setting max_length via fix_length_and_charset() 
  instead of direct assignment.
- Adding tests

mysql-test/r/func_digest.result

@@ -1405,3 +1405,24 @@ LENGTH(SHA2( 'size', 384 )) / 2 * 8       = 384
 SELECT LENGTH(SHA2( 'computed', 512 )) / 2 * 8   = 512;
 LENGTH(SHA2( 'computed', 512 )) / 2 * 8   = 512
 1
+#
+# Bug#54661 sha2() returns BINARY result
+#
+SET NAMES binary;
+SELECT sha2('1',224);
+Catalog	Database	Table	Table_alias	Column	Column_alias	Type	Length	Max length	Is_null	Flags	Decimals	Charsetnr
+def					sha2('1',224)	253	56	56	Y	128	31	63
+sha2('1',224)
+e25388fde8290dc286a6164fa2d97e551b53498dcbf7bc378eb1f178
+SET NAMES utf8;
+SELECT sha2('1',224);
+Catalog	Database	Table	Table_alias	Column	Column_alias	Type	Length	Max length	Is_null	Flags	Decimals	Charsetnr
+def					sha2('1',224)	253	168	56	Y	0	31	33
+sha2('1',224)
+e25388fde8290dc286a6164fa2d97e551b53498dcbf7bc378eb1f178
+SET NAMES latin1;
+SELECT sha2('1',224);
+Catalog	Database	Table	Table_alias	Column	Column_alias	Type	Length	Max length	Is_null	Flags	Decimals	Charsetnr
+def					sha2('1',224)	253	56	56	Y	0	31	8
+sha2('1',224)
+e25388fde8290dc286a6164fa2d97e551b53498dcbf7bc378eb1f178

mysql-test/t/func_digest.test

@@ -481,3 +481,16 @@ SELECT LENGTH(SHA2( '', 224 )) / 2 * 8           = 224;
 SELECT LENGTH(SHA2( 'any', 256 )) / 2 * 8        = 256;
 SELECT LENGTH(SHA2( 'size', 384 )) / 2 * 8       = 384;
 SELECT LENGTH(SHA2( 'computed', 512 )) / 2 * 8   = 512;
+
+--echo #
+--echo # Bug#54661 sha2() returns BINARY result
+--echo #
+--enable_metadata
+SET NAMES binary;
+SELECT sha2('1',224);
+SET NAMES utf8;
+SELECT sha2('1',224);
+SET NAMES latin1;
+SELECT sha2('1',224);
+--disable_metadata
+

sql/item_strfunc.cc

@@ -242,7 +242,7 @@ void Item_func_sha::fix_length_and_dec()
   fix_length_and_charset(SHA1_HASH_SIZE * 2, default_charset());
 }
 
-String *Item_func_sha2::val_str(String *str)
+String *Item_func_sha2::val_str_ascii(String *str)
 {
   DBUG_ASSERT(fixed == 1);
 #if defined(HAVE_OPENSSL) && !defined(EMBEDDED_LIBRARY)
@@ -338,19 +338,19 @@ void Item_func_sha2::fix_length_and_dec()
   switch (sha_variant) {
 #ifndef OPENSSL_NO_SHA512
   case 512:
-    max_length= SHA512_DIGEST_LENGTH*2;
+    fix_length_and_charset(SHA512_DIGEST_LENGTH * 2, default_charset());
     break;
   case 384:
-    max_length= SHA384_DIGEST_LENGTH*2;
+    fix_length_and_charset(SHA384_DIGEST_LENGTH * 2, default_charset());
     break;
 #endif
 #ifndef OPENSSL_NO_SHA256
   case 256:
   case 0: // SHA-256 is the default
-    max_length= SHA256_DIGEST_LENGTH*2;
+    fix_length_and_charset(SHA256_DIGEST_LENGTH * 2, default_charset());
     break;
   case 224:
-    max_length= SHA224_DIGEST_LENGTH*2;
+    fix_length_and_charset(SHA224_DIGEST_LENGTH * 2, default_charset());
     break;
 #endif
   default:

sql/item_strfunc.h

@@ -82,16 +82,13 @@ public:
   const char *func_name() const { return "sha"; }	
 };
 
-class Item_func_sha2 :public Item_str_func
+class Item_func_sha2 :public Item_str_ascii_func
 {
 public:
-  Item_func_sha2(Item *a, Item *b) :Item_str_func(a, b)
-  {
-    collation.set(&my_charset_bin);
-  }
-  String *val_str(String *);    
-  void fix_length_and_dec();      
-  const char *func_name() const { return "sha2"; }	
+  Item_func_sha2(Item *a, Item *b) :Item_str_ascii_func(a, b) {}
+  String *val_str_ascii(String *);
+  void fix_length_and_dec();
+  const char *func_name() const { return "sha2"; }
 };
 
 class Item_func_aes_encrypt :public Item_str_func