MDEV-12607 Hybrid functions create wrong VARBINARY length when mixing character and binary data

mysql-test/r/ctype_utf8.result

@@ -11184,5 +11184,21 @@ SET NAMES utf8;
 CREATE TABLE t1 (a SET('a,bü'));
 ERROR 22007: Illegal set 'a,bü' value found during parsing
 #
+# MDEV-12607 Hybrid functions create wrong VARBINARY length when mixing character and binary data
+#
+SET sql_mode='';
+SET NAMES utf8;
+CREATE OR REPLACE TABLE t1 AS SELECT COALESCE('ßa',_binary 'a');
+SELECT * FROM t1;
+COALESCE('ßa',_binary 'a')
+ßa
+SHOW CREATE TABLE t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `COALESCE('ßa',_binary 'a')` varbinary(6) NOT NULL
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+DROP TABLE t1;
+SET sql_mode=DEFAULT;
+#
 # End of 10.3 tests
 #

mysql-test/t/ctype_utf8.test

@@ -2109,6 +2109,18 @@ SET NAMES utf8;
 --error ER_ILLEGAL_VALUE_FOR_TYPE
 CREATE TABLE t1 (a SET('a,bü'));
 
+--echo #
+--echo # MDEV-12607 Hybrid functions create wrong VARBINARY length when mixing character and binary data
+--echo #
+SET sql_mode='';
+SET NAMES utf8;
+CREATE OR REPLACE TABLE t1 AS SELECT COALESCE('ßa',_binary 'a');
+SELECT * FROM t1;
+SHOW CREATE TABLE t1;
+DROP TABLE t1;
+SET sql_mode=DEFAULT;
+
+
 --echo #
 --echo # End of 10.3 tests
 --echo #

sql/item_func.cc

@@ -632,6 +632,18 @@ void Item_func::count_only_length(Item **item, uint nitems)
 }
 
 
+void Item_func::count_octet_length(Item **item, uint nitems)
+{
+  max_length= 0;
+  unsigned_flag= 0;
+  for (uint i= 0; i < nitems ; i++)
+  {
+    set_if_bigger(max_length, item[i]->max_length);
+    set_if_bigger(unsigned_flag, item[i]->unsigned_flag);
+  }
+}
+
+
 /**
   Set max_length/decimals of function if function is floating point and
   result length/precision depends on argument ones.
@@ -681,6 +693,9 @@ bool Item_func::count_string_length(Item **items, uint nitems)
   DBUG_ASSERT(!is_temporal_type(field_type()));
   if (agg_arg_charsets_for_string_result(collation, items, nitems, 1))
     return true;
+  if (collation.collation == &my_charset_bin)
+    count_octet_length(items, nitems);
+  else
     count_only_length(items, nitems);
   decimals= max_length ? NOT_FIXED_DEC : 0;
   return false;

sql/item_func.h

@@ -43,6 +43,7 @@ class Item_func :public Item_func_or_sum
   String *val_str_from_val_str_ascii(String *str, String *str2);
 
   void count_only_length(Item **item, uint nitems);
+  void count_octet_length(Item **item, uint nitems);
   void count_real_length(Item **item, uint nitems);
   void count_decimal_length(Item **item, uint nitems);
   bool count_string_length(Item **item, uint nitems);