MDEV-4842 STR_TO_DATE does not work with UCS2/UTF16/UTF32

mysql-test/include/ctype_str_to_date.inc

+--echo #
+--echo # MDEV-4842 STR_TO_DATE does not work with UCS2/UTF16/UTF32
+--echo #
+
+SELECT @@character_set_connection, HEX(CAST(_utf8'÷' AS CHAR));
+SELECT STR_TO_DATE(CAST(_utf8'2001÷01÷01' AS CHAR),CAST(_utf8'%Y÷%m÷%d' AS CHAR));
+CREATE TABLE t1 AS SELECT REPEAT(' ', 64) AS subject, REPEAT(' ',64) AS pattern LIMIT 0;
+SHOW COLUMNS FROM t1;
+INSERT INTO t1 VALUES (_utf8'2001÷01÷01',_utf8'%Y÷%m÷%d');
+SELECT HEX(subject),HEX(pattern),STR_TO_DATE(subject, pattern) FROM t1;
+DROP TABLE t1;
+

mysql-test/r/ctype_latin1.result

@@ -336,6 +336,25 @@ select c1 as c2h from t1 where c1 like 'ab#_def' escape '#';
 c2h
 ab_def
 drop table t1;
+#
+# MDEV-4842 STR_TO_DATE does not work with UCS2/UTF16/UTF32
+#
+SELECT @@character_set_connection, HEX(CAST(_utf8'÷' AS CHAR));
+@@character_set_connection	HEX(CAST(_utf8'÷' AS CHAR))
+latin1	F7
+SELECT STR_TO_DATE(CAST(_utf8'2001÷01÷01' AS CHAR),CAST(_utf8'%Y÷%m÷%d' AS CHAR));
+STR_TO_DATE(CAST(_utf8'2001÷01÷01' AS CHAR),CAST(_utf8'%Y÷%m÷%d' AS CHAR))
+2001-01-01
+CREATE TABLE t1 AS SELECT REPEAT(' ', 64) AS subject, REPEAT(' ',64) AS pattern LIMIT 0;
+SHOW COLUMNS FROM t1;
+Field	Type	Null	Key	Default	Extra
+subject	varchar(64)	NO			
+pattern	varchar(64)	NO			
+INSERT INTO t1 VALUES (_utf8'2001÷01÷01',_utf8'%Y÷%m÷%d');
+SELECT HEX(subject),HEX(pattern),STR_TO_DATE(subject, pattern) FROM t1;
+HEX(subject)	HEX(pattern)	STR_TO_DATE(subject, pattern)
+32303031F73031F73031	2559F7256DF72564	2001-01-01 00:00:00
+DROP TABLE t1;
 SET collation_connection='latin1_bin';
 create table t1 select repeat('a',4000) a;
 delete from t1;

mysql-test/r/ctype_sjis.result

@@ -138,6 +138,25 @@ a	hex(b)	c
 3	F1F2F1F2F1F2F1F2F1F2F1F2F1F2F1F2F1F2F1F2F1F2	
 4	F1F2F1F2F1F2F1F2F1F2F1F2F1F2F1F2F1F2F1F2F1F2F1F2	
 DROP TABLE t1;
+#
+# MDEV-4842 STR_TO_DATE does not work with UCS2/UTF16/UTF32
+#
+SELECT @@character_set_connection, HEX(CAST(_utf8'ﾃｷ' AS CHAR));
+@@character_set_connection	HEX(CAST(_utf8'ﾃｷ' AS CHAR))
+sjis	8180
+SELECT STR_TO_DATE(CAST(_utf8'2001ﾃｷ01ﾃｷ01' AS CHAR),CAST(_utf8'%Yﾃｷ%mﾃｷ%d' AS CHAR));
+STR_TO_DATE(CAST(_utf8'2001ﾃｷ01ﾃｷ01' AS CHAR),CAST(_utf8'%Yﾃｷ%mﾃｷ%d' AS CHAR))
+2001-01-01
+CREATE TABLE t1 AS SELECT REPEAT(' ', 64) AS subject, REPEAT(' ',64) AS pattern LIMIT 0;
+SHOW COLUMNS FROM t1;
+Field	Type	Null	Key	Default	Extra
+subject	varchar(64)	NO			
+pattern	varchar(64)	NO			
+INSERT INTO t1 VALUES (_utf8'2001ﾃｷ01ﾃｷ01',_utf8'%Yﾃｷ%mﾃｷ%d');
+SELECT HEX(subject),HEX(pattern),STR_TO_DATE(subject, pattern) FROM t1;
+HEX(subject)	HEX(pattern)	STR_TO_DATE(subject, pattern)
+323030318180303181803031	25598180256D81802564	2001-01-01 00:00:00
+DROP TABLE t1;
 SET collation_connection='sjis_bin';
 create table t1 select repeat('a',4000) a;
 delete from t1;

mysql-test/r/ctype_ucs.result

@@ -804,6 +804,25 @@ a	hex(b)	c
 3	F1F2F1F2F1F2F1F2F1F2F1F2F1F2F1F2F1F2F1F2F1F2	
 4	F1F2F1F2F1F2F1F2F1F2F1F2F1F2F1F2F1F2F1F2F1F2F1F2	
 DROP TABLE t1;
+#
+# MDEV-4842 STR_TO_DATE does not work with UCS2/UTF16/UTF32
+#
+SELECT @@character_set_connection, HEX(CAST(_utf8'÷' AS CHAR));
+@@character_set_connection	HEX(CAST(_utf8'÷' AS CHAR))
+ucs2	00F7
+SELECT STR_TO_DATE(CAST(_utf8'2001÷01÷01' AS CHAR),CAST(_utf8'%Y÷%m÷%d' AS CHAR));
+STR_TO_DATE(CAST(_utf8'2001÷01÷01' AS CHAR),CAST(_utf8'%Y÷%m÷%d' AS CHAR))
+2001-01-01
+CREATE TABLE t1 AS SELECT REPEAT(' ', 64) AS subject, REPEAT(' ',64) AS pattern LIMIT 0;
+SHOW COLUMNS FROM t1;
+Field	Type	Null	Key	Default	Extra
+subject	varchar(64)	NO			
+pattern	varchar(64)	NO			
+INSERT INTO t1 VALUES (_utf8'2001÷01÷01',_utf8'%Y÷%m÷%d');
+SELECT HEX(subject),HEX(pattern),STR_TO_DATE(subject, pattern) FROM t1;
+HEX(subject)	HEX(pattern)	STR_TO_DATE(subject, pattern)
+003200300030003100F70030003100F700300031	0025005900F70025006D00F700250064	2001-01-01 00:00:00
+DROP TABLE t1;
 SET NAMES latin1;
 SET collation_connection='ucs2_bin';
 create table t1 select repeat('a',4000) a;

mysql-test/r/ctype_utf8.result

@@ -976,6 +976,25 @@ ss
 u,ü
 ue
 drop table t1;
+#
+# MDEV-4842 STR_TO_DATE does not work with UCS2/UTF16/UTF32
+#
+SELECT @@character_set_connection, HEX(CAST(_utf8'÷' AS CHAR));
+@@character_set_connection	HEX(CAST(_utf8'÷' AS CHAR))
+utf8	C3B7
+SELECT STR_TO_DATE(CAST(_utf8'2001÷01÷01' AS CHAR),CAST(_utf8'%Y÷%m÷%d' AS CHAR));
+STR_TO_DATE(CAST(_utf8'2001÷01÷01' AS CHAR),CAST(_utf8'%Y÷%m÷%d' AS CHAR))
+2001-01-01
+CREATE TABLE t1 AS SELECT REPEAT(' ', 64) AS subject, REPEAT(' ',64) AS pattern LIMIT 0;
+SHOW COLUMNS FROM t1;
+Field	Type	Null	Key	Default	Extra
+subject	varchar(64)	NO			
+pattern	varchar(64)	NO			
+INSERT INTO t1 VALUES (_utf8'2001÷01÷01',_utf8'%Y÷%m÷%d');
+SELECT HEX(subject),HEX(pattern),STR_TO_DATE(subject, pattern) FROM t1;
+HEX(subject)	HEX(pattern)	STR_TO_DATE(subject, pattern)
+32303031C3B73031C3B73031	2559C3B7256DC3B72564	2001-01-01 00:00:00
+DROP TABLE t1;
 SET collation_connection='utf8_bin';
 create table t1 select repeat('a',4000) a;
 delete from t1;

mysql-test/t/ctype_latin1.test

@@ -82,6 +82,7 @@ select 'a' regexp 'A' collate latin1_bin;
 SET collation_connection='latin1_swedish_ci';
 -- source include/ctype_filesort.inc
 -- source include/ctype_like_escape.inc
+-- source include/ctype_str_to_date.inc
 SET collation_connection='latin1_bin';
 -- source include/ctype_filesort.inc
 -- source include/ctype_like_escape.inc

mysql-test/t/ctype_sjis.test

@@ -69,6 +69,7 @@ SET collation_connection='sjis_japanese_ci';
 -- source include/ctype_innodb_like.inc
 -- source include/ctype_like_escape.inc
 -- source include/ctype_like_range_f1f2.inc
+-- source include/ctype_str_to_date.inc
 SET collation_connection='sjis_bin';
 -- source include/ctype_filesort.inc
 -- source include/ctype_innodb_like.inc

mysql-test/t/ctype_ucs.test

@@ -421,6 +421,7 @@ SET collation_connection='ucs2_general_ci';
 -- source include/ctype_like_escape.inc
 -- source include/ctype_german.inc
 -- source include/ctype_like_range_f1f2.inc
+-- source include/ctype_str_to_date.inc
 SET NAMES latin1;
 SET collation_connection='ucs2_bin';
 -- source include/ctype_filesort.inc

mysql-test/t/ctype_utf8.test

@@ -731,6 +731,7 @@ SET collation_connection='utf8_general_ci';
 -- source include/ctype_filesort.inc
 -- source include/ctype_like_escape.inc
 -- source include/ctype_german.inc
+-- source include/ctype_str_to_date.inc
 SET collation_connection='utf8_bin';
 -- source include/ctype_filesort.inc
 -- source include/ctype_like_escape.inc

sql/item.cc

@@ -218,6 +218,24 @@ bool Item::val_bool()
 }
 
 
+String *Item::val_str(String *str, String *converter, CHARSET_INFO *cs)
+{
+  String *res= val_str(str);
+  if (null_value)
+    return (String *) 0;
+
+  if (!cs)
+    return res;
+
+  uint errors;
+  if ((null_value= converter->copy(res->ptr(), res->length(),
+                                   collation.collation, cs,  &errors)))
+    return (String *) 0;
+
+  return converter;
+}
+
+
 String *Item::val_string_from_real(String *str)
 {
   double nr= val_real();

sql/item.h

@@ -793,6 +793,10 @@ class Item {
       If value is not null null_value flag will be reset to FALSE.
   */
   virtual String *val_str(String *str)=0;
+  /*
+    Returns the val_str() value converted to the given character set.
+  */
+  String *val_str(String *str, String *converter, CHARSET_INFO *to);
   /*
     Return decimal representation of item with fixed point.
 

sql/item_timefunc.cc

@@ -2980,13 +2980,25 @@ get_date_time_result_type(const char *format, uint length)
 
 void Item_func_str_to_date::fix_length_and_dec()
 {
+  if (agg_arg_charsets(collation, args, 2, MY_COLL_ALLOW_CONV, 1))
+    return;
+  if (collation.collation->mbminlen > 1)
+  {
+#if MYSQL_VERSION_ID > 50500
+    internal_charset= &my_charset_utf8mb4_general_ci;
+#else
+    internal_charset= &my_charset_utf8_general_ci;
+#endif
+  }
+
   cached_field_type= MYSQL_TYPE_DATETIME;
   decimals= NOT_FIXED_DEC;
   if ((const_item= args[1]->const_item()))
   {
     char format_buff[64];
     String format_str(format_buff, sizeof(format_buff), &my_charset_bin);
-    String *format= args[1]->val_str(&format_str);
+    String *format= args[1]->val_str(&format_str, &format_converter,
+                                     internal_charset);
     decimals= 0;
     if (!args[1]->null_value)
     {
@@ -3024,8 +3036,8 @@ bool Item_func_str_to_date::get_date(MYSQL_TIME *ltime, uint fuzzy_date)
   String format_str(format_buff, sizeof(format_buff), &my_charset_bin),
     *format;
 
-  val=    args[0]->val_str(&val_string);
-  format= args[1]->val_str(&format_str);
+  val=    args[0]->val_str(&val_string, &subject_converter, internal_charset);
+  format= args[1]->val_str(&format_str, &format_converter, internal_charset);
   if (args[0]->null_value || args[1]->null_value)
     goto null_date;
 

sql/item_timefunc.h

@@ -998,9 +998,13 @@ class Item_func_str_to_date :public Item_temporal_func
   enum_field_types cached_field_type;
   timestamp_type cached_timestamp_type;
   bool const_item;
+  String subject_converter;
+  String format_converter;
+  CHARSET_INFO *internal_charset;
 public:
   Item_func_str_to_date(Item *a, Item *b)
-    :Item_temporal_func(a, b), const_item(false)
+    :Item_temporal_func(a, b), const_item(false),
+    internal_charset(NULL)
   {}
   bool get_date(MYSQL_TIME *ltime, uint fuzzy_date);
   const char *func_name() const { return "str_to_date"; }