Fixed a few bugs in hex string generation, in call to val_str for partition...

Fixed a few bugs in hex string generation, in call to val_str for partition expressions, also made code reusable for DEFAULT handling to fix BUG#48464 by introducing function get_cs_converted_string_value, added partition_utf8 test case for UTF8 outputs

BUILD/build_mccge.sh

@@ -1274,7 +1274,7 @@ set_bsd_configs()
   if test "x$fast_flag" != "xno" ; then
     compiler_flags="$compiler_flags -O3"
   else
-    compiler_flags="$compiler_flags -O"
+    compiler_flags="$compiler_flags -O0"
   fi
   set_cc_and_cxx_for_gcc
 }
@@ -1305,7 +1305,7 @@ set_linux_configs()
     if test "x$fast_flag" != "xno" ; then
       compiler_flags="$compiler_flags -O2"
     else
-      compiler_flags="$compiler_flags -O"
+      compiler_flags="$compiler_flags -O0"
     fi
 # configure will set proper compiler flags for gcc on Linux
   elif test "x$compiler" = "xicc" ; then
@@ -1375,8 +1375,8 @@ set_solaris_configs()
         LDFLAGS="$LDFLAGS -O2"
         compiler_flags="$compiler_flags -O2"
       else
-        LDFLAGS="$LDFLAGS -O"
-        compiler_flags="$compiler_flags -O"
+        LDFLAGS="$LDFLAGS -O0"
+        compiler_flags="$compiler_flags -O0"
       fi
     fi
   else
@@ -1407,7 +1407,7 @@ set_solaris_configs()
       elif test "x$fast_flag" = "xgeneric" ; then
         compiler_flags="$compiler_flags -xO2"
       else
-        compiler_flags="$compiler_flags -xO"
+        compiler_flags="$compiler_flags -xO0"
       fi
     else
 #Using SPARC cpu with SunStudio (Forte) compiler
@@ -1421,7 +1421,7 @@ set_solaris_configs()
       elif test "x$fast_flag" = "xgeneric" ; then
         compiler_flags="$compiler_flags -xO2"
       else
-        compiler_flags="$compiler_flags -xO"
+        compiler_flags="$compiler_flags -xO0"
       fi
     fi
   fi
@@ -1452,7 +1452,7 @@ set_macosx_configs()
   if test "x$fast_flag" != "xno" ; then
     compiler_flags="$compiler_flags -Os"
   else
-    compiler_flags="$compiler_flags -O"
+    compiler_flags="$compiler_flags -O0"
   fi
   set_cc_and_cxx_for_gcc
 }

mysql-test/r/partition_column.result

 drop table if exists t1;
-create table t1 (a varchar(1500), b varchar(1570))
-partition by list columns(a,b)
-( partition p0 values in (('a','b')));
-ERROR HY000: The total length of the partitioning fields is too large
-create table t1 (a varchar(1023) character set utf8 collate utf8_spanish2_ci)
-partition by range columns(a)
-( partition p0 values less than ('CZ'),
-partition p1 values less than ('CH'),
-partition p2 values less than ('D'));
-insert into t1 values ('czz'),('chi'),('ci'),('cg');
-select * from t1 where a between 'cg' AND 'ci';
-a
-ci
-cg
+create table t1 (a varchar(5))
+partition by list columns(a)
+( partition p0 values in ('\''),
+  partition p1 values in ('\\'),
+  partition p2 values in ('\0'));
+show create table t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `a` varchar(5) DEFAULT NULL
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+/*!50100 PARTITION BY LIST  COLUMNS(a)
+(PARTITION p0 VALUES IN ('''') ENGINE = MyISAM,
+ PARTITION p1 VALUES IN ('\\') ENGINE = MyISAM,
+ PARTITION p2 VALUES IN ('\0') ENGINE = MyISAM) */
 drop table t1;
 set @@sql_mode=allow_invalid_dates;
 create table t1 (a char, b char, c date)
@@ -24,22 +24,6 @@ partition by range columns (a,b,c)
 ( partition p0 values less than (0,0,'3000-11-31'));
 ERROR HY000: Partition column values of incorrect type
 set @@sql_mode='';
-create table t1 (a varchar(2) character set ucs2)
-partition by list columns (a)
-(partition p0 values in (0x2020),
-partition p1 values in (''));
-set names utf8;
-show create table t1;
-Table	Create Table
-t1	CREATE TABLE `t1` (
-  `a` varchar(2) CHARACTER SET ucs2 DEFAULT NULL
-) ENGINE=MyISAM DEFAULT CHARSET=latin1
-/*!50100 PARTITION BY LIST  COLUMNS(a)
-(PARTITION p0 VALUES IN ('†') ENGINE = MyISAM,
- PARTITION p1 VALUES IN ('') ENGINE = MyISAM) */
-insert into t1 values ('');
-insert into t1 values (_ucs2 0x2020);
-drop table t1;
 create table t1 (a int, b char(10), c varchar(25), d datetime)
 partition by range columns(a,b,c,d)
 subpartition by hash (to_seconds(d))

mysql-test/r/partition_utf8.result

+set names utf8;
+create table t1 (a varchar(2) character set cp1250)
+partition by list columns (a)
+( partition p0 values in (0x81));
+show create table t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `a` varchar(2) CHARACTER SET cp1250 DEFAULT NULL
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+/*!50100 PARTITION BY LIST  COLUMNS(a)
+(PARTITION p0 VALUES IN (_cp1250 0x81) ENGINE = MyISAM) */
+drop table t1;
+create table t1 (a varchar(2) character set cp1250)
+partition by list columns (a)
+( partition p0 values in (0x80));
+show create table t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `a` varchar(2) CHARACTER SET cp1250 DEFAULT NULL
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+/*!50100 PARTITION BY LIST  COLUMNS(a)
+(PARTITION p0 VALUES IN ('€') ENGINE = MyISAM) */
+drop table t1;
+create table t1 (a varchar(1500), b varchar(1570))
+partition by list columns(a,b)
+( partition p0 values in (('a','b')));
+ERROR HY000: The total length of the partitioning fields is too large
+create table t1 (a varchar(1023) character set utf8 collate utf8_spanish2_ci)
+partition by range columns(a)
+( partition p0 values less than ('CZ'),
+partition p1 values less than ('CH'),
+partition p2 values less than ('D'));
+insert into t1 values ('czz'),('chi'),('ci'),('cg');
+select * from t1 where a between 'cg' AND 'ci';
+a
+ci
+cg
+drop table t1;
+create table t1 (a varchar(2) character set ucs2)
+partition by list columns (a)
+(partition p0 values in (0x2020),
+partition p1 values in (''));
+show create table t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `a` varchar(2) CHARACTER SET ucs2 DEFAULT NULL
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+/*!50100 PARTITION BY LIST  COLUMNS(a)
+(PARTITION p0 VALUES IN ('†') ENGINE = MyISAM,
+ PARTITION p1 VALUES IN ('') ENGINE = MyISAM) */
+insert into t1 values ('');
+insert into t1 values (_ucs2 0x2020);
+drop table t1;

mysql-test/t/partition_column.test

@@ -8,21 +8,12 @@
 drop table if exists t1;
 --enable_warnings
 
-#
-# BUG#48164, too long partition fields causes crash
-#
---error ER_PARTITION_FIELDS_TOO_LONG
-create table t1 (a varchar(1500), b varchar(1570))
-partition by list columns(a,b)
-( partition p0 values in (('a','b')));
-
-create table t1 (a varchar(1023) character set utf8 collate utf8_spanish2_ci)
-partition by range columns(a)
-( partition p0 values less than ('CZ'),
-  partition p1 values less than ('CH'),
-  partition p2 values less than ('D'));
-insert into t1 values ('czz'),('chi'),('ci'),('cg');
-select * from t1 where a between 'cg' AND 'ci';
+create table t1 (a varchar(5))
+partition by list columns(a)
+( partition p0 values in ('\''),
+  partition p1 values in ('\\'),
+  partition p2 values in ('\0'));
+show create table t1;
 drop table t1;
 
 #
@@ -40,19 +31,6 @@ partition by range columns (a,b,c)
 ( partition p0 values less than (0,0,'3000-11-31'));
 set @@sql_mode='';
 
-#
-# BUG#48163, Dagger in UCS2 not working as partition value
-#
-create table t1 (a varchar(2) character set ucs2)
-partition by list columns (a)
-(partition p0 values in (0x2020),
- partition p1 values in (''));
-set names utf8;
-show create table t1;
-insert into t1 values ('');
-insert into t1 values (_ucs2 0x2020);
-drop table t1;
-
 --error ER_WRONG_TYPE_COLUMN_VALUE_ERROR
 create table t1 (a int, b char(10), c varchar(25), d datetime)
 partition by range columns(a,b,c,d)

mysql-test/t/partition_utf8.test

+# Tests for Column list which requires utf8 output
+--source include/have_partition.inc
+set names utf8;
+create table t1 (a varchar(2) character set cp1250)
+partition by list columns (a)
+( partition p0 values in (0x81));
+show create table t1;
+drop table t1;
+create table t1 (a varchar(2) character set cp1250)
+partition by list columns (a)
+( partition p0 values in (0x80));
+show create table t1;
+drop table t1;
+
+#
+# BUG#48164, too long partition fields causes crash
+#
+--error ER_PARTITION_FIELDS_TOO_LONG
+create table t1 (a varchar(1500), b varchar(1570))
+partition by list columns(a,b)
+( partition p0 values in (('a','b')));
+
+create table t1 (a varchar(1023) character set utf8 collate utf8_spanish2_ci)
+partition by range columns(a)
+( partition p0 values less than ('CZ'),
+  partition p1 values less than ('CH'),
+  partition p2 values less than ('D'));
+insert into t1 values ('czz'),('chi'),('ci'),('cg');
+select * from t1 where a between 'cg' AND 'ci';
+drop table t1;
+
+#
+# BUG#48163, Dagger in UCS2 not working as partition value
+#
+create table t1 (a varchar(2) character set ucs2)
+partition by list columns (a)
+(partition p0 values in (0x2020),
+ partition p1 values in (''));
+show create table t1;
+insert into t1 values ('');
+insert into t1 values (_ucs2 0x2020);
+drop table t1;

sql/mysql_priv.h

@@ -1326,7 +1326,6 @@ void remove_status_vars(SHOW_VAR *list);
 void init_status_vars();
 void free_status_vars();
 void reset_status_vars();
-
 /* information schema */
 extern LEX_STRING INFORMATION_SCHEMA_NAME;
 /* log tables */

sql/sql_partition.cc

@@ -2132,92 +2132,6 @@ static Create_field* get_sql_field(char *field_name,
 }
 
 
-/*
-  Convert a string in a given character set to a string which can be
-  used for FRM file storage in which case use_hex is TRUE and we store
-  the character constants as hex strings in the character set encoding
-  their field have. In the case of SHOW CREATE TABLE and the
-  PARTITIONS information schema table we instead provide utf8 strings
-  to the user and convert to the utf8 character set.
-
-  SYNOPSIS
-    get_converted_part_value_from_string()
-    item                           Item from which constant comes
-    res                            String as provided by val_str after
-                                   conversion to character set
-    field_cs                       Character set string is encoded in
-                                   NULL for INT_RESULT's here
-    val_conv                       Out value: The string created
-    use_hex                        TRUE => hex string created
-                                   FALSE => utf8 constant string created
-  RETURN VALUES
-    TRUE                           Error
-    FALSE                          Ok
-*/
-
-int get_converted_part_value_from_string(Item *item,
-                                         String *res,
-                                         CHARSET_INFO *field_cs,
-                                         String *val_conv,
-                                         bool use_hex)
-{
-  String val;
-  uint dummy_errors;
-  uint len, high, low, i;
-  const char *ptr;
-  char buf[3];
-
-  if (item->result_type() == INT_RESULT)
-  {
-    longlong value= item->val_int();
-    val_conv->set(value, system_charset_info);
-    return FALSE;
-  }
-  if (!res)
-  {
-    my_error(ER_PARTITION_FUNCTION_IS_NOT_ALLOWED, MYF(0));
-    return TRUE;
-  }
-  val_conv->length(0);
-  if (!field_cs || res->length() == 0)
-  {
-    val_conv->append("'");
-    if (res->length() != 0)
-      val_conv->append(*res);
-    val_conv->append("'");
-    return FALSE;
-  }
-  if (field_cs && use_hex)
-  {
-    val_conv->append("_");
-    val_conv->append(field_cs->csname);
-    val_conv->append(" ");
-  }
-  if (use_hex)
-  {
-    val_conv->append("0x");
-    len= res->length();
-    ptr= res->ptr();
-    for (i= 0; i < len; i++)
-    {
-      high= (*ptr) >> 4;
-      low= (*ptr) & 0x0F;
-      buf[0]= _dig_vec_upper[high];
-      buf[1]= _dig_vec_upper[low];
-      buf[2]= 0;
-      val_conv->append((const char*)buf);
-      ptr++;
-    }
-  }
-  else
-  {
-    val.copy(res->ptr(), res->length(), field_cs,
-             system_charset_info, &dummy_errors);
-    append_unescaped(val_conv, val.ptr(), val.length());
-  }
-  return FALSE;
-}
-
 static int add_column_list_values(File fptr, partition_info *part_info,
                                   part_elem_value *list_value,
                                   HA_CREATE_INFO *create_info,
@@ -2311,9 +2225,11 @@ static int add_column_list_values(File fptr, partition_info *part_info,
         }
         {
           String val_conv;
+          val_conv.set_charset(system_charset_info);
           res= item_expr->val_str(&str);
-          if (get_converted_part_value_from_string(item_expr, res,
-                                                   field_cs, &val_conv,
+          if (get_cs_converted_part_value_from_string(current_thd,
+                                                      item_expr, res,
+                                                      &val_conv, field_cs,
                                                    (bool)(alter_info != NULL)))
             return 1;
           err+= add_string_object(fptr, &val_conv);

sql/sql_partition.h

@@ -75,11 +75,12 @@ void get_partition_set(const TABLE *table, uchar *buf, const uint index,
                        const key_range *key_spec,
                        part_id_range *part_spec);
 uint get_partition_field_store_length(Field *field);
-int get_converted_part_value_from_string(Item *item,
-                                         String *res,
-                                         CHARSET_INFO *cs,
-                                         String *val_conv,
-                                         bool use_hex);
+int get_cs_converted_part_value_from_string(THD *thd,
+                                            Item *item,
+                                            String *input_str,
+                                            String *output_str,
+                                            CHARSET_INFO *cs,
+                                            bool use_hex);
 void get_full_part_id_from_key(const TABLE *table, uchar *buf,
                                KEY *key_info,
                                const key_range *key_spec,

sql/sql_show.cc

@@ -78,6 +78,12 @@ static TYPELIB grant_types = { sizeof(grant_names)/sizeof(char **),
 static void store_key_options(THD *thd, String *packet, TABLE *table,
                               KEY *key_info);
 
+static void get_cs_converted_string_value(THD *thd,
+                                          String *input_str,
+                                          String *output_str,
+                                          CHARSET_INFO *cs,
+                                          bool use_hex);
+
 static void
 append_algorithm(TABLE_LIST *table, String *buff);
 
@@ -4795,6 +4801,57 @@ static void collect_partition_expr(List<char> &field_list, String *str)
   }
   return;
 }
+
+
+/*
+  Convert a string in a given character set to a string which can be
+  used for FRM file storage in which case use_hex is TRUE and we store
+  the character constants as hex strings in the character set encoding
+  their field have. In the case of SHOW CREATE TABLE and the
+  PARTITIONS information schema table we instead provide utf8 strings
+  to the user and convert to the utf8 character set.
+
+  SYNOPSIS
+    get_cs_converted_part_value_from_string()
+    item                           Item from which constant comes
+    input_str                      String as provided by val_str after
+                                   conversion to character set
+    output_str                     Out value: The string created
+    cs                             Character set string is encoded in
+                                   NULL for INT_RESULT's here
+    use_hex                        TRUE => hex string created
+                                   FALSE => utf8 constant string created
+
+  RETURN VALUES
+    TRUE                           Error
+    FALSE                          Ok
+*/
+
+int get_cs_converted_part_value_from_string(THD *thd,
+                                            Item *item,
+                                            String *input_str,
+                                            String *output_str,
+                                            CHARSET_INFO *cs,
+                                            bool use_hex)
+{
+  if (item->result_type() == INT_RESULT)
+  {
+    longlong value= item->val_int();
+    output_str->set(value, system_charset_info);
+    return FALSE;
+  }
+  if (!input_str)
+  {
+    my_error(ER_PARTITION_FUNCTION_IS_NOT_ALLOWED, MYF(0));
+    return TRUE;
+  }
+  get_cs_converted_string_value(thd,
+                                input_str,
+                                output_str,
+                                cs,
+                                use_hex);
+  return FALSE;
+}
 #endif
 
 
@@ -4877,7 +4934,8 @@ static void store_schema_partitions_record(THD *thd, TABLE *schema_table,
 }
 
 static int
-get_partition_column_description(partition_info *part_info,
+get_partition_column_description(THD *thd,
+                                 partition_info *part_info,
                                  part_elem_value *list_value,
                                  String &tmp_str)
 {
@@ -4905,9 +4963,9 @@ get_partition_column_description(partition_info *part_info,
         DBUG_RETURN(1);
       }
       String *res= item->val_str(&str);
-      if (get_converted_part_value_from_string(item, res,
+      if (get_cs_converted_part_value_from_string(thd, item, res, &val_conv,
                               part_info->part_field_array[i]->charset(),
-                              &val_conv, FALSE))
+                              FALSE))
       {
         DBUG_RETURN(1);
       }
@@ -5055,7 +5113,8 @@ static int get_schema_partitions_record(THD *thd, TABLE_LIST *tables,
           List_iterator<part_elem_value> list_val_it(part_elem->list_val_list);
           part_elem_value *list_value= list_val_it++;
           tmp_str.length(0);
-          if (get_partition_column_description(part_info,
+          if (get_partition_column_description(thd,
+                                               part_info,
                                                list_value,
                                                tmp_str))
           {
@@ -5092,7 +5151,8 @@ static int get_schema_partitions_record(THD *thd, TABLE_LIST *tables,
           {
             if (part_info->part_field_list.elements > 1U)
               tmp_str.append("(");
-            if (get_partition_column_description(part_info,
+            if (get_partition_column_description(thd,
+                                                 part_info,
                                                  list_value,
                                                  tmp_str))
             {
@@ -7231,3 +7291,95 @@ bool show_create_trigger(THD *thd, const sp_name *trg_name)
     status and client connection will be closed.
   */
 }
+
+/*
+  Convert a string in character set in column character set format
+  to utf8 character set if possible, the utf8 character set string
+  will later possibly be converted to character set used by client.
+  Thus we attempt conversion from column character set to both
+  utf8 and to character set client.
+
+  Examples of strings that should fail conversion to utf8 are unassigned
+  characters as e.g. 0x81 in cp1250 (Windows character set for for countries
+  like Czech and Poland). Example of string that should fail conversion to
+  character set on client (e.g. if this is latin1) is 0x2020 (daggger) in
+  ucs2.
+
+  If the conversion fails we will as a fall back convert the string to
+  hex encoded format. The caller of the function can also ask for hex
+  encoded format of output string unconditionally.
+
+  SYNOPSIS
+    get_cs_converted_string_value()
+    thd                             Thread object
+    input_str                       Input string in cs character set
+    output_str                      Output string to be produced in utf8
+    cs                              Character set of input string
+    use_hex                         Use hex string unconditionally
+ 
+
+  RETURN VALUES
+    No return value
+*/
+
+static void get_cs_converted_string_value(THD *thd,
+                                          String *input_str,
+                                          String *output_str,
+                                          CHARSET_INFO *cs,
+                                          bool use_hex)
+{
+
+  output_str->length(0);
+  if (input_str->length() == 0)
+  {
+    output_str->append("''");
+    return;
+  }
+  if (!use_hex)
+  {
+    String try_val;
+    uint try_conv_error= 0;
+
+    try_val.copy(input_str->ptr(), input_str->length(), cs,
+                 thd->variables.character_set_client, &try_conv_error);
+    if (!try_conv_error)
+    {
+      String val;
+      uint conv_error= 0;
+
+      val.copy(input_str->ptr(), input_str->length(), cs,
+               system_charset_info, &conv_error);
+      if (!conv_error)
+      {
+        append_unescaped(output_str, val.ptr(), val.length());
+        return;
+      }
+    }
+    /* We had a conversion error, use hex encoded string for safety */
+  }
+  {
+    const uchar *ptr;
+    uint i, len;
+    char buf[3];
+
+    output_str->append("_");
+    output_str->append(cs->csname);
+    output_str->append(" ");
+    output_str->append("0x");
+    len= input_str->length();
+    ptr= (uchar*)input_str->ptr();
+    for (i= 0; i < len; i++)
+    {
+      uint high, low;
+
+      high= (*ptr) >> 4;
+      low= (*ptr) & 0x0F;
+      buf[0]= _dig_vec_upper[high];
+      buf[1]= _dig_vec_upper[low];
+      buf[2]= 0;
+      output_str->append((const char*)buf);
+      ptr++;
+    }
+  }
+  return;
+}