MDEV-12849 Out-of-range errors when casting hex-hybrid to SIGNED and UNSIGNED

mysql-test/t/cast.test

@@ -514,3 +514,48 @@ CREATE TABLE t1 AS SELECT
   CONCAT(CONVERT('',UNSIGNED));
 SHOW CREATE TABLE t1;
 DROP TABLE t1;
+
+
+--echo #
+--echo # MDEV-12849 Out-of-range errors when casting hex-hybrid to SIGNED and UNSIGNED
+--echo #
+
+DELIMITER $$;
+CREATE PROCEDURE p1(hh TEXT)
+BEGIN
+  EXECUTE IMMEDIATE
+    CONCAT('CREATE OR REPLACE TABLE t1 AS SELECT CAST(0x', hh, ' AS UNSIGNED) AS c');
+  DESCRIBE t1;
+  SELECT c, LENGTH(c) FROM t1;
+  DROP TABLE t1;
+  EXECUTE IMMEDIATE
+    CONCAT('CREATE OR REPLACE TABLE t1 AS SELECT CAST(0x', hh, ' AS SIGNED) AS c');
+  DESCRIBE t1;
+  SELECT c, LENGTH(c) FROM t1;
+  DROP TABLE t1;
+  SELECT '' AS `------`;
+END
+$$
+DELIMITER ;$$
+CALL p1('FF');
+CALL p1('FFFF');
+CALL p1('FFFFFF');
+CALL p1('FFFFFFFF');
+CALL p1('FFFFFFFFFF');
+CALL p1('FFFFFFFFFFFF');
+CALL p1('FFFFFFFFFFFFFF');
+CALL p1('FFFFFFFFFFFFFFFF');
+CALL p1('FFFFFFFFFFFFFFFFFF');
+CALL p1('FFFFFFFFFFFFFFFFFFFF');
+
+CALL p1('8000000000000000');
+CALL p1('80000000000000FF');
+CALL p1('800000000000FFFF');
+CALL p1('8000000000FFFFFF');
+CALL p1('80000000FFFFFFFF');
+CALL p1('800000FFFFFFFFFF');
+CALL p1('8000FFFFFFFFFFFF');
+CALL p1('80FFFFFFFFFFFFFF');
+CALL p1('8FFFFFFFFFFFFFFF');
+
+DROP PROCEDURE p1;

sql/item.cc

@@ -6718,6 +6718,22 @@ void Item_hex_hybrid::print(String *str, enum_query_type query_type)
 }
 
 
+uint Item_hex_hybrid::decimal_precision() const
+{
+  switch (max_length) {// HEX                                 DEC
+  case 0:              // ----                                ---
+  case 1: return 3;    // 0xFF                                255
+  case 2: return 5;    // 0xFFFF                            65535
+  case 3: return 8;    // 0xFFFFFF                       16777215
+  case 4: return 10;   // 0xFFFFFFFF                   4294967295
+  case 5: return 13;   // 0xFFFFFFFFFF              1099511627775
+  case 6: return 15;   // 0xFFFFFFFFFFFF          281474976710655
+  case 7: return 17;   // 0xFFFFFFFFFFFFFF      72057594037927935
+  }
+  return 20;           // 0xFFFFFFFFFFFFFFFF 18446744073709551615
+}
+
+
 void Item_hex_string::print(String *str, enum_query_type query_type)
 {
   str->append("X'");

sql/item.h

@@ -3810,6 +3810,7 @@ class Item_hex_hybrid: public Item_hex_constant
   Item_hex_hybrid(THD *thd): Item_hex_constant(thd) {}
   Item_hex_hybrid(THD *thd, const char *str, uint str_length):
     Item_hex_constant(thd, str, str_length) {}
+  uint decimal_precision() const;
   double val_real()
   { 
     DBUG_ASSERT(fixed == 1); 

sql/item_func.h

@@ -852,6 +852,19 @@ class Item_func_signed :public Item_int_func
     set_if_bigger(char_length, 1U + (unsigned_flag ? 0 : 1));
     fix_char_length(char_length);
   }
+  void fix_length_and_dec_string()
+  {
+    /*
+      For strings, use decimal_int_part() instead of max_char_length().
+      This is important for Item_hex_hybrid:
+        SELECT CAST(0x1FFFFFFFF AS SIGNED);
+      Length is 5, decimal_int_part() is 13.
+    */
+    uint32 char_length= MY_MIN(args[0]->decimal_int_part(),
+                               MY_INT64_NUM_DECIMAL_DIGITS);
+    set_if_bigger(char_length, 1U + (unsigned_flag ? 0 : 1));
+    fix_char_length(char_length);
+  }
   void fix_length_and_dec()
   {
     args[0]->type_handler()->Item_func_signed_fix_length_and_dec(this);

sql/sql_type.cc

@@ -4168,6 +4168,22 @@ bool Type_handler::
 }
 
 
+bool Type_handler_string_result::
+       Item_func_signed_fix_length_and_dec(Item_func_signed *item) const
+{
+  item->fix_length_and_dec_string();
+  return false;
+}
+
+
+bool Type_handler_string_result::
+       Item_func_unsigned_fix_length_and_dec(Item_func_unsigned *item) const
+{
+  item->fix_length_and_dec_string();
+  return false;
+}
+
+
 bool Type_handler::
        Item_double_typecast_fix_length_and_dec(Item_double_typecast *item) const
 {

sql/sql_type.h

@@ -1615,6 +1615,8 @@ class Type_handler_string_result: public Type_handler
   bool Item_sum_sum_fix_length_and_dec(Item_sum_sum *) const;
   bool Item_sum_avg_fix_length_and_dec(Item_sum_avg *) const;
   bool Item_sum_variance_fix_length_and_dec(Item_sum_variance *) const;
+  bool Item_func_signed_fix_length_and_dec(Item_func_signed *item) const;
+  bool Item_func_unsigned_fix_length_and_dec(Item_func_unsigned *item) const;
   longlong Item_val_int_signed_typecast(Item *item) const;
   longlong Item_val_int_unsigned_typecast(Item *item) const;
   String *Item_func_hex_val_str_ascii(Item_func_hex *item, String *str) const;