Bug #27957 cast as decimal does not check overflow, also inconsistent with group, subselect

Missing check for overflow added to the Item_decimal_typecast::val_decimal

include/decimal.h

@@ -47,6 +47,7 @@ int decimal_bin_size(int precision, int scale);
 int decimal_result_size(decimal_t *from1, decimal_t *from2, char op,
                         int param);
 
+int decimal_intg(decimal_t *from);
 int decimal_add(decimal_t *from1, decimal_t *from2, decimal_t *to);
 int decimal_sub(decimal_t *from1, decimal_t *from2, decimal_t *to);
 int decimal_cmp(decimal_t *from1, decimal_t *from2);

mysql-test/r/cast.result

@@ -378,7 +378,9 @@ create table t1(s1 time);
 insert into t1 values ('11:11:11');
 select cast(s1 as decimal(7,2)) from t1;
 cast(s1 as decimal(7,2))
-111111.00
+99999.99
+Warnings:
+Error	1264	Out of range value adjusted for column 'cast(s1 as decimal(7,2))' at row 1
 drop table t1;
 CREATE TABLE t1 (v varchar(10), tt tinytext, t text,
 mt mediumtext, lt longtext);

mysql-test/r/type_newdecimal.result

@@ -1430,4 +1430,39 @@ select * from t1;
 a
 123456789012345678
 drop table t1;
+select cast(11.1234 as DECIMAL(3,2));
+cast(11.1234 as DECIMAL(3,2))
+9.99
+Warnings:
+Error	1264	Out of range value adjusted for column 'cast(11.1234 as DECIMAL(3,2))' at row 1
+select * from (select cast(11.1234 as DECIMAL(3,2))) t;
+cast(11.1234 as DECIMAL(3,2))
+9.99
+Warnings:
+Error	1264	Out of range value adjusted for column 'cast(11.1234 as DECIMAL(3,2))' at row 1
+select cast(a as DECIMAL(3,2))
+from (select 11.1233 as a
+UNION select 11.1234
+UNION select 12.1234
+) t;
+cast(a as DECIMAL(3,2))
+9.99
+9.99
+9.99
+Warnings:
+Error	1264	Out of range value adjusted for column 'cast(a as DECIMAL(3,2))' at row 1
+Error	1264	Out of range value adjusted for column 'cast(a as DECIMAL(3,2))' at row 1
+Error	1264	Out of range value adjusted for column 'cast(a as DECIMAL(3,2))' at row 1
+select cast(a as DECIMAL(3,2)), count(*)
+from (select 11.1233 as a
+UNION select 11.1234
+UNION select 12.1234
+) t group by 1;
+cast(a as DECIMAL(3,2))	count(*)
+9.99	3
+Warnings:
+Error	1264	Out of range value adjusted for column 'cast(a as DECIMAL(3,2))' at row 1
+Error	1264	Out of range value adjusted for column 'cast(a as DECIMAL(3,2))' at row 1
+Error	1264	Out of range value adjusted for column 'cast(a as DECIMAL(3,2))' at row 1
+Error	1264	Out of range value adjusted for column 'cast(a as DECIMAL(3,2))' at row 1
 End of 5.0 tests

mysql-test/t/type_newdecimal.test

@@ -1130,4 +1130,23 @@ alter table t1 modify column a decimal(19);
 select * from t1;
 drop table t1;
 
+#
+# Bug #27957 cast as decimal does not check overflow, also inconsistent with group, subselect 
+#
+
+select cast(11.1234 as DECIMAL(3,2));
+select * from (select cast(11.1234 as DECIMAL(3,2))) t;
+
+select cast(a as DECIMAL(3,2))
+ from (select 11.1233 as a
+  UNION select 11.1234
+  UNION select 12.1234
+ ) t;
+
+select cast(a as DECIMAL(3,2)), count(*)
+ from (select 11.1233 as a
+  UNION select 11.1234
+  UNION select 12.1234
+ ) t group by 1;
+
 --echo End of 5.0 tests

sql/item_func.cc

@@ -1050,9 +1050,32 @@ longlong Item_decimal_typecast::val_int()
 my_decimal *Item_decimal_typecast::val_decimal(my_decimal *dec)
 {
   my_decimal tmp_buf, *tmp= args[0]->val_decimal(&tmp_buf);
+  bool sign;
   if ((null_value= args[0]->null_value))
     return NULL;
   my_decimal_round(E_DEC_FATAL_ERROR, tmp, decimals, FALSE, dec);
+  sign= dec->sign();
+  if (unsigned_flag)
+  {
+    if (sign)
+    {
+      my_decimal_set_zero(dec);
+      goto err;
+    }
+  }
+  if (max_length - 2 - decimals < (uint) my_decimal_intg(dec))
+  {
+    max_my_decimal(dec, max_length - 2, decimals);
+    dec->sign(sign);
+    goto err;
+  }
+  return dec;
+
+err:
+  push_warning_printf(current_thd, MYSQL_ERROR::WARN_LEVEL_ERROR,
+                      ER_WARN_DATA_OUT_OF_RANGE,
+                      ER(ER_WARN_DATA_OUT_OF_RANGE),
+                      name, 1);
   return dec;
 }
 

sql/my_decimal.h

@@ -387,5 +387,13 @@ int my_decimal_cmp(const my_decimal *a, const my_decimal *b)
   return decimal_cmp((decimal_t*) a, (decimal_t*) b);
 }
 
+
+inline
+int my_decimal_intg(const my_decimal *a)
+{
+    return decimal_intg((decimal_t*) a);
+}
+
+
 #endif /*my_decimal_h*/
 

strings/decimal.c

@@ -1911,6 +1911,14 @@ static int do_sub(decimal_t *from1, decimal_t *from2, decimal_t *to)
   return error;
 }
 
+int decimal_intg(decimal_t *from)
+{
+  int res;
+  dec1 *tmp_res;
+  tmp_res= remove_leading_zeroes(from, &res);
+  return res;
+}
+
 int decimal_add(decimal_t *from1, decimal_t *from2, decimal_t *to)
 {
   if (likely(from1->sign == from2->sign))