auto-merge conservative fix for Bug#55436/Bug#11762799

mysql-test/r/type_newdecimal.result

@@ -1913,4 +1913,17 @@ group by PAY.id + 1;
 mult	v_net_with_discount	v_total
 1.0000	27.18	27.180000
 DROP TABLE currencies, payments, sub_tasks;
+#
+# Bug#55436: buffer overflow in debug binary of dbug_buff in
+#            Field_new_decimal::store_value
+#
+SET SQL_MODE='';
+CREATE TABLE t1(f1 DECIMAL(44,24)) ENGINE=MYISAM;
+INSERT INTO t1 SET f1 = -64878E-85;
+Warnings:
+Note	1265	Data truncated for column 'f1' at row 1
+SELECT f1 FROM t1;
+f1
+0.000000000000000000000000
+DROP TABLE IF EXISTS t1;
 End of 5.1 tests

mysql-test/t/type_newdecimal.test

@@ -1510,5 +1510,19 @@ group by PAY.id + 1;
 
 DROP TABLE currencies, payments, sub_tasks;
 
+--echo #
+--echo # Bug#55436: buffer overflow in debug binary of dbug_buff in
+--echo #            Field_new_decimal::store_value
+--echo #
+
+# this threw memory warnings on Windows. Also make sure future changes
+# don't change these results, as per usual.
+SET SQL_MODE='';
+CREATE TABLE t1(f1 DECIMAL(44,24)) ENGINE=MYISAM;
+INSERT INTO t1 SET f1 = -64878E-85;
+SELECT f1 FROM t1;
+DROP TABLE IF EXISTS t1;
+
+
 
 --echo End of 5.1 tests

sql/field.cc

@@ -2583,7 +2583,7 @@ bool Field_new_decimal::store_value(const my_decimal *decimal_value)
   DBUG_ENTER("Field_new_decimal::store_value");
 #ifndef DBUG_OFF
   {
-    char dbug_buff[DECIMAL_MAX_STR_LENGTH+1];
+    char dbug_buff[DECIMAL_MAX_STR_LENGTH+2];
     DBUG_PRINT("enter", ("value: %s", dbug_decimal_as_string(dbug_buff, decimal_value)));
   }
 #endif
@@ -2598,7 +2598,7 @@ bool Field_new_decimal::store_value(const my_decimal *decimal_value)
   }
 #ifndef DBUG_OFF
   {
-    char dbug_buff[DECIMAL_MAX_STR_LENGTH+1];
+    char dbug_buff[DECIMAL_MAX_STR_LENGTH+2];
     DBUG_PRINT("info", ("saving with precision %d  scale: %d  value %s",
                         (int)precision, (int)dec,
                         dbug_decimal_as_string(dbug_buff, decimal_value)));
@@ -2673,7 +2673,7 @@ int Field_new_decimal::store(const char *from, uint length,
   }
 
 #ifndef DBUG_OFF
-  char dbug_buff[DECIMAL_MAX_STR_LENGTH+1];
+  char dbug_buff[DECIMAL_MAX_STR_LENGTH+2];
   DBUG_PRINT("enter", ("value: %s",
                        dbug_decimal_as_string(dbug_buff, &decimal_value)));
 #endif

sql/my_decimal.cc

@@ -95,10 +95,11 @@ int my_decimal2string(uint mask, const my_decimal *d,
     UNSIGNED. Hence the buffer for a ZEROFILLed value is the length
     the user requested, plus one for a possible decimal point, plus
     one if the user only wanted decimal places, but we force a leading
-    zero on them. Because the type is implicitly UNSIGNED, we do not
-    need to reserve a character for the sign. For all other cases,
-    fixed_prec will be 0, and my_decimal_string_length() will be called
-    instead to calculate the required size of the buffer.
+    zero on them, plus one for the '\0' terminator. Because the type
+    is implicitly UNSIGNED, we do not need to reserve a character for
+    the sign. For all other cases, fixed_prec will be 0, and
+    my_decimal_string_length() will be called instead to calculate the
+    required size of the buffer.
   */
   int length= (fixed_prec
                ? (fixed_prec + ((fixed_prec == fixed_dec) ? 1 : 0) + 1)
@@ -276,7 +277,7 @@ print_decimal_buff(const my_decimal *dec, const uchar* ptr, int length)
 
 const char *dbug_decimal_as_string(char *buff, const my_decimal *val)
 {
-  int length= DECIMAL_MAX_STR_LENGTH;
+  int length= DECIMAL_MAX_STR_LENGTH + 1;     /* minimum size for buff */
   if (!val)
     return "NULL";
   (void)decimal2string((decimal_t*) val, buff, &length, 0,0,0);

sql/my_decimal.h

@@ -55,7 +55,7 @@ C_MODE_END
 
 /**
   maximum length of string representation (number of maximum decimal
-  digits + 1 position for sign + 1 position for decimal point)
+  digits + 1 position for sign + 1 position for decimal point, no terminator)
 */
 #define DECIMAL_MAX_STR_LENGTH (DECIMAL_MAX_POSSIBLE_PRECISION + 2)
 
@@ -212,6 +212,7 @@ inline uint32 my_decimal_precision_to_length(uint precision, uint8 scale,
 inline
 int my_decimal_string_length(const my_decimal *d)
 {
+  /* length of string representation including terminating '\0' */
   return decimal_string_size(d);
 }
 

strings/decimal.c

@@ -320,8 +320,8 @@ int decimal_actual_fraction(decimal_t *from)
       from            - value to convert
       to              - points to buffer where string representation
                         should be stored
-      *to_len         - in:  size of to buffer
-                        out: length of the actually written string
+      *to_len         - in:  size of to buffer (incl. terminating '\0')
+                        out: length of the actually written string (excl. '\0')
       fixed_precision - 0 if representation can be variable length and
                         fixed_decimals will not be checked in this case.
                         Put number as with fixed point position with this
@@ -338,6 +338,7 @@ int decimal2string(decimal_t *from, char *to, int *to_len,
                    int fixed_precision, int fixed_decimals,
                    char filler)
 {
+  /* {intg_len, frac_len} output widths; {intg, frac} places in input */
   int len, intg, frac= from->frac, i, intg_len, frac_len, fill;
   /* number digits before decimal point */
   int fixed_intg= (fixed_precision ?