MDEV-23415 Server crash or Assertion `dec_length <= str_length' failed in...

MDEV-23415 Server crash or Assertion `dec_length <= str_length' failed in Item_func_format::val_str_ascii

Problem:

The crash happened in FORMAT(double, dec>=31, 'de_DE').

The patch for MDEV-23118 (commit 0041dacc)
did not take into account that String::set_real() has a limit of 31
(FLOATING_POINT_DECIMALS) fractional digits. So for the range of 31..38
digits, set_real() switches to use:
- my_fcvt() - decimal point notation, e.g. 1.9999999999
- my_gcvt() - scientific notation,    e.g. 1e22

my_gcvt() returned a shorter string than Item_func_format::val_str_ascii()
expected to get after the my_fcvt() call, so it crashed on assert.

Solution:

We cannot extend set_real() to use the my_fcvt() mode for the range of
31..38 fractional digits, because set_real() is used in a lot of places
and such a change will break everything.

Introducing String::set_fcvt() which always prints using my_fcvt()
for the whole range of decimals 0..38, supported by the FORMAT() function.

mysql-test/main/type_float.result

@@ -969,5 +969,175 @@ Warnings:
 Note	1003	select `test`.`t1`.`a` AS `a` from `test`.`t1` where 1
 DROP TABLE t1;
 #
+# MDEV-23415 Server crash or Assertion `dec_length <= str_length' failed in Item_func_format::val_str_ascii
+#
+SELECT FORMAT('0', 50, 'de_DE');
+FORMAT('0', 50, 'de_DE')
+0,00000000000000000000000000000000000000
+SELECT FORMAT(0e0, 50, 'de_DE');
+FORMAT(0e0, 50, 'de_DE')
+0,00000000000000000000000000000000000000
+FOR d IN 0..50
+DO
+SELECT
+d,
+FORMAT(123456789.123456789e0, d, 'de_DE') AS fdbl,
+FORMAT(123456789.123456789, d, 'de_DE') AS fdec;
+END FOR;
+$$
+d	0
+fdbl	123.456.789
+fdec	123.456.789
+d	1
+fdbl	123.456.789,1
+fdec	123.456.789,1
+d	2
+fdbl	123.456.789,12
+fdec	123.456.789,12
+d	3
+fdbl	123.456.789,123
+fdec	123.456.789,123
+d	4
+fdbl	123.456.789,1235
+fdec	123.456.789,1235
+d	5
+fdbl	123.456.789,12346
+fdec	123.456.789,12346
+d	6
+fdbl	123.456.789,123457
+fdec	123.456.789,123457
+d	7
+fdbl	123.456.789,1234568
+fdec	123.456.789,1234568
+d	8
+fdbl	123.456.789,12345680
+fdec	123.456.789,12345679
+d	9
+fdbl	123.456.789,123456790
+fdec	123.456.789,123456789
+d	10
+fdbl	123.456.789,1234567900
+fdec	123.456.789,1234567890
+d	11
+fdbl	123.456.789,12345680000
+fdec	123.456.789,12345678900
+d	12
+fdbl	123.456.789,123456790000
+fdec	123.456.789,123456789000
+d	13
+fdbl	123.456.789,1234568000000
+fdec	123.456.789,1234567890000
+d	14
+fdbl	123.456.789,12345679000000
+fdec	123.456.789,12345678900000
+d	15
+fdbl	123.456.789,123456790000000
+fdec	123.456.789,123456789000000
+d	16
+fdbl	123.456.789,1234567800000000
+fdec	123.456.789,1234567890000000
+d	17
+fdbl	123.456.789,12345679000000000
+fdec	123.456.789,12345678900000000
+d	18
+fdbl	123.456.789,123456790000000000
+fdec	123.456.789,123456789000000000
+d	19
+fdbl	123.456.789,1234567900000000000
+fdec	123.456.789,1234567890000000000
+d	20
+fdbl	123.456.789,12345679000000000000
+fdec	123.456.789,12345678900000000000
+d	21
+fdbl	123.456.789,123456800000000000000
+fdec	123.456.789,123456789000000000000
+d	22
+fdbl	123.456.789,1234567900000000000000
+fdec	123.456.789,1234567890000000000000
+d	23
+fdbl	123.456.789,12345680000000000000000
+fdec	123.456.789,12345678900000000000000
+d	24
+fdbl	123.456.789,123456790000000000000000
+fdec	123.456.789,123456789000000000000000
+d	25
+fdbl	123.456.789,1234567900000000000000000
+fdec	123.456.789,1234567890000000000000000
+d	26
+fdbl	123.456.789,12345679000000000000000000
+fdec	123.456.789,12345678900000000000000000
+d	27
+fdbl	123.456.789,123456780000000000000000000
+fdec	123.456.789,123456789000000000000000000
+d	28
+fdbl	123.456.789,1234567900000000000000000000
+fdec	123.456.789,1234567890000000000000000000
+d	29
+fdbl	123.456.789,12345678000000000000000000000
+fdec	123.456.789,12345678900000000000000000000
+d	30
+fdbl	123.456.789,123456790000000000000000000000
+fdec	123.456.789,123456789000000000000000000000
+d	31
+fdbl	123.456.789,1234567900000000000000000000000
+fdec	123.456.789,1234567890000000000000000000000
+d	32
+fdbl	123.456.789,12345679000000000000000000000000
+fdec	123.456.789,12345678900000000000000000000000
+d	33
+fdbl	123.456.789,123456790000000000000000000000000
+fdec	123.456.789,123456789000000000000000000000000
+d	34
+fdbl	123.456.789,1234567900000000000000000000000000
+fdec	123.456.789,1234567890000000000000000000000000
+d	35
+fdbl	123.456.789,12345679000000000000000000000000000
+fdec	123.456.789,12345678900000000000000000000000000
+d	36
+fdbl	123.456.789,123456790000000000000000000000000000
+fdec	123.456.789,123456789000000000000000000000000000
+d	37
+fdbl	123.456.789,1234567900000000000000000000000000000
+fdec	123.456.789,1234567890000000000000000000000000000
+d	38
+fdbl	123.456.789,12345678000000000000000000000000000000
+fdec	123.456.789,12345678900000000000000000000000000000
+d	39
+fdbl	123.456.789,12345678000000000000000000000000000000
+fdec	123.456.789,12345678900000000000000000000000000000
+d	40
+fdbl	123.456.789,12345678000000000000000000000000000000
+fdec	123.456.789,12345678900000000000000000000000000000
+d	41
+fdbl	123.456.789,12345678000000000000000000000000000000
+fdec	123.456.789,12345678900000000000000000000000000000
+d	42
+fdbl	123.456.789,12345678000000000000000000000000000000
+fdec	123.456.789,12345678900000000000000000000000000000
+d	43
+fdbl	123.456.789,12345678000000000000000000000000000000
+fdec	123.456.789,12345678900000000000000000000000000000
+d	44
+fdbl	123.456.789,12345678000000000000000000000000000000
+fdec	123.456.789,12345678900000000000000000000000000000
+d	45
+fdbl	123.456.789,12345678000000000000000000000000000000
+fdec	123.456.789,12345678900000000000000000000000000000
+d	46
+fdbl	123.456.789,12345678000000000000000000000000000000
+fdec	123.456.789,12345678900000000000000000000000000000
+d	47
+fdbl	123.456.789,12345678000000000000000000000000000000
+fdec	123.456.789,12345678900000000000000000000000000000
+d	48
+fdbl	123.456.789,12345678000000000000000000000000000000
+fdec	123.456.789,12345678900000000000000000000000000000
+d	49
+fdbl	123.456.789,12345678000000000000000000000000000000
+fdec	123.456.789,12345678900000000000000000000000000000
+d	50
+fdbl	123.456.789,12345678000000000000000000000000000000
+fdec	123.456.789,12345678900000000000000000000000000000
+#
 # End of 10.4 tests
 #

mysql-test/main/type_float.test

@@ -666,6 +666,26 @@ EXECUTE IMMEDIATE 'EXPLAIN EXTENDED SELECT * FROM t1 WHERE ?+a<=>1e0+a' USING 1e
 EXECUTE IMMEDIATE 'EXPLAIN EXTENDED SELECT * FROM t1 WHERE 1e0+a<=>?+a' USING 1e0;
 DROP TABLE t1;
 
+--echo #
+--echo # MDEV-23415 Server crash or Assertion `dec_length <= str_length' failed in Item_func_format::val_str_ascii
+--echo #
+
+SELECT FORMAT('0', 50, 'de_DE');
+SELECT FORMAT(0e0, 50, 'de_DE');
+
+--vertical_results
+DELIMITER $$;
+FOR d IN 0..50
+DO
+  SELECT
+    d,
+    FORMAT(123456789.123456789e0, d, 'de_DE') AS fdbl,
+    FORMAT(123456789.123456789, d, 'de_DE') AS fdec;
+END FOR;
+$$
+DELIMITER ;$$
+--horizontal_results
+
 --echo #
 --echo # End of 10.4 tests
 --echo #

sql/item_strfunc.cc

@@ -2752,7 +2752,7 @@ String *Item_func_format::val_str_ascii(String *str)
     if ((null_value=args[0]->null_value))
       return 0; /* purecov: inspected */
     nr= my_double_round(nr, (longlong) dec, FALSE, FALSE);
-    str->set_real(nr, dec, &my_charset_numeric);
+    str->set_fcvt(nr, dec);
     if (!std::isfinite(nr))
       return str;
     str_length=str->length();

sql/sql_string.cc

@@ -183,6 +183,17 @@ bool Binary_string::set_hex(const char *str, uint32 len)
 }
 
 
+bool Binary_string::set_fcvt(double num, uint decimals)
+{
+  // Assert that `decimals` is small enough to fit into FLOATING_POINT_BUFFER
+  DBUG_ASSERT(decimals < DECIMAL_NOT_SPECIFIED);
+  if (alloc(FLOATING_POINT_BUFFER))
+    return true;
+  length(my_fcvt(num, decimals, Ptr, NULL));
+  return false;
+}
+
+
 bool String::set_real(double num,uint decimals, CHARSET_INFO *cs)
 {
   char buff[FLOATING_POINT_BUFFER];

sql/sql_string.h

@@ -517,6 +517,7 @@ class Binary_string: public Static_binary_string
 
   bool set_hex(ulonglong num);
   bool set_hex(const char *str, uint32 len);
+  bool set_fcvt(double num, uint decimals);
 
   bool copy();                                  // Alloc string if not alloced
   bool copy(const Binary_string &s);            // Allocate new string
@@ -781,6 +782,11 @@ class String: public Charset, public Binary_string
   bool set(longlong num, CHARSET_INFO *cs) { return set_int(num, false, cs); }
   bool set(ulonglong num, CHARSET_INFO *cs) { return set_int((longlong)num, true, cs); }
   bool set_real(double num,uint decimals, CHARSET_INFO *cs);
+  bool set_fcvt(double num, uint decimals)
+  {
+    set_charset(&my_charset_latin1);
+    return Binary_string::set_fcvt(num, decimals);
+  }
 
   bool set_hex(ulonglong num)
   {