MDEV-28490 Strange result truncation with group_concat_max_len=1GB.

Arythmetic can overrun the uint type when possible group_concat_max_len
is multiplied to collation.mbmaxlen (can easily be like 4).
So use ulonglong there for calculations.

mysql-test/suite/sys_vars/r/group_concat_max_len_func.result

@@ -94,4 +94,58 @@ DROP TABLE t1;
 disconnect test_con2;
 disconnect test_con1;
 connection default;
+CREATE TABLE t1(val VARCHAR(100) PRIMARY KEY) CHARACTER SET utf8mb4 COLLATE utf8mb4_danish_ci;
+INSERT INTO t1 VALUES('bar');
+INSERT INTO t1 VALUES('foo');
+SET group_concat_max_len = 1073741823;
+SHOW VARIABLES LIKE 'group_concat_max_len';
+Variable_name	Value
+group_concat_max_len	1073741823
+SELECT GROUP_CONCAT(val) AS simple FROM t1;
+simple
+bar,foo
+SELECT * FROM ( SELECT GROUP_CONCAT(val) AS nested FROM t1) As tmp;
+nested
+bar,foo
+SET group_concat_max_len = 1073741824;
+SHOW VARIABLES LIKE 'group_concat_max_len';
+Variable_name	Value
+group_concat_max_len	1073741824
+SELECT GROUP_CONCAT(val) AS simple FROM t1;
+simple
+bar,foo
+SELECT * FROM ( SELECT GROUP_CONCAT(val) AS nested FROM t1) As tmp;
+nested
+bar,foo
+SET group_concat_max_len = 1073741825;
+SHOW VARIABLES LIKE 'group_concat_max_len';
+Variable_name	Value
+group_concat_max_len	1073741825
+SELECT GROUP_CONCAT(val) AS simple FROM t1;
+simple
+bar,foo
+SELECT * FROM ( SELECT GROUP_CONCAT(val) AS nested FROM t1) As tmp;
+nested
+bar,foo
+SET group_concat_max_len = 1073741826;
+SHOW VARIABLES LIKE 'group_concat_max_len';
+Variable_name	Value
+group_concat_max_len	1073741826
+SELECT GROUP_CONCAT(val) AS simple FROM t1;
+simple
+bar,foo
+SELECT * FROM ( SELECT GROUP_CONCAT(val) AS nested FROM t1) As tmp;
+nested
+bar,foo
+SET group_concat_max_len = 2147483649;
+SHOW VARIABLES LIKE 'group_concat_max_len';
+Variable_name	Value
+group_concat_max_len	2147483649
+SELECT GROUP_CONCAT(val) AS simple FROM t1;
+simple
+bar,foo
+SELECT * FROM ( SELECT GROUP_CONCAT(val) AS nested FROM t1) As tmp;
+nested
+bar,foo
+DROP TABLE t1;
 SET @@global.group_concat_max_len = @save;

mysql-test/suite/sys_vars/t/group_concat_max_len_func.test

@@ -132,4 +132,35 @@ disconnect test_con1;
 
 connection default;
 
+CREATE TABLE t1(val VARCHAR(100) PRIMARY KEY) CHARACTER SET utf8mb4 COLLATE utf8mb4_danish_ci;
+INSERT INTO t1 VALUES('bar');
+INSERT INTO t1 VALUES('foo');
+
+SET group_concat_max_len = 1073741823;
+SHOW VARIABLES LIKE 'group_concat_max_len';
+SELECT GROUP_CONCAT(val) AS simple FROM t1;
+SELECT * FROM ( SELECT GROUP_CONCAT(val) AS nested FROM t1) As tmp;
+
+SET group_concat_max_len = 1073741824;
+SHOW VARIABLES LIKE 'group_concat_max_len';
+SELECT GROUP_CONCAT(val) AS simple FROM t1;
+SELECT * FROM ( SELECT GROUP_CONCAT(val) AS nested FROM t1) As tmp;
+
+SET group_concat_max_len = 1073741825;
+SHOW VARIABLES LIKE 'group_concat_max_len';
+SELECT GROUP_CONCAT(val) AS simple FROM t1;
+SELECT * FROM ( SELECT GROUP_CONCAT(val) AS nested FROM t1) As tmp;
+
+SET group_concat_max_len = 1073741826;
+SHOW VARIABLES LIKE 'group_concat_max_len';
+SELECT GROUP_CONCAT(val) AS simple FROM t1;
+SELECT * FROM ( SELECT GROUP_CONCAT(val) AS nested FROM t1) As tmp;
+
+SET group_concat_max_len = 2147483649;
+SHOW VARIABLES LIKE 'group_concat_max_len';
+SELECT GROUP_CONCAT(val) AS simple FROM t1;
+SELECT * FROM ( SELECT GROUP_CONCAT(val) AS nested FROM t1) As tmp;
+
+DROP TABLE t1;
+
 SET @@global.group_concat_max_len = @save;

sql/item_sum.cc

@@ -4242,7 +4242,7 @@ Item_func_group_concat::fix_fields(THD *thd, Item **ref)
   result.set_charset(collation.collation);
   result_field= 0;
   null_value= 1;
-  max_length= (uint32)MY_MIN(thd->variables.group_concat_max_len
+  max_length= (uint32) MY_MIN((ulonglong) thd->variables.group_concat_max_len
                               / collation.collation->mbminlen
                               * collation.collation->mbmaxlen, UINT_MAX32);