MDEV-22715: SIGSEGV in radixsort_for_str_ptr and in native_compare/my_qsort2 (optimized builds)

For DECIMAL[(M[,D])] datatype max_sort_length was not being honoured which was leading to buffer
overflow while making the sort key. The fix to this problem would be to create sort keys for decimals
with atmost max_sort_key bytes

Important:
The minimum value of max_sort_length has been raised to 8 (previously was 4),
so fixed size datatypes like DOUBLE and BIGINIT are not truncated for
lower values of max_sort_length.

mysql-test/include/ctype_utf8mb4.inc

@@ -1576,7 +1576,7 @@ drop table t1;
 --echo #
 --echo # Check strnxfrm() with odd length
 --echo #
-set max_sort_length=5;
+set max_sort_length=9;
 select @@max_sort_length;
 eval create table t1 (a varchar(128) character set utf8mb4 collate utf8mb4_general_ci) engine $engine;
 insert into t1 values ('a'),('b'),('c');

mysql-test/r/ctype_utf16.result

@@ -1489,7 +1489,7 @@ ab
 ab
 AE
 AE
-SET max_sort_length=4;
+SET max_sort_length=8;
 SELECT * FROM t1 ORDER BY s1;
 s1
 ab

mysql-test/r/ctype_utf16le.result

@@ -1762,7 +1762,7 @@ ab
 ab
 AE
 AE
-SET max_sort_length=4;
+SET max_sort_length=8;
 SELECT * FROM t1 ORDER BY s1;
 s1
 ab

mysql-test/r/ctype_utf32.result

@@ -1503,7 +1503,7 @@ ab
 ab
 AE
 AE
-SET max_sort_length=4;
+SET max_sort_length=8;
 SELECT * FROM t1 ORDER BY s1;
 s1
 ab

mysql-test/r/ctype_utf8.result

@@ -6807,10 +6807,10 @@ DFFFDFFF9CFF9DFF9EFF
 #
 # Checking strnxfrm() with odd length
 #
-set max_sort_length=5;
+set max_sort_length=9;
 select @@max_sort_length;
 @@max_sort_length
-5
+9
 create table t1 (a varchar(128) character set utf8 collate utf8_general_ci);
 insert into t1 values ('a'),('b'),('c');
 select * from t1 order by a;

mysql-test/r/ctype_utf8mb4.result

@@ -2364,10 +2364,10 @@ drop table t1;
 #
 # Check strnxfrm() with odd length
 #
-set max_sort_length=5;
+set max_sort_length=9;
 select @@max_sort_length;
 @@max_sort_length
-5
+9
 create table t1 (a varchar(128) character set utf8mb4 collate utf8mb4_general_ci);
 insert into t1 values ('a'),('b'),('c');
 select * from t1 order by a;

mysql-test/r/ctype_utf8mb4_heap.result

@@ -2196,10 +2196,10 @@ drop table t1;
 #
 # Check strnxfrm() with odd length
 #
-set max_sort_length=5;
+set max_sort_length=9;
 select @@max_sort_length;
 @@max_sort_length
-5
+9
 create table t1 (a varchar(128) character set utf8mb4 collate utf8mb4_general_ci) engine heap;
 insert into t1 values ('a'),('b'),('c');
 select * from t1 order by a;

mysql-test/r/ctype_utf8mb4_innodb.result

@@ -2324,10 +2324,10 @@ drop table t1;
 #
 # Check strnxfrm() with odd length
 #
-set max_sort_length=5;
+set max_sort_length=9;
 select @@max_sort_length;
 @@max_sort_length
-5
+9
 create table t1 (a varchar(128) character set utf8mb4 collate utf8mb4_general_ci) engine InnoDB;
 insert into t1 values ('a'),('b'),('c');
 select * from t1 order by a;

mysql-test/r/ctype_utf8mb4_myisam.result

@@ -2324,10 +2324,10 @@ drop table t1;
 #
 # Check strnxfrm() with odd length
 #
-set max_sort_length=5;
+set max_sort_length=9;
 select @@max_sort_length;
 @@max_sort_length
-5
+9
 create table t1 (a varchar(128) character set utf8mb4 collate utf8mb4_general_ci) engine MyISAM;
 insert into t1 values ('a'),('b'),('c');
 select * from t1 order by a;

mysql-test/r/order_by.result

@@ -3237,3 +3237,68 @@ o	15
 p	16
 set @@sort_buffer_size= @save_sort_buffer_size;
 drop table t1;
+#
+# MDEV-22715: SIGSEGV in radixsort_for_str_ptr and in native_compare/my_qsort2 (optimized builds)
+#
+SET @save_sort_buffer_size= @@sort_buffer_size;
+SET @save_max_sort_length= @@max_sort_length;
+SET max_sort_length=8;
+SET sort_buffer_size=1024;
+CREATE TABLE t1(a INT, b DECIMAL(65), c BLOB);
+INSERT INTO t1 SELECT seq, seq, seq from seq_1_to_25;
+INSERT INTO t1 SELECT seq, seq, seq from seq_1_to_25;
+SELECT * FROM t1 ORDER BY a,b;
+a	b	c
+1	1	1
+1	1	1
+2	2	2
+2	2	2
+3	3	3
+3	3	3
+4	4	4
+4	4	4
+5	5	5
+5	5	5
+6	6	6
+6	6	6
+7	7	7
+7	7	7
+8	8	8
+8	8	8
+9	9	9
+9	9	9
+10	10	10
+10	10	10
+11	11	11
+11	11	11
+12	12	12
+12	12	12
+13	13	13
+13	13	13
+14	14	14
+14	14	14
+15	15	15
+15	15	15
+16	16	16
+16	16	16
+17	17	17
+17	17	17
+18	18	18
+18	18	18
+19	19	19
+19	19	19
+20	20	20
+20	20	20
+21	21	21
+21	21	21
+22	22	22
+22	22	22
+23	23	23
+23	23	23
+24	24	24
+24	24	24
+25	25	25
+25	25	25
+SET @@sort_buffer_size= @save_sort_buffer_size;
+SET @@max_sort_length= @save_max_sort_length;
+DROP TABLE t1;

mysql-test/suite/sys_vars/r/max_sort_length_basic.result

@@ -27,14 +27,14 @@ SELECT @@session.max_sort_length = 1024;
 @@session.max_sort_length = 1024
 1
 '#--------------------FN_DYNVARS_084_03-------------------------#'
-SET @@global.max_sort_length = 4;
+SET @@global.max_sort_length = 8;
 SELECT @@global.max_sort_length;
 @@global.max_sort_length
-4
-SET @@global.max_sort_length = 5;
+8
+SET @@global.max_sort_length = 9;
 SELECT @@global.max_sort_length;
 @@global.max_sort_length
-5
+9
 SET @@global.max_sort_length = 8388608;
 SELECT @@global.max_sort_length;
 @@global.max_sort_length
@@ -48,14 +48,14 @@ SELECT @@global.max_sort_length;
 @@global.max_sort_length
 65536
 '#--------------------FN_DYNVARS_084_04-------------------------#'
-SET @@session.max_sort_length = 4;
+SET @@session.max_sort_length = 8;
 SELECT @@session.max_sort_length;
 @@session.max_sort_length
-4
-SET @@session.max_sort_length = 5;
+8
+SET @@session.max_sort_length = 9;
 SELECT @@session.max_sort_length;
 @@session.max_sort_length
-5
+9
 SET @@session.max_sort_length = 8388608;
 SELECT @@session.max_sort_length;
 @@session.max_sort_length
@@ -74,13 +74,13 @@ Warnings:
 Warning	1292	Truncated incorrect max_sort_length value: '-1024'
 SELECT @@global.max_sort_length;
 @@global.max_sort_length
-4
+8
 SET @@global.max_sort_length = 3;
 Warnings:
 Warning	1292	Truncated incorrect max_sort_length value: '3'
 SELECT @@global.max_sort_length;
 @@global.max_sort_length
-4
+8
 SET @@global.max_sort_length = 8388609;
 Warnings:
 Warning	1292	Truncated incorrect max_sort_length value: '8388609'
@@ -92,17 +92,17 @@ Warnings:
 Warning	1292	Truncated incorrect max_sort_length value: '0'
 SELECT @@global.max_sort_length;
 @@global.max_sort_length
-4
+8
 SET @@global.max_sort_length = 65530.34;
 ERROR 42000: Incorrect argument type to variable 'max_sort_length'
 SELECT @@global.max_sort_length;
 @@global.max_sort_length
-4
+8
 SET @@global.max_sort_length = test;
 ERROR 42000: Incorrect argument type to variable 'max_sort_length'
 SELECT @@global.max_sort_length;
 @@global.max_sort_length
-4
+8
 SET @@session.max_sort_length = 8388610;
 Warnings:
 Warning	1292	Truncated incorrect max_sort_length value: '8388610'
@@ -114,19 +114,19 @@ Warnings:
 Warning	1292	Truncated incorrect max_sort_length value: '-1'
 SELECT @@session.max_sort_length;
 @@session.max_sort_length
-4
+8
 SET @@session.max_sort_length = 3;
 Warnings:
 Warning	1292	Truncated incorrect max_sort_length value: '3'
 SELECT @@session.max_sort_length;
 @@session.max_sort_length
-4
+8
 SET @@session.max_sort_length = 0;
 Warnings:
 Warning	1292	Truncated incorrect max_sort_length value: '0'
 SELECT @@session.max_sort_length;
 @@session.max_sort_length
-4
+8
 SET @@session.max_sort_length = 65530.34;
 ERROR 42000: Incorrect argument type to variable 'max_sort_length'
 SET @@session.max_sort_length = 10737418241;
@@ -158,13 +158,13 @@ Warnings:
 Warning	1292	Truncated incorrect max_sort_length value: '1'
 SELECT @@global.max_sort_length;
 @@global.max_sort_length
-4
+8
 SET @@global.max_sort_length = FALSE;
 Warnings:
 Warning	1292	Truncated incorrect max_sort_length value: '0'
 SELECT @@global.max_sort_length;
 @@global.max_sort_length
-4
+8
 '#---------------------FN_DYNVARS_084_09----------------------#'
 SET @@global.max_sort_length = 2048;
 SELECT @@max_sort_length = @@global.max_sort_length;

mysql-test/suite/sys_vars/r/sysvars_server_embedded.result

@@ -2033,7 +2033,7 @@ DEFAULT_VALUE	1024
 VARIABLE_SCOPE	SESSION
 VARIABLE_TYPE	BIGINT UNSIGNED
 VARIABLE_COMMENT	The number of bytes to use when sorting BLOB or TEXT values (only the first max_sort_length bytes of each value are used; the rest are ignored)
-NUMERIC_MIN_VALUE	4
+NUMERIC_MIN_VALUE	8
 NUMERIC_MAX_VALUE	8388608
 NUMERIC_BLOCK_SIZE	1
 ENUM_VALUE_LIST	NULL

mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result

@@ -2229,7 +2229,7 @@ DEFAULT_VALUE	1024
 VARIABLE_SCOPE	SESSION
 VARIABLE_TYPE	BIGINT UNSIGNED
 VARIABLE_COMMENT	The number of bytes to use when sorting BLOB or TEXT values (only the first max_sort_length bytes of each value are used; the rest are ignored)
-NUMERIC_MIN_VALUE	4
+NUMERIC_MIN_VALUE	8
 NUMERIC_MAX_VALUE	8388608
 NUMERIC_BLOCK_SIZE	1
 ENUM_VALUE_LIST	NULL

mysql-test/suite/sys_vars/t/max_sort_length_basic.test

@@ -74,9 +74,9 @@ SELECT @@session.max_sort_length = 1024;
 # Change the value of max_sort_length to a valid value for GLOBAL Scope #
 #########################################################################
 
-SET @@global.max_sort_length = 4;
+SET @@global.max_sort_length = 8;
 SELECT @@global.max_sort_length;
-SET @@global.max_sort_length = 5;
+SET @@global.max_sort_length = 9;
 SELECT @@global.max_sort_length;
 SET @@global.max_sort_length = 8388608;
 SELECT @@global.max_sort_length;
@@ -90,10 +90,10 @@ SELECT @@global.max_sort_length;
 # Change the value of max_sort_length to a valid value for SESSION Scope #
 ##########################################################################
 
-SET @@session.max_sort_length = 4;
+SET @@session.max_sort_length = 8;
 SELECT @@session.max_sort_length;
 
-SET @@session.max_sort_length = 5;
+SET @@session.max_sort_length = 9;
 SELECT @@session.max_sort_length;
 
 SET @@session.max_sort_length = 8388608;

mysql-test/t/ctype_utf16.test

@@ -721,7 +721,7 @@ CREATE TABLE t1 AS SELECT repeat('a',2) as s1 LIMIT 0;
 SHOW CREATE TABLE t1;
 INSERT INTO t1 VALUES ('ab'),('AE'),('ab'),('AE');
 SELECT * FROM t1 ORDER BY s1;
-SET max_sort_length=4;
+SET max_sort_length=8;
 SELECT * FROM t1 ORDER BY s1;
 DROP TABLE t1;
 SET max_sort_length=DEFAULT;

mysql-test/t/ctype_utf16le.test

@@ -683,7 +683,7 @@ CREATE TABLE t1 AS SELECT REPEAT('a',2) as s1 LIMIT 0;
 SHOW CREATE TABLE t1;
 INSERT INTO t1 VALUES ('ab'),('AE'),('ab'),('AE');
 SELECT * FROM t1 ORDER BY s1;
-SET max_sort_length=4;
+SET max_sort_length=8;
 SELECT * FROM t1 ORDER BY s1;
 DROP TABLE t1;
 SET max_sort_length=DEFAULT;

mysql-test/t/ctype_utf32.test

@@ -778,7 +778,7 @@ CREATE TABLE t1 AS SELECT repeat('a',2) as s1 LIMIT 0;
 SHOW CREATE TABLE t1;
 INSERT INTO t1 VALUES ('ab'),('AE'),('ab'),('AE');
 SELECT * FROM t1 ORDER BY s1;
-SET max_sort_length=4;
+SET max_sort_length=8;
 SELECT * FROM t1 ORDER BY s1;
 DROP TABLE t1;
 SET max_sort_length=DEFAULT;

mysql-test/t/ctype_utf8.test

@@ -1756,7 +1756,7 @@ set @@collation_connection=utf8_bin;
 --echo #
 --echo # Checking strnxfrm() with odd length
 --echo #
-set max_sort_length=5;
+set max_sort_length=9;
 select @@max_sort_length;
 create table t1 (a varchar(128) character set utf8 collate utf8_general_ci);
 insert into t1 values ('a'),('b'),('c');

mysql-test/t/ctype_utf8mb4.test

@@ -1511,7 +1511,7 @@ drop table t1;
 --echo #
 --echo # Check strnxfrm() with odd length
 --echo #
-set max_sort_length=5;
+set max_sort_length=9;
 select @@max_sort_length;
 create table t1 (a varchar(128) character set utf8mb4 collate utf8mb4_general_ci);
 insert into t1 values ('a'),('b'),('c');

mysql-test/t/order_by.test

@@ -12,6 +12,8 @@ call mtr.add_suppression("Out of sort memory; increase server sort buffer size")
 # Test old ORDER BY bug
 #
 
+--source include/have_sequence.inc
+
 CREATE TABLE t1 (
   id int(6) DEFAULT '0' NOT NULL,
   idservice int(5),
@@ -2157,3 +2159,18 @@ select * from t1 order  by b;
 set @@sort_buffer_size= @save_sort_buffer_size;
 drop table t1;
 
+--echo #
+--echo # MDEV-22715: SIGSEGV in radixsort_for_str_ptr and in native_compare/my_qsort2 (optimized builds)
+--echo #
+
+SET @save_sort_buffer_size= @@sort_buffer_size;
+SET @save_max_sort_length= @@max_sort_length;
+SET max_sort_length=8;
+SET sort_buffer_size=1024;
+CREATE TABLE t1(a INT, b DECIMAL(65), c BLOB);
+INSERT INTO t1 SELECT seq, seq, seq from seq_1_to_25;
+INSERT INTO t1 SELECT seq, seq, seq from seq_1_to_25;
+SELECT * FROM t1 ORDER BY a,b;
+SET @@sort_buffer_size= @save_sort_buffer_size;
+SET @@max_sort_length= @save_max_sort_length;
+DROP TABLE t1;

sql/field.cc

@@ -3295,10 +3295,9 @@ int Field_new_decimal::cmp(const uchar *a,const uchar*b)
 }
 
 
-void Field_new_decimal::sort_string(uchar *buff,
-                                    uint length __attribute__((unused)))
+void Field_new_decimal::sort_string(uchar *buff, uint length)
 {
-  memcpy(buff, ptr, bin_size);
+  memcpy(buff, ptr, length);
 }
 
 

sql/field.h

@@ -1099,6 +1099,13 @@ class Field: public Value_source
 
   void make_sort_key(uchar *buff, uint length);
   virtual void make_field(Send_field *);
+
+  /*
+    Some implementations actually may write up to 8 bytes regardless of what
+    size was requested. This is due to the minimum value of the system variable
+    max_sort_length.
+  */
+
   virtual void sort_string(uchar *buff,uint length)=0;
   virtual bool optimize_range(uint idx, uint part);
   virtual void free() {}

sql/sys_vars.cc

@@ -2178,7 +2178,7 @@ static Sys_var_ulong Sys_max_sort_length(
        "the first max_sort_length bytes of each value are used; the rest "
        "are ignored)",
        SESSION_VAR(max_sort_length), CMD_LINE(REQUIRED_ARG),
-       VALID_RANGE(4, 8192*1024L), DEFAULT(1024), BLOCK_SIZE(1));
+       VALID_RANGE(8, 8192*1024L), DEFAULT(1024), BLOCK_SIZE(1));
 
 static Sys_var_ulong Sys_max_sp_recursion_depth(
        "max_sp_recursion_depth",