Commit 68ad4090 authored by Oleg Smirnov's avatar Oleg Smirnov Committed by Sergei Golubchik

MDEV-27277 Add a warning when max_sort_length is reached

During a query execution some sorting and grouping operations
on strings may be involved. System variable max_sort_length defines
the maximum number of bytes to use when comparing strings during
sorting/grouping. Thus, the comparable parts of strings may be less
than their actual size, so the results of the query may be not
sorted/grouped properly.
To indicate that some comparisons were done on a truncated lengths,
a new warning has been introduced with this commit.
parent 882aa0a2
......@@ -438,11 +438,19 @@ typedef struct
#define MY_STRNXFRM_TRUNCATED_WEIGHT_TRAILING_SPACE 1
#define MY_STRNXFRM_TRUNCATED_WEIGHT_REAL_CHAR 2
typedef struct
typedef struct my_strnxfrm_ret_t
{
size_t m_result_length;
size_t m_source_length_used;
uint m_warnings;
#ifdef __cplusplus
/*
Allow casting this type to size_t for backward compatibility
with external code, e.g. ColumnStore SE.
*/
operator size_t() const { return m_result_length; }
#endif
} my_strnxfrm_ret_t;
......
......@@ -146,6 +146,11 @@ while ($_dt_tables)
--echo # increasing group_concat_max_len from $_dt_old_group_concat_max_len to $_dt_order_by_length
}
}
# Increase sorting buffers.
--let $_dt_old_max_sort_length= `SELECT @@SESSION.max_sort_length`
--let $_dt_old_sort_buffer_size= `SELECT @@SESSION.sort_buffer_size`
--eval SET SESSION max_sort_length = 100000;
--eval SET SESSION sort_buffer_size = 10000000;
# Generate ORDER BY clause.
# It would be better to do GROUP_CONCAT(CONCAT('`', column_name, '`')) but
# BUG#58087 prevents us from returning strings that begin with backticks.
......@@ -170,6 +175,9 @@ while ($_dt_tables)
--let $_dt_outfile= $_dt_outfile/diff_table-$_dt_connection-$_dt_database-$_dt_table
eval SELECT * INTO OUTFILE '$_dt_outfile' FROM $_dt_database.$_dt_table ORDER BY `$_dt_column_list`;
--enable_ps2_protocol
# Restore sorting buffers.
--eval SET SESSION max_sort_length = $_dt_old_max_sort_length;
--eval SET SESSION sort_buffer_size = $_dt_old_sort_buffer_size;
# Compare files.
if ($_dt_prev_outfile)
......
......@@ -680,7 +680,11 @@ b MEDIUMTEXT CHARACTER SET big5);
INSERT INTO t1 VALUES
(REPEAT(0x1125,200000), REPEAT(0x1125,200000)), ('', ''), ('', '');
SELECT a FROM t1 GROUP BY 1 LIMIT 1 INTO @nullll;
Warnings:
Warning 4201 1 values were longer than max_sort_length. Sorting used only the first 1024 bytes
SELECT b FROM t1 GROUP BY 1 LIMIT 1 INTO @nullll;
Warnings:
Warning 4201 1 values were longer than max_sort_length. Sorting used only the first 1024 bytes
DROP TABLES t1;
End of 5.0 tests
#
......
......@@ -1918,6 +1918,8 @@ Extra Using filesort
SELECT SUBSTRING(a,1,10), LENGTH(a), GROUP_CONCAT(b) FROM t1 GROUP BY a;
SUBSTRING(a,1,10) LENGTH(a) GROUP_CONCAT(b)
1111111111 1300 one,two
Warnings:
Warning 4201 2 values were longer than max_sort_length. Sorting used only the first 1024 bytes
EXPLAIN
SELECT SUBSTRING(a,1,10), LENGTH(a) FROM t1 GROUP BY a;
id 1
......@@ -1933,6 +1935,8 @@ Extra Using temporary; Using filesort
SELECT SUBSTRING(a,1,10), LENGTH(a) FROM t1 GROUP BY a;
SUBSTRING(a,1,10) LENGTH(a)
1111111111 1300
Warnings:
Warning 4201 1 values were longer than max_sort_length. Sorting used only the first 1024 bytes
DROP TABLE t1;
#
# Bug#57688 Assertion `!table || (!table->write_set || bitmap_is_set(table->write_set, field
......@@ -2728,6 +2732,8 @@ f1 f2
NULL
NULL NULL
Warnings:
Warning 4201 116 values were longer than max_sort_length. Sorting used only the first 1024 bytes
SET @@sort_buffer_size = @old_sort_buff_size;
DROP TABLE t1;
#
......
......@@ -859,6 +859,8 @@ xxxxxxxxxxxxxxxxxxxx
xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxab
xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxaa
xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxz
Warnings:
Warning 4201 2 values were longer than max_sort_length. Sorting used only the first 64 bytes
drop table t1;
create table t1 (
`sid` decimal(8,0) default null,
......@@ -4122,6 +4124,8 @@ a substr(b, @save_max_sort_length+1)
3 ABE
2 AB
1 A
Warnings:
Warning 4201 5 values were longer than max_sort_length. Sorting used only the first 1024 bytes
analyze format=json
select a, substr(b, @save_max_sort_length+1) from t1 order by b desc;
ANALYZE
......@@ -4167,6 +4171,8 @@ ANALYZE
]
}
}
Warnings:
Warning 4201 5 values were longer than max_sort_length. Sorting used only the first 1024 bytes
drop table t1;
#
# Packing sort keys with complex collations
......
......@@ -834,6 +834,8 @@ xxxxxxxxxxxxxxxxxxxxxxxx
xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxw
xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxy
Warnings:
Warning 4201 3 values were longer than max_sort_length. Sorting used only the first 64 bytes
set max_sort_length=200;
select c from t1 order by c, id;
c
......@@ -877,7 +879,7 @@ Variable_name Value
Qcache_queries_in_cache 0
show status like "Qcache_inserts";
Variable_name Value
Qcache_inserts 19
Qcache_inserts 18
show status like "Qcache_hits";
Variable_name Value
Qcache_hits 6
......@@ -890,7 +892,7 @@ Variable_name Value
Qcache_queries_in_cache 1
show status like "Qcache_inserts";
Variable_name Value
Qcache_inserts 20
Qcache_inserts 19
show status like "Qcache_hits";
Variable_name Value
Qcache_hits 7
......
......@@ -899,6 +899,7 @@ Warnings:
Warning 1260 Row 1 was cut by group_concat()
Warning 1260 Row 2 was cut by group_concat()
Warning 1260 Row 3 was cut by group_concat()
Warning 4201 3 values were longer than max_sort_length. Sorting used only the first 1024 bytes
set @@group_concat_max_len = 256;
explain extended select left(a1,7), left(a2,7)
from t1_1024
......@@ -916,6 +917,7 @@ Warnings:
Warning 1260 Row 1 was cut by group_concat()
Warning 1260 Row 2 was cut by group_concat()
Warning 1260 Row 3 was cut by group_concat()
Warning 4201 3 values were longer than max_sort_length. Sorting used only the first 1024 bytes
drop table t1_1024, t2_1024, t3_1024;
set @blob_len = 1025;
set @suffix_len = @blob_len - @prefix_len;
......@@ -1000,6 +1002,7 @@ Warnings:
Warning 1260 Row 1 was cut by group_concat()
Warning 1260 Row 2 was cut by group_concat()
Warning 1260 Row 3 was cut by group_concat()
Warning 4201 3 values were longer than max_sort_length. Sorting used only the first 1024 bytes
set @@group_concat_max_len = 256;
explain extended select left(a1,7), left(a2,7)
from t1_1025
......@@ -1017,6 +1020,7 @@ Warnings:
Warning 1260 Row 1 was cut by group_concat()
Warning 1260 Row 2 was cut by group_concat()
Warning 1260 Row 3 was cut by group_concat()
Warning 4201 3 values were longer than max_sort_length. Sorting used only the first 1024 bytes
drop table t1_1025, t2_1025, t3_1025;
create table t1bit (a1 bit(3), a2 bit(3));
create table t2bit (b1 bit(3), b2 bit(3));
......
......@@ -919,6 +919,7 @@ Warnings:
Warning 1260 Row 1 was cut by group_concat()
Warning 1260 Row 2 was cut by group_concat()
Warning 1260 Row 3 was cut by group_concat()
Warning 4201 3 values were longer than max_sort_length. Sorting used only the first 1024 bytes
set @@group_concat_max_len = 256;
explain extended select left(a1,7), left(a2,7)
from t1_1024
......@@ -937,6 +938,7 @@ Warnings:
Warning 1260 Row 1 was cut by group_concat()
Warning 1260 Row 2 was cut by group_concat()
Warning 1260 Row 3 was cut by group_concat()
Warning 4201 3 values were longer than max_sort_length. Sorting used only the first 1024 bytes
drop table t1_1024, t2_1024, t3_1024;
set @blob_len = 1025;
set @suffix_len = @blob_len - @prefix_len;
......@@ -1022,6 +1024,7 @@ Warnings:
Warning 1260 Row 1 was cut by group_concat()
Warning 1260 Row 2 was cut by group_concat()
Warning 1260 Row 3 was cut by group_concat()
Warning 4201 3 values were longer than max_sort_length. Sorting used only the first 1024 bytes
set @@group_concat_max_len = 256;
explain extended select left(a1,7), left(a2,7)
from t1_1025
......@@ -1040,6 +1043,7 @@ Warnings:
Warning 1260 Row 1 was cut by group_concat()
Warning 1260 Row 2 was cut by group_concat()
Warning 1260 Row 3 was cut by group_concat()
Warning 4201 3 values were longer than max_sort_length. Sorting used only the first 1024 bytes
drop table t1_1025, t2_1025, t3_1025;
create table t1bit (a1 bit(3), a2 bit(3));
create table t2bit (b1 bit(3), b2 bit(3));
......
......@@ -767,6 +767,8 @@ SELECT col_1_text = REPEAT("क", 4000) FROM worklog5743 ORDER BY col_1_text;
col_1_text = REPEAT("क", 4000)
1
0
Warnings:
Warning 4201 2 values were longer than max_sort_length. Sorting used only the first 1024 bytes
DROP TABLE worklog5743;
CREATE TABLE worklog5743 (
col_1_text TEXT(4000) , col_2_text TEXT(4000) ,
......
......@@ -195,6 +195,93 @@ INSERT INTO t2 set c = concat(repeat('x',68),'g','w');
SELECT c from t2 ORDER BY c, id;
--echo ** Results should not be sorted **
DROP TABLE t, t1, t2;
--echo #
--echo # MDEV-27277 Add a warning when max_sort_length is reached
--echo #
set max_sort_length = 70;
--echo # Table having a fixed-length string field
create table t1(a char(100));
insert into t1 values
('ShortStr1'), ('ShortStr2'),
(concat(repeat('Str', 25), 'zzz')), (concat(repeat('Str', 25), 'yyy')),
(concat(repeat('Str', 25), 'xxx'));
--echo # Strings are not sorted properly due to max_sort_length limitation
select a from t1 order by a;
--echo # Make sure there are warnings when a string function is used:
select a from t1 order by coalesce(a);
select a from t1 order by concat(a, '1');
select a from t1 order by binary(a);
--echo # Table having a variable-length string field and UTF16 encoding (2 bytes per char):
create table t2(a varchar(100)) character set 'utf16';
insert into t2 values
(concat(repeat('Str', 15), 'zzz')), (concat(repeat('Str', 15), 'yyy')),
(concat(repeat('Str', 15), 'xxx')),
('shortString89'),
('shortString51');
set max_sort_length = 64;
select * from t2 order by a;
--echo # Table having text blobs
create table t3(a text, b mediumtext, c longtext) character set 'utf16';
insert into t3 values
(concat(repeat('Text', 20), '999'), concat(repeat('Medium', 15), '99'), concat(repeat('Long', 20), '99')),
(concat(repeat('Text', 20), '888'), concat(repeat('Medium', 15), '88'), concat(repeat('Long', 20), '88')),
(concat(repeat('Text', 20), '777'), concat(repeat('Medium', 15), '77'), concat(repeat('Long', 20), '77')),
('shortString89', 'short123', 'short456'),
('shortString51', 'short777', 'short897');
select * from t3 order by a;
select * from t3 order by b desc;
select * from t3 order by coalesce(b) desc;
select * from t3 order by c;
select * from t3 order by c, a desc;
--echo # Packing of sort keys will be applied here:
select * from t3 order by a, c, b;
select * from t3 order by a;
--echo # Test a prepared statement re-execution (expecting warnings at both executions)
prepare stmt from "select * from t2 order by a";
execute stmt;
execute stmt;
--echo # Test a stored procedure
create procedure p1() select * from t2 order by a;
call p1();
call p1();
drop procedure p1;
--echo # Test a stored function
create function f1 () returns char(100) return (select a from t1 order by a limit 1);
--disable_view_protocol # Because warnings display different number of strings
select f1() as f1_res from t1 order by f1_res;
--enable_view_protocol
--echo # Test a view
create view v1 as select f1() as f1_res from t1 order by f1_res;
select * from v1;
drop function f1;
drop view v1;
drop table t1, t2, t3;
#
# Cleanup
#
......@@ -204,6 +291,4 @@ connection default;
disconnect test_con1;
disconnect test_con2;
DROP TABLE t, t1, t2;
SET @@global.max_sort_length= @start_value;
......@@ -91,6 +91,8 @@ x left(b, 10) left(v, 10)
44 zzzzzzzzzz zzzzzzzzzz
50 zzzzzzzzzz zzzzzzzzzz
56 zzzzzzzzzz zzzzzzzzzz
Warnings:
Warning 4201 29 values were longer than max_sort_length. Sorting used only the first 1024 bytes
update t1 set b= 'bar' where v > 'a' limit 20;
drop table t1;
# Cover return_top_record() in ha_partition::handle_ordered_index_scan()
......
......@@ -257,12 +257,16 @@ select x, left(y, 4), length(y), check_row(row_start, row_end) from t1 for syste
x left(y, 4) length(y) check_row(row_start, row_end)
1 LONG 8192 HISTORICAL ROW
2 LONG 8192 CURRENT ROW
Warnings:
Warning 4201 2 values were longer than max_sort_length. Sorting used only the first 1024 bytes
update t1 set y= 'SHORT';
select x, left(y, 4), length(y), check_row(row_start, row_end) from t1 for system_time all order by x, y;
x left(y, 4) length(y) check_row(row_start, row_end)
1 LONG 8192 HISTORICAL ROW
2 LONG 8192 HISTORICAL ROW
2 SHOR 5 CURRENT ROW
Warnings:
Warning 4201 2 values were longer than max_sort_length. Sorting used only the first 1024 bytes
drop tables t1;
### Issue tempesta-tech/mariadb#365, bug 7 (duplicate of historical row)
create or replace table t1 (a int primary key, b int)
......
......@@ -1058,8 +1058,8 @@ CPP_UNNAMED_NS_END
@brief
Create a fixed size sort key part
@param buff buffer where values are written
@param length fixed size of the sort column
@param buff buffer where values are written
@param length fixed size of the sort column
*/
void Field::make_sort_key_part(uchar *buff,uint length)
......@@ -7810,13 +7810,21 @@ int Field_string::cmp_prefix(const uchar *a_ptr, const uchar *b_ptr,
void Field_string::sort_string(uchar *to,uint length)
{
/*
Let's find the real value length to truncate trailing padding spaces.
This is needed to avoid redundant WARN_SORTING_ON_TRUNCATED_LENGTH
warnings.
*/
const LEX_CSTRING str= to_lex_cstring();
my_strnxfrm_ret_t rc=
field_charset()->strnxfrm(to, length,
char_length() * field_charset()->strxfrm_multiply,
ptr, field_length,
(const uchar *) str.str, str.length,
MY_STRXFRM_PAD_WITH_SPACE |
MY_STRXFRM_PAD_TO_MAXLEN);
DBUG_ASSERT(rc.m_result_length == length);
if (rc.m_warnings & MY_STRNXFRM_TRUNCATED_WEIGHT_REAL_CHAR)
get_thd()->num_of_strings_sorted_on_truncated_length++;
}
......@@ -8265,12 +8273,14 @@ void Field_varstring::sort_string(uchar *to,uint length)
}
my_strnxfrm_ret_t rc=
field_charset()->strnxfrm(to, length,
char_length() * field_charset()->strxfrm_multiply,
(const uchar *) buf.ptr(), buf.length(),
MY_STRXFRM_PAD_WITH_SPACE |
MY_STRXFRM_PAD_TO_MAXLEN);
field_charset()->strnxfrm(to, length,
char_length() * field_charset()->strxfrm_multiply,
(const uchar *) buf.ptr(), buf.length(),
MY_STRXFRM_PAD_WITH_SPACE |
MY_STRXFRM_PAD_TO_MAXLEN);
DBUG_ASSERT(rc.m_result_length == length);
if (rc.m_warnings & MY_STRNXFRM_TRUNCATED_WEIGHT_REAL_CHAR)
get_thd()->num_of_strings_sorted_on_truncated_length++;
}
......@@ -9153,11 +9163,13 @@ void Field_blob::sort_string(uchar *to,uint length)
}
my_strnxfrm_ret_t rc=
field_charset()->strnxfrm(to, length, length,
(const uchar *) buf.ptr(), buf.length(),
MY_STRXFRM_PAD_WITH_SPACE |
MY_STRXFRM_PAD_TO_MAXLEN);
field_charset()->strnxfrm(to, length, length,
(const uchar *) buf.ptr(), buf.length(),
MY_STRXFRM_PAD_WITH_SPACE |
MY_STRXFRM_PAD_TO_MAXLEN);
DBUG_ASSERT(rc.m_result_length == length);
if (rc.m_warnings & MY_STRNXFRM_TRUNCATED_WEIGHT_REAL_CHAR)
get_thd()->num_of_strings_sorted_on_truncated_length++;
}
}
......
......@@ -1240,15 +1240,15 @@ Type_handler_string_result::make_sort_key_part(uchar *to, Item *item,
if (use_strnxfrm(cs))
{
#ifdef DBUG_ASSERT_EXISTS
my_strnxfrm_ret_t rc=
#endif
cs->strnxfrm(to, sort_field->length,
item->max_char_length() * cs->strxfrm_multiply,
(uchar*) res->ptr(), res->length(),
MY_STRXFRM_PAD_WITH_SPACE |
MY_STRXFRM_PAD_TO_MAXLEN);
cs->strnxfrm(to, sort_field->length,
item->max_char_length() * cs->strxfrm_multiply,
(uchar*) res->ptr(), res->length(),
MY_STRXFRM_PAD_WITH_SPACE |
MY_STRXFRM_PAD_TO_MAXLEN);
DBUG_ASSERT(rc.m_result_length == sort_field->length);
if (rc.m_warnings & MY_STRNXFRM_TRUNCATED_WEIGHT_REAL_CHAR)
current_thd->num_of_strings_sorted_on_truncated_length++;
}
else
{
......@@ -1268,7 +1268,10 @@ Type_handler_string_result::make_sort_key_part(uchar *to, Item *item,
store_length(to + sort_field_length, length, sort_field->suffix_length);
}
/* apply cs->sort_order for case-insensitive comparison if needed */
cs->strnxfrm((uchar*)to, length, (const uchar*) res->ptr(), length);
my_strnxfrm_ret_t rc= cs->strnxfrm((uchar*)to, length,
(const uchar*) res->ptr(), res->length());
if (rc.m_warnings & MY_STRNXFRM_TRUNCATED_WEIGHT_REAL_CHAR)
current_thd->num_of_strings_sorted_on_truncated_length++;
char fill_char= ((cs->state & MY_CS_BINSORT) ? (char) 0 : ' ');
cs->fill((char *) to + length, diff, fill_char);
}
......@@ -2937,9 +2940,14 @@ SORT_FIELD_ATTR::pack_sort_string(uchar *to, const Binary_string *str,
length= (uint32) str->length();
if (length + suffix_length <= original_length)
{
data_length= length;
}
else
{
data_length= original_length - suffix_length;
current_thd->num_of_strings_sorted_on_truncated_length++;
}
// length stored in lowendian form
store_key_part_length(data_length + suffix_length, to, length_bytes);
......
......@@ -12283,3 +12283,8 @@ ER_VECTOR_BINARY_FORMAT_INVALID
eng "Invalid binary vector format. Must use IEEE standard float representation in little-endian format. Use VEC_FromText() to generate it."
ER_VECTOR_FORMAT_INVALID
eng "Invalid vector format at offset: %d for '%-.100s'. Must be a valid JSON array of numbers."
WARN_SORTING_ON_TRUNCATED_LENGTH
eng "%llu values were longer than max_sort_length. Sorting used only the first %lu bytes"
ger "%llu Werte waren länger als max_sort_length. Sie wurden anhand der ersten %lu Bytes verglichen"
rus "%llu значений были длиннее, чем max_sort_length. Их сортировка проведена по первым %lu байтам"
swe "%llu värden var längre än max_sort_length=%lu"
......@@ -1382,6 +1382,7 @@ void THD::init()
first_successful_insert_id_in_cur_stmt= 0;
current_backup_stage= BACKUP_FINISHED;
backup_commit_lock= 0;
num_of_strings_sorted_on_truncated_length= 0;
#ifdef WITH_WSREP
wsrep_last_query_id= 0;
wsrep_xid.null();
......
......@@ -5868,6 +5868,13 @@ class THD: public THD_count, /* this must be first */
/* Handling of timeouts for commands */
thr_timer_t query_timer;
/*
Number of strings which were involved in sorting or grouping and whose
lengths were truncated according to the max_sort_length system variable
setting
*/
ulonglong num_of_strings_sorted_on_truncated_length;
public:
void set_query_timer()
{
......@@ -6068,6 +6075,23 @@ class THD: public THD_count, /* this must be first */
bool need_report_unit_results();
bool report_collected_unit_results();
bool init_collecting_unit_results();
/*
Push post-execution warnings, which may be some kinds of aggregate messages
like number of times max_sort_length was reached during sorting/grouping
*/
void push_final_warnings()
{
if (num_of_strings_sorted_on_truncated_length)
{
push_warning_printf(this, Sql_condition::WARN_LEVEL_WARN,
WARN_SORTING_ON_TRUNCATED_LENGTH,
ER_THD(this, WARN_SORTING_ON_TRUNCATED_LENGTH),
num_of_strings_sorted_on_truncated_length,
variables.max_sort_length);
num_of_strings_sorted_on_truncated_length= 0;
}
}
};
......
......@@ -5348,7 +5348,7 @@ mysql_select(THD *thd, TABLE_LIST *tables, List<Item> &fields, COND *conds,
}
err:
thd->push_final_warnings();
if (select_lex->pushdown_select)
{
delete select_lex->pushdown_select;
......@@ -694,7 +694,7 @@ MY_FUNCTION_NAME(strnxfrm_internal)(CHARSET_INFO *cs __attribute__((unused)),
}
return my_strnxfrm_ret_construct(dst - dst0, src - src0,
warnings |
(src - se ? MY_STRNXFRM_TRUNCATED_WEIGHT_REAL_CHAR : 0));
((src - se) ? MY_STRNXFRM_TRUNCATED_WEIGHT_REAL_CHAR : 0));
}
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment