MDEV-21922: Allow packing addon fields even if they don't honour max_length_for_sort_data

Addon fields will be packed if the length of addon fields is greater
than max_length_for_sort_data.

mysql-test/main/order_by.result

@@ -3797,3 +3797,56 @@ efg	5	5
 fgh	6	6
 set max_sort_length= @save_max_sort_length;
 drop table t1;
+#
+# MDEV-21922: Allow packing addon fields even if they don't honour max_length_for_sort_data
+#
+create table t1 (a varchar(200) character set utf8, b int);
+insert into t1 select seq, seq from seq_1_to_10;
+select * from t1 order by a;
+a	b
+1	1
+10	10
+2	2
+3	3
+4	4
+5	5
+6	6
+7	7
+8	8
+9	9
+#
+# r_sort_mode should show both packed_sort_key and packed_addon_fields
+#
+analyze format=json select * from t1 order by a;
+ANALYZE
+{
+  "query_block": {
+    "select_id": 1,
+    "r_loops": 1,
+    "r_total_time_ms": "REPLACED",
+    "read_sorted_file": {
+      "r_rows": 10,
+      "filesort": {
+        "sort_key": "t1.a",
+        "r_loops": 1,
+        "r_total_time_ms": "REPLACED",
+        "r_used_priority_queue": false,
+        "r_output_rows": 10,
+        "r_buffer_size": "REPLACED",
+        "r_sort_mode": "packed_sort_key,packed_addon_fields",
+        "table": {
+          "table_name": "t1",
+          "access_type": "ALL",
+          "r_loops": 1,
+          "rows": 10,
+          "r_rows": 10,
+          "r_table_time_ms": "REPLACED",
+          "r_other_time_ms": "REPLACED",
+          "filtered": 100,
+          "r_filtered": 100
+        }
+      }
+    }
+  }
+}
+drop table t1;

mysql-test/main/order_by.test

@@ -2386,3 +2386,17 @@ select a, b, c from t1 order by a, b;
 set max_sort_length= @save_max_sort_length;
 
 drop table t1;
+
+--echo #
+--echo # MDEV-21922: Allow packing addon fields even if they don't honour max_length_for_sort_data
+--echo #
+
+create table t1 (a varchar(200) character set utf8, b int);
+insert into t1 select seq, seq from seq_1_to_10;
+select * from t1 order by a;
+--echo #
+--echo # r_sort_mode should show both packed_sort_key and packed_addon_fields
+--echo #
+--source include/analyze-format.inc
+analyze format=json select * from t1 order by a;
+drop table t1;

sql/filesort.cc

@@ -139,8 +139,6 @@ void Sort_param::try_to_pack_addons(ulong max_length_for_sort_data)
     return;
 
   const uint sz= Addon_fields::size_of_length_field;
-  if (rec_length + sz > max_length_for_sort_data)
-    return;
 
   // Heuristic: skip packing if potential savings are less than 10 bytes.
   if (m_packable_length < (10 + sz))
@@ -2281,7 +2279,7 @@ sortlength(THD *thd, Sort_keys *sort_keys, bool *multi_byte_charset,
   Check whether addon fields can be used or not.
 
   @param table                  Table structure
-  @param sortlength             Length of sort key
+  @param sortlength             Length of sort key [strxfrm form]
   @param length [OUT]           Max length of addon fields
   @param fields [OUT]           Number of addon fields
   @param null_fields [OUT]      Number of nullable addon fields
@@ -2321,6 +2319,15 @@ bool filesort_use_addons(TABLE *table, uint sortlength,
     return false;
   (*length)+= (*null_fields+7)/8;
 
+  /*
+    sortlength used here is unpacked key length (the strxfrm form). This is
+    done because unpacked key length is a good upper bound for packed sort
+    key length.
+    But for some collations the max packed length may be greater than the
+    length obtained from the strxfrm form.
+    Example: for utf8_general_ci, the original string form can be longer than
+    its mem-comparable form (note that this is rarely achieved in practice).
+  */
   return *length + sortlength <
          table->in_use->variables.max_length_for_sort_data;
 }