MDEV-31743 Server crash in store_length, assertion failure in...

MDEV-31743 Server crash in store_length, assertion failure in Type_handler_string_result::sort_length

After MDEV-21580 the truncation of SORT_FIELD::length
  set_if_smaller(sortorder->length, thd->variables.max_sort_length)

became conditional:
  if (is_variable_sized())
    set_if_smaller(length, thd->variables.max_sort_length)

To provide correct functioning of is_variable_sized() SORT_FIELD::type
must be set properly. This commit adds the necessary initialization
of SORT_FIELD::type to JOIN_TAB::remove_duplicates() as it is done
in filesort's sortlength() function.

DBUG_ASSERT is added to sortlength() just in case to prevent
a possible uint32 overflow

mysql-test/main/distinct.result

@@ -1180,5 +1180,16 @@ a
 1
 drop table t1,t2;
 #
+# MDEV-31743 Server crash in store_length, assertion failure in
+#            Type_handler_string_result::sort_length
+#
+create table t1 (a int, b longtext, c varchar(18));
+insert into t1 values (1, 'Aa123456', 'abc'), (2, 'Bb7897777', 'def'),
+(3, 'Cc01287', 'xyz'), (5, 'd12345', 'efg');
+select distinct if(sum(a), b, 0) from t1 group by value(c) with rollup;
+if(sum(a), b, 0)
+Aa123456
+drop table t1;
+#
 # end of 10.5 tests
 #

mysql-test/main/distinct.test

@@ -914,6 +914,18 @@ select distinct a from t1 where t1.a=1 and t1.a in (select a+0 from t2 where t2.
 select distinct a from t1 where t1.a=1 and t1.a in (select a+0 from t2 where t2.b in (1,2)) limit 0,1;
 drop table t1,t2;
 
+--echo #
+--echo # MDEV-31743 Server crash in store_length, assertion failure in
+--echo #            Type_handler_string_result::sort_length
+--echo #
+
+create table t1 (a int, b longtext, c varchar(18));
+insert into t1 values (1, 'Aa123456', 'abc'), (2, 'Bb7897777', 'def'),
+  (3, 'Cc01287', 'xyz'), (5, 'd12345', 'efg');
+
+select distinct if(sum(a), b, 0) from t1 group by value(c) with rollup;
+drop table t1;
+
 --echo #
 --echo # end of 10.5 tests
 --echo #

sql/filesort.cc

@@ -2275,7 +2275,8 @@ sortlength(THD *thd, Sort_keys *sort_keys, bool *allow_packing_for_sortkeys)
       set_if_smaller(sortorder->length, thd->variables.max_sort_length);
       set_if_smaller(sortorder->original_length, thd->variables.max_sort_length);
     }
-    length+=sortorder->length;
+    DBUG_ASSERT(length < UINT_MAX32 - sortorder->length);
+    length+= sortorder->length;
 
     sort_keys->increment_size_of_packable_fields(sortorder->length_bytes);
     sort_keys->increment_original_sort_length(sortorder->original_length);

sql/sql_select.cc

@@ -24629,6 +24629,9 @@ JOIN_TAB::remove_duplicates()
       {
         /* Item is not stored in temporary table, remember it */
         sorder->item= item;
+        sorder->type= sorder->item->type_handler()->is_packable() ?
+                      SORT_FIELD_ATTR::VARIABLE_SIZE :
+                      SORT_FIELD_ATTR::FIXED_SIZE;
         /* Calculate sorder->length */
         item->type_handler()->sort_length(thd, item, sorder);
         sorder++;