MDEV-27309 Server crash or ASAN memcpy-param-overlap upon INSERT into...

MDEV-27309 Server crash or ASAN memcpy-param-overlap upon INSERT into Aria/MyISAM table with DESC key

MyiSAM and Aria, indexes with prefix compression, where the first keypart
could be NULL - in this case they didn't expect the next key after the
not NULL key to be NULL.

Expect the first keypart of the next key to have zero length even
if store_not_null==1, this combination means keypart is NULL,
don't pack it.

also fixes MDEV-27340

mysql-test/main/myisam.result

@@ -2732,6 +2732,13 @@ Table	Op	Msg_type	Msg_text
 test.t1	check	status	OK
 drop table t1;
 #
+# MDEV-27309 Server crash or ASAN memcpy-param-overlap upon INSERT into Aria/MyISAM table with DESC key
+#
+CREATE TABLE t1 (id INT, c BINARY(80), PRIMARY KEY(id));
+ALTER  TABLE t1 ADD KEY(c DESC, id);
+INSERT INTO t1 VALUES (1,NULL),(2,''),(3,'');
+DROP TABLE t1;
+#
 # MDEV-27330 Wrong sorting order with DESC index and empty strings in MyISAM/Aria table
 #
 create table t (id int, c char(128) not null, key (c desc));
@@ -2744,5 +2751,30 @@ bar
 foo
 drop table t;
 #
+# MDEV-27340 NULL gets lost (becomes empty string), SELECT hangs with DESC index on MyISAM/Aria table
+#
+create table t (c char(8), key(c desc)) character set utf8mb4;
+insert into t values (''),('foo'),(null),(''),('bar');
+check table t;
+Table	Op	Msg_type	Msg_text
+test.t	check	status	OK
+check table t extended;
+Table	Op	Msg_type	Msg_text
+test.t	check	status	OK
+select distinct c from t;
+c
+NULL
+
+bar
+foo
+select c from t;
+c
+foo
+bar
+
+
+NULL
+drop table t;
+#
 # End of 10.8 tests
 #

mysql-test/main/myisam.test

@@ -1855,6 +1855,14 @@ insert into t1 (c) values
 check table t1 extended;
 drop table t1;
 
+--echo #
+--echo # MDEV-27309 Server crash or ASAN memcpy-param-overlap upon INSERT into Aria/MyISAM table with DESC key
+--echo #
+CREATE TABLE t1 (id INT, c BINARY(80), PRIMARY KEY(id));
+ALTER  TABLE t1 ADD KEY(c DESC, id);
+INSERT INTO t1 VALUES (1,NULL),(2,''),(3,'');
+DROP TABLE t1;
+
 --echo #
 --echo # MDEV-27330 Wrong sorting order with DESC index and empty strings in MyISAM/Aria table
 --echo #
@@ -1863,6 +1871,17 @@ insert into t values (1,''),(2,'foo'),(3,''),(4,'bar');
 select c from t order by c;
 drop table t;
 
+--echo #
+--echo # MDEV-27340 NULL gets lost (becomes empty string), SELECT hangs with DESC index on MyISAM/Aria table
+--echo #
+create table t (c char(8), key(c desc)) character set utf8mb4;
+insert into t values (''),('foo'),(null),(''),('bar');
+check table t;
+check table t extended;
+select distinct c from t;
+select c from t;
+drop table t;
+
 --echo #
 --echo # End of 10.8 tests
 --echo #

mysql-test/suite/maria/maria2.result

@@ -97,6 +97,13 @@ Table	Op	Msg_type	Msg_text
 test.t1	check	status	OK
 drop table t1;
 #
+# MDEV-27309 Server crash or ASAN memcpy-param-overlap upon INSERT into Aria/MyISAM table with DESC key
+#
+CREATE TABLE t1 (id INT, c BINARY(80), PRIMARY KEY(id));
+ALTER  TABLE t1 ADD KEY(c DESC, id);
+INSERT INTO t1 VALUES (1,NULL),(2,''),(3,'');
+DROP TABLE t1;
+#
 # MDEV-27330 Wrong sorting order with DESC index and empty strings in MyISAM/Aria table
 #
 create table t (id int, c char(128) not null, key (c desc)) engine=aria;
@@ -109,5 +116,30 @@ bar
 foo
 drop table t;
 #
+# MDEV-27340 NULL gets lost (becomes empty string), SELECT hangs with DESC index on MyISAM/Aria table
+#
+create table t (c char(8), key(c desc)) engine=aria character set utf8mb4;
+insert into t values (''),('foo'),(null),(''),('bar');
+check table t;
+Table	Op	Msg_type	Msg_text
+test.t	check	status	OK
+check table t extended;
+Table	Op	Msg_type	Msg_text
+test.t	check	status	OK
+select distinct c from t;
+c
+NULL
+
+bar
+foo
+select c from t;
+c
+foo
+bar
+
+
+NULL
+drop table t;
+#
 # End of 10.8 tests
 #

mysql-test/suite/maria/maria2.test

@@ -138,6 +138,14 @@ insert into t1 (c) values
 check table t1 extended;
 drop table t1;
 
+--echo #
+--echo # MDEV-27309 Server crash or ASAN memcpy-param-overlap upon INSERT into Aria/MyISAM table with DESC key
+--echo #
+CREATE TABLE t1 (id INT, c BINARY(80), PRIMARY KEY(id));
+ALTER  TABLE t1 ADD KEY(c DESC, id);
+INSERT INTO t1 VALUES (1,NULL),(2,''),(3,'');
+DROP TABLE t1;
+
 --echo #
 --echo # MDEV-27330 Wrong sorting order with DESC index and empty strings in MyISAM/Aria table
 --echo #
@@ -146,6 +154,17 @@ insert into t values (1,''),(2,'foo'),(3,''),(4,'bar');
 select c from t order by c;
 drop table t;
 
+--echo #
+--echo # MDEV-27340 NULL gets lost (becomes empty string), SELECT hangs with DESC index on MyISAM/Aria table
+--echo #
+create table t (c char(8), key(c desc)) engine=aria character set utf8mb4;
+insert into t values (''),('foo'),(null),(''),('bar');
+check table t;
+check table t extended;
+select distinct c from t;
+select c from t;
+drop table t;
+
 --echo #
 --echo # End of 10.8 tests
 --echo #

storage/maria/ma_search.c

@@ -2039,7 +2039,7 @@ _ma_calc_var_pack_key_length(const MARIA_KEY *int_key, uint nod_flag,
   DBUG_PRINT("test",("tot_length: %u  length: %d  uniq_key_length: %u",
                      key_length, length, s_temp->key_length));
 
-        /* If something after that hasn't length=0, test if we can combine */
+  /* If something after that hasn't length=0, test if we can combine */
   if ((s_temp->next_key_pos=next_key))
   {
     uint packed,n_length;
@@ -2052,7 +2052,7 @@ _ma_calc_var_pack_key_length(const MARIA_KEY *int_key, uint nod_flag,
     }
     else
       n_length= *next_key++ & 127;
-    if (!packed)
+    if (!packed && n_length)
       n_length-= s_temp->store_not_null;
 
     if (n_length || packed)             /* Don't pack 0 length keys */
@@ -2352,7 +2352,7 @@ void _ma_store_var_pack_key(MARIA_KEYDEF *keyinfo  __attribute__((unused)),
       store_key_length_inc(key_pos,s_temp->n_length);
     }
   }
-  else
+  else if (s_temp->n_length)
   {
     s_temp->n_length+= s_temp->store_not_null;
     store_pack_length(s_temp->pack_marker == 128,key_pos,s_temp->n_length);

storage/myisam/mi_search.c

@@ -1599,7 +1599,7 @@ _mi_calc_var_pack_key_length(MI_KEYDEF *keyinfo,uint nod_flag,uchar *next_key,
     }
     else
       n_length= *next_key++ & 127;
-    if (!packed)
+    if (!packed && n_length)
       n_length-= s_temp->store_not_null;
 
     if (n_length || packed)             /* Don't pack 0 length keys */
@@ -1888,7 +1888,7 @@ void _mi_store_var_pack_key(MI_KEYDEF *keyinfo  __attribute__((unused)),
       return;                                   /* Identical key */
     store_key_length(key_pos,s_temp->n_length);
   }
-  else
+  else if (s_temp->n_length)
   {
     s_temp->n_length+= s_temp->store_not_null;
     store_pack_length(s_temp->pack_marker == 128,key_pos,s_temp->n_length);