Fixed bug #35993: memory corruption and crash with multibyte conversion.

Grouping or ordering of long values in not indexed BLOB/TEXT columns
with GBK or BIG5 charsets crashes the server.

MySQL server uses sorting (the filesort procedure) in the temporary
table to evaluate the GROUP BY clause in case of lack of suitable index.
That procedure takes into account only first @max_sort_length bytes
(system variable, usually 1024) of TEXT/BLOB sorting key string.
The my_strnxfrm_gbk and my_strnxfrm_big5 fill temporary keys
with data of whole blob length instead of @max_sort_length bytes
length. That buffer overrun has been fixed.


mysql-test/r/ctype_gbk.result:
  Added test case for bug #35993.
mysql-test/t/ctype_gbk.test:
  Added test case for bug #35993.
strings/ctype-big5.c:
  Fixed bug #35993: memory corruption and crash with multibyte conversion.
  
  Buffer overrun has been fixed in the my_strnxfrm_big5 function.
strings/ctype-gbk.c:
  Fixed bug #35993: memory corruption and crash with multibyte conversion.
  
  Buffer overrun has been fixed in the my_strnxfrm_gbk function.

mysql-test/r/ctype_gbk.result

@@ -247,4 +247,11 @@ t1	CREATE TABLE `t1` (
   `c2` text NOT NULL
 ) ENGINE=MyISAM DEFAULT CHARSET=gbk
 drop table t1;
+CREATE TABLE t1(a MEDIUMTEXT CHARACTER SET gbk,
+b MEDIUMTEXT CHARACTER SET big5);
+INSERT INTO t1 VALUES
+(REPEAT(0x1125,200000), REPEAT(0x1125,200000)), ('', ''), ('', '');
+SELECT a FROM t1 GROUP BY 1 LIMIT 1 INTO @nullll;
+SELECT b FROM t1 GROUP BY 1 LIMIT 1 INTO @nullll;
+DROP TABLES t1;
 End of 5.0 tests

mysql-test/t/ctype_gbk.test

@@ -53,4 +53,18 @@ alter table t1 change c1 c1 mediumtext  character set gbk not null;
 show create table t1;
 drop table t1;
 
+#
+# Bug#35993: severe memory corruption and crash with multibyte conversion
+#
+
+CREATE TABLE t1(a MEDIUMTEXT CHARACTER SET gbk,
+                b MEDIUMTEXT CHARACTER SET big5);
+INSERT INTO t1 VALUES
+  (REPEAT(0x1125,200000), REPEAT(0x1125,200000)), ('', ''), ('', '');
+
+SELECT a FROM t1 GROUP BY 1 LIMIT 1 INTO @nullll;
+SELECT b FROM t1 GROUP BY 1 LIMIT 1 INTO @nullll;
+
+DROP TABLES t1;
+
 --echo End of 5.0 tests

strings/ctype-big5.c

@@ -307,14 +307,16 @@ static int my_strnxfrm_big5(CHARSET_INFO *cs __attribute__((unused)),
 {
   uint16 e;
   uint dstlen= len;
+  uchar *dest_end= dest + dstlen;
 
   len = srclen;
-  while (len--)
+  while (len-- && dest < dest_end)
   {
     if ((len > 0) && isbig5code(*src, *(src+1)))
     {
       e = big5strokexfrm((uint16) big5code(*src, *(src+1)));
       *dest++ = big5head(e);
+      if (dest < dest_end)
         *dest++ = big5tail(e);
       src +=2;
       len--;

strings/ctype-gbk.c

@@ -2668,14 +2668,16 @@ static int my_strnxfrm_gbk(CHARSET_INFO *cs __attribute__((unused)),
 {
   uint16 e;
   uint dstlen= len;
+  uchar *dest_end= dest + dstlen;
 
   len = srclen;
-  while (len--)
+  while (len-- && dest < dest_end)
   {
     if ((len > 0) && isgbkcode(*src, *(src+1)))
     {
       e = gbksortorder((uint16) gbkcode(*src, *(src+1)));
       *dest++ = gbkhead(e);
+      if (dest < dest_end)
         *dest++ = gbktail(e);
       src+=2;
       len--;