Bug#13580775 ASSERTION FAILED: RECORD_LENGTH == M_RECORD_LENGTH

Bug#13011410 CRASH IN FILESORT CODE WITH GROUP BY/ROLLUP

The assert in 13580775 is visible in 5.6 only, 
but shows that all versions are vulnerable.
13011410 crashes in all versions.

filesort tries to re-use the sort buffer between invocations in order to save
malloc/free overhead.
The fix for Bug 11748783 - 37359: FILESORT CAN BE MORE EFFICIENT.
added an assert that buffer properties (num_records, record_length) are
consistent between invocations. Indeed, they are not necessarily consistent.
  
Fix: re-allocate the sort buffer if properties change.

mysql-test/r/partition.result

@@ -2206,4 +2206,44 @@ TRUNCATE TABLE t1;
 INSERT INTO t1 VALUES(0);
 DROP TABLE t1;
 SET GLOBAL myisam_use_mmap=default;
+#
+# Bug#13580775 ASSERTION FAILED: RECORD_LENGTH == M_RECORD_LENGTH,
+# FILE FILESORT_UTILS.CC
+#
+CREATE TABLE t1 (
+a INT PRIMARY KEY,
+b INT,
+c CHAR(1),
+d INT,
+KEY (c,d)
+) PARTITION BY KEY () PARTITIONS 1;
+INSERT INTO t1 VALUES (1,1,'a',1), (2,2,'a',1);
+SELECT 1 FROM t1 WHERE 1 IN
+(SELECT  group_concat(b)
+FROM t1
+WHERE c > geomfromtext('point(1 1)')
+GROUP BY b
+);
+1
+1
+1
+DROP TABLE t1;
+#
+# Bug#13011410 CRASH IN FILESORT CODE WITH GROUP BY/ROLLUP
+#
+CREATE TABLE t1 (
+a INT,
+b MEDIUMINT,
+c VARCHAR(300) CHARACTER SET hp8 COLLATE hp8_bin,
+PRIMARY KEY (a,c(299))) 
+ENGINE=myisam
+PARTITION BY LINEAR KEY () PARTITIONS 2;
+INSERT INTO t1 VALUES (1,2,'test'), (2,3,'hi'), (4,5,'bye');
+SELECT 1 FROM t1 WHERE b < SOME
+( SELECT 1 FROM t1 WHERE a >= 1
+GROUP BY b WITH ROLLUP
+HAVING b > geomfromtext("")
+);
+1
+DROP TABLE t1;
 End of 5.1 tests

mysql-test/t/partition.test

@@ -2218,4 +2218,49 @@ INSERT INTO t1 VALUES(0);
 DROP TABLE t1;
 SET GLOBAL myisam_use_mmap=default;
 
+--echo #
+--echo # Bug#13580775 ASSERTION FAILED: RECORD_LENGTH == M_RECORD_LENGTH,
+--echo # FILE FILESORT_UTILS.CC
+--echo #
+
+CREATE TABLE t1 (
+  a INT PRIMARY KEY,
+  b INT,
+  c CHAR(1),
+  d INT,
+  KEY (c,d)
+) PARTITION BY KEY () PARTITIONS 1;
+
+INSERT INTO t1 VALUES (1,1,'a',1), (2,2,'a',1);
+
+SELECT 1 FROM t1 WHERE 1 IN
+(SELECT  group_concat(b)
+ FROM t1
+ WHERE c > geomfromtext('point(1 1)')
+ GROUP BY b
+);
+
+DROP TABLE t1;
+
+--echo #
+--echo # Bug#13011410 CRASH IN FILESORT CODE WITH GROUP BY/ROLLUP
+--echo #
+
+CREATE TABLE t1 (
+  a INT,
+  b MEDIUMINT,
+  c VARCHAR(300) CHARACTER SET hp8 COLLATE hp8_bin,
+  PRIMARY KEY (a,c(299))) 
+ENGINE=myisam
+PARTITION BY LINEAR KEY () PARTITIONS 2;
+
+INSERT INTO t1 VALUES (1,2,'test'), (2,3,'hi'), (4,5,'bye');
+SELECT 1 FROM t1 WHERE b < SOME
+( SELECT 1 FROM t1 WHERE a >= 1
+  GROUP BY b WITH ROLLUP
+  HAVING b > geomfromtext("")
+);
+
+DROP TABLE t1;
+
 --echo End of 5.1 tests

sql/filesort.cc

@@ -41,7 +41,8 @@ if (my_b_write((file),(uchar*) (from),param->ref_length)) \
 
 	/* functions defined in this file */
 
-static char **make_char_array(char **old_pos, register uint fields,
+static size_t char_array_size(uint fields, uint length);
+static uchar **make_char_array(uchar **old_pos, uint fields,
                                uint length, myf my_flag);
 static uchar *read_buffpek_from_file(IO_CACHE *buffer_file, uint count,
                                      uchar *buf);
@@ -221,10 +222,22 @@ ha_rows filesort(THD *thd, TABLE *table, SORT_FIELD *sortorder, uint s_length,
     ulong old_memavl;
     ulong keys= memavl/(param.rec_length+sizeof(char*));
     param.keys=(uint) min(records+1, keys);
+
+    if (table_sort.sort_keys &&
+        table_sort.sort_keys_size != char_array_size(param.keys,
+                                                     param.rec_length))
+    {
+      x_free((uchar*) table_sort.sort_keys);
+      table_sort.sort_keys= NULL;
+      table_sort.sort_keys_size= 0;
+    }
     if ((table_sort.sort_keys=
-	 (uchar **) make_char_array((char **) table_sort.sort_keys,
+         make_char_array(table_sort.sort_keys,
                          param.keys, param.rec_length, MYF(0))))
+    {
+      table_sort.sort_keys_size= char_array_size(param.keys, param.rec_length);
       break;
+    }
     old_memavl=memavl;
     if ((memavl=memavl/4*3) < min_sort_memory && old_memavl > min_sort_memory)
       memavl= min_sort_memory;
@@ -306,6 +319,7 @@ ha_rows filesort(THD *thd, TABLE *table, SORT_FIELD *sortorder, uint s_length,
   {
     x_free((uchar*) sort_keys);
     table_sort.sort_keys= 0;
+    table_sort.sort_keys_size= 0;
     x_free((uchar*) buffpek);
     table_sort.buffpek= 0;
     table_sort.buffpek_len= 0;
@@ -353,6 +367,7 @@ void filesort_free_buffers(TABLE *table, bool full)
     {
       x_free((uchar*) table->sort.sort_keys);
       table->sort.sort_keys= 0;
+      table->sort.sort_keys_size= 0;
     }
     if (table->sort.buffpek)
     {
@@ -372,19 +387,28 @@ void filesort_free_buffers(TABLE *table, bool full)
 
 /** Make a array of string pointers. */
 
-static char **make_char_array(char **old_pos, register uint fields,
+static size_t char_array_size(uint fields, uint length)
+{
+  return fields * (length + sizeof(uchar*));
+}
+
+
+static uchar **make_char_array(uchar **old_pos, uint fields,
                                uint length, myf my_flag)
 {
-  register char **pos;
-  char *char_pos;
+  register uchar **pos;
+  uchar *char_pos;
   DBUG_ENTER("make_char_array");
 
   if (old_pos ||
-      (old_pos= (char**) my_malloc((uint) fields*(length+sizeof(char*)),
-				   my_flag)))
+      (old_pos= (uchar**) my_malloc(char_array_size(fields, length), my_flag)))
   {
-    pos=old_pos; char_pos=((char*) (pos+fields)) -length;
-    while (fields--) *(pos++) = (char_pos+= length);
+    pos=old_pos;
+    char_pos= ((uchar*)(pos+fields)) -length;
+    while (fields--)
+    {
+      *(pos++) = (char_pos+= length);
+    }
   }
 
   DBUG_RETURN(old_pos);

sql/table.h

@@ -157,6 +157,7 @@ typedef struct st_filesort_info
 {
   IO_CACHE *io_cache;           /* If sorted through filesort */
   uchar     **sort_keys;        /* Buffer for sorting keys */
+  size_t    sort_keys_size;     /* Number of bytes allocated */
   uchar     *buffpek;           /* Buffer for buffpek structures */
   uint      buffpek_len;        /* Max number of buffpeks in the buffer */
   uchar     *addon_buf;         /* Pointer to a buffer if sorted with fields */