Merge 5.1-security => 5.5-security

mysql-test/r/partition.result

@@ -2348,6 +2348,46 @@ TRUNCATE TABLE t1;
 INSERT INTO t1 VALUES(0);
 DROP TABLE t1;
 SET GLOBAL myisam_use_mmap=default;
+#
+# Bug#13580775 ASSERTION FAILED: RECORD_LENGTH == M_RECORD_LENGTH,
+# FILE FILESORT_UTILS.CC
+#
+CREATE TABLE t1 (
+a INT PRIMARY KEY,
+b INT,
+c CHAR(1),
+d INT,
+KEY (c,d)
+) PARTITION BY KEY () PARTITIONS 1;
+INSERT INTO t1 VALUES (1,1,'a',1), (2,2,'a',1);
+SELECT 1 FROM t1 WHERE 1 IN
+(SELECT  group_concat(b)
+FROM t1
+WHERE c > geomfromtext('point(1 1)')
+GROUP BY b
+);
+1
+1
+1
+DROP TABLE t1;
+#
+# Bug#13011410 CRASH IN FILESORT CODE WITH GROUP BY/ROLLUP
+#
+CREATE TABLE t1 (
+a INT,
+b MEDIUMINT,
+c VARCHAR(300) CHARACTER SET hp8 COLLATE hp8_bin,
+PRIMARY KEY (a,c(299))) 
+ENGINE=myisam
+PARTITION BY LINEAR KEY () PARTITIONS 2;
+INSERT INTO t1 VALUES (1,2,'test'), (2,3,'hi'), (4,5,'bye');
+SELECT 1 FROM t1 WHERE b < SOME
+( SELECT 1 FROM t1 WHERE a >= 1
+GROUP BY b WITH ROLLUP
+HAVING b > geomfromtext("")
+);
+1
+DROP TABLE t1;
 End of 5.1 tests
 #
 # BUG#55385: UPDATE statement throws an error, but still updates

mysql-test/t/partition.test

@@ -2346,6 +2346,51 @@ INSERT INTO t1 VALUES(0);
 DROP TABLE t1;
 SET GLOBAL myisam_use_mmap=default;
 
+--echo #
+--echo # Bug#13580775 ASSERTION FAILED: RECORD_LENGTH == M_RECORD_LENGTH,
+--echo # FILE FILESORT_UTILS.CC
+--echo #
+
+CREATE TABLE t1 (
+  a INT PRIMARY KEY,
+  b INT,
+  c CHAR(1),
+  d INT,
+  KEY (c,d)
+) PARTITION BY KEY () PARTITIONS 1;
+
+INSERT INTO t1 VALUES (1,1,'a',1), (2,2,'a',1);
+
+SELECT 1 FROM t1 WHERE 1 IN
+(SELECT  group_concat(b)
+ FROM t1
+ WHERE c > geomfromtext('point(1 1)')
+ GROUP BY b
+);
+
+DROP TABLE t1;
+
+--echo #
+--echo # Bug#13011410 CRASH IN FILESORT CODE WITH GROUP BY/ROLLUP
+--echo #
+
+CREATE TABLE t1 (
+  a INT,
+  b MEDIUMINT,
+  c VARCHAR(300) CHARACTER SET hp8 COLLATE hp8_bin,
+  PRIMARY KEY (a,c(299))) 
+ENGINE=myisam
+PARTITION BY LINEAR KEY () PARTITIONS 2;
+
+INSERT INTO t1 VALUES (1,2,'test'), (2,3,'hi'), (4,5,'bye');
+SELECT 1 FROM t1 WHERE b < SOME
+( SELECT 1 FROM t1 WHERE a >= 1
+  GROUP BY b WITH ROLLUP
+  HAVING b > geomfromtext("")
+);
+
+DROP TABLE t1;
+
 --echo End of 5.1 tests
 
 --echo #

sql/filesort.cc

@@ -41,7 +41,8 @@ if (my_b_write((file),(uchar*) (from),param->ref_length)) \
 
 	/* functions defined in this file */
 
-static char **make_char_array(char **old_pos, register uint fields,
+static size_t char_array_size(uint fields, uint length);
+static uchar **make_char_array(uchar **old_pos, uint fields,
                                uint length, myf my_flag);
 static uchar *read_buffpek_from_file(IO_CACHE *buffer_file, uint count,
                                      uchar *buf);
@@ -224,10 +225,22 @@ ha_rows filesort(THD *thd, TABLE *table, SORT_FIELD *sortorder, uint s_length,
     ulong old_memavl;
     ulong keys= memavl/(param.rec_length+sizeof(char*));
     param.keys=(uint) min(records+1, keys);
+
+    if (table_sort.sort_keys &&
+        table_sort.sort_keys_size != char_array_size(param.keys,
+                                                     param.rec_length))
+    {
+      my_free(table_sort.sort_keys);
+      table_sort.sort_keys= NULL;
+      table_sort.sort_keys_size= 0;
+    }
     if ((table_sort.sort_keys=
-	 (uchar **) make_char_array((char **) table_sort.sort_keys,
+         make_char_array(table_sort.sort_keys,
                          param.keys, param.rec_length, MYF(0))))
+    {
+      table_sort.sort_keys_size= char_array_size(param.keys, param.rec_length);
       break;
+    }
     old_memavl=memavl;
     if ((memavl=memavl/4*3) < min_sort_memory && old_memavl > min_sort_memory)
       memavl= min_sort_memory;
@@ -308,6 +321,7 @@ ha_rows filesort(THD *thd, TABLE *table, SORT_FIELD *sortorder, uint s_length,
   {
     my_free(sort_keys);
     table_sort.sort_keys= 0;
+    table_sort.sort_keys_size= 0;
     my_free(buffpek);
     table_sort.buffpek= 0;
     table_sort.buffpek_len= 0;
@@ -369,6 +383,7 @@ void filesort_free_buffers(TABLE *table, bool full)
   {
     my_free(table->sort.sort_keys);
     table->sort.sort_keys= NULL;
+    table->sort.sort_keys_size= 0;
     my_free(table->sort.buffpek);
     table->sort.buffpek= NULL;
     table->sort.buffpek_len= 0;
@@ -382,22 +397,31 @@ void filesort_free_buffers(TABLE *table, bool full)
 
 /** Make a array of string pointers. */
 
-static char **make_char_array(char **old_pos, register uint fields,
+static size_t char_array_size(uint fields, uint length)
+{
+  return fields * (length + sizeof(uchar*));
+}
+
+
+static uchar **make_char_array(uchar **old_pos, uint fields,
                                uint length, myf my_flag)
 {
-  register char **pos;
-  char *char_pos;
+  register uchar **pos;
+  uchar *char_pos;
   DBUG_ENTER("make_char_array");
 
   DBUG_EXECUTE_IF("make_char_array_fail",
                   DBUG_SET("+d,simulate_out_of_memory"););
 
   if (old_pos ||
-      (old_pos= (char**) my_malloc((uint) fields*(length+sizeof(char*)),
-				   my_flag)))
+      (old_pos= (uchar**) my_malloc(char_array_size(fields, length), my_flag)))
   {
-    pos=old_pos; char_pos=((char*) (pos+fields)) -length;
-    while (fields--) *(pos++) = (char_pos+= length);
+    pos=old_pos;
+    char_pos= ((uchar*)(pos+fields)) -length;
+    while (fields--)
+    {
+      *(pos++) = (char_pos+= length);
+    }
   }
 
   DBUG_RETURN(old_pos);

sql/table.h

@@ -298,6 +298,7 @@ typedef struct st_filesort_info
 {
   IO_CACHE *io_cache;           /* If sorted through filesort */
   uchar     **sort_keys;        /* Buffer for sorting keys */
+  size_t    sort_keys_size;     /* Number of bytes allocated */
   uchar     *buffpek;           /* Buffer for buffpek structures */
   uint      buffpek_len;        /* Max number of buffpeks in the buffer */
   uchar     *addon_buf;         /* Pointer to a buffer if sorted with fields */