Fixed bug mdev-449.

The bug could caused a crash when the server executed a query with
ORDER by and sort_buffer_size was set to a small enough number.
It happened because the small sort buffer did not allow to allocate
all merge buffers in it.
Made sure that the allocated sort buffer would be big enough
to contain all possible merge buffers.  

mysql-test/r/order_by.result

@@ -1783,3 +1783,135 @@ Warnings:
 Note	1003	select `test`.`t2`.`b` AS `b` from `test`.`t1` join `test`.`t2` where (`test`.`t2`.`a` = `test`.`t1`.`b`) group by `test`.`t2`.`b` order by `test`.`t1`.`b`
 drop table t1,t2;
 End of 5.2 tests
+#
+# Bug mdev-449: ORDER BY with small sort_buffer_size
+#
+CREATE TABLE t1(f0 int auto_increment primary key, f1 int, f2 varchar(200));
+INSERT INTO t1(f1, f2) VALUES 
+(0,"0"),(1,"1"),(2,"2"),(3,"3"),(4,"4"),(5,"5"),
+(6,"6"),(7,"7"),(8,"8"),(9,"9"),(10,"10"),
+(11,"11"),(12,"12"),(13,"13"),(14,"14"),(15,"15"),
+(16,"16"),(17,"17"),(18,"18"),(19,"19"),(20,"20"),
+(21,"21"),(22,"22"),(23,"23"),(24,"24"),(25,"25"),
+(26,"26"),(27,"27"),(28,"28"),(29,"29"),(30,"30"),
+(31,"31"),(32,"32"),(33,"33"),(34,"34"),(35,"35"),
+(36,"36"),(37,"37"),(38,"38"),(39,"39"),(40,"40"),
+(41,"41"),(42,"42"),(43,"43"),(44,"44"),(45,"45"),
+(46,"46"),(47,"47"),(48,"48"),(49,"49"),(50,"50"),
+(51,"51"),(52,"52"),(53,"53"),(54,"54"),(55,"55"),
+(56,"56"),(57,"57"),(58,"58"),(59,"59"),(60,"60"),
+(61,"61"),(62,"62"),(63,"63"),(64,"64"),(65,"65"),
+(66,"66"),(67,"67"),(68,"68"),(69,"69"),(70,"70"),
+(71,"71"),(72,"72"),(73,"73"),(74,"74"),(75,"75"),
+(76,"76"),(77,"77"),(78,"78"),(79,"79"),(80,"80"),
+(81,"81"),(82,"82"),(83,"83"),(84,"84"),(85,"85"),
+(86,"86"),(87,"87"),(88,"88"),(89,"89"),(90,"90"),
+(91,"91"),(92,"92"),(93,"93"),(94,"94"),(95,"95"),
+(96,"96"),(97,"97"),(98,"98"),(99,"99");
+set @save_sort_buffer_size= @@sort_buffer_size;
+set sort_buffer_size= 2000;
+SELECT * FROM t1 ORDER BY f1 DESC, f0;
+f0	f1	f2
+100	99	99
+99	98	98
+98	97	97
+97	96	96
+96	95	95
+95	94	94
+94	93	93
+93	92	92
+92	91	91
+91	90	90
+90	89	89
+89	88	88
+88	87	87
+87	86	86
+86	85	85
+85	84	84
+84	83	83
+83	82	82
+82	81	81
+81	80	80
+80	79	79
+79	78	78
+78	77	77
+77	76	76
+76	75	75
+75	74	74
+74	73	73
+73	72	72
+72	71	71
+71	70	70
+70	69	69
+69	68	68
+68	67	67
+67	66	66
+66	65	65
+65	64	64
+64	63	63
+63	62	62
+62	61	61
+61	60	60
+60	59	59
+59	58	58
+58	57	57
+57	56	56
+56	55	55
+55	54	54
+54	53	53
+53	52	52
+52	51	51
+51	50	50
+50	49	49
+49	48	48
+48	47	47
+47	46	46
+46	45	45
+45	44	44
+44	43	43
+43	42	42
+42	41	41
+41	40	40
+40	39	39
+39	38	38
+38	37	37
+37	36	36
+36	35	35
+35	34	34
+34	33	33
+33	32	32
+32	31	31
+31	30	30
+30	29	29
+29	28	28
+28	27	27
+27	26	26
+26	25	25
+25	24	24
+24	23	23
+23	22	22
+22	21	21
+21	20	20
+20	19	19
+19	18	18
+18	17	17
+17	16	16
+16	15	15
+15	14	14
+14	13	13
+13	12	12
+12	11	11
+11	10	10
+10	9	9
+9	8	8
+8	7	7
+7	6	6
+6	5	5
+5	4	4
+4	3	3
+3	2	2
+2	1	1
+1	0	0
+set sort_buffer_size= @save_sort_buffer_size;
+DROP TABLE t1;
+End of 5.3 tests

mysql-test/t/order_by.test

@@ -1589,3 +1589,42 @@ SELECT t2.b FROM t1, t2 WHERE t1.b = t2.a GROUP BY t2.b ORDER BY t1.b;
 drop table t1,t2;
 
 --echo End of 5.2 tests
+
+--echo #
+--echo # Bug mdev-449: ORDER BY with small sort_buffer_size
+--echo #
+
+CREATE TABLE t1(f0 int auto_increment primary key, f1 int, f2 varchar(200));
+INSERT INTO t1(f1, f2) VALUES 
+(0,"0"),(1,"1"),(2,"2"),(3,"3"),(4,"4"),(5,"5"),
+(6,"6"),(7,"7"),(8,"8"),(9,"9"),(10,"10"),
+(11,"11"),(12,"12"),(13,"13"),(14,"14"),(15,"15"),
+(16,"16"),(17,"17"),(18,"18"),(19,"19"),(20,"20"),
+(21,"21"),(22,"22"),(23,"23"),(24,"24"),(25,"25"),
+(26,"26"),(27,"27"),(28,"28"),(29,"29"),(30,"30"),
+(31,"31"),(32,"32"),(33,"33"),(34,"34"),(35,"35"),
+(36,"36"),(37,"37"),(38,"38"),(39,"39"),(40,"40"),
+(41,"41"),(42,"42"),(43,"43"),(44,"44"),(45,"45"),
+(46,"46"),(47,"47"),(48,"48"),(49,"49"),(50,"50"),
+(51,"51"),(52,"52"),(53,"53"),(54,"54"),(55,"55"),
+(56,"56"),(57,"57"),(58,"58"),(59,"59"),(60,"60"),
+(61,"61"),(62,"62"),(63,"63"),(64,"64"),(65,"65"),
+(66,"66"),(67,"67"),(68,"68"),(69,"69"),(70,"70"),
+(71,"71"),(72,"72"),(73,"73"),(74,"74"),(75,"75"),
+(76,"76"),(77,"77"),(78,"78"),(79,"79"),(80,"80"),
+(81,"81"),(82,"82"),(83,"83"),(84,"84"),(85,"85"),
+(86,"86"),(87,"87"),(88,"88"),(89,"89"),(90,"90"),
+(91,"91"),(92,"92"),(93,"93"),(94,"94"),(95,"95"),
+(96,"96"),(97,"97"),(98,"98"),(99,"99");
+
+set @save_sort_buffer_size= @@sort_buffer_size;
+set sort_buffer_size= 2000;
+
+SELECT * FROM t1 ORDER BY f1 DESC, f0;
+
+set sort_buffer_size= @save_sort_buffer_size; 
+
+DROP TABLE t1;
+
+--echo End of 5.3 tests
+

sql/filesort.cc

@@ -102,6 +102,7 @@ ha_rows filesort(THD *thd, TABLE *table, SORT_FIELD *sortorder, uint s_length,
 {
   int error;
   ulong memavl, min_sort_memory;
+  ulong sort_buff_sz;
   uint maxbuffer;
   BUFFPEK *buffpek;
   ha_rows records= HA_POS_ERROR;
@@ -210,6 +211,7 @@ ha_rows filesort(THD *thd, TABLE *table, SORT_FIELD *sortorder, uint s_length,
 
   memavl= thd->variables.sortbuff_size;
   min_sort_memory= max(MIN_SORT_MEMORY, param.sort_length*MERGEBUFF2);
+  set_if_bigger(min_sort_memory, sizeof(BUFFPEK*)*MERGEBUFF2);
   if (!table_sort.sort_keys)
   {
     while (memavl >= min_sort_memory)
@@ -217,9 +219,10 @@ ha_rows filesort(THD *thd, TABLE *table, SORT_FIELD *sortorder, uint s_length,
       ulong old_memavl;
       ulong keys= memavl/(param.rec_length+sizeof(char*));
       table_sort.keys= (uint) min(records+1, keys);
+      sort_buff_sz= table_sort.keys*(param.rec_length+sizeof(char*));
+      set_if_bigger(sort_buff_sz, param.rec_length * MERGEBUFF2);   
       if ((table_sort.sort_keys=
-           (uchar**) my_malloc(table_sort.keys*(param.rec_length+sizeof(char*)),
-                               MYF(0))))
+           (uchar**) my_malloc(sort_buff_sz, MYF(0))))
         break;
       old_memavl=memavl;
       if ((memavl=memavl/4*3) < min_sort_memory &&
@@ -282,8 +285,7 @@ ha_rows filesort(THD *thd, TABLE *table, SORT_FIELD *sortorder, uint s_length,
       Use also the space previously used by string pointers in sort_buffer
       for temporary key storage.
     */
-    param.keys=((param.keys*(param.rec_length+sizeof(char*))) /
-		param.rec_length-1);
+    param.keys= sort_buff_sz / param.rec_length;
     maxbuffer--;				// Offset from 0
     if (merge_many_buff(&param,(uchar*) sort_keys,buffpek,&maxbuffer,
 			&tempfile))
@@ -1257,7 +1259,7 @@ int merge_buffers(SORTPARAM *param, IO_CACHE *from_file,
   to_start_filepos= my_b_tell(to_file);
   strpos= sort_buffer;
   org_max_rows=max_rows= param->max_rows;
-
+  
   /* The following will fire if there is not enough space in sort_buffer */
   DBUG_ASSERT(maxcount!=0);