filesort.cc, order_by.result:

   Fixed bug 263
order_by.test:
  Fixed bug 263

mysql-test/r/order_by.result

@@ -517,3 +517,28 @@ SELECT titre,t1.numeropost,auteur,icone,nbrep,'0',date,vue,ouvert,lastauteur,des
 titre	numeropost	auteur	icone	nbrep	0	date	vue	ouvert	lastauteur	dest
 test	1	joce	0	0	0	0000-00-00 00:00:00	0	1		bug
 drop table t1,t2;
+CREATE TABLE t1 (a int, b int);
+INSERT INTO t1 VALUES (1, 2);
+INSERT INTO t1 VALUES (3, 4);
+INSERT INTO t1 VALUES (5, NULL);
+SELECT * FROM t1 ORDER BY b;
+a	b
+5	NULL
+1	2
+3	4
+SELECT * FROM t1 ORDER BY b DESC;
+a	b
+3	4
+1	2
+5	NULL
+SELECT * FROM t1 ORDER BY (a + b);
+a	b
+5	NULL
+1	2
+3	4
+SELECT * FROM t1 ORDER BY (a + b) DESC;
+a	b
+3	4
+1	2
+5	NULL
+DROP TABLE t1;

mysql-test/t/order_by.test

@@ -331,3 +331,17 @@ INSERT INTO t2 (numeropost,pseudo) VALUES (1,'joce'),(1,'bug');
 SELECT titre,t1.numeropost,auteur,icone,nbrep,0,date,vue,ouvert,lastauteur,dest FROM t2 LEFT JOIN t1 USING(numeropost) WHERE t2.pseudo='joce' ORDER BY date DESC LIMIT 0,30;
 SELECT titre,t1.numeropost,auteur,icone,nbrep,'0',date,vue,ouvert,lastauteur,dest FROM t2 LEFT JOIN t1 USING(numeropost) WHERE t2.pseudo='joce' ORDER BY date DESC LIMIT 0,30;
 drop table t1,t2;
+
+#
+# Test order by with NULL values
+#
+CREATE TABLE t1 (a int, b int);
+INSERT INTO t1 VALUES (1, 2);
+INSERT INTO t1 VALUES (3, 4);
+INSERT INTO t1 VALUES (5, NULL);
+SELECT * FROM t1 ORDER BY b;
+SELECT * FROM t1 ORDER BY b DESC;
+SELECT * FROM t1 ORDER BY (a + b);
+SELECT * FROM t1 ORDER BY (a + b) DESC;
+DROP TABLE t1;
+

sql/filesort.cc

@@ -456,6 +456,7 @@ static void make_sortkey(register SORTPARAM *param,
        sort_field != param->end ;
        sort_field++)
   {
+    bool maybe_null=0;
     if ((field=sort_field->field))
     {						// Field
       if (field->maybe_null())
@@ -480,7 +481,7 @@ static void make_sortkey(register SORTPARAM *param,
       switch (sort_field->result_type) {
       case STRING_RESULT:
 	{
-	  if (item->maybe_null)
+	  if ((maybe_null=item->maybe_null))
 	    *to++=1;
 	  /* All item->str() to use some extra byte for end null.. */
 	  String tmp((char*) to,sort_field->length+4);
@@ -546,7 +547,7 @@ static void make_sortkey(register SORTPARAM *param,
       case INT_RESULT:
 	{
 	  longlong value=item->val_int();
-	  if (item->maybe_null)
+	  if ((maybe_null=item->maybe_null))
 	    *to++=1;				/* purecov: inspected */
 	  if (item->null_value)
 	  {
@@ -580,13 +581,13 @@ static void make_sortkey(register SORTPARAM *param,
       case REAL_RESULT:
 	{
 	  double value=item->val();
-	  if (item->null_value)
+	  if ((maybe_null=item->null_value))
 	  {
 	    bzero((char*) to,sort_field->length+1);
 	    to++;
 	    break;
 	  }
-	  if (item->maybe_null)
+	  if ((maybe_null=item->maybe_null))
 	    *to++=1;
 	  change_double_for_sort(value,(byte*) to);
 	  break;
@@ -595,6 +596,8 @@ static void make_sortkey(register SORTPARAM *param,
     }
     if (sort_field->reverse)
     {							/* Revers key */
+      if (maybe_null)
+        to[-1]= ~to[-1];
       length=sort_field->length;
       while (length--)
       {