func_group.test, func_group.result:

  Added test case for bug #5406.
opt_sum.cc:
  Fixed bug #5406.

mysql-test/r/func_group.result

@@ -684,3 +684,12 @@ max(a)
 2
 deallocate prepare stmt1;
 drop table t1;
+CREATE TABLE t1 (a int primary key);
+INSERT INTO t1 VALUES (1),(2),(3),(4);
+SELECT MAX(a) FROM t1 WHERE a > 5;
+MAX(a)
+NULL
+SELECT MIN(a) FROM t1 WHERE a < 0;
+MIN(a)
+NULL
+DROP TABLE t1;

mysql-test/t/func_group.test

@@ -418,3 +418,19 @@ execute stmt1;
 execute stmt1;
 deallocate prepare stmt1;
 drop table t1;
+
+#
+# Bug #5406 min/max optimization for empty set
+#
+
+CREATE TABLE t1 (a int primary key);
+INSERT INTO t1 VALUES (1),(2),(3),(4);
+
+SELECT MAX(a) FROM t1 WHERE a > 5;
+SELECT MIN(a) FROM t1 WHERE a < 0;
+
+DROP TABLE t1;
+
+
+
+

sql/opt_sum.cc

@@ -186,16 +186,15 @@ int opt_sum_query(TABLE_LIST *tables, List<Item> &all_fields,COND *conds)
           if (!ref.key_length)
             error= table->file->index_first(table->record[0]);
           else
-	  {
 	    error= table->file->index_read(table->record[0],key_buff,
 					   ref.key_length,
 					   range_fl & NEAR_MIN ?
 					   HA_READ_AFTER_KEY :
 					   HA_READ_KEY_OR_NEXT);
-	    if (!error && reckey_in_range(0, &ref, item_field->field, 
+	  if ((!error || error == HA_ERR_KEY_NOT_FOUND) &&
+               reckey_in_range(0, &ref, item_field->field, 
 			       conds, range_fl, prefix_len))
 	    error= HA_ERR_KEY_NOT_FOUND;
-	  }
           if (table->key_read)
           {
             table->key_read= 0;
@@ -260,16 +259,15 @@ int opt_sum_query(TABLE_LIST *tables, List<Item> &all_fields,COND *conds)
           if (!ref.key_length)
             error= table->file->index_last(table->record[0]);
           else
-	  {
 	    error= table->file->index_read(table->record[0], key_buff,
 					   ref.key_length,
 					   range_fl & NEAR_MAX ?
 					   HA_READ_BEFORE_KEY :
 					   HA_READ_PREFIX_LAST_OR_PREV);
-	    if (!error && reckey_in_range(1, &ref, item_field->field, 
+	  if ((!error || error == HA_ERR_KEY_NOT_FOUND) &&
+               reckey_in_range(1, &ref, item_field->field, 
 			       conds, range_fl, prefix_len))
 	    error= HA_ERR_KEY_NOT_FOUND;
-	  }
           if (table->key_read)
           {
             table->key_read=0;