Merge mysql.com:/home/timka/mysql/src/4.1-virgin

into mysql.com:/home/timka/mysql/src/4.1-bug-11185

mysql-test/r/range.result

@@ -556,11 +556,42 @@ count(*)
 0
 select count(*) from t1 where x > -16;
 count(*)
-1
+2
 select count(*) from t1 where x = 18446744073709551601;
 count(*)
 1
 drop table t1;
+create table t1 (a bigint unsigned);
+create index t1i on t1(a);
+insert into t1 select 18446744073709551615;
+insert into t1 select 18446744073709551614;
+explain select * from t1 where a <> -1;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	index	t1i	t1i	9	NULL	2	Using where; Using index
+select * from t1 where a <> -1;
+a
+18446744073709551614
+18446744073709551615
+explain select * from t1 where a > -1 or a < -1;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	index	t1i	t1i	9	NULL	2	Using where; Using index
+select * from t1 where a > -1 or a < -1;
+a
+18446744073709551614
+18446744073709551615
+explain select * from t1 where a > -1;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	index	t1i	t1i	9	NULL	2	Using where; Using index
+select * from t1 where a > -1;
+a
+18446744073709551614
+18446744073709551615
+explain select * from t1 where a < -1;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE noticed after reading const tables
+select * from t1 where a < -1;
+a
+drop table t1;
 set names latin1;
 create table t1 (a char(10), b text, key (a)) character set latin1;
 INSERT INTO t1 (a) VALUES

mysql-test/t/range.test

@@ -423,14 +423,30 @@ select count(*) from t1 where x=0;
 select count(*) from t1 where x<0;
 select count(*) from t1 where x < -16;
 select count(*) from t1 where x = -16;
-# The following query returns wrong value because the range optimizer can't
-# handle search on a signed value for an unsigned parameter. This will be fixed in
-# 5.0
 select count(*) from t1 where x > -16;
 select count(*) from t1 where x = 18446744073709551601;
 
 drop table t1;
 
+#
+# Bug #11185 incorrect comparison of unsigned int to signed constant
+#
+create table t1 (a bigint unsigned);
+create index t1i on t1(a);
+insert into t1 select 18446744073709551615;
+insert into t1 select 18446744073709551614;
+
+explain select * from t1 where a <> -1;
+select * from t1 where a <> -1;
+explain select * from t1 where a > -1 or a < -1;
+select * from t1 where a > -1 or a < -1;
+explain select * from t1 where a > -1;
+select * from t1 where a > -1;
+explain select * from t1 where a < -1;
+select * from t1 where a < -1;
+
+drop table t1;
+
 #
 # Bug #6045: Binary Comparison regression in MySQL 4.1
 # Binary searches didn't use a case insensitive index.

sql/opt_range.cc

@@ -960,7 +960,9 @@ get_mm_parts(PARAM *param, COND *cond_func, Field *field,
 	if (sel_arg->type == SEL_ARG::IMPOSSIBLE)
 	{
 	  tree->type=SEL_TREE::IMPOSSIBLE;
-	  DBUG_RETURN(tree);
+          /* If this is an NE_FUNC, we still need to check GT_FUNC. */
+          if (!ne_func)
+            DBUG_RETURN(tree);
 	}
       }
       else
@@ -979,8 +981,9 @@ get_mm_parts(PARAM *param, COND *cond_func, Field *field,
     SEL_TREE *tree2= get_mm_parts(param, cond_func,
 				  field, Item_func::GT_FUNC,
                                   value, cmp_type);
-    if (tree2)
-      tree= tree_or(param,tree,tree2);
+    if (!tree2)
+      DBUG_RETURN(0)
+    tree= tree_or(param,tree,tree2);
   }
   DBUG_RETURN(tree);
 }
@@ -1159,6 +1162,35 @@ get_mm_leaf(PARAM *param, COND *conf_func, Field *field, KEY_PART *key_part,
   if (!(tree=new SEL_ARG(field,str,str2)))
     DBUG_RETURN(0);		// out of memory
 
+  /*
+    Check if we are comparing an UNSIGNED integer with a negative constant.
+    In this case we know that:
+    (a) (unsigned_int [< | <=] negative_constant) == FALSE
+    (b) (unsigned_int [> | >=] negative_constant) == TRUE
+    In case (a) the condition is false for all values, and in case (b) it
+    is true for all values, so we can avoid unnecessary retrieval and condition
+    testing, and we also get correct comparison of unsinged integers with
+    negative integers (which otherwise fails because at query execution time
+    negative integers are cast to unsigned if compared with unsigned).
+   */
+  Item_result field_result_type= field->result_type();
+  Item_result value_result_type= value->result_type();
+  if (field_result_type == INT_RESULT && value_result_type == INT_RESULT &&
+      ((Field_num*)field)->unsigned_flag && !((Item_int*)value)->unsigned_flag)
+  {
+    longlong item_val= value->val_int();
+    if (item_val < 0)
+    {
+      if (type == Item_func::LT_FUNC || type == Item_func::LE_FUNC)
+      {
+        tree->type= SEL_ARG::IMPOSSIBLE;
+        DBUG_RETURN(tree);
+      }
+      if (type == Item_func::GT_FUNC || type == Item_func::GE_FUNC)
+        DBUG_RETURN(0);
+    }
+  }
+
   switch (type) {
   case Item_func::LT_FUNC:
     if (field_is_equal_to_item(field,value))