Fix bug #10802 Index is not used if table using BDB engine on HP-UX

HP-UX compiler was generating code which makes wrong conversion from double to
ulonglong which results in not using index in some cases.

Changed type cast operators odrer with which generated code makes proper 
conversion.


sql/ha_berkeley.cc:
  Fix bug #10802 Index is not used if table using BDB engine on HP-UX
mysql-test/include/varchar.inc:
  Test case for bug #10802 "Index is not used if table using BDB engine on HP-UX" is same as for bug #9489

mysql-test/include/varchar.inc

@@ -229,6 +229,7 @@ drop table t1;
 
 #
 # Bug #9489: problem with hash indexes
+# Bug #10802: Index is not used if table using BDB engine on HP-UX
 #
 
 create table t1(a int, b varchar(12), key ba(b, a));

sql/ha_berkeley.cc

@@ -2150,7 +2150,7 @@ ha_rows ha_berkeley::records_in_range(uint keynr, key_range *start_key,
     end_pos=end_range.less+end_range.equal;
   rows=(end_pos-start_pos)*records;
   DBUG_PRINT("exit",("rows: %g",rows));
-  DBUG_RETURN(rows <= 1.0 ? (ha_rows) 1 : (ha_rows) rows);
+  DBUG_RETURN((ha_rows)(rows <= 1.0 ? 1 : rows));
 }