Fixed Bug#1002564: Wrong result for a lookup query from a heap table

mysql-test/suite/heap/heap_hash.result:
  Added test case
mysql-test/suite/heap/heap_hash.test:
  Added test case
storage/heap/hp_hash.c:
  Limit key data length to max key length

mysql-test/suite/heap/heap_hash.result

@@ -427,4 +427,23 @@ INDEX(col_int_key) USING HASH) ENGINE = HEAP;
 INSERT INTO t1 (col_int_nokey, col_int_key) VALUES (3, 0), (4, 0), (3, 1);
 DELETE FROM t1 WHERE col_int_nokey = 5 ORDER BY col_int_key LIMIT 2;
 DROP TABLE t1;
+#
+# Bug #1002564: Wrong result for a lookup query from a heap table
+#
+CREATE TABLE t1 (c1 VARCHAR(10) NOT NULL, KEY i1 (c1(3))) ENGINE=MEMORY DEFAULT CHARSET=latin1;
+INSERT INTO t1 VALUES ('foo1'), ('bar2'), ('baz3');
+explain SELECT * FROM t1 WHERE c1='bar2';
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	ref	i1	i1	5	const	2	Using where
+SELECT * FROM t1 WHERE c1='bar2';
+c1
+bar2
+ALTER TABLE t1 DROP KEY i1, ADD KEY il (c1(3)) using btree;
+explain SELECT * FROM t1 WHERE c1='bar2';
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	ref	il	il	5	const	1	Using where
+SELECT * FROM t1 WHERE c1='bar2';
+c1
+bar2
+DROP TABLE t1;
 End of 5.5 tests

mysql-test/suite/heap/heap_hash.test

@@ -316,4 +316,17 @@ DELETE FROM t1 WHERE col_int_nokey = 5 ORDER BY col_int_key LIMIT 2;
 
 DROP TABLE t1;
 
+--echo #
+--echo # Bug #1002564: Wrong result for a lookup query from a heap table
+--echo #
+
+CREATE TABLE t1 (c1 VARCHAR(10) NOT NULL, KEY i1 (c1(3))) ENGINE=MEMORY DEFAULT CHARSET=latin1;
+INSERT INTO t1 VALUES ('foo1'), ('bar2'), ('baz3');
+explain SELECT * FROM t1 WHERE c1='bar2';
+SELECT * FROM t1 WHERE c1='bar2';
+ALTER TABLE t1 DROP KEY i1, ADD KEY il (c1(3)) using btree;
+explain SELECT * FROM t1 WHERE c1='bar2';
+SELECT * FROM t1 WHERE c1='bar2';
+DROP TABLE t1;
+
 --echo End of 5.5 tests

storage/heap/hp_hash.c

@@ -348,6 +348,8 @@ ulong hp_rec_hashnr(register HP_KEYDEF *keydef, register const uchar *rec)
                                 seg->length/cs->mbmaxlen);
         set_if_smaller(length, char_length);
       }
+      else
+        set_if_smaller(length, seg->length);
       cs->coll->hash_sort(cs, pos+pack_length, length, &nr, &nr2);
     }
     else
@@ -593,6 +595,11 @@ int hp_rec_key_cmp(HP_KEYDEF *keydef, const uchar *rec1, const uchar *rec2,
         char_length2= my_charpos(cs, pos2, pos2 + char_length2, char_length);
         set_if_smaller(char_length2, safe_length2);
       }
+      else
+      {
+        set_if_smaller(char_length1, seg->length);
+        set_if_smaller(char_length2, seg->length);
+      }
 
       if (cs->coll->strnncollsp(seg->charset,
                                 pos1, char_length1,
@@ -689,6 +696,8 @@ int hp_key_cmp(HP_KEYDEF *keydef, const uchar *rec, const uchar *key)
         char_length2= my_charpos(cs, pos, pos + char_length_rec, char_length2);
         set_if_smaller(char_length_rec, char_length2);
       }
+      else
+        set_if_smaller(char_length_rec, seg->length);
 
       if (cs->coll->strnncollsp(seg->charset,
                                 (uchar*) pos, char_length_rec,