Merge fix for BUG51877 to mysql-5.1-bugteam.

mysql-test/r/gis-rtree.result

@@ -1540,5 +1540,12 @@ a
 HANDLER t1 READ a LAST;
 a
 HANDLER t1 CLOSE;
+HANDLER t1 OPEN;
+HANDLER t1 READ a FIRST;
+a
+INSERT INTO t1 VALUES (GeomFromText('Polygon((40 40,60 40,60 60,40 60,40 40))'));
+# should not crash
+HANDLER t1 READ a NEXT;
+HANDLER t1 CLOSE;
 DROP TABLE t1;
 End of 5.0 tests.

mysql-test/r/handler_myisam.result

@@ -756,4 +756,17 @@ TRUNCATE t1;
 HANDLER t1 READ FIRST;
 ERROR 42S02: Unknown table 't1' in HANDLER
 DROP TABLE t1;
+#
+# BUG#51877 - HANDLER interface causes invalid memory read
+#
+CREATE TABLE t1(a INT, KEY(a));
+HANDLER t1 OPEN;
+HANDLER t1 READ a FIRST;
+a
+INSERT INTO t1 VALUES(1);
+HANDLER t1 READ a NEXT;
+a
+1
+HANDLER t1 CLOSE;
+DROP TABLE t1;
 End of 5.1 tests

mysql-test/t/gis-rtree.test

@@ -914,14 +914,15 @@ HANDLER t1 READ a PREV;
 HANDLER t1 READ a LAST;
 HANDLER t1 CLOSE;
 
-#TODO: re-enable this test please when bug #51877 is solved
 # second crash fixed when the tree has changed since the last search.
-#HANDLER t1 OPEN;
-#HANDLER t1 READ a FIRST;
-#INSERT INTO t1 VALUES (GeomFromText('Polygon((40 40,60 40,60 60,40 60,40 40))'));
-#HANDLER t1 READ a NEXT;
-#HANDLER t1 CLOSE;
-#TODO: end of the 51877 dependent section
+HANDLER t1 OPEN;
+HANDLER t1 READ a FIRST;
+INSERT INTO t1 VALUES (GeomFromText('Polygon((40 40,60 40,60 60,40 60,40 40))'));
+--echo # should not crash
+--disable_result_log
+HANDLER t1 READ a NEXT;
+--enable_result_log
+HANDLER t1 CLOSE;
 
 DROP TABLE t1;
 

mysql-test/t/handler_myisam.test

@@ -37,4 +37,15 @@ TRUNCATE t1;
 HANDLER t1 READ FIRST;
 DROP TABLE t1;
 
+--echo #
+--echo # BUG#51877 - HANDLER interface causes invalid memory read
+--echo #
+CREATE TABLE t1(a INT, KEY(a));
+HANDLER t1 OPEN;
+HANDLER t1 READ a FIRST;
+INSERT INTO t1 VALUES(1);
+HANDLER t1 READ a NEXT;
+HANDLER t1 CLOSE;
+DROP TABLE t1;
+
 --echo End of 5.1 tests

storage/myisam/mi_rnext.c

@@ -28,6 +28,7 @@ int mi_rnext(MI_INFO *info, uchar *buf, int inx)
 {
   int error,changed;
   uint flag;
+  uint update_mask= HA_STATE_NEXT_FOUND;
   DBUG_ENTER("mi_rnext");
 
   if ((inx = _mi_check_index(info,inx)) < 0)
@@ -55,6 +56,20 @@ int mi_rnext(MI_INFO *info, uchar *buf, int inx)
 			   info->s->state.key_root[inx]);
       break;
     }
+    /*
+      "search first" failed. This means we have no pivot for
+      "search next", or in other words MI_INFO::lastkey is
+      likely uninitialized.
+
+      Normally SQL layer would never request "search next" if
+      "search first" failed. But HANDLER may do anything.
+
+      As mi_rnext() without preceeding mi_rkey()/mi_rfirst()
+      equals to mi_rfirst(), we must restore original state
+      as if failing mi_rfirst() was not called.
+    */
+    if (error)
+      update_mask|= HA_STATE_PREV_FOUND;
   }
   else
   {
@@ -100,7 +115,7 @@ int mi_rnext(MI_INFO *info, uchar *buf, int inx)
   }
 	/* Don't clear if database-changed */
   info->update&= (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
-  info->update|= HA_STATE_NEXT_FOUND;
+  info->update|= update_mask;
 
   if (error)
   {