Fix Bug#32125 (http://bugs.mysql.com/32125)

"Database crash due to ha_innodb.cc:3896: ulint convert_search_mode_to_innobase":

When unknown find_flag is encountered in convert_search_mode_to_innobase()
do not call assert(0); instead queue a MySQL error using my_error() and
return the error code PAGE_CUR_UNSUPP. Change the functions that call
convert_search_mode_to_innobase() to handle that error code by "canceling"
execution and returning appropriate error code further upstream.

Approved by:	Heikki (via IM)

handler/ha_innodb.cc

@@ -685,6 +685,9 @@ convert_error_code_to_mysql(
 		return(HA_ERR_RECORD_FILE_FULL);
 #endif
 
+	} else if (error == DB_UNSUPPORTED) {
+
+		return(HA_ERR_UNSUPPORTED);
     	} else {
     		return(-1);			// Unknown error
     	}
@@ -3995,11 +3998,21 @@ convert_search_mode_to_innobase(
 		  and comparison of non-latin1 char type fields in
 		  innobase_mysql_cmp() to get PAGE_CUR_LE_OR_EXTENDS to
 		  work correctly. */
-
-		default:			assert(0);
+		case HA_READ_MBR_CONTAIN:
+		case HA_READ_MBR_INTERSECT:
+		case HA_READ_MBR_WITHIN:
+		case HA_READ_MBR_DISJOINT:
+			my_error(ER_TABLE_CANT_HANDLE_SPKEYS, MYF(0));
+			return(PAGE_CUR_UNSUPP);
+		/* do not use "default:" in order to produce a gcc warning:
+		enumeration value '...' not handled in switch
+		(if -Wswitch or -Wall is used)
+		*/
 	}
 
-	return(0);
+	my_error(ER_CHECK_NOT_IMPLEMENTED, MYF(0), "this functionality");
+
+	return(PAGE_CUR_UNSUPP);
 }
 
 /*
@@ -4127,11 +4140,18 @@ ha_innobase::index_read(
 
 	last_match_mode = (uint) match_mode;
 
+	if (mode != PAGE_CUR_UNSUPP) {
+
 		innodb_srv_conc_enter_innodb(prebuilt->trx);
 
-	ret = row_search_for_mysql((byte*) buf, mode, prebuilt, match_mode, 0);
+		ret = row_search_for_mysql((byte*) buf, mode, prebuilt,
+					   match_mode, 0);
 
 		innodb_srv_conc_exit_innodb(prebuilt->trx);
+	} else {
+
+		ret = DB_UNSUPPORTED;
+	}
 
 	if (ret == DB_SUCCESS) {
 		error = 0;
@@ -5481,8 +5501,16 @@ ha_innobase::records_in_range(
 	mode2 = convert_search_mode_to_innobase(max_key ? max_key->flag :
 						HA_READ_KEY_EXACT);
 
+	if (mode1 != PAGE_CUR_UNSUPP && mode2 != PAGE_CUR_UNSUPP) {
+
 		n_rows = btr_estimate_n_rows_in_range(index, range_start,
-						mode1, range_end, mode2);
+						      mode1, range_end,
+						      mode2);
+	} else {
+
+		n_rows = 0;
+	}
+
 	dtuple_free_for_mysql(heap1);
 	dtuple_free_for_mysql(heap2);
 

include/page0cur.h

@@ -22,6 +22,7 @@ Created 10/4/1994 Heikki Tuuri
 
 /* Page cursor search modes; the values must be in this order! */
 
+#define	PAGE_CUR_UNSUPP	0
 #define	PAGE_CUR_G	1
 #define	PAGE_CUR_GE	2
 #define	PAGE_CUR_L	3