MDEV-19634: Assertion `0' failed in row_sel_convert_mysql_key_to_innobase,...

MDEV-19634: Assertion `0' failed in row_sel_convert_mysql_key_to_innobase, [Warning] InnoDB: Using a partial-field key prefix in search

For a key with keyparts (k1,k2,k3) , if we are building a range over the keyparts
we should make sure that if min_value/max_value for a keypart is not added to
key buffer then the keyparts following should also not be allowed.

mysql-test/main/range_innodb.result

@@ -80,3 +80,21 @@ ERROR HY000: Table definition has changed, please retry transaction
 DROP TABLE t0,t1;
 set @@global.debug_dbug="-d";
 set @@optimizer_switch= @optimizer_switch_save;
+#
+# MDEV-19634: Assertion `0' failed in row_sel_convert_mysql_key_to_innobase,
+# [Warning] InnoDB: Using a partial-field key prefix in search
+#
+CREATE TABLE t1 (
+pk INT,
+a VARCHAR(1),
+b INT,
+PRIMARY KEY (pk),
+KEY (a,b)
+) ENGINE=InnoDB;
+INSERT INTO t1 VALUES (1,'a',1),(2,'b',2);
+explain SELECT a FROM t1 WHERE pk < 0 AND a <= 'w' and b > 0;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	index	PRIMARY,a	a	9	NULL	2	Using where; Using index
+SELECT a FROM t1 WHERE pk < 0 AND a <= 'w' and b > 0;
+a
+drop table t1;

mysql-test/main/range_innodb.test

@@ -87,3 +87,22 @@ select * from t1 where a=10 and b=10;
 DROP TABLE t0,t1;
 set @@global.debug_dbug="-d";
 set @@optimizer_switch= @optimizer_switch_save;
+
+--echo #
+--echo # MDEV-19634: Assertion `0' failed in row_sel_convert_mysql_key_to_innobase,
+--echo # [Warning] InnoDB: Using a partial-field key prefix in search
+--echo #
+
+CREATE TABLE t1 (
+    pk INT,
+    a VARCHAR(1),
+    b INT,
+    PRIMARY KEY (pk),
+    KEY (a,b)
+) ENGINE=InnoDB;
+
+INSERT INTO t1 VALUES (1,'a',1),(2,'b',2);
+
+explain SELECT a FROM t1 WHERE pk < 0 AND a <= 'w' and b > 0;
+SELECT a FROM t1 WHERE pk < 0 AND a <= 'w' and b > 0;
+drop table t1;

sql/opt_range.h

@@ -459,7 +459,8 @@ class SEL_ARG :public Sql_alloc
     uint res= key_tree->store_min(key[key_tree->part].store_length,
                                   range_key, *range_key_flag);
     // add flags only if a key_part is written to the buffer
-    if (res)
+    if (!res)
+      return 0;
     *range_key_flag|= key_tree->min_flag;
     if (key_tree->next_key_part &&
 	key_tree->next_key_part->type == SEL_ARG::KEY_RANGE &&
@@ -482,8 +483,9 @@ class SEL_ARG :public Sql_alloc
     SEL_ARG *key_tree= last();
     uint res=key_tree->store_max(key[key_tree->part].store_length,
                                  range_key, *range_key_flag);
-    if (res)
-      (*range_key_flag)|= key_tree->max_flag;
+    if (!res)
+      return 0;
+    *range_key_flag|= key_tree->max_flag;
     if (key_tree->next_key_part &&
 	key_tree->next_key_part->type == SEL_ARG::KEY_RANGE &&
         key_tree->part != last_part &&