MDEV-34580 Assertion `(key_part->key_part_flag & 4) == 0' failed key_hashnr

The patch for MDEV-34417 erroneously assumed in DBUG_ASSERT() that BNLH joins
cannot be used on prefix indexes. In fact, BNHL can work with prefix indexes,
but the tests did not cover this scenario.

Fixing the asserts from "the key part does not have prefix segments" to
"the key value does not need truncation".

include/m_ctype.h

@@ -685,6 +685,10 @@ struct charset_info_st
   {
     return (cset->numchars)(this, b, e);
   }
+  size_t numchars(const char *str, size_t length) const
+  {
+    return (cset->numchars)(this, str, str + length);
+  }
 
   size_t charpos(const char *b, const char *e, size_t pos) const
   {

mysql-test/main/join_cache.result

@@ -6420,3 +6420,47 @@ DROP TABLE t1,t2,t3;
 #
 # End of 10.4 tests
 #
+#
+# Start of 10.5 tests
+#
+#
+# MDEV-34580 Assertion `(key_part->key_part_flag & 4) == 0' failed key_hashnr 
+#
+SET join_cache_level=3;
+CREATE TABLE t1 (
+a TIMESTAMP,
+b varchar(100),
+c varchar(10),
+KEY b (b(66)),
+KEY a (a));
+INSERT INTO `t1` VALUES
+('0000-00-00 00:00:00','GHOBS','EMLCG'),
+('0000-00-00 00:00:00','t','p');
+CREATE TABLE t2 (
+a varchar(100),
+b varchar(100),
+c varchar(10),
+KEY b (b(66)),
+KEY a (a)) ;
+INSERT INTO `t2` VALUES
+('a','CLUSK','FMJIK'),('d','b','folk'),('YVTAH','MSRXN','NZHIW'),
+('LOKJB','RJZAB','admission'),('ICCMG','girlfriend','JICGI'),
+('EBZEZ','p','fjzhuwxgb'),('m','KKAXM','NGJWJ'),('FHWPN','v','k'),
+('KBNOR','ALWLV','PUAIZ'),('u','d','below'),('NJCXE','always','a'),
+('e','minister','i'),('b','c','jxhojyrze'),('j','l','o'),('i','p','kikirjxho'),
+('r','y','l'),('YPUJI','p','IREVK'),('YPAMT','defender','ersfjzhuw'),
+('FLNLS','x','horror'),('q','h','difficult'),('m','z','e'),('n','cut','y'),
+('u','PLPLJ','k');
+SELECT t1.a FROM t1 JOIN t2 ON t1.b = t2.b
+WHERE t1.b > 'e' OR t1.c != t2.c GROUP BY t1.a;
+a
+INSERT INTO t2 VALUES ('a','t','p');
+SELECT t1.a FROM t1 JOIN t2 ON t1.b = t2.b
+WHERE t1.b > 'e' OR t1.c != t2.c GROUP BY t1.a;
+a
+0000-00-00 00:00:00
+DROP TABLE t1, t2;
+SET join_cache_level=DEFAULT;
+#
+# End of 10.5 tests
+#

mysql-test/main/join_cache.test

@@ -4300,3 +4300,53 @@ DROP TABLE t1,t2,t3;
 --echo #
 --echo # End of 10.4 tests
 --echo #
+
+--echo #
+--echo # Start of 10.5 tests
+--echo #
+
+--echo #
+--echo # MDEV-34580 Assertion `(key_part->key_part_flag & 4) == 0' failed key_hashnr 
+--echo #
+
+SET join_cache_level=3;
+CREATE TABLE t1 (
+  a TIMESTAMP,
+  b varchar(100),
+  c varchar(10),
+  KEY b (b(66)),
+  KEY a (a));
+INSERT INTO `t1` VALUES
+('0000-00-00 00:00:00','GHOBS','EMLCG'),
+('0000-00-00 00:00:00','t','p');
+CREATE TABLE t2 (
+  a varchar(100),
+  b varchar(100),
+  c varchar(10),
+  KEY b (b(66)),
+  KEY a (a)) ;
+INSERT INTO `t2` VALUES
+('a','CLUSK','FMJIK'),('d','b','folk'),('YVTAH','MSRXN','NZHIW'),
+('LOKJB','RJZAB','admission'),('ICCMG','girlfriend','JICGI'),
+('EBZEZ','p','fjzhuwxgb'),('m','KKAXM','NGJWJ'),('FHWPN','v','k'),
+('KBNOR','ALWLV','PUAIZ'),('u','d','below'),('NJCXE','always','a'),
+('e','minister','i'),('b','c','jxhojyrze'),('j','l','o'),('i','p','kikirjxho'),
+('r','y','l'),('YPUJI','p','IREVK'),('YPAMT','defender','ersfjzhuw'),
+('FLNLS','x','horror'),('q','h','difficult'),('m','z','e'),('n','cut','y'),
+('u','PLPLJ','k');
+
+SELECT t1.a FROM t1 JOIN t2 ON t1.b = t2.b
+WHERE t1.b > 'e' OR t1.c != t2.c GROUP BY t1.a;
+
+INSERT INTO t2 VALUES ('a','t','p');
+
+SELECT t1.a FROM t1 JOIN t2 ON t1.b = t2.b
+WHERE t1.b > 'e' OR t1.c != t2.c GROUP BY t1.a;
+
+DROP TABLE t1, t2;
+
+SET join_cache_level=DEFAULT;
+
+--echo #
+--echo # End of 10.5 tests
+--echo #

sql/key.cc

@@ -755,10 +755,12 @@ ulong key_hashnr(KEY *key_info, uint used_key_parts, const uchar *key)
     if (is_string)
     {
       /*
-        Prefix keys are not possible in BNLH joins.
-        Use the whole string to calculate the hash.
+        Let's assert that no truncation is needed,
+        so we can pass the whole string into hash_sort().
       */
-      DBUG_ASSERT((key_part->key_part_flag & HA_PART_KEY_SEG) == 0);
+      DBUG_ASSERT(length <= key_part->length);
+      DBUG_ASSERT(cs->numchars((const char *) pos + pack_length, length) <=
+                  key_part->length / cs->mbmaxlen);
       cs->hash_sort(pos+pack_length, length, &nr, &nr2);
       key+= pack_length;
     }
@@ -862,10 +864,15 @@ bool key_buf_cmp(KEY *key_info, uint used_key_parts,
     if (is_string)
     {
       /*
-        Prefix keys are not possible in BNLH joins.
-        Compare whole strings.
+        Let's assert that no truncation is needed,
+        so we can pass the whole string into strnncollsp().
       */
-      DBUG_ASSERT((key_part->key_part_flag & HA_PART_KEY_SEG) == 0);
+      DBUG_ASSERT(length1 <= key_part->length);
+      DBUG_ASSERT(length2 <= key_part->length);
+      DBUG_ASSERT(cs->numchars((const char *) pos1 + pack_length, length1) <=
+                  key_part->length / cs->mbmaxlen);
+      DBUG_ASSERT(cs->numchars((const char *) pos2 + pack_length, length2) <=
+                  key_part->length / cs->mbmaxlen);
       if (cs->strnncollsp(pos1 + pack_length, length1,
                           pos2 + pack_length, length2))
         return true;