MDEV-18039 Assertion failed in btr_node_ptr_max_size for VARCHAR(0)

btr_node_ptr_max_size(): Do not reserve extra space for indexed VARCHAR(0)
columns.

mysql-test/suite/innodb/r/data_types.result

@@ -77,7 +77,9 @@ t1_VARCHAR_500_BINARY VARCHAR(500) BINARY,
 t1_YEAR_2 YEAR(2),
 t1_YEAR_4 YEAR(4),
 t1_CHAR_0 CHAR(0),
-t1_MYSQL_0 CHAR(0) CHARACTER SET utf8
+t1_MYSQL_0 CHAR(0) CHARACTER SET utf8,
+t1_VARCHAR_0 VARCHAR(0),
+t1_VARMYSQL_0 VARCHAR(0) CHARACTER SET utf8
 ) ENGINE=InnoDB;
 Warnings:
 Note	1287	'YEAR(2)' is deprecated and will be removed in a future release. Please use YEAR(4) instead
@@ -151,10 +153,12 @@ t1_TINYINT	DATA_INT
 t1_TINYINT_UNSIGNED	DATA_INT	UNSIGNED
 t1_TINYTEXT	DATA_BLOB	
 t1_VARBINARY_100	DATA_BINARY	
+t1_VARCHAR_0	DATA_VARCHAR	
 t1_VARCHAR_10	DATA_VARCHAR	
 t1_VARCHAR_10_BINARY	DATA_VARMYSQL	
 t1_VARCHAR_500	DATA_VARCHAR	
 t1_VARCHAR_500_BINARY	DATA_VARMYSQL	
+t1_VARMYSQL_0	DATA_VARMYSQL	
 t1_YEAR_2	DATA_INT	UNSIGNED
 t1_YEAR_4	DATA_INT	UNSIGNED
 DROP TABLE t1;
@@ -164,3 +168,9 @@ DROP TABLE t1;
 CREATE TABLE t1 (c CHAR(0), KEY(c)) ENGINE=InnoDB;
 INSERT INTO t1 VALUES ('');
 DROP TABLE t1;
+#
+# MDEV-18039 Assertion failed in btr_node_ptr_max_size for VARCHAR(0)
+#
+CREATE TABLE t1 (c VARCHAR(0), KEY(c)) ENGINE=InnoDB;
+INSERT INTO t1 VALUES ('');
+DROP TABLE t1;

mysql-test/suite/innodb/t/data_types.test

@@ -90,7 +90,9 @@ CREATE TABLE t1
   t1_YEAR_2 YEAR(2),
   t1_YEAR_4 YEAR(4),
   t1_CHAR_0 CHAR(0),
-  t1_MYSQL_0 CHAR(0) CHARACTER SET utf8
+  t1_MYSQL_0 CHAR(0) CHARACTER SET utf8,
+  t1_VARCHAR_0 VARCHAR(0),
+  t1_VARMYSQL_0 VARCHAR(0) CHARACTER SET utf8
 ) ENGINE=InnoDB;
 
 INSERT INTO t1 () VALUES ();
@@ -127,3 +129,10 @@ DROP TABLE t1;
 CREATE TABLE t1 (c CHAR(0), KEY(c)) ENGINE=InnoDB;
 INSERT INTO t1 VALUES ('');
 DROP TABLE t1;
+
+--echo #
+--echo # MDEV-18039 Assertion failed in btr_node_ptr_max_size for VARCHAR(0)
+--echo #
+CREATE TABLE t1 (c VARCHAR(0), KEY(c)) ENGINE=InnoDB;
+INSERT INTO t1 VALUES ('');
+DROP TABLE t1;

storage/innobase/btr/btr0cur.cc

@@ -754,9 +754,20 @@ static ulint btr_node_ptr_max_size(const dict_index_t* index)
 		field_max_size = dict_col_get_max_size(col);
 		if (UNIV_UNLIKELY(!field_max_size)) {
 			switch (col->mtype) {
+			case DATA_VARCHAR:
+				if (!comp
+				    && (!strcmp(index->table->name.m_name,
+						"SYS_FOREIGN")
+					|| !strcmp(index->table->name.m_name,
+						   "SYS_FOREIGN_COLS"))) {
+					break;
+				}
+				/* fall through */
+			case DATA_VARMYSQL:
 			case DATA_CHAR:
 			case DATA_MYSQL:
-				/* CHAR(0) is a possible data type.
+				/* CHAR(0) and VARCHAR(0) are possible
+				data type definitions in MariaDB.
 				The InnoDB internal SQL parser maps
 				CHAR to DATA_VARCHAR, so DATA_CHAR (or
 				DATA_MYSQL) is only coming from the
@@ -772,6 +783,7 @@ static ulint btr_node_ptr_max_size(const dict_index_t* index)
 				}
 				continue;
 			}
+
 			/* SYS_FOREIGN.ID is defined as CHAR in the
 			InnoDB internal SQL parser, which translates
 			into the incorrect VARCHAR(0).  InnoDB does
@@ -788,6 +800,7 @@ static ulint btr_node_ptr_max_size(const dict_index_t* index)
 			      || !strcmp(index->table->name.m_name,
 					 "SYS_FOREIGN_COLS"));
 			ut_ad(!comp);
+			ut_ad(col->mtype == DATA_VARCHAR);
 
 			rec_max_size += (srv_page_size == UNIV_PAGE_SIZE_MAX)
 				? REDUNDANT_REC_MAX_DATA_SIZE