Check index column sizes in a better way (bug #13315).

handler/ha_innodb.cc

@@ -2517,6 +2517,12 @@ ha_innobase::open(
   	DBUG_RETURN(0);
 }
 
+uint
+ha_innobase::max_supported_key_part_length() const
+{
+	return(DICT_MAX_INDEX_COL_LEN - 1);
+}
+
 /**********************************************************************
 Closes a handle to an InnoDB table. */
 
@@ -4674,6 +4680,9 @@ create_index(
 				0, prefix_len);
 	}
 
+	/* Even though we've defined max_supported_key_part_length, we
+	still do our own checking using field_lengths to be absolutely
+	sure we don't create too long indexes. */
 	error = row_create_index_for_mysql(index, trx, field_lengths);
 
 	error = convert_error_code_to_mysql(error, NULL);

handler/ha_innodb.h

@@ -110,7 +110,7 @@ class ha_innobase: public handler
 				but currently MySQL does not work with keys
 				whose size is > MAX_KEY_LENGTH */
   	uint max_supported_key_length() const { return 3500; }
-  	uint max_supported_key_part_length() const { return 3500; }
+  	uint max_supported_key_part_length() const;
 	const key_map *keys_to_use_for_scanning() { return &key_map_full; }
   	bool has_transactions()  { return 1;}
 

mysql-test/innodb.result

@@ -2437,7 +2437,9 @@ a	b
 20	NULL
 drop table t1;
 create table t1 (v varchar(65530), key(v));
-ERROR HY000: Can't create table './test/t1' (errno: 139)
+Warnings:
+Warning	1071	Specified key was too long; max key length is 767 bytes
+drop table t1;
 create table t1 (v varchar(65536));
 Warnings:
 Note	1246	Converting column 'v' from VARCHAR to TEXT
@@ -2577,22 +2579,49 @@ create table t8 (col1 blob, index(col1(767)))
 character set = latin1 engine = innodb;
 create table t9 (col1 varchar(512), col2 varchar(512), index(col1, col2))
 character set = latin1 engine = innodb;
+show create table t9;
+Table	Create Table
+t9	CREATE TABLE `t9` (
+  `col1` varchar(512) default NULL,
+  `col2` varchar(512) default NULL,
+  KEY `col1` (`col1`,`col2`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1
 drop table t1, t2, t3, t4, t5, t6, t7, t8, t9;
-create table t1 (col1 varchar(768), index (col1))
+create table t1 (col1 varchar(768), index(col1))
+character set = latin1 engine = innodb;
+Warnings:
+Warning	1071	Specified key was too long; max key length is 767 bytes
+create table t2 (col1 varbinary(768), index(col1))
+character set = latin1 engine = innodb;
+Warnings:
+Warning	1071	Specified key was too long; max key length is 767 bytes
+create table t3 (col1 text, index(col1(768)))
 character set = latin1 engine = innodb;
-ERROR HY000: Can't create table './test/t1.frm' (errno: 139)
-create table t2 (col1 varchar(768) primary key)
+Warnings:
+Warning	1071	Specified key was too long; max key length is 767 bytes
+create table t4 (col1 blob, index(col1(768)))
+character set = latin1 engine = innodb;
+Warnings:
+Warning	1071	Specified key was too long; max key length is 767 bytes
+show create table t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `col1` varchar(768) default NULL,
+  KEY `col1` (`col1`(767))
+) ENGINE=InnoDB DEFAULT CHARSET=latin1
+drop table t1, t2, t3, t4;
+create table t1 (col1 varchar(768) primary key)
 character set = latin1 engine = innodb;
-ERROR HY000: Can't create table './test/t2.frm' (errno: 139)
-create table t3 (col1 varbinary(768) primary key)
+ERROR 42000: Specified key was too long; max key length is 767 bytes
+create table t2 (col1 varbinary(768) primary key)
 character set = latin1 engine = innodb;
-ERROR HY000: Can't create table './test/t3.frm' (errno: 139)
-create table t4 (col1 text, index(col1(768)))
+ERROR 42000: Specified key was too long; max key length is 767 bytes
+create table t3 (col1 text, primary key(col1(768)))
 character set = latin1 engine = innodb;
-ERROR HY000: Can't create table './test/t4.frm' (errno: 139)
-create table t5 (col1 blob, index(col1(768)))
+ERROR 42000: Specified key was too long; max key length is 767 bytes
+create table t4 (col1 blob, primary key(col1(768)))
 character set = latin1 engine = innodb;
-ERROR HY000: Can't create table './test/t5.frm' (errno: 139)
+ERROR 42000: Specified key was too long; max key length is 767 bytes
 CREATE TABLE t1
 (
 id INT PRIMARY KEY

mysql-test/innodb.test

@@ -1356,8 +1356,8 @@ source include/varchar.inc;
 # Clean up filename -- embedded server reports whole path without .frm,
 # regular server reports relative path with .frm (argh!)
 --replace_result \\ / $MYSQL_TEST_DIR . /var/master-data/ / t1.frm t1
---error 1005
 create table t1 (v varchar(65530), key(v));
+drop table t1;
 create table t1 (v varchar(65536));
 show create table t1;
 drop table t1;
@@ -1485,7 +1485,7 @@ CREATE TEMPORARY TABLE t2
 DROP TABLE t1;
 
 #
-# Test that index column max sizes are checked (bug #13315)
+# Test that index column max sizes are honored (bug #13315)
 #
 
 # prefix index
@@ -1512,22 +1512,36 @@ create table t8 (col1 blob, index(col1(767)))
 create table t9 (col1 varchar(512), col2 varchar(512), index(col1, col2))
  character set = latin1 engine = innodb;
 
+show create table t9;
+
 drop table t1, t2, t3, t4, t5, t6, t7, t8, t9;
 
---error 1005
-create table t1 (col1 varchar(768), index (col1))
+# these should have their index length trimmed
+create table t1 (col1 varchar(768), index(col1))
  character set = latin1 engine = innodb;
---error 1005
-create table t2 (col1 varchar(768) primary key)
+create table t2 (col1 varbinary(768), index(col1))
  character set = latin1 engine = innodb;
---error 1005
-create table t3 (col1 varbinary(768) primary key)
+create table t3 (col1 text, index(col1(768)))
  character set = latin1 engine = innodb;
---error 1005
-create table t4 (col1 text, index(col1(768)))
+create table t4 (col1 blob, index(col1(768)))
  character set = latin1 engine = innodb;
---error 1005
-create table t5 (col1 blob, index(col1(768)))
+
+show create table t1;
+
+drop table t1, t2, t3, t4;
+
+# these should be refused
+--error 1071
+create table t1 (col1 varchar(768) primary key)
+ character set = latin1 engine = innodb;
+--error 1071
+create table t2 (col1 varbinary(768) primary key)
+ character set = latin1 engine = innodb;
+--error 1071
+create table t3 (col1 text, primary key(col1(768)))
+ character set = latin1 engine = innodb;
+--error 1071
+create table t4 (col1 blob, primary key(col1(768)))
  character set = latin1 engine = innodb;
 
 #