Fixed bug mdev-5721.

Do not define a look-up key for a temporary table if its length
exceeds the maximum length of such keys. 

include/myisam.h

@@ -48,6 +48,12 @@ extern "C" {
 #endif
 
 #define MI_MAX_POSSIBLE_KEY_BUFF    HA_MAX_POSSIBLE_KEY_BUFF
+/*
+  The following defines can be increased if necessary.
+  But beware the dependency of MI_MAX_POSSIBLE_KEY_BUFF and MI_MAX_KEY_LENGTH.
+*/
+#define MI_MAX_KEY_LENGTH           1000            /* Max length in bytes */
+#define MI_MAX_KEY_SEG              16              /* Max segments for key */
 
 #define MI_NAME_IEXT	".MYI"
 #define MI_NAME_DEXT	".MYD"

mysql-test/r/derived_view.result

@@ -2396,6 +2396,94 @@ deallocate prepare stmt;
 drop table t1,t2;
 set optimizer_switch=@save_optimizer_switch5740;
 #
+# Bug mdev-5721: possible long key access to a materialized derived table
+# (see also the test case for Bug#13261277 that is actually the same bug)
+#
+CREATE TABLE t1 (
+id varchar(255) NOT NULL DEFAULT '',
+familyid int(11) DEFAULT NULL,
+withdrawndate date DEFAULT NULL,
+KEY index_td_familyid_id (familyid,id)
+) ENGINE=MyISAM DEFAULT CHARSET=utf8;
+CREATE TABLE t2 (
+id int(11) NOT NULL AUTO_INCREMENT,
+activefromts datetime NOT NULL DEFAULT '0000-00-00 00:00:00',
+shortdescription text,
+useraccessfamily varchar(512) DEFAULT NULL,
+serialized longtext,
+PRIMARY KEY (id)
+) ENGINE=MyISAM DEFAULT CHARSET=utf8;
+insert into t1 values ('picture/89/1369722032695.pmd',89,NULL);
+insert into t1 values ('picture/90/1369832057370.pmd',90,NULL);
+insert into t2 values (38,'2013-03-04 07:49:22','desc','CODE','string');
+EXPLAIN 
+SELECT * FROM t2 x,
+(SELECT t2.useraccessfamily, t2.serialized AS picturesubuser, COUNT(*)
+FROM t2, t1 GROUP BY t2.useraccessfamily, picturesubuser) y
+WHERE x.useraccessfamily = y.useraccessfamily;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	x	system	NULL	NULL	NULL	NULL	1	
+1	PRIMARY	<derived2>	ALL	NULL	NULL	NULL	NULL	2	Using where
+2	DERIVED	t2	system	NULL	NULL	NULL	NULL	1	
+2	DERIVED	t1	index	NULL	index_td_familyid_id	772	NULL	2	Using index
+SELECT * FROM t2 x,
+(SELECT t2.useraccessfamily, t2.serialized AS picturesubuser, COUNT(*)
+FROM t2, t1 GROUP BY t2.useraccessfamily, picturesubuser) y
+WHERE x.useraccessfamily = y.useraccessfamily;
+id	activefromts	shortdescription	useraccessfamily	serialized	useraccessfamily	picturesubuser	COUNT(*)
+38	2013-03-04 07:49:22	desc	CODE	string	CODE	string	2
+DROP TABLE t1,t2;
+#
+# Bug#13261277: Unchecked key length caused missing records.
+#
+CREATE TABLE t1 (  
+col_varchar varchar(1024) CHARACTER SET utf8 DEFAULT NULL,
+stub1 varchar(1024) CHARACTER SET utf8 DEFAULT NULL,
+stub2 varchar(1024) CHARACTER SET utf8 DEFAULT NULL,
+stub3 varchar(1024) CHARACTER SET utf8 DEFAULT NULL
+);
+INSERT INTO t1 VALUES
+('d','d','l','ther'),
+(NULL,'s','NJBIQ','trzetuchv'),
+(-715390976,'coul','MYWFB','cfhtrzetu'),
+(1696792576,'f','i\'s','c'),
+  (1,'i','ltpemcfhtr','gsltpemcf'),
+  (-663027712,'mgsltpemcf','sa','amgsltpem'),
+  (-1686700032,'JPRVK','i','vamgsltpe'),
+  (NULL,'STUNB','UNVJV','u'),
+  (5,'oka','qyihvamgsl','AXSMD'),
+  (NULL,'tqwmqyihva','h','yntqwmqyi'),
+  (3,'EGMJN','e','e');
+CREATE TABLE t2 (
+col_varchar varchar(10) DEFAULT NULL,
+col_int INT DEFAULT NULL
+);
+INSERT INTO t2 VALUES ('d',9);
+set optimizer_switch='derived_merge=off,derived_with_keys=on';
+SET @save_heap_size= @@max_heap_table_size;
+SET @@max_heap_table_size= 16384;
+SELECT t2.col_int
+FROM t2
+RIGHT JOIN ( SELECT * FROM t1 ) AS dt 
+ON t2.col_varchar = dt.col_varchar
+WHERE t2.col_int IS NOT NULL ;
+col_int
+9
+# Shouldn't use auto_key0 for derived table
+EXPLAIN 
+SELECT t2.col_int
+FROM t2
+RIGHT JOIN ( SELECT * FROM t1 ) AS dt
+ON t2.col_varchar = dt.col_varchar
+WHERE t2.col_int IS NOT NULL ;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	t2	system	NULL	NULL	NULL	NULL	1	
+1	PRIMARY	<derived2>	ALL	NULL	NULL	NULL	NULL	11	Using where
+2	DERIVED	t1	ALL	NULL	NULL	NULL	NULL	11	
+SET @@max_heap_table_size= @save_heap_size;
+SET optimizer_switch=@save_optimizer_switch;
+DROP TABLE t1,t2;
+#
 # end of 5.3 tests
 #
 set optimizer_switch=@exit_optimizer_switch;

mysql-test/t/derived_view.test

@@ -1730,6 +1730,99 @@ deallocate prepare stmt;
 drop table t1,t2;
 set optimizer_switch=@save_optimizer_switch5740;
 
+--echo #
+--echo # Bug mdev-5721: possible long key access to a materialized derived table
+--echo # (see also the test case for Bug#13261277 that is actually the same bug)
+--echo #
+
+CREATE TABLE t1 (
+  id varchar(255) NOT NULL DEFAULT '',
+  familyid int(11) DEFAULT NULL,
+  withdrawndate date DEFAULT NULL,
+  KEY index_td_familyid_id (familyid,id)
+) ENGINE=MyISAM DEFAULT CHARSET=utf8;
+
+CREATE TABLE t2 (
+  id int(11) NOT NULL AUTO_INCREMENT,
+  activefromts datetime NOT NULL DEFAULT '0000-00-00 00:00:00',
+  shortdescription text,
+  useraccessfamily varchar(512) DEFAULT NULL,
+  serialized longtext,
+  PRIMARY KEY (id)
+) ENGINE=MyISAM DEFAULT CHARSET=utf8;
+
+insert into t1 values ('picture/89/1369722032695.pmd',89,NULL);
+insert into t1 values ('picture/90/1369832057370.pmd',90,NULL);
+insert into t2 values (38,'2013-03-04 07:49:22','desc','CODE','string');
+
+EXPLAIN 
+SELECT * FROM t2 x,
+(SELECT t2.useraccessfamily, t2.serialized AS picturesubuser, COUNT(*)
+  FROM t2, t1 GROUP BY t2.useraccessfamily, picturesubuser) y
+WHERE x.useraccessfamily = y.useraccessfamily;
+
+SELECT * FROM t2 x,
+(SELECT t2.useraccessfamily, t2.serialized AS picturesubuser, COUNT(*)
+  FROM t2, t1 GROUP BY t2.useraccessfamily, picturesubuser) y
+WHERE x.useraccessfamily = y.useraccessfamily;
+
+DROP TABLE t1,t2;
+
+--echo #
+--echo # Bug#13261277: Unchecked key length caused missing records.
+--echo #
+
+CREATE TABLE t1 (  
+  col_varchar varchar(1024) CHARACTER SET utf8 DEFAULT NULL,
+  stub1 varchar(1024) CHARACTER SET utf8 DEFAULT NULL,
+  stub2 varchar(1024) CHARACTER SET utf8 DEFAULT NULL,
+  stub3 varchar(1024) CHARACTER SET utf8 DEFAULT NULL
+);
+
+INSERT INTO t1 VALUES
+  ('d','d','l','ther'),
+  (NULL,'s','NJBIQ','trzetuchv'),
+  (-715390976,'coul','MYWFB','cfhtrzetu'),
+  (1696792576,'f','i\'s','c'),
+  (1,'i','ltpemcfhtr','gsltpemcf'),
+  (-663027712,'mgsltpemcf','sa','amgsltpem'),
+  (-1686700032,'JPRVK','i','vamgsltpe'),
+  (NULL,'STUNB','UNVJV','u'),
+  (5,'oka','qyihvamgsl','AXSMD'),
+  (NULL,'tqwmqyihva','h','yntqwmqyi'),
+  (3,'EGMJN','e','e');
+
+CREATE TABLE t2 (
+  col_varchar varchar(10) DEFAULT NULL,
+  col_int INT DEFAULT NULL
+);
+
+INSERT INTO t2 VALUES ('d',9);
+
+set optimizer_switch='derived_merge=off,derived_with_keys=on';
+
+SET @save_heap_size= @@max_heap_table_size;
+SET @@max_heap_table_size= 16384;
+
+SELECT t2.col_int
+FROM t2
+    RIGHT JOIN ( SELECT * FROM t1 ) AS dt 
+  ON t2.col_varchar = dt.col_varchar
+WHERE t2.col_int IS NOT NULL ;
+
+--echo # Shouldn't use auto_key0 for derived table
+EXPLAIN 
+SELECT t2.col_int
+FROM t2
+    RIGHT JOIN ( SELECT * FROM t1 ) AS dt
+  ON t2.col_varchar = dt.col_varchar
+WHERE t2.col_int IS NOT NULL ;
+
+SET @@max_heap_table_size= @save_heap_size;
+SET optimizer_switch=@save_optimizer_switch;
+
+DROP TABLE t1,t2;
+
 --echo #
 --echo # end of 5.3 tests
 --echo #

sql/sql_select.cc

@@ -8977,6 +8977,25 @@ uint get_next_field_for_derived_key(uchar *arg)
 }
 
 
+static
+uint get_next_field_for_derived_key_simple(uchar *arg)
+{
+  KEYUSE *keyuse= *(KEYUSE **) arg;
+  if (!keyuse)
+    return (uint) (-1);
+  TABLE *table= keyuse->table;
+  uint key= keyuse->key;
+  uint fldno= keyuse->keypart; 
+  for ( ; 
+        keyuse->table == table && keyuse->key == key && keyuse->keypart == fldno;
+        keyuse++)
+    ;
+  if (keyuse->key != key)
+    keyuse= 0;
+  *((KEYUSE **) arg)= keyuse;
+  return fldno;
+}
+
 static 
 bool generate_derived_keys_for_table(KEYUSE *keyuse, uint count, uint keys)
 {
@@ -9007,12 +9026,28 @@ bool generate_derived_keys_for_table(KEYUSE *keyuse, uint count, uint keys)
     }
     else
     {
-      if (table->add_tmp_key(table->s->keys, parts, 
-                             get_next_field_for_derived_key, 
-                             (uchar *) &first_keyuse,
-                             FALSE))
-        return TRUE;
-      table->reginfo.join_tab->keys.set_bit(table->s->keys);
+      KEYUSE *save_first_keyuse= first_keyuse;
+      if (table->check_tmp_key(table->s->keys, parts,
+                               get_next_field_for_derived_key_simple, 
+                               (uchar *) &first_keyuse))
+ 
+      {
+        first_keyuse= save_first_keyuse;
+        if (table->add_tmp_key(table->s->keys, parts, 
+                               get_next_field_for_derived_key, 
+                               (uchar *) &first_keyuse,
+                               FALSE))
+          return TRUE;
+        table->reginfo.join_tab->keys.set_bit(table->s->keys);
+      }
+      else
+      {
+        /* Mark keyuses for this key to be excluded */
+        for (KEYUSE *curr=save_first_keyuse; curr < first_keyuse; curr++)
+	{
+          curr->key= MAX_KEY;
+        }
+      }
       first_keyuse= keyuse;
       key_count++;
       parts= 0;

sql/table.cc

@@ -5439,6 +5439,52 @@ void TABLE::create_key_part_by_field(KEY *keyinfo,
 }
 
 
+/**
+  @brief
+  Check validity of a possible key for the derived table
+
+  @param key            the number of the key
+  @param key_parts      number of components of the key
+  @param next_field_no  the call-back function that returns the number of
+                        the field used as the next component of the key
+  @param arg            the argument for the above function
+
+  @details
+  The function checks whether a possible key satisfies the constraints
+  imposed on the keys of any temporary table.
+
+  @return TRUE if the key is valid
+  @return FALSE otherwise
+*/
+
+bool TABLE::check_tmp_key(uint key, uint key_parts,
+                          uint (*next_field_no) (uchar *), uchar *arg)
+{
+  Field **reg_field;
+  uint i;
+  uint key_len= 0;
+
+  for (i= 0; i < key_parts; i++)
+  {
+    uint fld_idx= next_field_no(arg);
+    reg_field= field + fld_idx;
+    uint fld_store_len= (uint16) (*reg_field)->key_length();
+    if ((*reg_field)->real_maybe_null())
+      fld_store_len+= HA_KEY_NULL_LENGTH;
+    if ((*reg_field)->type() == MYSQL_TYPE_BLOB ||
+        (*reg_field)->real_type() == MYSQL_TYPE_VARCHAR ||
+        (*reg_field)->type() == MYSQL_TYPE_GEOMETRY)
+      fld_store_len+= HA_KEY_BLOB_LENGTH;
+    key_len+= fld_store_len;
+  }
+  /*
+    We use MI_MAX_KEY_LENGTH (myisam's default) below because it is
+    smaller than MAX_KEY_LENGTH (heap's default) and it's unknown whether
+    myisam or heap will be used for the temporary table.
+  */
+  return key_len <= MI_MAX_KEY_LENGTH;
+}
+
 /**
   @brief
   Add one key to a temporary table
@@ -5470,6 +5516,7 @@ bool TABLE::add_tmp_key(uint key, uint key_parts,
   KEY* keyinfo;
   Field **reg_field;
   uint i;
+  
   bool key_start= TRUE;
   KEY_PART_INFO* key_part_info=
       (KEY_PART_INFO*) alloc_root(&mem_root, sizeof(KEY_PART_INFO)*key_parts);

sql/table.h

@@ -986,6 +986,8 @@ struct st_table {
   inline bool needs_reopen_or_name_lock()
   { return s->version != refresh_version; }
   bool alloc_keys(uint key_count);
+  bool check_tmp_key(uint key, uint key_parts,
+                     uint (*next_field_no) (uchar *), uchar *arg);
   bool add_tmp_key(uint key, uint key_parts,
                    uint (*next_field_no) (uchar *), uchar *arg,
                    bool unique);