new format length calculation check added.

mysql-test/r/dyncol.result

@@ -1413,6 +1413,13 @@ Warnings:
 Warning	1265	Data truncated for column 'dyn' at row 1
 SELECT COLUMN_ADD( dyn, 6, REPEAT('x',80), 4, REPEAT('y',215) AS INTEGER ) FROM t1;
 ERROR HY000: Encountered illegal format of dynamic column string
+delete from t1;
+#above test with 10.0 format
+INSERT INTO t1 SET dyn = COLUMN_CREATE( 'a', REPEAT('a', 250), 'b', REPEAT('b', 322) );
+Warnings:
+Warning	1265	Data truncated for column 'dyn' at row 1
+SELECT COLUMN_ADD( dyn, 'c', REPEAT('x',80), 'b', REPEAT('y',215) AS INTEGER ) FROM t1;
+ERROR HY000: Encountered illegal format of dynamic column string
 DROP table t1;
 #
 # MDEV-4812: Valgrind warnings (Invalid write) in
@@ -1423,6 +1430,11 @@ INSERT INTO t1 SET dyncol = COLUMN_CREATE( 7, REPEAT('k',487), 209, REPEAT('x',4
 Warnings:
 Warning	1265	Data truncated for column 'dyncol' at row 1
 SELECT COLUMN_ADD( dyncol, 7, '22:22:22', 8, REPEAT('x',270) AS CHAR ) FROM t1;
+delete from t1;
+INSERT INTO t1 SET dyncol = COLUMN_CREATE( 'a', REPEAT('k',487), 'b', REPEAT('x',464) );
+Warnings:
+Warning	1265	Data truncated for column 'dyncol' at row 1
+SELECT COLUMN_ADD( dyncol, 'a', '22:22:22', 'c', REPEAT('x',270) AS CHAR ) FROM t1;
 DROP table t1;
 #
 # end of 5.3 tests

mysql-test/t/dyncol.test

@@ -612,6 +612,12 @@ CREATE TABLE t1 (dyn TINYBLOB) ENGINE=MyISAM;
 INSERT INTO t1 SET dyn = COLUMN_CREATE( 40, REPEAT('a', 233), 4, REPEAT('b', 322) );
 --error ER_DYN_COL_WRONG_FORMAT
 SELECT COLUMN_ADD( dyn, 6, REPEAT('x',80), 4, REPEAT('y',215) AS INTEGER ) FROM t1;
+delete from t1;
+--echo #above test with 10.0 format
+INSERT INTO t1 SET dyn = COLUMN_CREATE( 'a', REPEAT('a', 250), 'b', REPEAT('b', 322) );
+--error ER_DYN_COL_WRONG_FORMAT
+SELECT COLUMN_ADD( dyn, 'c', REPEAT('x',80), 'b', REPEAT('y',215) AS INTEGER ) FROM t1;
+
 
 DROP table t1;
 
@@ -624,6 +630,11 @@ CREATE TABLE t1 (dyncol TINYBLOB) ENGINE=MyISAM;
 INSERT INTO t1 SET dyncol = COLUMN_CREATE( 7, REPEAT('k',487), 209, REPEAT('x',464) );
 --error 0,ER_DYN_COL_WRONG_FORMAT
 SELECT COLUMN_ADD( dyncol, 7, '22:22:22', 8, REPEAT('x',270) AS CHAR ) FROM t1;
+delete from t1;
+INSERT INTO t1 SET dyncol = COLUMN_CREATE( 'a', REPEAT('k',487), 'b', REPEAT('x',464) );
+--error 0,ER_DYN_COL_WRONG_FORMAT
+SELECT COLUMN_ADD( dyncol, 'a', '22:22:22', 'c', REPEAT('x',270) AS CHAR ) FROM t1;
+
 DROP table t1;
 
 --echo #

mysys/ma_dyncol.c

@@ -1941,13 +1941,15 @@ static size_t hdr_interval_length(DYN_HEADER *hdr, uchar *next_entry)
 
   if ((*fmt->type_and_offset_read)(&hdr->type, &hdr->offset,
                                    hdr->entry + fmt->fixed_hdr_entry,
-                                   hdr->offset_size))
+                                   hdr->offset_size) ||
+      hdr->data_size < hdr->offset)
     return DYNCOL_OFFSET_ERROR;
   if (next_entry == hdr->header + hdr->header_size)
     return hdr->data_size - hdr->offset;
   if ((*fmt->type_and_offset_read)(&next_entry_type, &next_entry_offset,
                                    next_entry + fmt->fixed_hdr_entry,
-                                   hdr->offset_size))
+                                   hdr->offset_size) ||
+      hdr->data_size < next_entry_offset)
     return DYNCOL_OFFSET_ERROR;
   return (next_entry_offset - hdr->offset);
 }