MDEV-4811 Assertion `offset < 0x1f' fails in type_and_offset_store on COLUMN_ADD

MDEV-4812 Valgrind warnings (Invalid write) in dynamic_column_update_many on COLUMN_ADD

Fixed problem of working on wrong data (do not allow offset to out of string length).

mysql-test/r/dyncol.result

@@ -1404,5 +1404,26 @@ v1	CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VI
 drop view v1;
 drop table t1;
 #
+# MDEV-4811: Assertion `offset < 0x1f' fails in type_and_offset_store 
+# on COLUMN_ADD
+#
+CREATE TABLE t1 (dyn TINYBLOB) ENGINE=MyISAM;
+INSERT INTO t1 SET dyn = COLUMN_CREATE( 40, REPEAT('a', 233), 4, REPEAT('b', 322) );
+Warnings:
+Warning	1265	Data truncated for column 'dyn' at row 1
+SELECT COLUMN_ADD( dyn, 6, REPEAT('x',80), 4, REPEAT('y',215) AS INTEGER ) FROM t1;
+ERROR HY000: Encountered illegal format of dynamic column string
+DROP table t1;
+#
+# MDEV-4812: Valgrind warnings (Invalid write) in
+# dynamic_column_update_many on COLUMN_ADD
+#
+CREATE TABLE t1 (dyncol TINYBLOB) ENGINE=MyISAM;
+INSERT INTO t1 SET dyncol = COLUMN_CREATE( 7, REPEAT('k',487), 209, REPEAT('x',464) );
+Warnings:
+Warning	1265	Data truncated for column 'dyncol' at row 1
+SELECT COLUMN_ADD( dyncol, 7, '22:22:22', 8, REPEAT('x',270) AS CHAR ) FROM t1;
+DROP table t1;
+#
 # end of 5.3 tests
 #

mysql-test/t/dyncol.test

@@ -599,6 +599,29 @@ drop view v1;
 
 drop table t1;
 
+--echo #
+--echo # MDEV-4811: Assertion `offset < 0x1f' fails in type_and_offset_store 
+--echo # on COLUMN_ADD
+--echo #
+
+CREATE TABLE t1 (dyn TINYBLOB) ENGINE=MyISAM;
+INSERT INTO t1 SET dyn = COLUMN_CREATE( 40, REPEAT('a', 233), 4, REPEAT('b', 322) );
+--error ER_DYN_COL_WRONG_FORMAT
+SELECT COLUMN_ADD( dyn, 6, REPEAT('x',80), 4, REPEAT('y',215) AS INTEGER ) FROM t1;
+
+DROP table t1;
+
+--echo #
+--echo # MDEV-4812: Valgrind warnings (Invalid write) in
+--echo # dynamic_column_update_many on COLUMN_ADD
+--echo #
+CREATE TABLE t1 (dyncol TINYBLOB) ENGINE=MyISAM;
+
+INSERT INTO t1 SET dyncol = COLUMN_CREATE( 7, REPEAT('k',487), 209, REPEAT('x',464) );
+--error 0,ER_DYN_COL_WRONG_FORMAT
+SELECT COLUMN_ADD( dyncol, 7, '22:22:22', 8, REPEAT('x',270) AS CHAR ) FROM t1;
+DROP table t1;
+
 --echo #
 --echo # end of 5.3 tests
 --echo #

mysys/ma_dyncol.c

@@ -1228,13 +1228,14 @@ dynamic_column_create(DYNAMIC_COLUMN *str, uint column_nr,
   @param header_end      Pointer to the header end
   @param offset_size     Size of offset field in bytes
   @param last_offset     Size of the data segment
+  @param error           Set in case of error
 
   @return number of bytes
 */
 
 static size_t get_length_interval(uchar *entry, uchar *entry_next,
                                   uchar *header_end, size_t offset_size,
-                                  size_t last_offset)
+                                  size_t last_offset, my_bool *error)
 {
   size_t offset, offset_next;
   DYNAMIC_COLUMN_TYPE type, type_next;
@@ -1242,8 +1243,12 @@ static size_t get_length_interval(uchar *entry, uchar *entry_next,
 
   type_and_offset_read(&type, &offset, entry, offset_size);
   if (entry_next >= header_end)
+  {
+    *error= 0;
     return (last_offset - offset);
+  }
   type_and_offset_read(&type_next, &offset_next, entry_next, offset_size);
+  *error= (offset_next > last_offset);
   return (offset_next - offset);
 }
 
@@ -1255,17 +1260,18 @@ static size_t get_length_interval(uchar *entry, uchar *entry_next,
   @param header_end      Pointer to the header end
   @param offset_size     Size of offset field in bytes
   @param last_offset     Size of the data segment
+  @param error           Set in case of error
 
   @return number of bytes
 */
 
 static size_t get_length(uchar *entry, uchar *header_end,
                          size_t offset_size,
-                         size_t last_offset)
+                         size_t last_offset, my_bool *error)
 {
   return get_length_interval(entry,
                              entry + offset_size + COLUMN_NUMBER_SIZE,
-                             header_end, offset_size, last_offset);
+                             header_end, offset_size, last_offset, error);
 }
 
 
@@ -1304,6 +1310,7 @@ find_column(DYNAMIC_COLUMN_TYPE *type, uchar **data, size_t *length,
   uchar *entry;
   size_t offset, total_data, header_size, entry_size;
   uchar key[2+4];
+  my_bool error;
 
   if (!entry_pos)
     entry_pos= &entry;
@@ -1329,12 +1336,12 @@ find_column(DYNAMIC_COLUMN_TYPE *type, uchar **data, size_t *length,
     return 1;
   *data= header + header_size + offset;
   *length= get_length(entry, header + header_size, offset_size,
-                      total_data);
+                      total_data, &error);
   /*
     Check that the found data is withing the ranges. This can happen if
     we get data with wrong offsets.
   */
-  if ((long) *length < 0 || offset + *length > total_data)
+  if (error || (long) *length < 0 || offset + *length > total_data)
     return 1;
 
   *entry_pos= entry;
@@ -1837,12 +1844,13 @@ dynamic_column_update_many(DYNAMIC_COLUMN *str,
                    entry_size, column_count, &entry))
     {
       size_t entry_data_size;
+      my_bool error;
 
       /* Data existed; We have to replace or delete it */
 
       entry_data_size= get_length(entry, header_end,
-                                  offset_size, max_offset);
-      if ((long) entry_data_size < 0)
+                                  offset_size, max_offset, &error);
+      if (error || (long) entry_data_size < 0)
       {
         rc= ER_DYNCOL_FORMAT;
         goto end;
@@ -2038,12 +2046,13 @@ dynamic_column_update_many(DYNAMIC_COLUMN *str,
       /* copy first the data that was not replaced in original packed data */
       if (start < end)
       {
+        my_bool error;
         /* Add old data last in 'tmp' */
         size_t data_size=
           get_length_interval(header_base + start * entry_size,
                               header_base + end * entry_size,
-                              header_end, offset_size, max_offset);
-        if ((long) data_size < 0 ||
+                              header_end, offset_size, max_offset, &error);
+        if (error || (long) data_size < 0 ||
             data_size > max_offset - first_offset)
         {
           dynamic_column_column_free(&tmp);