http://bugs.mysql.com/bug.php?id=2368

Multibyte charsets do not check that incoming data is well-formed

BUILD/compile-pentium-debug-max-no-embedded

+#! /bin/sh
+
+path=`dirname $0`
+. "$path/SETUP.sh"
+
+extra_flags="$pentium_cflags $debug_cflags"
+c_warnings="$c_warnings $debug_extra_warnings"
+cxx_warnings="$cxx_warnings $debug_extra_warnings"
+extra_configs="$pentium_configs $debug_configs"
+
+extra_configs="$extra_configs --with-berkeley-db --with-innodb --without-isam --with-openssl --with-raid"
+
+. "$path/FINISH.sh"

mysql-test/r/ctype_utf8.result

@@ -88,3 +88,58 @@ select s1,hex(s1),char_length(s1),octet_length(s1) from t1;
 s1	hex(s1)	char_length(s1)	octet_length(s1)
 	D0B0	1	2
 drop table t1;
+create table t1 (s1 tinytext character set utf8);
+insert into t1 select repeat('a',300);
+Warnings:
+Warning	1264	Data truncated for column 's1' at row 1
+insert into t1 select repeat('',300);
+Warnings:
+Warning	1264	Data truncated for column 's1' at row 1
+insert into t1 select repeat('a',300);
+Warnings:
+Warning	1264	Data truncated for column 's1' at row 1
+insert into t1 select repeat('a',300);
+Warnings:
+Warning	1264	Data truncated for column 's1' at row 1
+insert into t1 select repeat('',300);
+Warnings:
+Warning	1264	Data truncated for column 's1' at row 1
+select hex(s1) from t1;
+hex(s1)
+616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161
+D18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18F
+61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F
+D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61D18F61
+D18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18FD18F
+select length(s1),char_length(s1) from t1;
+length(s1)	char_length(s1)
+255	255
+254	127
+255	170
+255	170
+254	127
+drop table t1;
+create table t1 (s1 text character set utf8);
+insert into t1 select repeat('a',66000);
+Warnings:
+Warning	1264	Data truncated for column 's1' at row 1
+insert into t1 select repeat('',66000);
+Warnings:
+Warning	1264	Data truncated for column 's1' at row 1
+insert into t1 select repeat('a',66000);
+Warnings:
+Warning	1264	Data truncated for column 's1' at row 1
+insert into t1 select repeat('a',66000);
+Warnings:
+Warning	1264	Data truncated for column 's1' at row 1
+insert into t1 select repeat('',66000);
+Warnings:
+Warning	1264	Data truncated for column 's1' at row 1
+select length(s1),char_length(s1) from t1;
+length(s1)	char_length(s1)
+65535	65535
+65534	32767
+65535	43690
+65535	43690
+65534	32767
+drop table t1;

mysql-test/t/ctype_utf8.test

@@ -61,3 +61,22 @@ create table t1 (s1 char(1) character set utf8);
 insert into t1 values (_koi8r'');
 select s1,hex(s1),char_length(s1),octet_length(s1) from t1;
 drop table t1;
+
+create table t1 (s1 tinytext character set utf8);
+insert into t1 select repeat('a',300);
+insert into t1 select repeat('',300);
+insert into t1 select repeat('a',300);
+insert into t1 select repeat('a',300);
+insert into t1 select repeat('',300);
+select hex(s1) from t1;
+select length(s1),char_length(s1) from t1;
+drop table t1;
+
+create table t1 (s1 text character set utf8);
+insert into t1 select repeat('a',66000);
+insert into t1 select repeat('',66000);
+insert into t1 select repeat('a',66000);
+insert into t1 select repeat('a',66000);
+insert into t1 select repeat('',66000);
+select length(s1),char_length(s1) from t1;
+drop table t1;

sql/field.cc

@@ -4485,19 +4485,9 @@ void Field_blob::store_length(uint32 number)
 {
   switch (packlength) {
   case 1:
-    if (number > 255)
-    {
-      number=255;
-      set_warning(MYSQL_ERROR::WARN_LEVEL_WARN, ER_WARN_DATA_TRUNCATED);
-    }
     ptr[0]= (uchar) number;
     break;
   case 2:
-    if (number > (uint16) ~0)
-    {
-      number= (uint16) ~0;
-      set_warning(MYSQL_ERROR::WARN_LEVEL_WARN, ER_WARN_DATA_TRUNCATED);
-    }
 #ifdef WORDS_BIGENDIAN
     if (table->db_low_byte_first)
     {
@@ -4508,11 +4498,6 @@ void Field_blob::store_length(uint32 number)
       shortstore(ptr,(unsigned short) number);
     break;
   case 3:
-    if (number > (uint32) (1L << 24))
-    {
-      number= (uint32) (1L << 24)-1L;
-      set_warning(MYSQL_ERROR::WARN_LEVEL_WARN, ER_WARN_DATA_TRUNCATED);
-    }
     int3store(ptr,number);
     break;
   case 4:
@@ -4573,6 +4558,7 @@ int Field_blob::store(const char *from,uint length,CHARSET_INFO *cs)
     bool was_conversion;
     char buff[80];
     String tmpstr(buff,sizeof(buff), &my_charset_bin);
+    uint copy_length;
     uint32 not_used;
 
     /* Convert character set if nesessary */
@@ -4583,12 +4569,22 @@ int Field_blob::store(const char *from,uint length,CHARSET_INFO *cs)
       from= tmpstr.ptr();
       length=  tmpstr.length();
     }
-    Field_blob::store_length(length);
-    if (was_conversion || table->copy_blobs || length <= MAX_FIELD_WIDTH)
+    
+    copy_length= max_data_length();
+    if (copy_length > length)
+      copy_length= length;
+    copy_length= field_charset->cset->wellformedlen(field_charset,
+                                                    from,from+copy_length,
+                                                    field_length);
+    if (copy_length < length)
+      set_warning(MYSQL_ERROR::WARN_LEVEL_WARN, ER_WARN_DATA_TRUNCATED);
+    
+    Field_blob::store_length(copy_length);
+    if (was_conversion || table->copy_blobs || copy_length <= MAX_FIELD_WIDTH)
     {						// Must make a copy
       if (from != value.ptr())			// For valgrind
       {
-	value.copy(from,length,charset());
+	value.copy(from,copy_length,charset());
 	from=value.ptr();
       }
     }

sql/field.h

@@ -949,6 +949,15 @@ class Field_blob :public Field_str {
   void sort_string(char *buff,uint length);
   uint32 pack_length() const
   { return (uint32) (packlength+table->blob_ptr_size); }
+  uint32 max_data_length() const
+  {
+    switch (packlength) {
+    case 1: return 255;
+    case 2: return (uint32) 0xFFFFL;
+    case 3: return (uint32) 0xFFFFFF;
+    default: return (uint32) 0xFFFFFFFF;
+    }
+  }
   void reset(void) { bzero(ptr, packlength+sizeof(char*)); }
   void reset_fields() { bzero((char*) &value,sizeof(value)); }
   void store_length(uint32 number);

strings/ctype-mb.c

@@ -274,18 +274,21 @@ uint my_charpos_mb(CHARSET_INFO *cs __attribute__((unused)),
   return pos ? e+2-b0 : b-b0;
 }
 
-uint my_wellformedlen_mb(CHARSET_INFO *cs __attribute__((unused)),
-		     const char *b, const char *e, uint pos)
+uint my_wellformedlen_mb(CHARSET_INFO *cs,
+			 const char *b, const char *e, uint pos)
 {
-  uint mblen;
-  const char *b0=b;
+  my_wc_t wc;
+  int mblen;
+  const char *b0= b;
   
-  while (pos && b<e)
+  while (pos)
   {
-    b+= (mblen= my_ismbchar(cs,b,e)) ? mblen : 1;
+    if ((mblen= cs->cset->mb_wc(cs, &wc, b, e)) <0)
+      break;
+    b+= mblen;
     pos--;
   }
-  return b-b0;
+  return b - b0;
 }