Merge abotchkov@bk-internal.mysql.com:/home/bk/mysql-4.1

into deer.(none):/home/hf/work/mysql-4.1.6117

mysql-test/r/alter_table.result

@@ -349,6 +349,7 @@ alter table t1 change a a char(10) character set koi8r;
 select a,hex(a) from t1;
 a	hex(a)
 	D4C5D3D4
+delete from t1;
 show create table t1;
 Table	Create Table
 t1	CREATE TABLE `t1` (

mysql-test/r/ctype_recoding.result

@@ -166,3 +166,11 @@ ERROR HY000: Invalid utf8 character string: '
 SET NAMES utf8;
 CREATE TABLE `good` (a int);
 ERROR HY000: Invalid utf8 character string: '` (a int)'
+set names latin1;
+create table t1 (a char(10) character set koi8r, b text character set koi8r);
+insert into t1 values ('test','test');
+insert into t1 values ('','');
+Warnings:
+Warning	1265	Data truncated for column 'a' at row 1
+Warning	1265	Data truncated for column 'b' at row 1
+drop table t1;

mysql-test/r/fulltext.result

@@ -356,6 +356,11 @@ SELECT t, collation(t),MATCH t AGAINST ('Osnabruck') FROM t1 WHERE MATCH t AGAIN
 t	collation(t)	MATCH t AGAINST ('Osnabruck')
 aus Osnabrck	utf8_general_ci	1.591139793396
 alter table t1 modify t varchar(200) collate latin1_german2_ci not null;
+Warnings:
+Warning	1265	Data truncated for column 't' at row 3
+Warning	1265	Data truncated for column 't' at row 4
+Warning	1265	Data truncated for column 't' at row 5
+Warning	1265	Data truncated for column 't' at row 6
 SELECT t, collation(t) FROM t1 WHERE MATCH t AGAINST ('Osnabrck');
 t	collation(t)
 aus Osnabrck	latin1_german2_ci

mysql-test/t/alter_table.test

@@ -207,6 +207,7 @@ alter table t1 change a a text character set cp1251;
 select a,hex(a) from t1;
 alter table t1 change a a char(10) character set koi8r;
 select a,hex(a) from t1;
+delete from t1;
 
 #
 # Test ALTER TABLE .. CHARACTER SET ..

mysql-test/t/ctype_recoding.test

@@ -121,3 +121,13 @@ CREATE TABLE `good
 SET NAMES utf8;
 --error 1300
 CREATE TABLE `good` (a int);
+
+
+#
+# Test that we produce a warnign when conversion loses data.
+#
+set names latin1;
+create table t1 (a char(10) character set koi8r, b text character set koi8r);
+insert into t1 values ('test','test');
+insert into t1 values ('','');
+drop table t1;

scripts/make_win_src_distribution.sh

@@ -6,7 +6,6 @@
 
 version=@VERSION@
 export version
-SOURCE=`pwd`
 CP="cp -p"
 
 DEBUG=0
@@ -24,6 +23,7 @@ if [ ! -f scripts/make_win_src_distribution ]; then
   echo "ERROR : You must run this script from the MySQL top-level directory"
   exit 1
 fi
+SOURCE=`pwd`
 
 #
 # Check for source compilation/configuration
@@ -119,7 +119,7 @@ unix_to_dos()
   for arg do
     print_debug "Replacing LF -> CRLF from '$arg'"
 
-    cat $arg | awk '{sub(/$/,"\r");print}' > $arg.tmp
+    awk '{sub(/$/,"\r");print}' < $arg   > $arg.tmp
     rm -f $arg
     mv $arg.tmp $arg
   done
@@ -138,14 +138,6 @@ if [ -d $BASE ] ; then
 fi
 
 $CP -r $SOURCE/VC++Files $BASE
-(
-find $BASE \( -name "*.dsp" -o -name "*.dsw" \) -and -not -path \*SCCS\* -print
-)|(
-  while read v
-  do
-    unix_to_dos $v
-  done
-)
 
 #
 # Process version tags in InstallShield files
@@ -281,7 +273,6 @@ for i in COPYING ChangeLog README EXCEPTIONS-CLIENT\
          Docs/manual_toc.html  Docs/manual.html \
          Docs/manual.txt Docs/mysqld_error.txt \
          Docs/INSTALL-BINARY Docs/internals.texi
-
 do
   print_debug "Copying file '$i'"
   if [ -f $i ]
@@ -322,7 +313,18 @@ done
 ./extra/replace std:: "" < $BASE/sql/sql_yacc.cpp | sed '/^ *switch (yytype)$/ { N; /\n *{$/ { N; /\n *default:$/ { N; /\n *break;$/ { N; /\n *}$/ d; };};};} ' > $BASE/sql/sql_yacc.cpp-new
 mv $BASE/sql/sql_yacc.cpp-new $BASE/sql/sql_yacc.cpp
 
-unix_to_dos $BASE/README
+#
+# Search the tree for plain text files and adapt the line end marker
+#
+find $BASE \( -name "*.dsp" -o -name "*.dsw" -o -name "*.cnf" -o -name "*.ini" \
+           -o -name COPYING -o -name ChangeLog -o -name EXCEPTIONS-CLIENT -o -name "INSTALL*" -o -name LICENSE -o -name "README*" \) -type f -print \
+| while read v
+  do
+    unix_to_dos $v
+  done
+# File extension '.txt' matches too many other files, error messages etc.
+unix_to_dos $BASE/Docs/*.txt 
+
 mv $BASE/README $BASE/README.txt
 
 #

sql/field.cc

@@ -4275,9 +4275,12 @@ int Field_string::store(const char *from,uint length,CHARSET_INFO *cs)
   /* Convert character set if nesessary */
   if (String::needs_conversion(length, cs, field_charset, &not_used))
   { 
-    tmpstr.copy(from, length, cs, field_charset);
+    uint conv_errors;
+    tmpstr.copy(from, length, cs, field_charset, &conv_errors);
     from= tmpstr.ptr();
     length=  tmpstr.length();
+    if (conv_errors)
+      error= 1;
   }
 
   /* 
@@ -4300,11 +4303,11 @@ int Field_string::store(const char *from,uint length,CHARSET_INFO *cs)
     from+= field_charset->cset->scan(field_charset, from, end,
 				     MY_SEQ_SPACES);
     if (from != end)
-    {
-      set_warning(MYSQL_ERROR::WARN_LEVEL_WARN, ER_WARN_DATA_TRUNCATED, 1);
-      error=1;
-    }
+      error= 1;
   }
+  if (error)
+    set_warning(MYSQL_ERROR::WARN_LEVEL_WARN, ER_WARN_DATA_TRUNCATED, 1);
+
   return error;
 }
 
@@ -4528,16 +4531,20 @@ int Field_varstring::store(const char *from,uint length,CHARSET_INFO *cs)
   /* Convert character set if nesessary */
   if (String::needs_conversion(length, cs, field_charset, &not_used))
   { 
-    tmpstr.copy(from, length, cs, field_charset);
+    uint conv_errors;
+    tmpstr.copy(from, length, cs, field_charset, &conv_errors);
     from= tmpstr.ptr();
     length=  tmpstr.length();
+    if (conv_errors)
+      error= 1;
   }
   if (length > field_length)
   {
     length=field_length;
-    set_warning(MYSQL_ERROR::WARN_LEVEL_WARN, ER_WARN_DATA_TRUNCATED, 1);
     error= 1;
   }
+  if (error)
+    set_warning(MYSQL_ERROR::WARN_LEVEL_WARN, ER_WARN_DATA_TRUNCATED, 1);
   memcpy(ptr+HA_KEY_BLOB_LENGTH,from,length);
   int2store(ptr, length);
   return error;
@@ -4865,6 +4872,7 @@ void Field_blob::put_length(char *pos, uint32 length)
 
 int Field_blob::store(const char *from,uint length,CHARSET_INFO *cs)
 {
+  int error= 0;
   if (!length)
   {
     bzero(ptr,Field_blob::pack_length());
@@ -4881,9 +4889,12 @@ int Field_blob::store(const char *from,uint length,CHARSET_INFO *cs)
     if ((was_conversion= String::needs_conversion(length, cs, field_charset,
 						  &not_used)))
     { 
-      tmpstr.copy(from, length, cs, field_charset);
+      uint conv_errors;
+      tmpstr.copy(from, length, cs, field_charset, &conv_errors);
       from= tmpstr.ptr();
       length=  tmpstr.length();
+      if (conv_errors)
+        error= 1;
     }
     
     copy_length= max_data_length();
@@ -4897,8 +4908,7 @@ int Field_blob::store(const char *from,uint length,CHARSET_INFO *cs)
 						      min(length, copy_length),
 						      copy_length);
     if (copy_length < length)
-      set_warning(MYSQL_ERROR::WARN_LEVEL_WARN, ER_WARN_DATA_TRUNCATED, 1);
-    
+      error= 1;
     Field_blob::store_length(copy_length);
     if (was_conversion || table->copy_blobs || copy_length <= MAX_FIELD_WIDTH)
     {						// Must make a copy
@@ -4910,6 +4920,8 @@ int Field_blob::store(const char *from,uint length,CHARSET_INFO *cs)
     }
     bmove(ptr+packlength,(char*) &from,sizeof(char*));
   }
+  if (error)
+    set_warning(MYSQL_ERROR::WARN_LEVEL_WARN, ER_WARN_DATA_TRUNCATED, 1);
   return 0;
 }
 

sql/item_func.cc

@@ -2873,10 +2873,10 @@ void Item_func_match::init_search(bool no_order)
   if (key == NO_SUCH_KEY)
   {
     List<Item> fields;
+    fields.push_back(new Item_string(" ",1, cmp_collation.collation));
     for (uint i=1; i < arg_count; i++)
       fields.push_back(args[i]);
-    concat=new Item_func_concat_ws(new Item_string(" ",1,
-                                   cmp_collation.collation), fields);
+    concat=new Item_func_concat_ws(fields);
     /*
       Above function used only to get value and do not need fix_fields for it:
       Item_string - basic constant

sql/item_strfunc.cc

@@ -532,7 +532,7 @@ String *Item_func_concat_ws::val_str(String *str)
   uint i;
 
   null_value=0;
-  if (!(sep_str= separator->val_str(&tmp_sep_str)))
+  if (!(sep_str= args[0]->val_str(&tmp_sep_str)))
     goto null;
 
   use_as_buff= &tmp_value;
@@ -541,7 +541,7 @@ String *Item_func_concat_ws::val_str(String *str)
 
   // Skip until non-null argument is found.
   // If not, return the empty string
-  for (i=0; i < arg_count; i++)
+  for (i=1; i < arg_count; i++)
     if ((res= args[i]->val_str(str)))
       break;
   if (i ==  arg_count)
@@ -635,67 +635,25 @@ null:
   return 0;
 }
 
-void Item_func_concat_ws::split_sum_func(THD *thd, Item **ref_pointer_array,
-					 List<Item> &fields)
-{
-  if (separator->with_sum_func && separator->type() != SUM_FUNC_ITEM)
-    separator->split_sum_func(thd, ref_pointer_array, fields);
-  else if (separator->used_tables() || separator->type() == SUM_FUNC_ITEM)
-  {
-    uint el= fields.elements;
-    Item *new_item= new Item_ref(ref_pointer_array + el, 0, separator->name);
-    fields.push_front(separator);
-    ref_pointer_array[el]= separator;
-    thd->change_item_tree(&separator, new_item);
-  }
-  Item_str_func::split_sum_func(thd, ref_pointer_array, fields);
-}
 
 void Item_func_concat_ws::fix_length_and_dec()
 {
-  collation.set(separator->collation);
-  max_length=separator->max_length*(arg_count-1);
-  for (uint i=0 ; i < arg_count ; i++)
-  {
-    DTCollation tmp(collation.collation, collation.derivation);
+  max_length=0;
+
+  if (agg_arg_collations(collation, args, arg_count))
+    return;
+
+  max_length= arg_count > 1 ? args[0]->max_length * (arg_count - 2) : 0;
+  for (uint i=1 ; i < arg_count ; i++)
     max_length+=args[i]->max_length;
-    if (collation.aggregate(args[i]->collation))
-    {
-      collation.set(tmp); // Restore the previous value
-      my_coll_agg_error(collation, args[i]->collation, func_name());
-      break;
-    }
-  }
+
   if (max_length > MAX_BLOB_WIDTH)
   {
     max_length=MAX_BLOB_WIDTH;
     maybe_null=1;
   }
-  used_tables_cache|=     separator->used_tables();
-  not_null_tables_cache&= separator->not_null_tables();
-  const_item_cache&=	  separator->const_item();
-  with_sum_func=	  with_sum_func || separator->with_sum_func;
-}
-
-void Item_func_concat_ws::update_used_tables()
-{
-  Item_func::update_used_tables();
-  separator->update_used_tables();
-  used_tables_cache|=separator->used_tables();
-  const_item_cache&=separator->const_item();
 }
 
-void Item_func_concat_ws::print(String *str)
-{
-  str->append("concat_ws(", 10);
-  separator->print(str);
-  if (arg_count)
-  {
-    str->append(',');
-    print_args(str, 0);
-  }
-  str->append(')');
-}
 
 String *Item_func_reverse::val_str(String *str)
 {

sql/item_strfunc.h

@@ -89,30 +89,12 @@ public:
 
 class Item_func_concat_ws :public Item_str_func
 {
-  Item *separator;
   String tmp_value;
-
 public:
-  Item_func_concat_ws(Item *a,List<Item> &list) 
-    :Item_str_func(list),separator(a) {}
+  Item_func_concat_ws(List<Item> &list) :Item_str_func(list) {}
   String *val_str(String *);
   void fix_length_and_dec();
-  void update_used_tables();
-  bool fix_fields(THD *thd, TABLE_LIST *tlist, Item **ref)
-  {
-    DBUG_ASSERT(fixed == 0);
-    return (separator->fix_fields(thd, tlist, &separator) ||
-	    separator->check_cols(1) ||
-	    Item_func::fix_fields(thd, tlist, ref));
-  }
-  void split_sum_func(THD *thd, Item **ref_pointer_array, List<Item> &fields);
   const char *func_name() const { return "concat_ws"; }
-  bool walk(Item_processor processor, byte *arg)
-  {
-    return separator->walk(processor, arg) ||
-      Item_str_func::walk(processor, arg);
-  }
-  void print(String *str);
 };
 
 class Item_func_reverse :public Item_str_func

sql/sql_string.cc

@@ -331,19 +331,26 @@ bool String::set_or_copy_aligned(const char *str,uint32 arg_length,
 	/* Copy with charset convertion */
 
 bool String::copy(const char *str, uint32 arg_length,
-		  CHARSET_INFO *from_cs, CHARSET_INFO *to_cs)
+		  CHARSET_INFO *from_cs, CHARSET_INFO *to_cs, uint *errors)
 {
   uint32 offset;
   if (!needs_conversion(arg_length, from_cs, to_cs, &offset))
+  {
+    if (errors)
+      *errors= 0;
     return copy(str, arg_length, to_cs);
+  }
   if ((from_cs == &my_charset_bin) && offset)
+  {
+    if (errors)
+      *errors= 0;
     return copy_aligned(str, arg_length, offset, to_cs);
-  
+  }
   uint32 new_length= to_cs->mbmaxlen*arg_length;
   if (alloc(new_length))
     return TRUE;
   str_length=copy_and_convert((char*) Ptr, new_length, to_cs,
-			      str, arg_length, from_cs);
+                              str, arg_length, from_cs, errors);
   str_charset=to_cs;
   return FALSE;
 }
@@ -769,7 +776,8 @@ String *copy_if_not_alloced(String *to,String *from,uint32 from_length)
 
 uint32
 copy_and_convert(char *to, uint32 to_length, CHARSET_INFO *to_cs, 
-		 const char *from, uint32 from_length, CHARSET_INFO *from_cs)
+                 const char *from, uint32 from_length, CHARSET_INFO *from_cs,
+                 uint *errors)
 {
   int         cnvres;
   my_wc_t     wc;
@@ -780,6 +788,7 @@ copy_and_convert(char *to, uint32 to_length, CHARSET_INFO *to_cs,
 	       const uchar *) = from_cs->cset->mb_wc;
   int (*wc_mb)(struct charset_info_st *, my_wc_t, uchar *s, uchar *e)=
     to_cs->cset->wc_mb;
+  uint error_count= 0;
 
   while (1)
   {
@@ -788,6 +797,7 @@ copy_and_convert(char *to, uint32 to_length, CHARSET_INFO *to_cs,
       from+= cnvres;
     else if (cnvres == MY_CS_ILSEQ)
     {
+      error_count++;
       from++;
       wc= '?';
     }
@@ -799,12 +809,15 @@ outp:
       to+= cnvres;
     else if (cnvres == MY_CS_ILUNI && wc != '?')
     {
+      error_count++;
       wc= '?';
       goto outp;
     }
     else
       break;
   }
+  if (errors)
+    *errors= error_count;
   return (uint32) (to - to_start);
 }
 

sql/sql_string.h

@@ -29,7 +29,7 @@ int sortcmp(const String *a,const String *b, CHARSET_INFO *cs);
 String *copy_if_not_alloced(String *a,String *b,uint32 arg_length);
 uint32 copy_and_convert(char *to, uint32 to_length, CHARSET_INFO *to_cs,
 			const char *from, uint32 from_length,
-			CHARSET_INFO *from_cs);
+			CHARSET_INFO *from_cs, uint *errors= 0);
 
 class String
 {
@@ -199,7 +199,7 @@ public:
 		    CHARSET_INFO *cs);
   bool set_or_copy_aligned(const char *s, uint32 arg_length, CHARSET_INFO *cs);
   bool copy(const char*s,uint32 arg_length, CHARSET_INFO *csfrom,
-	    CHARSET_INFO *csto);
+	    CHARSET_INFO *csto, uint *errors= 0);
   bool append(const String &s);
   bool append(const char *s);
   bool append(const char *s,uint32 arg_length);

sql/sql_yacc.yy

@@ -2889,7 +2889,7 @@ simple_expr:
 	| CONCAT '(' expr_list ')'
 	  { $$= new Item_func_concat(* $3); }
 	| CONCAT_WS '(' expr ',' expr_list ')'
-	  { $$= new Item_func_concat_ws($3, *$5); }
+	  { $5->push_front($3); $$= new Item_func_concat_ws(*$5); }
 	| CONVERT_TZ_SYM '(' expr ',' expr ',' expr ')'
 	  {
 	    Lex->time_zone_tables_used= &fake_time_zone_tables_list;