Fix for bug#16678 FORMAT gives wrong result if client run with default-character-set=utf8

  calculate Item_func_format::max_length using charset->mbmaxlen

mysql-test/r/func_math.result

@@ -218,3 +218,23 @@ truncate(-5000111000111000155,-1)
 select truncate(15000111000111000155,-1);
 truncate(15000111000111000155,-1)
 15000111000111000150
+set names utf8;
+create table t1
+(f1 varchar(32) not null,
+f2 smallint(5) unsigned not null,
+f3 int(10) unsigned not null default '0')
+engine=myisam default charset=utf8;
+insert into t1 values ('zombie',0,0),('gold',1,10000),('silver',2,10000);
+create table t2
+(f1 int(10) unsigned not null,
+f2 int(10) unsigned not null,
+f3 smallint(5) unsigned not null)
+engine=myisam default charset=utf8;
+insert into t2 values (16777216,16787215,1),(33554432,33564431,2);
+select format(t2.f2-t2.f1+1,0) from t1,t2
+where t1.f2 = t2.f3 order by t1.f1;
+format(t2.f2-t2.f1+1,0)
+10,000
+10,000
+drop table t1, t2;
+set names default;

mysql-test/t/func_math.test

@@ -155,3 +155,25 @@ select truncate(-5000111000111000155,-1);
 # truncate on unsigned bigint
 select truncate(15000111000111000155,-1);
 
+#
+# Bug#16678 FORMAT gives wrong result if client run with default-character-set=utf8
+#
+set names utf8;
+create table t1
+(f1 varchar(32) not null,
+ f2 smallint(5) unsigned not null,
+ f3 int(10) unsigned not null default '0')
+engine=myisam default charset=utf8;
+insert into t1 values ('zombie',0,0),('gold',1,10000),('silver',2,10000);
+
+create table t2
+(f1 int(10) unsigned not null,
+ f2 int(10) unsigned not null,
+ f3 smallint(5) unsigned not null)
+engine=myisam default charset=utf8;
+insert into t2 values (16777216,16787215,1),(33554432,33564431,2);
+
+select format(t2.f2-t2.f1+1,0) from t1,t2
+where t1.f2 = t2.f3 order by t1.f1;
+drop table t1, t2;
+set names default;

sql/item_strfunc.h

@@ -471,7 +471,9 @@ class Item_func_format :public Item_str_func
   void fix_length_and_dec()
   {
     collation.set(default_charset());
-    max_length=args[0]->max_length+(args[0]->max_length-args[0]->decimals)/3;
+    uint char_length= args[0]->max_length/args[0]->collation.collation->mbmaxlen;
+    max_length= ((char_length + (char_length-args[0]->decimals)/3) *
+                 collation.collation->mbmaxlen);
   }
   const char *func_name() const { return "format"; }
   void print(String *);