Bug#12406055 BUFFER OVERFLOW OF VARIABLE 'BUFF' IN STRING::SET_REAL

The buffer was simply too small.
In 5.5 and trunk, the size is 311 + 31,
in 5.1 and below, the size is 331

client/sql_string.cc

@@ -119,7 +119,7 @@ bool String::set(ulonglong num, CHARSET_INFO *cs)
 
 bool String::set(double num,uint decimals, CHARSET_INFO *cs)
 {
-  char buff[331];
+  char buff[FLOATING_POINT_BUFFER];
   uint dummy_errors;
 
   str_charset=cs;
@@ -188,7 +188,9 @@ bool String::set(double num,uint decimals, CHARSET_INFO *cs)
 #else
 #ifdef HAVE_SNPRINTF
   buff[sizeof(buff)-1]=0;			// Safety
-  snprintf(buff,sizeof(buff)-1, "%.*f",(int) decimals,num);
+  int num_chars= snprintf(buff, sizeof(buff)-1, "%.*f",(int) decimals, num);
+  DBUG_ASSERT(num_chars > 0);
+  DBUG_ASSERT(num_chars < (int) sizeof(buff));
 #else
   sprintf(buff,"%.*f",(int) decimals,num);
 #endif

include/m_string.h

@@ -216,6 +216,15 @@ extern int is_prefix(const char *, const char *);
 double my_strtod(const char *str, char **end, int *error);
 double my_atof(const char *nptr);
 
+#ifndef NOT_FIXED_DEC
+#define NOT_FIXED_DEC			31
+#endif
+
+/*
+  Max length of a floating point number.
+ */
+#define FLOATING_POINT_BUFFER (311 + NOT_FIXED_DEC)
+
 extern char *llstr(longlong value,char *buff);
 extern char *ullstr(longlong value,char *buff);
 #ifndef HAVE_STRTOUL

mysql-test/r/type_float.result

@@ -407,4 +407,16 @@ SELECT f1 FROM t1;
 f1
 -1.79769313486231e+308
 DROP TABLE t1;
+#
+# Bug#12406055 BUFFER OVERFLOW OF VARIABLE 'BUFF' IN STRING::SET_REAL
+# 
+select format(-1.7976931348623157E+307,256) as foo;
+foo
+ignore_float_result
+select least(-1.1111111111111111111111111,
+- group_concat(1.7976931348623157E+308)) as foo;
+foo
+ignore_float_result
+select concat((truncate((-1.7976931348623157E+307),(0x1e))),
+(99999999999999999999999999999999999999999999999999999999999999999)) into @a;
 End of 5.0 tests

mysql-test/t/type_float.test

@@ -276,4 +276,19 @@ INSERT INTO t1 VALUES(-1.79769313486231e+308);
 SELECT f1 FROM t1;
 DROP TABLE t1;
 
+--echo #
+--echo # Bug#12406055 BUFFER OVERFLOW OF VARIABLE 'BUFF' IN STRING::SET_REAL
+--echo # 
+
+let $nine_65=
+99999999999999999999999999999999999999999999999999999999999999999;
+
+--replace_column 1 ignore_float_result
+select format(-1.7976931348623157E+307,256) as foo;
+--replace_column 1 ignore_float_result
+select least(-1.1111111111111111111111111,
+             - group_concat(1.7976931348623157E+308)) as foo;
+eval select concat((truncate((-1.7976931348623157E+307),(0x1e))),
+                   ($nine_65)) into @a;
+
 --echo End of 5.0 tests

sql/sql_string.cc

@@ -117,7 +117,7 @@ bool String::set(ulonglong num, CHARSET_INFO *cs)
 
 bool String::set(double num,uint decimals, CHARSET_INFO *cs)
 {
-  char buff[331];
+  char buff[FLOATING_POINT_BUFFER];
   uint dummy_errors;
 
   str_charset=cs;
@@ -186,7 +186,9 @@ bool String::set(double num,uint decimals, CHARSET_INFO *cs)
 #else
 #ifdef HAVE_SNPRINTF
   buff[sizeof(buff)-1]=0;			// Safety
-  snprintf(buff,sizeof(buff)-1, "%.*f",(int) decimals,num);
+  int num_chars= snprintf(buff, sizeof(buff)-1, "%.*f",(int) decimals, num);
+  DBUG_ASSERT(num_chars > 0);
+  DBUG_ASSERT(num_chars < (int) sizeof(buff));
 #else
   sprintf(buff,"%.*f",(int) decimals,num);
 #endif

sql/unireg.h

@@ -175,7 +175,6 @@
 */
 
 #define BIN_LOG_HEADER_SIZE    4 
-#define FLOATING_POINT_BUFFER 331
 
 #define DEFAULT_KEY_CACHE_NAME "default"