Bug#15757: Wrong SUBSTRING() result when a tmp table was employed.

When the SUBSTRING() function was used over a LONGTEXT field the max_length of
the SUBSTRING() result was wrongly calculated and set to 0. As the max_length
parameter is used while tmp field creation it limits the length of the result
field and leads to printing an empty string instead of the correct result.

Now the Item_func_substr::fix_length_and_dec() function correctly calculates
the max_length parameter.

mysql-test/r/func_str.result

@@ -1946,4 +1946,15 @@ NULL
 SELECT UNHEX('G') IS NULL;
 UNHEX('G') IS NULL
 1
+create table t1(f1 longtext);
+insert into t1 values ("123"),("456");
+select substring(f1,1,1) from t1 group by 1;
+substring(f1,1,1)
+1
+4
+create table t2(f1 varchar(3));
+insert into t1 values ("123"),("456");
+select substring(f1,4,1), substring(f1,-4,1) from t2;
+substring(f1,4,1)	substring(f1,-4,1)
+drop table t1,t2;
 End of 5.0 tests

mysql-test/t/func_str.test

@@ -1014,4 +1014,15 @@ select lpad('abc', cast(5 as unsigned integer), 'x');
 SELECT UNHEX('G');
 SELECT UNHEX('G') IS NULL;
 
+#
+# Bug#15757: Wrong SUBSTRING() result when a tmp table was employed.
+#
+create table t1(f1 longtext);
+insert into t1 values ("123"),("456");
+select substring(f1,1,1) from t1 group by 1;
+create table t2(f1 varchar(3));
+insert into t1 values ("123"),("456");
+select substring(f1,4,1), substring(f1,-4,1) from t2;
+drop table t1,t2;
+
 --echo End of 5.0 tests

sql/item_strfunc.cc

@@ -1184,11 +1184,10 @@ void Item_func_substr::fix_length_and_dec()
   if (args[1]->const_item())
   {
     int32 start= (int32) args[1]->val_int();
-    start= (int32)((start < 0) ? max_length + start : start - 1);
-    if (start < 0 || start >= (int32) max_length)
-      max_length=0; /* purecov: inspected */
+    if (start < 0)
+      max_length= ((uint)(-start) > max_length) ? 0 : (uint)(-start);
     else
-      max_length-= (uint) start;
+      max_length-= min((uint)(start - 1), max_length);
   }
   if (arg_count == 3 && args[2]->const_item())
   {