Bug#12537160 ASSERTION FAILED: STOP0 <= &TO->BUF[TO->LEN] WITH LARGE NUMBER.

Turns out the DBUG_ASSERT added by fix for Bug#11792200 was overly pessimistic:
'stop0' is used in the main loop of do_div_mod, but we only dereference 'buf0'
for div operations, not for mod.


mysql-test/r/func_math.result:
  New test case.
mysql-test/t/func_math.test:
  New test case.
strings/decimal.c:
  Move DBUG_ASSERT down to where we actually dereference the loop pointer.

mysql-test/r/func_math.result

@@ -699,3 +699,13 @@ select (1.175494351E-37 div 1.7976931348623157E+308);
 0
 Warnings:
 Warning	1292	Truncated incorrect DECIMAL value: ''
+#
+# Bug#12537160 ASSERTION FAILED:
+# STOP0 <= &TO->BUF[TO->LEN] WITH LARGE NUMBER.
+#
+select 999999999999999999999999999999999999999999999999999999999999999999999999999999999 % 0.1 as foo;
+foo
+0.0
+select 999999999999999999999999999999999999999999999999999999999999999999999999999999999 % 0.0 as foo;
+foo
+NULL

mysql-test/t/func_math.test

@@ -536,3 +536,14 @@ SELECT 1 div null;
 --echo # Bug #11792200 - DIVIDING LARGE NUMBERS CAUSES STACK CORRUPTIONS
 --echo #
 select (1.175494351E-37 div 1.7976931348623157E+308);
+
+--echo #
+--echo # Bug#12537160 ASSERTION FAILED:
+--echo # STOP0 <= &TO->BUF[TO->LEN] WITH LARGE NUMBER.
+--echo #
+
+let $nine_81=
+999999999999999999999999999999999999999999999999999999999999999999999999999999999;
+
+eval select $nine_81 % 0.1 as foo;
+eval select $nine_81 % 0.0 as foo;

strings/decimal.c

@@ -2182,7 +2182,6 @@ static int do_div_mod(const decimal_t *from1, const decimal_t *from2,
   }
   buf0=to->buf;
   stop0=buf0+intg0+frac0;
-  DBUG_ASSERT(stop0 <= &to->buf[to->len]);
   if (likely(div_mod))
     while (dintg++ < 0 && buf0 < &to->buf[to->len])
     {
@@ -2277,7 +2276,10 @@ static int do_div_mod(const decimal_t *from1, const decimal_t *from2,
       }
     }
     if (likely(div_mod))
+    {
+      DBUG_ASSERT(buf0 < to->buf + to->len);
       *buf0=(dec1)guess;
+    }
     dcarry= *start1;
     start1++;
   }