MDEV-17643 Assertion `nr >= 0.0' failed in Item_sum_std::val_real()

mysql-test/r/func_math.result

@@ -807,3 +807,46 @@ STDDEV_POP(ROUND(0,@A:=2009))
 #
 # End of 10.0 tests
 #
+#
+# Start of 10.1 tests
+#
+#
+# MDEV-17643 Assertion `nr >= 0.0' failed in Item_sum_std::val_real()
+#
+CREATE TABLE t1 ( pk int NOT NULL, i1 int NOT NULL, d1 date NOT NULL, t1 time);
+INSERT INTO t1 VALUES (7,9,'2007-08-15','03:55:02'),(8,7,'1993-06-05','04:17:51'),(9,7,'2034-07-01','17:31:12'),(10,0,'1998-08-24','08:09:27');
+SELECT DISTINCT  STDDEV_SAMP(EXPORT_SET(t1, -1379790335835635712, (i1 + 'o'), (MD5(d1)))) FROM t1;
+STDDEV_SAMP(EXPORT_SET(t1, -1379790335835635712, (i1 + 'o'), (MD5(d1))))
+NULL
+Warnings:
+Warning	1292	Truncated incorrect DOUBLE value: 'o'
+Warning	1292	Truncated incorrect DOUBLE value: '98e466c7ff40fe6b95cde24200f376303-13797903358356357128e466c7ff40fe6b95cde24200f376303-13797903358356357128e466c7ff40fe6b95cde242'
+Warning	1292	Truncated incorrect DOUBLE value: 'o'
+Warning	1292	Truncated incorrect DOUBLE value: '-1379790335835635712e315457d879863c6ccf2ddee5562fc24-1379790335835635712e315457d879863c6ccf2ddee5562fc24-1379790335835635712e315'
+Warning	1292	Truncated incorrect DOUBLE value: 'o'
+Warning	1292	Truncated incorrect DOUBLE value: '7b4dd517b633f1f6304b773523b5279747b4dd517b633f1f6304b773523b5279747b4dd517b633f1f6304b773523b527974-1379790335835635712b4dd517b6'
+Warning	1292	Truncated incorrect DOUBLE value: 'o'
+Warning	1292	Truncated incorrect DOUBLE value: '-1379790335835635712b0e107767ea830fd3318893e40412a43-1379790335835635712b0e107767ea830fd3318893e40412a43-1379790335835635712b0e1'
+DROP TABLE t1;
+CREATE TABLE t1 (a VARCHAR(128));
+INSERT INTO t1 VALUES ('1e310');
+INSERT INTO t1 VALUES ('-1e310');
+INSERT INTO t1 VALUES ('0');
+SELECT STDDEV_SAMP(a) FROM t1;
+STDDEV_SAMP(a)
+NULL
+Warnings:
+Warning	1292	Truncated incorrect DOUBLE value: '1e310'
+Warning	1292	Truncated incorrect DOUBLE value: '-1e310'
+DROP TABLE t1;
+CREATE OR REPLACE TABLE t1 (a DOUBLE);
+INSERT INTO t1 VALUES (1.7e+308);
+INSERT INTO t1 VALUES (-1.7e+308);
+INSERT INTO t1 VALUES (0);
+SELECT STDDEV_SAMP(a) FROM t1;
+STDDEV_SAMP(a)
+NULL
+DROP TABLE t1;
+#
+# End of 10.1 tests
+#

mysql-test/t/func_math.test

@@ -599,3 +599,36 @@ SELECT STDDEV_POP(ROUND(0,@A:=2009)) FROM (SELECT 1 UNION SELECT 2) fake_table;
 --echo #
 --echo # End of 10.0 tests
 --echo #
+
+
+--echo #
+--echo # Start of 10.1 tests
+--echo #
+
+--echo #
+--echo # MDEV-17643 Assertion `nr >= 0.0' failed in Item_sum_std::val_real()
+--echo #
+
+CREATE TABLE t1 ( pk int NOT NULL, i1 int NOT NULL, d1 date NOT NULL, t1 time);
+INSERT INTO t1 VALUES (7,9,'2007-08-15','03:55:02'),(8,7,'1993-06-05','04:17:51'),(9,7,'2034-07-01','17:31:12'),(10,0,'1998-08-24','08:09:27');
+SELECT DISTINCT  STDDEV_SAMP(EXPORT_SET(t1, -1379790335835635712, (i1 + 'o'), (MD5(d1)))) FROM t1;
+DROP TABLE t1;
+
+CREATE TABLE t1 (a VARCHAR(128));
+INSERT INTO t1 VALUES ('1e310');
+INSERT INTO t1 VALUES ('-1e310');
+INSERT INTO t1 VALUES ('0');
+SELECT STDDEV_SAMP(a) FROM t1;
+DROP TABLE t1;
+
+CREATE OR REPLACE TABLE t1 (a DOUBLE);
+INSERT INTO t1 VALUES (1.7e+308);
+INSERT INTO t1 VALUES (-1.7e+308);
+INSERT INTO t1 VALUES (0);
+SELECT STDDEV_SAMP(a) FROM t1;
+DROP TABLE t1;
+
+
+--echo #
+--echo # End of 10.1 tests
+--echo #

sql/item_sum.cc

@@ -1765,6 +1765,18 @@ double Item_sum_std::val_real()
 {
   DBUG_ASSERT(fixed == 1);
   double nr= Item_sum_variance::val_real();
+  if (isnan(nr))
+  {
+    /*
+      variance_fp_recurrence_next() can overflow in some cases and return "nan":
+
+      CREATE OR REPLACE TABLE t1 (a DOUBLE);
+      INSERT INTO t1 VALUES (1.7e+308), (-1.7e+308), (0);
+      SELECT STDDEV_SAMP(a) FROM t1;
+    */
+    null_value= true; // Convert "nan" to NULL
+    return 0;
+  }
   if (my_isinf(nr))
     return DBL_MAX;
   DBUG_ASSERT(nr >= 0.0);