MDEV-31221 UBSAN runtime error: negation of -9223372036854775808 cannot be...

MDEV-31221 UBSAN runtime error: negation of -9223372036854775808 cannot be represented in type 'long long int' in my_strtoll10_utf32

The code in my_strtoll10_mb2 and my_strtoll10_utf32
could hit undefinite behavior by negation of LONGLONG_MIN.
Fixing to avoid this.

Also, fixing my_strtoll10() in the same style.
The previous reduction produced a redundant warning on
CAST(_latin1'-9223372036854775808' AS SIGNED)

mysql-test/main/ctype_latin1.result

@@ -8975,5 +8975,11 @@ CAST(_latin1 0x61FF62 AS INT)
 Warnings:
 Warning	1292	Truncated incorrect INTEGER value: 'ab'
 #
+# MDEV-31221 UBSAN runtime error: negation of -9223372036854775808 cannot be represented in type 'long long int' in my_strtoll10_utf32
+#
+SELECT CAST(CONVERT('-9223372036854775808' USING latin1) AS SIGNED) AS c1;
+c1
+-9223372036854775808
+#
 # End of 10.5 tests
 #

mysql-test/main/ctype_latin1.test

@@ -501,6 +501,13 @@ SELECT CAST(_latin1 0x617E62 AS INT);
 SELECT CAST(_latin1 0x61FF62 AS INT);
 
 
+--echo #
+--echo # MDEV-31221 UBSAN runtime error: negation of -9223372036854775808 cannot be represented in type 'long long int' in my_strtoll10_utf32
+--echo #
+
+SELECT CAST(CONVERT('-9223372036854775808' USING latin1) AS SIGNED) AS c1;
+
+
 --echo #
 --echo # End of 10.5 tests
 --echo #

mysql-test/main/ctype_ucs.result

@@ -6549,5 +6549,11 @@ OCT(c)
 1000000000000000000000
 DROP TABLE t1;
 #
+# MDEV-31221 UBSAN runtime error: negation of -9223372036854775808 cannot be represented in type 'long long int' in my_strtoll10_utf32
+#
+SELECT CAST(CONVERT('-9223372036854775808' USING ucs2) AS SIGNED) AS c1;
+c1
+-9223372036854775808
+#
 # End of 10.5 tests
 #

mysql-test/main/ctype_ucs.test

@@ -1226,6 +1226,11 @@ INSERT INTO t1 VALUES ('-9223372036854775808.5');
 SELECT OCT(c) FROM t1;
 DROP TABLE t1;
 
+--echo #
+--echo # MDEV-31221 UBSAN runtime error: negation of -9223372036854775808 cannot be represented in type 'long long int' in my_strtoll10_utf32
+--echo #
+
+SELECT CAST(CONVERT('-9223372036854775808' USING ucs2) AS SIGNED) AS c1;
 
 --echo #
 --echo # End of 10.5 tests

mysql-test/main/ctype_utf32.result

@@ -3024,3 +3024,12 @@ HEX(DATE_FORMAT(TIME'11:22:33',@format))
 #
 # End of 10.4 tests
 #
+#
+# MDEV-31221 UBSAN runtime error: negation of -9223372036854775808 cannot be represented in type 'long long int' in my_strtoll10_utf32
+#
+SELECT CAST(CONVERT('-9223372036854775808' USING utf32) AS SIGNED) AS c1;
+c1
+-9223372036854775808
+#
+# End of 10.5 tests
+#

mysql-test/main/ctype_utf32.test

@@ -1162,3 +1162,14 @@ SELECT HEX(DATE_FORMAT(TIME'11:22:33',@format));
 --echo #
 
 --enable_service_connection
+
+
+--echo #
+--echo # MDEV-31221 UBSAN runtime error: negation of -9223372036854775808 cannot be represented in type 'long long int' in my_strtoll10_utf32
+--echo #
+
+SELECT CAST(CONVERT('-9223372036854775808' USING utf32) AS SIGNED) AS c1;
+
+--echo #
+--echo # End of 10.5 tests
+--echo #

strings/ctype-ucs2.c

@@ -1011,6 +1011,8 @@ my_strtoll10_mb2(CHARSET_INFO *cs __attribute__((unused)),
   {
    if (li > MAX_NEGATIVE_NUMBER)
      goto overflow;
+   if (li == MAX_NEGATIVE_NUMBER) // Avoid undefinite behavior in negation
+     return LONGLONG_MIN;
    return -((longlong) li);
   }
   return (longlong) li;
@@ -2574,6 +2576,8 @@ my_strtoll10_utf32(CHARSET_INFO *cs __attribute__((unused)),
   {
    if (li > MAX_NEGATIVE_NUMBER)
      goto overflow;
+   if (li == MAX_NEGATIVE_NUMBER) // Avoid undefinite behavior in negation
+     return LONGLONG_MIN;
    return -((longlong) li);
   }
   return (longlong) li;

strings/my_strtoll10.c

@@ -241,8 +241,10 @@ longlong my_strtoll10(const char *nptr, char **endptr, int *error)
   *endptr= (char*) s;
   if (negative)
   {
-   if (li >= MAX_NEGATIVE_NUMBER) // Avoid undefined behavior
+   if (li > MAX_NEGATIVE_NUMBER)
      goto overflow;
+   if (li == MAX_NEGATIVE_NUMBER) // Avoid undefined behavior
+     return LONGLONG_MIN;
    return -((longlong) li);
   }
   return (longlong) li;