MDEV-28387 UBSAN: runtime error: negation of -9223372036854775808 cannot be...

MDEV-28387 UBSAN: runtime error: negation of -9223372036854775808 cannot be represented in type 'long long int'; cast to an unsigned type to negate this value to itself in my_strtoll10 on SELECT

Fixing the condition to raise an overflow in the ulonglong
representation of the number is greater or equal to 0x8000000000000000ULL.
Before this change the condition did not catch -9223372036854775808
(the smallest possible signed negative longlong number).

mysql-test/main/func_str.result

@@ -5311,5 +5311,13 @@ NULL
 DROP TABLE t1;
 DROP VIEW v1;
 #
+# MDEV-28387 UBSAN: runtime error: negation of -9223372036854775808 cannot be represented in type 'long long int'; cast to an unsigned type to negate this value to itself in my_strtoll10 on SELECT
+#
+SET @a='-9223372036854775808';
+CREATE TABLE t (c1 INT,c2 CHAR);
+SELECT SUBSTR(0,@a) FROM t;
+SUBSTR(0,@a)
+DROP TABLE t;
+#
 # End of 10.5 tests
 #

mysql-test/main/func_str.test

@@ -2348,6 +2348,15 @@ DROP TABLE t1;
 DROP VIEW v1;
 
 
+--echo #
+--echo # MDEV-28387 UBSAN: runtime error: negation of -9223372036854775808 cannot be represented in type 'long long int'; cast to an unsigned type to negate this value to itself in my_strtoll10 on SELECT
+--echo #
+
+SET @a='-9223372036854775808';  # Quite specific value; considerably varying it will not work
+CREATE TABLE t (c1 INT,c2 CHAR);
+SELECT SUBSTR(0,@a) FROM t;
+DROP TABLE t;
+
 --echo #
 --echo # End of 10.5 tests
 --echo #

strings/my_strtoll10.c

@@ -241,7 +241,7 @@ longlong my_strtoll10(const char *nptr, char **endptr, int *error)
   *endptr= (char*) s;
   if (negative)
   {
-   if (li > MAX_NEGATIVE_NUMBER)
+   if (li >= MAX_NEGATIVE_NUMBER)
      goto overflow;
    return -((longlong) li);
   }