MDEV-30932 UBSAN: negation of -X cannot be represented in type ..

  'long long int'; cast to an unsigned type to negate this value ..
  to itself in Item_func_mul::int_op and Item_func_round::int_op

Problems:

  The code in multiple places in the following methods:
    - Item_func_mul::int_op()
    - longlong Item_func_int_div::val_int()
    - Item_func_mod::int_op()
    - Item_func_round::int_op()

  did not properly check for corner values LONGLONG_MIN
  and (LONGLONG_MAX+1) before doing negation.
  This cuased UBSAN to complain about undefined behaviour.

Fix summary:

  - Adding helper classes ULonglong, ULonglong_null, ULonglong_hybrid
    (in addition to their signed couterparts in sql/sql_type_int.h).

  - Moving the code performing multiplication of ulonglong numbers
    from Item_func_mul::int_op() to ULonglong_hybrid::ullmul().

  - Moving the code responsible for extracting absolute values
    from negative numbers to Longlong::abs().
    It makes sure to perform negation without undefinite behavior:
    LONGLONG_MIN is handled in a special way.

  - Moving negation related code to ULonglong::operator-().
    It makes sure to perform negation without undefinite behavior:
    (LONGLONG_MAX + 1) is handled in a special way.

  - Moving signed<=>unsigned conversion code to
    Longlong_hybrid::val_int() and ULonglong_hybrid::val_int().

  - Reusing old and new sql_type_int.h classes in multiple
    places in Item_func_xxx::int_op().

Fix details (explain how sql_type_int.h classes are reused):

  - Instead of straight negation of negative "longlong" arguments
    *before* performing unsigned multiplication,
    Item_func_mul::int_op() now calls ULonglong_null::ullmul()
    using Longlong_hybrid_null::abs() to pass arguments.
    This fixes undefined behavior N1.

  - Instead of straight negation of "ulonglong" result
    *after* performing unsigned multiplication,
    Item_func_mul::int_op() now calls ULonglong_hybrid::val_int(),
    which recursively calls ULonglong::operator-().
    This fixes undefined behavior N2.

  - Removing duplicate negating code from Item_func_mod::int_op().
    Using ULonglong_hybrid::val_int() instead.
    This fixes undefinite behavior N3.

  - Removing literal "longlong" negation from Item_func_round::int_op().
    Using Longlong::abs() instead, which correctly handler LONGLONG_MIN.
    This fixes undefinite behavior N4.

  - Removing the duplicate (negation related) code from
    Item_func_int_div::val_int(). Reusing class ULonglong_hybrid.
    There were no undefinite behavior in here.
    However, this change allowed to reveal a bug in
    "-9223372036854775808 DIV 1".
    The removed negation code appeared to be incorrect when
    negating +9223372036854775808. It returned the "out of range" error.
    ULonglong_hybrid::operator-() now handles all values correctly
    and returns +9223372036854775808 as a negation for -9223372036854775808.

    Re-recording wrong results for
      SELECT -9223372036854775808 DIV  1;
    Now instead of "out of range", it returns -9223372036854775808,
    which is the smallest possible value for the expression data type
    (signed) BIGINT.

  - Removing "no UBSAN" branch from Item_func_splus::int_opt()
    and Item_func_minus::int_opt(), as it made UBSAN happy but
    in RelWithDebInfo some MTR tests started to fail.

mysql-test/main/func_math.result

@@ -972,7 +972,8 @@ SELECT  9223372036854775808 DIV  1;
 SELECT  9223372036854775808 DIV -1;
 ERROR 22003: BIGINT UNSIGNED value is out of range in '9223372036854775808 DIV -1'
 SELECT -9223372036854775808 DIV  1;
-ERROR 22003: BIGINT value is out of range in '-9223372036854775808 DIV 1'
+-9223372036854775808 DIV  1
+-9223372036854775808
 SELECT -9223372036854775808 DIV -1;
 ERROR 22003: BIGINT value is out of range in '-9223372036854775808 DIV -1'
 SELECT  9223372036854775808 MOD  1;
@@ -3546,5 +3547,31 @@ t2	CREATE TABLE `t2` (
 ) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci
 DROP TABLE t1,t2;
 #
+# MDEV-30932 UBSAN: negation of -X cannot be represented in type 'long long int'; cast to an unsigned type to negate this value to itself in Item_func_mul::int_op and Item_func_round::int_op
+#
+SELECT (1 DIV(-1/POW(807,14))*1);
+ERROR 22003: BIGINT value is out of range in '1 DIV (-1 / pow(807,14))'
+DO((-9223372036854775808)*(1));
+SELECT (-9223372036854775808)*(1);
+(-9223372036854775808)*(1)
+-9223372036854775808
+SELECT (GET_FORMAT(TIME,'JIS'))DIV(POW(-40,65)DIV(1)*2);
+ERROR 22003: BIGINT value is out of range in 'pow(-40,65) DIV 1'
+SELECT -9223372036854775808 MOD 9223372036854775810;
+-9223372036854775808 MOD 9223372036854775810
+-9223372036854775808
+CREATE TABLE t1 (c INT);
+INSERT INTO t1 VALUES(TRUNCATE(0,-1.e+30));
+DROP TABLE t1;
+SELECT TRUNCATE(0, -9223372036854775808);
+TRUNCATE(0, -9223372036854775808)
+0
+SELECT GET_FORMAT(TIME,'JIS') DIV ATAN (TRUNCATE (0,'2000000000000000' DIV SIN(1500)*NOW(5)));
+GET_FORMAT(TIME,'JIS') DIV ATAN (TRUNCATE (0,'2000000000000000' DIV SIN(1500)*NOW(5)))
+NULL
+SELECT (GET_FORMAT(TIME,'JIS') DIV ATAN (TRUNCATE (0,'2000000000000000' DIV SIN(1500)*NOW(5))/ROUND(-1)))DIV(-1-LOG2(1))-(-1*POWER(-1,0));
+(GET_FORMAT(TIME,'JIS') DIV ATAN (TRUNCATE (0,'2000000000000000' DIV SIN(1500)*NOW(5))/ROUND(-1)))DIV(-1-LOG2(1))-(-1*POWER(-1,0))
+NULL
+#
 # End of 10.4 tests
 #

mysql-test/main/func_math.test

@@ -710,7 +710,6 @@ DROP TABLE t1;
 SELECT  9223372036854775808 DIV  1;
 --error ER_DATA_OUT_OF_RANGE
 SELECT  9223372036854775808 DIV -1;
---error ER_DATA_OUT_OF_RANGE
 SELECT -9223372036854775808 DIV  1;
 --error ER_DATA_OUT_OF_RANGE
 SELECT -9223372036854775808 DIV -1;
@@ -1867,6 +1866,32 @@ SELECT * FROM t2;
 SHOW CREATE TABLE t2;
 DROP TABLE t1,t2;
 
+--echo #
+--echo # MDEV-30932 UBSAN: negation of -X cannot be represented in type 'long long int'; cast to an unsigned type to negate this value to itself in Item_func_mul::int_op and Item_func_round::int_op
+--echo #
+
+--error ER_DATA_OUT_OF_RANGE
+SELECT (1 DIV(-1/POW(807,14))*1);
+
+DO((-9223372036854775808)*(1));
+
+SELECT (-9223372036854775808)*(1);
+
+--error ER_DATA_OUT_OF_RANGE
+SELECT (GET_FORMAT(TIME,'JIS'))DIV(POW(-40,65)DIV(1)*2);
+
+SELECT -9223372036854775808 MOD 9223372036854775810;
+
+CREATE TABLE t1 (c INT);
+INSERT INTO t1 VALUES(TRUNCATE(0,-1.e+30));
+DROP TABLE t1;
+SELECT TRUNCATE(0, -9223372036854775808);
+
+--disable_warnings
+SELECT GET_FORMAT(TIME,'JIS') DIV ATAN (TRUNCATE (0,'2000000000000000' DIV SIN(1500)*NOW(5)));
+SELECT (GET_FORMAT(TIME,'JIS') DIV ATAN (TRUNCATE (0,'2000000000000000' DIV SIN(1500)*NOW(5))/ROUND(-1)))DIV(-1-LOG2(1))-(-1*POWER(-1,0));
+--enable_warnings
+
 --echo #
 --echo # End of 10.4 tests
 --echo #

sql/item_func.cc

@@ -78,7 +78,7 @@ bool check_reserved_words(const LEX_CSTRING *name)
 */
 static inline bool test_if_sum_overflows_ull(ulonglong arg1, ulonglong arg2)
 {
-  return ULONGLONG_MAX - arg1 < arg2;
+  return ULonglong::test_if_sum_overflows_ull(arg1, arg2);
 }
 
 
@@ -1157,14 +1157,10 @@ longlong Item_func_plus::int_op()
     }
   }
 
-#ifndef WITH_UBSAN
-  res= val0 + val1;
-#else
   if (res_unsigned)
     res= (longlong) ((ulonglong) val0 + (ulonglong) val1);
   else
     res= val0+val1;
-#endif /* WITH_UBSAN */
 
   return check_integer_overflow(res, res_unsigned);
 
@@ -1325,14 +1321,10 @@ longlong Item_func_minus::int_op()
         goto err;
     }
   }
-#ifndef WITH_UBSAN
-  res= val0 - val1;
-#else
   if (res_unsigned)
     res= (longlong) ((ulonglong) val0 - (ulonglong) val1);
   else
     res= val0 - val1;
-#endif /* WITH_UBSAN */
 
   return check_integer_overflow(res, res_unsigned);
 
@@ -1375,79 +1367,23 @@ double Item_func_mul::real_op()
 longlong Item_func_mul::int_op()
 {
   DBUG_ASSERT(fixed == 1);
-  longlong a= args[0]->val_int();
-  longlong b= args[1]->val_int();
-  longlong res;
-  ulonglong res0, res1;
-  ulong a0, a1, b0, b1;
-  bool     res_unsigned= FALSE;
-  bool     a_negative= FALSE, b_negative= FALSE;
-
-  if ((null_value= args[0]->null_value || args[1]->null_value))
-    return 0;
-
   /*
-    First check whether the result can be represented as a
-    (bool unsigned_flag, longlong value) pair, then check if it is compatible
-    with this Item's unsigned_flag by calling check_integer_overflow().
-
-    Let a = a1 * 2^32 + a0 and b = b1 * 2^32 + b0. Then
-    a * b = (a1 * 2^32 + a0) * (b1 * 2^32 + b0) = a1 * b1 * 2^64 +
-            + (a1 * b0 + a0 * b1) * 2^32 + a0 * b0;
-    We can determine if the above sum overflows the ulonglong range by
-    sequentially checking the following conditions:
-    1. If both a1 and b1 are non-zero.
-    2. Otherwise, if (a1 * b0 + a0 * b1) is greater than ULONG_MAX.
-    3. Otherwise, if (a1 * b0 + a0 * b1) * 2^32 + a0 * b0 is greater than
-    ULONGLONG_MAX.
-
     Since we also have to take the unsigned_flag for a and b into account,
     it is easier to first work with absolute values and set the
     correct sign later.
   */
-  if (!args[0]->unsigned_flag && a < 0)
-  {
-    a_negative= TRUE;
-    a= -a;
-  }
-  if (!args[1]->unsigned_flag && b < 0)
-  {
-    b_negative= TRUE;
-    b= -b;
-  }
+  Longlong_hybrid_null ha= args[0]->to_longlong_hybrid_null();
+  Longlong_hybrid_null hb= args[1]->to_longlong_hybrid_null();
 
-  a0= 0xFFFFFFFFUL & a;
-  a1= ((ulonglong) a) >> 32;
-  b0= 0xFFFFFFFFUL & b;
-  b1= ((ulonglong) b) >> 32;
-
-  if (a1 && b1)
-    goto err;
-
-  res1= (ulonglong) a1 * b0 + (ulonglong) a0 * b1;
-  if (res1 > 0xFFFFFFFFUL)
-    goto err;
-
-  res1= res1 << 32;
-  res0= (ulonglong) a0 * b0;
-
-  if (test_if_sum_overflows_ull(res1, res0))
-    goto err;
-  res= res1 + res0;
-
-  if (a_negative != b_negative)
-  {
-    if ((ulonglong) res > (ulonglong) LONGLONG_MIN + 1)
-      goto err;
-    res= -res;
-  }
-  else
-    res_unsigned= TRUE;
+  if ((null_value= ha.is_null() || hb.is_null()))
+    return 0;
 
-  return check_integer_overflow(res, res_unsigned);
+  ULonglong_null ures= ULonglong_null::ullmul(ha.abs(), hb.abs());
+  if (ures.is_null())
+    return raise_integer_overflow();
 
-err:
-  return raise_integer_overflow();
+  return check_integer_overflow(ULonglong_hybrid(ures.value(),
+                                                 ha.neg() != hb.neg()));
 }
 
 
@@ -1645,15 +1581,8 @@ longlong Item_func_int_div::val_int()
     return 0;
   }
 
-  bool res_negative= val0.neg() != val1.neg();
-  ulonglong res= val0.abs() / val1.abs();
-  if (res_negative)
-  {
-    if (res > (ulonglong) LONGLONG_MAX)
-      return raise_integer_overflow();
-    res= (ulonglong) (-(longlong) res);
-  }
-  return check_integer_overflow(res, !res_negative);
+  return check_integer_overflow(ULonglong_hybrid(val0.abs() / val1.abs(),
+                                                 val0.neg() != val1.neg()));
 }
 
 
@@ -1687,9 +1616,8 @@ longlong Item_func_mod::int_op()
     LONGLONG_MIN by -1 generates SIGFPE, we calculate using unsigned values and
     then adjust the sign appropriately.
   */
-  ulonglong res= val0.abs() % val1.abs();
-  return check_integer_overflow(val0.neg() ? -(longlong) res : res,
-                                !val0.neg());
+  return check_integer_overflow(ULonglong_hybrid(val0.abs() % val1.abs(),
+                                                 val0.neg()));
 }
 
 double Item_func_mod::real_op()
@@ -2669,7 +2597,7 @@ longlong Item_func_round::int_op()
   if ((dec >= 0) || args[1]->unsigned_flag)
     return value; // integer have not digits after point
 
-  abs_dec= -dec;
+  abs_dec= Longlong(dec).abs(); // Avoid undefined behavior
   longlong tmp;
   
   if(abs_dec >= array_elements(log_10_int))

sql/item_func.h

@@ -252,12 +252,23 @@ class Item_func :public Item_func_or_sum,
   */
   inline longlong check_integer_overflow(longlong value, bool val_unsigned)
   {
-    if ((unsigned_flag && !val_unsigned && value < 0) ||
-        (!unsigned_flag && val_unsigned &&
-         (ulonglong) value > (ulonglong) LONGLONG_MAX))
-      return raise_integer_overflow();
-    return value;
+    return check_integer_overflow(Longlong_hybrid(value, val_unsigned));
+  }
+
+  // Check if the value is compatible with Item::unsigned_flag.
+  inline longlong check_integer_overflow(const Longlong_hybrid &sval)
+  {
+    Longlong_null res= sval.val_int(unsigned_flag);
+    return res.is_null() ? raise_integer_overflow() : res.value();
   }
+
+  // Check if the value is compatible with Item::unsigned_flag.
+  longlong check_integer_overflow(const ULonglong_hybrid &uval)
+  {
+    Longlong_null res= uval.val_int(unsigned_flag);
+    return res.is_null() ? raise_integer_overflow() : res.value();
+  }
+
   /**
      Throw an error if the error code of a DECIMAL operation is E_DEC_OVERFLOW.
   */

sql/sql_type_int.h

@@ -34,6 +34,12 @@ class Longlong
 public:
   longlong value() const { return m_value; }
   Longlong(longlong nr) :m_value(nr) { }
+  ulonglong abs()
+  {
+    if (m_value == LONGLONG_MIN) // avoid undefined behavior
+      return ((ulonglong) LONGLONG_MAX) + 1;
+    return m_value < 0 ? -m_value : m_value;
+  }
 };
 
 
@@ -46,6 +52,86 @@ class Longlong_null: public Longlong, public Null_flag
 };
 
 
+class ULonglong
+{
+protected:
+  ulonglong m_value;
+public:
+  ulonglong value() const { return m_value; }
+  explicit ULonglong(ulonglong nr) :m_value(nr) { }
+
+  static bool test_if_sum_overflows_ull(ulonglong arg1, ulonglong arg2)
+  {
+    return ULONGLONG_MAX - arg1 < arg2;
+  }
+
+  Longlong_null operator-() const
+  {
+    if (m_value > (ulonglong) LONGLONG_MAX) // Avoid undefined behaviour
+    {
+      return m_value == (ulonglong) LONGLONG_MAX + 1 ?
+             Longlong_null(LONGLONG_MIN, false) :
+             Longlong_null(0, true);
+    }
+    return Longlong_null(-(longlong) m_value, false);
+  }
+
+  // Convert to Longlong_null with the range check
+  Longlong_null to_longlong_null() const
+  {
+    if (m_value > (ulonglong) LONGLONG_MAX)
+      return Longlong_null(0, true);
+    return Longlong_null((longlong) m_value, false);
+  }
+
+};
+
+
+class ULonglong_null: public ULonglong, public Null_flag
+{
+public:
+  ULonglong_null(ulonglong nr, bool is_null)
+   :ULonglong(nr), Null_flag(is_null)
+  { }
+
+  /*
+    Multiply two ulonglong values.
+
+    Let a = a1 * 2^32 + a0 and b = b1 * 2^32 + b0. Then
+    a * b = (a1 * 2^32 + a0) * (b1 * 2^32 + b0) = a1 * b1 * 2^64 +
+            + (a1 * b0 + a0 * b1) * 2^32 + a0 * b0;
+    We can determine if the above sum overflows the ulonglong range by
+    sequentially checking the following conditions:
+    1. If both a1 and b1 are non-zero.
+    2. Otherwise, if (a1 * b0 + a0 * b1) is greater than ULONG_MAX.
+    3. Otherwise, if (a1 * b0 + a0 * b1) * 2^32 + a0 * b0 is greater than
+    ULONGLONG_MAX.
+  */
+  static ULonglong_null ullmul(ulonglong a, ulonglong b)
+  {
+    ulong a1= a >> 32;
+    ulong b1= b >> 32;
+
+    if (a1 && b1)
+      return ULonglong_null(0, true);
+
+    ulong a0= 0xFFFFFFFFUL & a;
+    ulong b0= 0xFFFFFFFFUL & b;
+
+    ulonglong res1= (ulonglong) a1 * b0 + (ulonglong) a0 * b1;
+    if (res1 > 0xFFFFFFFFUL)
+      return ULonglong_null(0, true);
+
+    res1= res1 << 32;
+    ulonglong res0= (ulonglong) a0 * b0;
+
+    if (test_if_sum_overflows_ull(res1, res0))
+      return ULonglong_null(0, true);
+    return ULonglong_null(res1 + res0, false);
+  }
+};
+
+
 // A longlong/ulonglong hybrid. Good to store results of val_int().
 class Longlong_hybrid: public Longlong
 {
@@ -74,9 +160,7 @@ class Longlong_hybrid: public Longlong
   {
     if (m_unsigned)
       return (ulonglong) m_value;
-    if (m_value == LONGLONG_MIN) // avoid undefined behavior
-      return ((ulonglong) LONGLONG_MAX) + 1;
-    return m_value < 0 ? -m_value : m_value;
+    return Longlong(m_value).abs();
   }
   /*
     Convert to an unsigned number:
@@ -94,6 +178,33 @@ class Longlong_hybrid: public Longlong
   {
     return (uint) to_ulonglong(upper_bound);
   }
+
+
+  Longlong_null val_int_signed() const
+  {
+    if (m_unsigned)
+      return ULonglong((ulonglong) m_value).to_longlong_null();
+    return Longlong_null(m_value, false);
+  }
+
+  Longlong_null val_int_unsigned() const
+  {
+    if (!m_unsigned && m_value < 0)
+      return Longlong_null(0, true);
+    return Longlong_null(m_value, false);
+  }
+
+  /*
+    Return in Item compatible val_int() format:
+    - signed numbers as a straight longlong value
+    - unsigned numbers as a ulonglong value reinterpreted to longlong
+  */
+  Longlong_null val_int(bool want_unsigned_value) const
+  {
+    return want_unsigned_value ? val_int_unsigned() :
+                                 val_int_signed();
+  }
+
   int cmp(const Longlong_hybrid& other) const
   {
     if (m_unsigned == other.m_unsigned)
@@ -143,4 +254,50 @@ class Longlong_hybrid_null: public Longlong_hybrid,
 };
 
 
+/*
+  Stores the absolute value of a number, and the sign.
+  Value range: -ULONGLONG_MAX .. +ULONGLONG_MAX.
+
+  Provides a wider range for negative numbers than Longlong_hybrid does.
+  Usefull to store intermediate results of an expression whose value
+  is further needed to be negated. For example, these methods:
+    - Item_func_mul::int_op()
+    - Item_func_int_div::val_int()
+    - Item_func_mod::int_op()
+  calculate the result of absolute values of the arguments,
+  then optionally negate the result.
+*/
+class ULonglong_hybrid: public ULonglong
+{
+  bool m_neg;
+public:
+  ULonglong_hybrid(ulonglong value, bool neg)
+   :ULonglong(value), m_neg(neg)
+  {
+    if (m_neg && !m_value)
+      m_neg= false;        // convert -0 to +0
+  }
+  Longlong_null val_int_unsigned() const
+  {
+    return m_neg ? Longlong_null(0, true) :
+                   Longlong_null((longlong) m_value, false);
+  }
+  Longlong_null val_int_signed() const
+  {
+    return m_neg ? -ULonglong(m_value) : ULonglong::to_longlong_null();
+  }
+
+  /*
+    Return in Item compatible val_int() format:
+    - signed numbers as a straight longlong value
+    - unsigned numbers as a ulonglong value reinterpreted to longlong
+  */
+  Longlong_null val_int(bool want_unsigned_value) const
+  {
+    return want_unsigned_value ? val_int_unsigned() :
+                                 val_int_signed();
+  }
+};
+
+
 #endif // SQL_TYPE_INT_INCLUDED