MDEV-31379 Undefined behavior in the reference Ed25519 implementation

apply the fix from MXS-4686

plugin/auth_ed25519/ref10/fe_mul.c

 #include "fe.h"
 #include "crypto_int64.h"
+#include "crypto_uint64.h"
 
 /*
 h = f * g
@@ -179,16 +180,16 @@ void fe_mul(fe h,const fe f,const fe g)
   crypto_int64 h7 = f0g7+f1g6   +f2g5   +f3g4   +f4g3   +f5g2   +f6g1   +f7g0   +f8g9_19+f9g8_19;
   crypto_int64 h8 = f0g8+f1g7_2 +f2g6   +f3g5_2 +f4g4   +f5g3_2 +f6g2   +f7g1_2 +f8g0   +f9g9_38;
   crypto_int64 h9 = f0g9+f1g8   +f2g7   +f3g6   +f4g5   +f5g4   +f6g3   +f7g2   +f8g1   +f9g0   ;
-  crypto_int64 carry0;
-  crypto_int64 carry1;
-  crypto_int64 carry2;
-  crypto_int64 carry3;
-  crypto_int64 carry4;
-  crypto_int64 carry5;
-  crypto_int64 carry6;
-  crypto_int64 carry7;
-  crypto_int64 carry8;
-  crypto_int64 carry9;
+  crypto_uint64 carry0;
+  crypto_uint64 carry1;
+  crypto_uint64 carry2;
+  crypto_uint64 carry3;
+  crypto_uint64 carry4;
+  crypto_uint64 carry5;
+  crypto_uint64 carry6;
+  crypto_uint64 carry7;
+  crypto_uint64 carry8;
+  crypto_uint64 carry9;
 
   /*
   |h0| <= (1.65*1.65*2^52*(1+19+19+19+19)+1.65*1.65*2^50*(38+38+38+38+38))

plugin/auth_ed25519/ref10/fe_sq.c

 #include "fe.h"
 #include "crypto_int64.h"
+#include "crypto_uint64.h"
 
 /*
 h = f * f
@@ -106,16 +107,16 @@ void fe_sq(fe h,const fe f)
   crypto_int64 h7 = f0f7_2+f1f6_2 +f2f5_2 +f3f4_2 +f8f9_38;
   crypto_int64 h8 = f0f8_2+f1f7_4 +f2f6_2 +f3f5_4 +f4f4   +f9f9_38;
   crypto_int64 h9 = f0f9_2+f1f8_2 +f2f7_2 +f3f6_2 +f4f5_2;
-  crypto_int64 carry0;
-  crypto_int64 carry1;
-  crypto_int64 carry2;
-  crypto_int64 carry3;
-  crypto_int64 carry4;
-  crypto_int64 carry5;
-  crypto_int64 carry6;
-  crypto_int64 carry7;
-  crypto_int64 carry8;
-  crypto_int64 carry9;
+  crypto_uint64 carry0;
+  crypto_uint64 carry1;
+  crypto_uint64 carry2;
+  crypto_uint64 carry3;
+  crypto_uint64 carry4;
+  crypto_uint64 carry5;
+  crypto_uint64 carry6;
+  crypto_uint64 carry7;
+  crypto_uint64 carry8;
+  crypto_uint64 carry9;
 
   carry0 = (h0 + (crypto_int64) (1<<25)) >> 26; h1 += carry0; h0 -= carry0 << 26;
   carry4 = (h4 + (crypto_int64) (1<<25)) >> 26; h5 += carry4; h4 -= carry4 << 26;

plugin/auth_ed25519/ref10/fe_sq2.c

 #include "fe.h"
 #include "crypto_int64.h"
+#include "crypto_uint64.h"
 
 /*
 h = 2 * f * f
@@ -106,16 +107,16 @@ void fe_sq2(fe h,const fe f)
   crypto_int64 h7 = f0f7_2+f1f6_2 +f2f5_2 +f3f4_2 +f8f9_38;
   crypto_int64 h8 = f0f8_2+f1f7_4 +f2f6_2 +f3f5_4 +f4f4   +f9f9_38;
   crypto_int64 h9 = f0f9_2+f1f8_2 +f2f7_2 +f3f6_2 +f4f5_2;
-  crypto_int64 carry0;
-  crypto_int64 carry1;
-  crypto_int64 carry2;
-  crypto_int64 carry3;
-  crypto_int64 carry4;
-  crypto_int64 carry5;
-  crypto_int64 carry6;
-  crypto_int64 carry7;
-  crypto_int64 carry8;
-  crypto_int64 carry9;
+  crypto_uint64 carry0;
+  crypto_uint64 carry1;
+  crypto_uint64 carry2;
+  crypto_uint64 carry3;
+  crypto_uint64 carry4;
+  crypto_uint64 carry5;
+  crypto_uint64 carry6;
+  crypto_uint64 carry7;
+  crypto_uint64 carry8;
+  crypto_uint64 carry9;
 
   h0 += h0;
   h1 += h1;

plugin/auth_ed25519/ref10/fe_tobytes.c

 #include "fe.h"
+#include "crypto_uint32.h"
 
 /*
 Preconditions:
@@ -38,16 +39,16 @@ void fe_tobytes(unsigned char *s,const fe h)
   crypto_int32 h8 = h[8];
   crypto_int32 h9 = h[9];
   crypto_int32 q;
-  crypto_int32 carry0;
-  crypto_int32 carry1;
-  crypto_int32 carry2;
-  crypto_int32 carry3;
-  crypto_int32 carry4;
-  crypto_int32 carry5;
-  crypto_int32 carry6;
-  crypto_int32 carry7;
-  crypto_int32 carry8;
-  crypto_int32 carry9;
+  crypto_uint32 carry0;
+  crypto_uint32 carry1;
+  crypto_uint32 carry2;
+  crypto_uint32 carry3;
+  crypto_uint32 carry4;
+  crypto_uint32 carry5;
+  crypto_uint32 carry6;
+  crypto_uint32 carry7;
+  crypto_uint32 carry8;
+  crypto_uint32 carry9;
 
   q = (19 * h9 + (((crypto_int32) 1) << 24)) >> 25;
   q = (h0 + q) >> 26;
@@ -87,32 +88,32 @@ void fe_tobytes(unsigned char *s,const fe h)
   s[0] = h0 >> 0;
   s[1] = h0 >> 8;
   s[2] = h0 >> 16;
-  s[3] = (h0 >> 24) | (h1 << 2);
+  s[3] = (h0 >> 24) | ((crypto_uint32)h1 << 2);
   s[4] = h1 >> 6;
   s[5] = h1 >> 14;
-  s[6] = (h1 >> 22) | (h2 << 3);
+  s[6] = (h1 >> 22) | ((crypto_uint32)h2 << 3);
   s[7] = h2 >> 5;
   s[8] = h2 >> 13;
-  s[9] = (h2 >> 21) | (h3 << 5);
+  s[9] = (h2 >> 21) | ((crypto_uint32)h3 << 5);
   s[10] = h3 >> 3;
   s[11] = h3 >> 11;
-  s[12] = (h3 >> 19) | (h4 << 6);
+  s[12] = (h3 >> 19) | ((crypto_uint32)h4 << 6);
   s[13] = h4 >> 2;
   s[14] = h4 >> 10;
   s[15] = h4 >> 18;
   s[16] = h5 >> 0;
   s[17] = h5 >> 8;
   s[18] = h5 >> 16;
-  s[19] = (h5 >> 24) | (h6 << 1);
+  s[19] = (h5 >> 24) | ((crypto_uint32)h6 << 1);
   s[20] = h6 >> 7;
   s[21] = h6 >> 15;
-  s[22] = (h6 >> 23) | (h7 << 3);
+  s[22] = (h6 >> 23) | ((crypto_uint32)h7 << 3);
   s[23] = h7 >> 5;
   s[24] = h7 >> 13;
-  s[25] = (h7 >> 21) | (h8 << 4);
+  s[25] = (h7 >> 21) | ((crypto_uint32)h8 << 4);
   s[26] = h8 >> 4;
   s[27] = h8 >> 12;
-  s[28] = (h8 >> 20) | (h9 << 6);
+  s[28] = (h8 >> 20) | ((crypto_uint32)h9 << 6);
   s[29] = h9 >> 2;
   s[30] = h9 >> 10;
   s[31] = h9 >> 18;

plugin/auth_ed25519/ref10/ge_scalarmult_base.c

@@ -35,7 +35,7 @@ static void select(ge_precomp *t,int pos,signed char b)
 {
   ge_precomp minust;
   unsigned char bnegative = negative(b);
-  unsigned char babs = b - (((-bnegative) & b) << 1);
+  unsigned char babs = b - ((unsigned char)((-bnegative) & b) << 1);
 
   ge_precomp_0(t);
   cmov(t,&base[pos][0],equal(babs,1));

plugin/auth_ed25519/ref10/sc_muladd.c

@@ -95,29 +95,29 @@ void sc_muladd(unsigned char *s,const unsigned char *a,const unsigned char *b,co
   crypto_int64 s21;
   crypto_int64 s22;
   crypto_int64 s23;
-  crypto_int64 carry0;
-  crypto_int64 carry1;
-  crypto_int64 carry2;
-  crypto_int64 carry3;
-  crypto_int64 carry4;
-  crypto_int64 carry5;
-  crypto_int64 carry6;
-  crypto_int64 carry7;
-  crypto_int64 carry8;
-  crypto_int64 carry9;
-  crypto_int64 carry10;
-  crypto_int64 carry11;
-  crypto_int64 carry12;
-  crypto_int64 carry13;
-  crypto_int64 carry14;
-  crypto_int64 carry15;
-  crypto_int64 carry16;
-  crypto_int64 carry17;
-  crypto_int64 carry18;
-  crypto_int64 carry19;
-  crypto_int64 carry20;
-  crypto_int64 carry21;
-  crypto_int64 carry22;
+  crypto_uint64 carry0;
+  crypto_uint64 carry1;
+  crypto_uint64 carry2;
+  crypto_uint64 carry3;
+  crypto_uint64 carry4;
+  crypto_uint64 carry5;
+  crypto_uint64 carry6;
+  crypto_uint64 carry7;
+  crypto_uint64 carry8;
+  crypto_uint64 carry9;
+  crypto_uint64 carry10;
+  crypto_uint64 carry11;
+  crypto_uint64 carry12;
+  crypto_uint64 carry13;
+  crypto_uint64 carry14;
+  crypto_uint64 carry15;
+  crypto_uint64 carry16;
+  crypto_uint64 carry17;
+  crypto_uint64 carry18;
+  crypto_uint64 carry19;
+  crypto_uint64 carry20;
+  crypto_uint64 carry21;
+  crypto_uint64 carry22;
 
   s0 = c0 + a0*b0;
   s1 = c1 + a0*b1 + a1*b0;

plugin/auth_ed25519/ref10/sc_reduce.c

@@ -58,23 +58,23 @@ void sc_reduce(unsigned char *s)
   crypto_int64 s21 = 2097151 & (load_3(s + 55) >> 1);
   crypto_int64 s22 = 2097151 & (load_4(s + 57) >> 6);
   crypto_int64 s23 = (load_4(s + 60) >> 3);
-  crypto_int64 carry0;
-  crypto_int64 carry1;
-  crypto_int64 carry2;
-  crypto_int64 carry3;
-  crypto_int64 carry4;
-  crypto_int64 carry5;
-  crypto_int64 carry6;
-  crypto_int64 carry7;
-  crypto_int64 carry8;
-  crypto_int64 carry9;
-  crypto_int64 carry10;
-  crypto_int64 carry11;
-  crypto_int64 carry12;
-  crypto_int64 carry13;
-  crypto_int64 carry14;
-  crypto_int64 carry15;
-  crypto_int64 carry16;
+  crypto_uint64 carry0;
+  crypto_uint64 carry1;
+  crypto_uint64 carry2;
+  crypto_uint64 carry3;
+  crypto_uint64 carry4;
+  crypto_uint64 carry5;
+  crypto_uint64 carry6;
+  crypto_uint64 carry7;
+  crypto_uint64 carry8;
+  crypto_uint64 carry9;
+  crypto_uint64 carry10;
+  crypto_uint64 carry11;
+  crypto_uint64 carry12;
+  crypto_uint64 carry13;
+  crypto_uint64 carry14;
+  crypto_uint64 carry15;
+  crypto_uint64 carry16;
 
   s11 += s23 * 666643;
   s12 += s23 * 470296;