- factored out 128-bit muladd and div into arith.go

- wrote corresponding fast versions in fast.arith.s
- implemented in-place operations for some routines
- updated existing code to be compatible with in-place
  routines

These changes allow the pidigits benchmark to run
approx. 30% faster. Enabling the assembly routines
in fast.arith.s will give another approx. 3%.

R=r
DELTA=486  (252 added, 68 deleted, 166 changed)
OCL=32980
CL=33003

src/pkg/bignum/Makefile

@@ -4,7 +4,7 @@
 
 
 # DO NOT EDIT.  Automatically generated by gobuild.
-# gobuild -m bignum.go integer.go rational.go >Makefile
+# gobuild -m arith.go bignum.go integer.go rational.go >Makefile
 
 D=
 
@@ -33,30 +33,37 @@ coverage: packages
 	$(AS) $*.s
 
 O1=\
-	bignum.$O\
+	arith.$O\
 
 O2=\
-	integer.$O\
+	bignum.$O\
 
 O3=\
+	integer.$O\
+
+O4=\
 	rational.$O\
 
 
-phases: a1 a2 a3
+phases: a1 a2 a3 a4
 _obj$D/bignum.a: phases
 
 a1: $(O1)
-	$(AR) grc _obj$D/bignum.a bignum.$O
+	$(AR) grc _obj$D/bignum.a arith.$O
 	rm -f $(O1)
 
 a2: $(O2)
-	$(AR) grc _obj$D/bignum.a integer.$O
+	$(AR) grc _obj$D/bignum.a bignum.$O
 	rm -f $(O2)
 
 a3: $(O3)
-	$(AR) grc _obj$D/bignum.a rational.$O
+	$(AR) grc _obj$D/bignum.a integer.$O
 	rm -f $(O3)
 
+a4: $(O4)
+	$(AR) grc _obj$D/bignum.a rational.$O
+	rm -f $(O4)
+
 
 newpkg: clean
 	mkdir -p _obj$D
@@ -66,6 +73,7 @@ $(O1): newpkg
 $(O2): a1
 $(O3): a2
 $(O4): a3
+$(O5): a4
 
 nuke: clean
 	rm -f $(GOROOT)/pkg/$(GOOS)_$(GOARCH)$D/bignum.a

src/pkg/bignum/arith.go

+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Fast versions of the routines in this file are in fast.arith.s.
+// Simply replace this file with arith.s (renamed from fast.arith.s)
+// and the bignum package will build and run on a platform that
+// supports the assembly routines.
+
+package bignum
+
+import "unsafe"
+
+// z1<<64 + z0 = x*y
+func Mul128(x, y uint64) (z1, z0 uint64) {
+	// Split x and y into 2 halfwords each, multiply
+	// the halfwords separately while avoiding overflow,
+	// and return the product as 2 words.
+
+	const (
+		W = uint(unsafe.Sizeof(x))*8;
+		W2 = W/2;
+		B2 = 1<<W2;
+		M2 = B2-1;
+	)
+
+	if x < y {
+		x, y = y, x
+	}
+
+	if x < B2 {
+		// y < B2 because y <= x
+		// sub-digits of x and y are (0, x) and (0, y)
+		// z = z[0] = x*y
+		z0 = x*y;
+		return;
+	}
+
+	if y < B2 {
+		// sub-digits of x and y are (x1, x0) and (0, y)
+		// x = (x1*B2 + x0)
+		// y = (y1*B2 + y0)
+		x1, x0 := x>>W2, x&M2;
+	
+		// x*y = t2*B2*B2 + t1*B2 + t0
+		t0 := x0*y;
+		t1 := x1*y;
+
+		// compute result digits but avoid overflow
+		// z = z[1]*B + z[0] = x*y
+		z0 = t1<<W2 + t0;
+		z1 = (t1 + t0>>W2) >> W2;
+		return;
+	}
+
+	// general case
+	// sub-digits of x and y are (x1, x0) and (y1, y0)
+	// x = (x1*B2 + x0)
+	// y = (y1*B2 + y0)
+	x1, x0 := x>>W2, x&M2;
+	y1, y0 := y>>W2, y&M2;
+
+	// x*y = t2*B2*B2 + t1*B2 + t0
+	t0 := x0*y0;
+	t1 := x1*y0 + x0*y1;
+	t2 := x1*y1;
+
+	// compute result digits but avoid overflow
+	// z = z[1]*B + z[0] = x*y
+	z0 = t1<<W2 + t0;
+	z1 = t2 + (t1 + t0>>W2) >> W2;
+	return;
+}
+
+
+// z1<<64 + z0 = x*y + c
+func MulAdd128(x, y, c uint64) (z1, z0 uint64) {
+	// Split x and y into 2 halfwords each, multiply
+	// the halfwords separately while avoiding overflow,
+	// and return the product as 2 words.
+
+	const (
+		W = uint(unsafe.Sizeof(x))*8;
+		W2 = W/2;
+		B2 = 1<<W2;
+		M2 = B2-1;
+	)
+
+	// TODO(gri) Should implement special cases for faster execution.
+
+	// general case
+	// sub-digits of x, y, and c are (x1, x0), (y1, y0), (c1, c0)
+	// x = (x1*B2 + x0)
+	// y = (y1*B2 + y0)
+	x1, x0 := x>>W2, x&M2;
+	y1, y0 := y>>W2, y&M2;
+	c1, c0 := c>>W2, c&M2;
+
+	// x*y + c = t2*B2*B2 + t1*B2 + t0
+	t0 := x0*y0 + c0;
+	t1 := x1*y0 + x0*y1 + c1;
+	t2 := x1*y1;
+
+	// compute result digits but avoid overflow
+	// z = z[1]*B + z[0] = x*y
+	z0 = t1<<W2 + t0;
+	z1 = t2 + (t1 + t0>>W2) >> W2;
+	return;
+}
+
+
+// q = (x1<<64 + x0)/y + r
+func Div128(x1, x0, y uint64) (q, r uint64) {
+	if x1 == 0 {
+		q, r = x0/y, x0%y;
+		return;
+	}
+
+	// TODO(gri) Implement general case.
+	panic("Div128 not implemented for x > 1<<64-1");
+}

src/pkg/bignum/fast.arith.s

+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This file provides fast assembly versions
+// of the routines in arith.go.
+
+// func Mul128(x, y uint64) (z1, z0 uint64)
+// z1<<64 + z0 = x*y
+//
+TEXT bignum·Mul128(SB),7,$0
+	MOVQ a+0(FP), AX
+	MULQ a+8(FP)
+	MOVQ DX, a+16(FP)
+	MOVQ AX, a+24(FP)
+	RET
+
+
+// func MulAdd128(x, y, c uint64) (z1, z0 uint64)
+// z1<<64 + z0 = x*y + c
+//
+TEXT bignum·MulAdd128(SB),7,$0
+	MOVQ a+0(FP), AX
+	MULQ a+8(FP)
+	ADDQ a+16(FP), AX
+	ADCQ $0, DX
+	MOVQ DX, a+24(FP)
+	MOVQ AX, a+32(FP)
+	RET
+
+
+// func Div128(x1, x0, y uint64) (q, r uint64)
+// q = (x1<<64 + x0)/y + r
+//
+TEXT bignum·Div128(SB),7,$0
+	MOVQ a+0(FP), DX
+	MOVQ a+8(FP), AX
+	DIVQ a+16(FP)
+	MOVQ AX, a+24(FP)
+	MOVQ DX, a+32(FP)
+	RET

src/pkg/bignum/integer.go

@@ -9,9 +9,12 @@
 
 package bignum
 
-import "bignum"
-import "fmt"
+import (
+	"bignum";
+	"fmt";
+)
 
+// TODO(gri) Complete the set of in-place operations.
 
 // Integer represents a signed integer value of arbitrary precision.
 //
@@ -113,61 +116,82 @@ func (x *Integer) Neg() *Integer {
 }
 
 
-// Add returns the sum x + y.
+// Iadd sets z to the sum x + y.
+// z must exist and may be x or y.
 //
-func (x *Integer) Add(y *Integer) *Integer {
-	var z *Integer;
+func Iadd(z, x, y *Integer) {
 	if x.sign == y.sign {
 		// x + y == x + y
 		// (-x) + (-y) == -(x + y)
-		z = MakeInt(x.sign, x.mant.Add(y.mant));
+		z.sign = x.sign;
+		Nadd(&z.mant, x.mant, y.mant);
 	} else {
 		// x + (-y) == x - y == -(y - x)
 		// (-x) + y == y - x == -(x - y)
 		if x.mant.Cmp(y.mant) >= 0 {
-			z = MakeInt(false, x.mant.Sub(y.mant));
+			z.sign = x.sign;
+			Nsub(&z.mant, x.mant, y.mant);
 		} else {
-			z = MakeInt(true, y.mant.Sub(x.mant));
+			z.sign = !x.sign;
+			Nsub(&z.mant, y.mant, x.mant);
 		}
 	}
-	if x.sign {
-		z.sign = !z.sign;
-	}
-	return z;
 }
 
 
-// Sub returns the difference x - y.
+// Add returns the sum x + y.
 //
-func (x *Integer) Sub(y *Integer) *Integer {
-	var z *Integer;
+func (x *Integer) Add(y *Integer) *Integer {
+	var z Integer;
+	Iadd(&z, x, y);
+	return &z;
+}
+
+
+func Isub(z, x, y *Integer) {
 	if x.sign != y.sign {
 		// x - (-y) == x + y
 		// (-x) - y == -(x + y)
-		z = MakeInt(false, x.mant.Add(y.mant));
+		z.sign = x.sign;
+		Nadd(&z.mant, x.mant, y.mant);
 	} else {
 		// x - y == x - y == -(y - x)
 		// (-x) - (-y) == y - x == -(x - y)
 		if x.mant.Cmp(y.mant) >= 0 {
-			z = MakeInt(false, x.mant.Sub(y.mant));
+			z.sign = x.sign;
+			Nsub(&z.mant, x.mant, y.mant);
 		} else {
-			z = MakeInt(true, y.mant.Sub(x.mant));
+			z.sign = !x.sign;
+			Nsub(&z.mant, y.mant, x.mant);
 		}
 	}
-	if x.sign {
-		z.sign = !z.sign;
+}
+
+
+// Sub returns the difference x - y.
+//
+func (x *Integer) Sub(y *Integer) *Integer {
+	var z Integer;
+	Isub(&z, x, y);
+	return &z;
+}
+
+
+// Nscale sets *z to the scaled value (*z) * d.
+//
+func Iscale(z *Integer, d int64) {
+	f := uint64(d);
+	if d < 0 {
+		f = uint64(-d);
 	}
-	return z;
+	z.sign = z.sign != (d < 0);
+	Nscale(&z.mant, f);
 }
 
 
 // Mul1 returns the product x * d.
 //
 func (x *Integer) Mul1(d int64) *Integer {
-	// x * y == x * y
-	// x * (-y) == -(x * y)
-	// (-x) * y == -(x * y)
-	// (-x) * (-y) == x * y
 	f := uint64(d);
 	if d < 0 {
 		f = uint64(-d);
@@ -326,7 +350,7 @@ func (x *Integer) Shl(s uint) *Integer {
 func (x *Integer) Shr(s uint) *Integer {
 	if x.sign {
 		// (-x) >> s == ^(x-1) >> s == ^((x-1) >> s) == -(((x-1) >> s) + 1)
-		return MakeInt(true, x.mant.Sub(nat[1]).Shr(s).Add(nat[1]));
+		return MakeInt(true, x.mant.Sub(Nat(1)).Shr(s).Add(Nat(1)));
 	}
 	
 	return MakeInt(false, x.mant.Shr(s));
@@ -337,11 +361,11 @@ func (x *Integer) Shr(s uint) *Integer {
 func (x *Integer) Not() *Integer {
 	if x.sign {
 		// ^(-x) == ^(^(x-1)) == x-1
-		return MakeInt(false, x.mant.Sub(nat[1]));
+		return MakeInt(false, x.mant.Sub(Nat(1)));
 	}
 
 	// ^x == -x-1 == -(x+1)
-	return MakeInt(true, x.mant.Add(nat[1]));
+	return MakeInt(true, x.mant.Add(Nat(1)));
 }
 
 
@@ -351,7 +375,7 @@ func (x *Integer) And(y *Integer) *Integer {
 	if x.sign == y.sign {
 		if x.sign {
 			// (-x) & (-y) == ^(x-1) & ^(y-1) == ^((x-1) | (y-1)) == -(((x-1) | (y-1)) + 1)
-			return MakeInt(true, x.mant.Sub(nat[1]).Or(y.mant.Sub(nat[1])).Add(nat[1]));
+			return MakeInt(true, x.mant.Sub(Nat(1)).Or(y.mant.Sub(Nat(1))).Add(Nat(1)));
 		}
 
 		// x & y == x & y
@@ -364,7 +388,7 @@ func (x *Integer) And(y *Integer) *Integer {
 	}
 
 	// x & (-y) == x & ^(y-1) == x &^ (y-1)
-	return MakeInt(false, x.mant.AndNot(y.mant.Sub(nat[1])));
+	return MakeInt(false, x.mant.AndNot(y.mant.Sub(Nat(1))));
 }
 
 
@@ -374,7 +398,7 @@ func (x *Integer) AndNot(y *Integer) *Integer {
 	if x.sign == y.sign {
 		if x.sign {
 			// (-x) &^ (-y) == ^(x-1) &^ ^(y-1) == ^(x-1) & (y-1) == (y-1) &^ (x-1)
-			return MakeInt(false, y.mant.Sub(nat[1]).AndNot(x.mant.Sub(nat[1])));
+			return MakeInt(false, y.mant.Sub(Nat(1)).AndNot(x.mant.Sub(Nat(1))));
 		}
 
 		// x &^ y == x &^ y
@@ -383,11 +407,11 @@ func (x *Integer) AndNot(y *Integer) *Integer {
 
 	if x.sign {
 		// (-x) &^ y == ^(x-1) &^ y == ^(x-1) & ^y == ^((x-1) | y) == -(((x-1) | y) + 1)
-		return MakeInt(true, x.mant.Sub(nat[1]).Or(y.mant).Add(nat[1]));
+		return MakeInt(true, x.mant.Sub(Nat(1)).Or(y.mant).Add(Nat(1)));
 	}
 
 	// x &^ (-y) == x &^ ^(y-1) == x & (y-1)
-	return MakeInt(false, x.mant.And(y.mant.Sub(nat[1])));
+	return MakeInt(false, x.mant.And(y.mant.Sub(Nat(1))));
 }
 
 
@@ -397,7 +421,7 @@ func (x *Integer) Or(y *Integer) *Integer {
 	if x.sign == y.sign {
 		if x.sign {
 			// (-x) | (-y) == ^(x-1) | ^(y-1) == ^((x-1) & (y-1)) == -(((x-1) & (y-1)) + 1)
-			return MakeInt(true, x.mant.Sub(nat[1]).And(y.mant.Sub(nat[1])).Add(nat[1]));
+			return MakeInt(true, x.mant.Sub(Nat(1)).And(y.mant.Sub(Nat(1))).Add(Nat(1)));
 		}
 
 		// x | y == x | y
@@ -410,7 +434,7 @@ func (x *Integer) Or(y *Integer) *Integer {
 	}
 
 	// x | (-y) == x | ^(y-1) == ^((y-1) &^ x) == -(^((y-1) &^ x) + 1)
-	return MakeInt(true, y.mant.Sub(nat[1]).AndNot(x.mant).Add(nat[1]));
+	return MakeInt(true, y.mant.Sub(Nat(1)).AndNot(x.mant).Add(Nat(1)));
 }
 
 
@@ -420,7 +444,7 @@ func (x *Integer) Xor(y *Integer) *Integer {
 	if x.sign == y.sign {
 		if x.sign {
 			// (-x) ^ (-y) == ^(x-1) ^ ^(y-1) == (x-1) ^ (y-1)
-			return MakeInt(false, x.mant.Sub(nat[1]).Xor(y.mant.Sub(nat[1])));
+			return MakeInt(false, x.mant.Sub(Nat(1)).Xor(y.mant.Sub(Nat(1))));
 		}
 
 		// x ^ y == x ^ y
@@ -433,7 +457,7 @@ func (x *Integer) Xor(y *Integer) *Integer {
 	}
 
 	// x ^ (-y) == x ^ ^(y-1) == ^(x ^ (y-1)) == -((x ^ (y-1)) + 1)
-	return MakeInt(true, x.mant.Xor(y.mant.Sub(nat[1])).Add(nat[1]));
+	return MakeInt(true, x.mant.Xor(y.mant.Sub(Nat(1))).Add(Nat(1)));
 }
 
 

src/pkg/bignum/rational.go

@@ -22,7 +22,7 @@ type Rational struct {
 //
 func MakeRat(a *Integer, b Natural) *Rational {
 	f := a.mant.Gcd(b);  // f > 0
-	if f.Cmp(nat[1]) != 0 {
+	if f.Cmp(Nat(1)) != 0 {
 		a = MakeInt(a.sign, a.mant.Div(f));
 		b = b.Div(f);
 	}
@@ -75,7 +75,7 @@ func (x *Rational) IsPos() bool {
 // in the form x == x'/1; i.e., if x is an integer value.
 //
 func (x *Rational) IsInt() bool {
-	return x.b.Cmp(nat[1]) == 0;
+	return x.b.Cmp(Nat(1)) == 0;
 }
 
 
@@ -184,7 +184,7 @@ func (x *Rational) Format(h fmt.State, c int) {
 func RatFromString(s string, base uint) (*Rational, uint, int) {
 	// read numerator
 	a, abase, alen := IntFromString(s, base);
-	b := nat[1];
+	b := Nat(1);
 
 	// read denominator or fraction, if any
 	var blen int;
@@ -211,7 +211,7 @@ func RatFromString(s string, base uint) (*Rational, uint, int) {
 			rlen++;
 			e, _, elen := IntFromString(s[rlen : len(s)], 10);
 			rlen += elen;
-			m := nat[10].Pow(uint(e.mant.Value()));
+			m := Nat(10).Pow(uint(e.mant.Value()));
 			if e.sign {
 				b = b.Mul(m);
 			} else {