Commit 950aa9f1 authored by Robert Griesemer's avatar Robert Griesemer

math/big: When result prec == 0, use at least prec == 64 for SetInt, SetRat.

This avoids surprises.

Change-Id: Iaae67da2d12e29c4e797ad6313e0895f7ce80cb1
Reviewed-on: https://go-review.googlesource.com/4480Reviewed-by: default avatarRob Pike <r@golang.org>
Reviewed-by: default avatarAlan Donovan <adonovan@google.com>
parent a15818fe
...@@ -567,15 +567,15 @@ func fnorm(m nat) uint { ...@@ -567,15 +567,15 @@ func fnorm(m nat) uint {
} }
// SetInt sets z to the (possibly rounded) value of x and returns z. // SetInt sets z to the (possibly rounded) value of x and returns z.
// If z's precision is 0, it is changed to x.BitLen() (and rounding will have // If z's precision is 0, it is changed to the larger of x.BitLen()
// no effect). // or 64 (and rounding will have no effect).
func (z *Float) SetInt(x *Int) *Float { func (z *Float) SetInt(x *Int) *Float {
// TODO(gri) can be more efficient if z.prec > 0 // TODO(gri) can be more efficient if z.prec > 0
// but small compared to the size of x, or if there // but small compared to the size of x, or if there
// are many trailing 0's. // are many trailing 0's.
bits := uint(x.BitLen()) bits := uint(x.BitLen())
if z.prec == 0 { if z.prec == 0 {
z.prec = bits z.prec = umax(bits, 64)
} }
z.acc = Exact z.acc = Exact
z.neg = x.neg z.neg = x.neg
...@@ -595,9 +595,8 @@ func (z *Float) SetInt(x *Int) *Float { ...@@ -595,9 +595,8 @@ func (z *Float) SetInt(x *Int) *Float {
} }
// SetRat sets z to the (possibly rounded) value of x and returns z. // SetRat sets z to the (possibly rounded) value of x and returns z.
// If z's precision is 0, it is changed to the larger of a.BitLen() // If z's precision is 0, it is changed to the largest of a.BitLen(),
// and b.BitLen(), where a and b are the numerator and denominator // b.BitLen(), or 64; with x = a/b.
// of x, respectively (x = a/b).
func (z *Float) SetRat(x *Rat) *Float { func (z *Float) SetRat(x *Rat) *Float {
// TODO(gri) can be more efficient if x is an integer // TODO(gri) can be more efficient if x is an integer
var a, b Float var a, b Float
...@@ -1110,6 +1109,7 @@ func (z *Float) Rsh(x *Float, s uint, mode RoundingMode) *Float { ...@@ -1110,6 +1109,7 @@ func (z *Float) Rsh(x *Float, s uint, mode RoundingMode) *Float {
// 0 if x == y (incl. -0 == 0) // 0 if x == y (incl. -0 == 0)
// +1 if x > y // +1 if x > y
// //
// Infinities with matching sign are equal.
func (x *Float) Cmp(y *Float) int { func (x *Float) Cmp(y *Float) int {
if debugFloat { if debugFloat {
x.validate() x.validate()
...@@ -1118,7 +1118,6 @@ func (x *Float) Cmp(y *Float) int { ...@@ -1118,7 +1118,6 @@ func (x *Float) Cmp(y *Float) int {
mx := x.mag() mx := x.mag()
my := y.mag() my := y.mag()
switch { switch {
case mx < my: case mx < my:
return -1 return -1
......
...@@ -490,8 +490,20 @@ func TestFloatSetInt(t *testing.T) { ...@@ -490,8 +490,20 @@ func TestFloatSetInt(t *testing.T) {
t.Errorf("invalid integer %s", want) t.Errorf("invalid integer %s", want)
continue continue
} }
n := x.BitLen()
var f Float var f Float
f.SetInt(&x) f.SetInt(&x)
// check precision
if n < 64 {
n = 64
}
if prec := f.Precision(); prec != uint(n) {
t.Errorf("got prec = %d; want %d", prec, n)
}
// check value
got := f.Format('g', 100) got := f.Format('g', 100)
if got != want { if got != want {
t.Errorf("got %s (%s); want %s", got, f.Format('p', 0), want) t.Errorf("got %s (%s); want %s", got, f.Format('p', 0), want)
...@@ -519,11 +531,24 @@ func TestFloatSetRat(t *testing.T) { ...@@ -519,11 +531,24 @@ func TestFloatSetRat(t *testing.T) {
t.Errorf("invalid fraction %s", want) t.Errorf("invalid fraction %s", want)
continue continue
} }
f := NewFloat(0, 1000, 0) // set a high precision - TODO(gri) find a cleaner way n := max(x.Num().BitLen(), x.Denom().BitLen())
f.SetRat(&x)
got := f.Format('g', 100) var f1 Float
var f2 = NewFloat(0, 1000, 0) // set a high precision - TODO(gri) find a cleaner way
f1.SetRat(&x)
f2.SetRat(&x)
// check precision when set automatically
if n < 64 {
n = 64
}
if prec := f1.Precision(); prec != uint(n) {
t.Errorf("got prec = %d; want %d", prec, n)
}
got := f2.Format('g', 100)
if got != want { if got != want {
t.Errorf("got %s (%s); want %s", got, f.Format('p', 0), want) t.Errorf("got %s (%s); want %s", got, f2.Format('p', 0), want)
} }
} }
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment