Commit 5781a00e authored by Anthony Martin's avatar Anthony Martin Committed by Robert Griesemer

big: fix panic and round correctly in Rat.FloatString

R=gri, rsc
CC=golang-dev
https://golang.org/cl/2212044
parent 1f6f5639
...@@ -294,25 +294,34 @@ func (z *Rat) FloatString(prec int) string { ...@@ -294,25 +294,34 @@ func (z *Rat) FloatString(prec int) string {
q, r := nat{}.div(nat{}, z.a.abs, z.b) q, r := nat{}.div(nat{}, z.a.abs, z.b)
s := q.string(10) p := natOne
if z.a.neg { if prec > 0 {
s = "-" + s p = nat{}.expNN(natTen, nat{}.setUint64(uint64(prec)), nil)
} }
p := nat{}.expNN(natTen, nat{Word(prec)}, nil)
r = r.mul(r, p) r = r.mul(r, p)
r, r2 := r.div(nat{}, r, z.b) r, r2 := r.div(nat{}, r, z.b)
// see if we need to round up // see if we need to round up
r2 = r2.mul(r2, natTwo) r2 = r2.add(r2, r2)
if z.b.cmp(r2) <= 0 { if z.b.cmp(r2) <= 0 {
r = r.add(r, natOne) r = r.add(r, natOne)
if r.cmp(p) >= 0 {
q = nat{}.add(q, natOne)
r = nat{}.sub(r, p)
}
}
s := q.string(10)
if z.a.neg {
s = "-" + s
} }
if prec > 0 {
rs := r.string(10) rs := r.string(10)
leadingZeros := prec - len(rs) leadingZeros := prec - len(rs)
s += "." + strings.Repeat("0", leadingZeros) + rs s += "." + strings.Repeat("0", leadingZeros) + rs
s = strings.TrimRight(s, "0") }
return s return s
} }
...@@ -71,6 +71,11 @@ var floatStringTests = []floatStringTest{ ...@@ -71,6 +71,11 @@ var floatStringTests = []floatStringTest{
floatStringTest{".25", 1, "0.3"}, floatStringTest{".25", 1, "0.3"},
floatStringTest{"-1/3", 3, "-0.333"}, floatStringTest{"-1/3", 3, "-0.333"},
floatStringTest{"-2/3", 4, "-0.6667"}, floatStringTest{"-2/3", 4, "-0.6667"},
floatStringTest{"0.96", 1, "1.0"},
floatStringTest{"0.999", 2, "1.00"},
floatStringTest{"0.9", 0, "1"},
floatStringTest{".25", -1, "0"},
floatStringTest{".55", -1, "1"},
} }
func TestFloatString(t *testing.T) { func TestFloatString(t *testing.T) {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment