Lines Matching full:prec
42 if prec := x.Prec(); prec != 0 {
43 t.Errorf("prec = %d; want 0", prec)
105 prec uint
109 // prec 0
117 // prec at upper limit
129 x := makeFloat(test.x).SetPrec(test.prec)
130 prec := test.prec
131 if prec > MaxPrec {
132 prec = MaxPrec
134 if got := x.Prec(); got != prec {
135 t.Errorf("%s.SetPrec(%d).Prec() == %d; want %d", test.x, test.prec, got, prec)
138 t.Errorf("%s.SetPrec(%d) = %s (%s); want %s (%s)", test.x, test.prec, got, acc, test.want, test.acc)
342 func testFloatRound(t *testing.T, x, r int64, prec uint, mode RoundingMode) {
368 t.Fatalf("incorrect test data for prec = %d, %s: x = %s, r = %s", prec, mode, toBinary(x), toBinary(r))
381 f := new(Float).SetMode(mode).SetInt64(x).SetPrec(prec)
385 p1 := f.Prec()
387 if r1 != r || p1 != prec || a1 != a {
389 toBinary(x), prec, mode,
391 toBinary(r), prec, a)
399 g := new(Float).SetMode(mode).SetPrec(prec).SetInt64(x)
402 toBinary(x), prec, mode,
412 h := new(Float).SetMode(mode).SetPrec(prec).Set(f)
415 toBinary(x), prec, mode,
427 prec uint
428 x, zero, neven, naway, away string // input, results rounded to prec bits
507 prec := test.prec
509 testFloatRound(t, x, z, prec, ToZero)
510 testFloatRound(t, x, e, prec, ToNearestEven)
511 testFloatRound(t, x, n, prec, ToNearestAway)
512 testFloatRound(t, x, a, prec, AwayFromZero)
514 testFloatRound(t, x, z, prec, ToNegativeInf)
515 testFloatRound(t, x, a, prec, ToPositiveInf)
517 testFloatRound(t, -x, -a, prec, ToNegativeInf)
518 testFloatRound(t, -x, -z, prec, ToPositiveInf)
558 for prec := uint(1); prec <= 64; prec++ {
559 f := new(Float).SetPrec(prec).SetMode(ToZero).SetUint64(x)
561 want := x &^ (1<<(64-prec) - 1) // cut off (round to zero) low 64-prec bits
593 for prec := uint(1); prec <= 63; prec++ {
594 f := new(Float).SetPrec(prec).SetMode(ToZero).SetInt64(x)
596 want := x &^ (1<<(63-prec) - 1) // cut off (round to zero) low 63-prec bits
636 for prec := uint(1); prec <= 52; prec++ {
637 f := new(Float).SetPrec(prec).SetMode(ToZero).SetFloat64(float64(x))
639 want := float64(x &^ (1<<(52-prec) - 1)) // cut off (round to zero) low 53-prec bits
681 if prec := f.Prec(); prec != uint(n) {
682 t.Errorf("got prec = %d; want %d", prec, n)
724 if prec := f1.Prec(); prec != uint(n) {
725 t.Errorf("got prec = %d; want %d", prec, n)
739 prec uint
747 x := f.SetPrec(test.prec).SetInf(test.signbit)
748 if got := x.String(); got != test.want || x.Prec() != test.prec {
749 t.Errorf("SetInf(%v) = %s (prec = %d); want %s (prec = %d)", test.signbit, got, x.Prec(), test.want, test.prec)
1122 for _, prec := range precList {
1123 if 1<<prec < n {
1124 continue // prec must be large enough to hold all numbers from 0 to n
1127 x.SetPrec(prec)
1133 t.Errorf("prec = %d: got %s; want %d", prec, &x, n)
1169 for _, prec := range precList {
1170 got := new(Float).SetPrec(prec).SetMode(mode)
1172 want := zbits.round(prec, mode)
1174 t.Errorf("i = %d, prec = %d, %s:\n\t %s %v\n\t+ %s %v\n\t= %s\n\twant %s",
1175 i, prec, mode, x, xbits, y, ybits, got, want)
1179 want = ybits.round(prec, mode)
1181 t.Errorf("i = %d, prec = %d, %s:\n\t %s %v\n\t- %s %v\n\t= %s\n\twant %s",
1182 i, prec, mode, z, zbits, x, xbits, got, want)
1271 for _, prec := range precList {
1272 got := new(Float).SetPrec(prec).SetMode(mode)
1274 want := zbits.round(prec, mode)
1276 t.Errorf("i = %d, prec = %d, %s:\n\t %s %v\n\t* %s %v\n\t= %s\n\twant %s",
1277 i, prec, mode, x, xbits, y, ybits, got, want)
1284 want = ybits.round(prec, mode)
1286 t.Errorf("i = %d, prec = %d, %s:\n\t %s %v\n\t/ %s %v\n\t= %s\n\twant %s",
1287 i, prec, mode, z, zbits, x, xbits, got, want)
1347 for _, prec := range precList {
1348 two := new(Float).SetPrec(prec).SetInt64(2)
1349 one := new(Float).SetPrec(prec).SetInt64(1)
1350 three := new(Float).SetPrec(prec).SetInt64(3)
1351 msix := new(Float).SetPrec(prec).SetInt64(-6)
1352 psix := new(Float).SetPrec(prec).SetInt64(+6)
1354 p := new(Float).SetPrec(prec)
1355 z1 := new(Float).SetPrec(prec)
1356 z2 := new(Float).SetPrec(prec)
1369 t.Fatalf("prec %d: got z1 = %s != z2 = %s; want z1 == z2\n", prec, z1, z2)
1372 t.Errorf("prec %d: got z1 = %s; want 0", prec, z1)
1375 t.Errorf("prec %d: got z2 = %s; want 0", prec, z2)
1402 x := new(Float).SetPrec(z.Prec() + y.Prec()).SetMode(ToZero)
1416 prec := uint(preci + d)
1417 got := new(Float).SetPrec(prec).SetMode(mode).Quo(x, y)
1418 want := bits.round(prec, mode)
1420 t.Errorf("i = %d, prec = %d, %s:\n\t %s\n\t/ %s\n\t= %s\n\twant %s",
1421 i, prec, mode, x, y, got, want)
1437 const prec = 10 + dprec // enough bits to hold n precisely
1451 A := new(Float).SetPrec(uint(prec + ad)).SetFloat64(a)
1452 B := new(Float).SetPrec(uint(prec + bd)).SetFloat64(b)
1547 prec uint
1585 z := new(Float).SetPrec(test.prec).SetMode(test.mode)
1600 "prec = %d (%s): %s %c %s = %s (%s); want %s (%s)",
1601 test.prec, test.mode, x.Text('p', 0), test.op, y.Text('p', 0), got, z.Acc(), test.want, test.acc,
1617 prec uint
1640 z.SetPrec(test.prec).SetMode(test.mode)
1655 test.mode, test.prec, test.x, test.op, test.y, got, acc, test.want,
