Lines Matching full:prec
59 prec uint32
96 // Float x are stored in a nat slice long enough to hold up to x.prec bits;
100 // trailing 0 bits or x.prec is not a multiple of the the Word size _W,
152 // SetPrec sets z's precision to prec and returns the (possibly) rounded
154 // cannot be represented in prec bits without loss of precision.
156 // If prec > MaxPrec, it is set to MaxPrec.
157 func (z *Float) SetPrec(prec uint) *Float {
161 if prec == 0 {
162 z.prec = 0
172 if prec > MaxPrec {
173 prec = MaxPrec
175 old := z.prec
176 z.prec = uint32(prec)
177 if z.prec < old {
199 // Prec returns the mantissa precision of x in bits.
201 func (x *Float) Prec() uint {
202 return uint(x.prec)
206 // (i.e., the smallest prec before x.SetPrec(prec) would start rounding x).
348 return x.prec <= uint32(x.exp) || x.MinPrec() <= uint(x.exp) // not enough bits for fractional mantissa
368 if x.prec == 0 {
373 // round rounds z according to z.mode to z.prec bits and sets z.acc accordingly.
392 // m > 0 implies z.prec > 0 (checked by validate)
396 if bits <= z.prec {
400 // bits > z.prec
402 n := (z.prec + (_W - 1)) / _W // mantissa length in words for desired precision
406 // z.prec leading mantissa bits (the "0.5"). The sbit is set if any
416 // bits > z.prec: mantissa too large => round
417 r := uint(bits - z.prec - 1) // rounding bit position; r >= 0
448 t := n*_W - z.prec // 0 <= t < _W
526 if z.prec == 0 {
527 z.prec = 64
540 if z.prec < 64 {
570 if z.prec == 0 {
571 z.prec = 53
591 if z.prec < 53 {
618 // TODO(gri) can be more efficient if z.prec > 0
622 if z.prec == 0 {
623 z.prec = umax32(bits, 64)
648 if z.prec == 0 {
649 z.prec = umax32(a.prec, b.prec)
683 if z.prec == 0 {
684 z.prec = x.prec
685 } else if z.prec < x.prec {
700 z.prec = x.prec
895 r.prec = uint32(p)
934 mant = msb32(r.mant) >> (fbits - r.prec)
1002 r.prec = uint32(p)
1041 mant = msb64(r.mant) >> (fbits - r.prec)
1318 n := int(z.prec/_W) + 1
1425 if z.prec == 0 {
1426 z.prec = umax32(x.prec, y.prec)
1488 if z.prec == 0 {
1489 z.prec = umax32(x.prec, y.prec)
1551 if z.prec == 0 {
1552 z.prec = umax32(x.prec, y.prec)
1596 if z.prec == 0 {
1597 z.prec = umax32(x.prec, y.prec)
