Lines Matching full:prec
18 // to the given format and precision prec. The format is one of:
30 // 'b' decimal integer mantissa using x.Prec() bits, or -0
36 // The precision prec controls the number of digits (excluding the exponent)
41 // The prec value is ignored for the 'b' or 'p' format.
44 func (x *Float) Text(format byte, prec int) string {
46 return string(x.Append(make([]byte, 0, prec+extra), format, prec))
56 func (x *Float) Append(buf []byte, fmt byte, prec int) []byte {
91 if prec < 0 {
99 prec = len(d.mant) - 1
101 prec = max(len(d.mant)-d.exp, 0)
103 prec = len(d.mant)
110 d.round(1 + prec)
113 d.round(d.exp + prec)
115 if prec == 0 {
116 prec = 1
118 d.round(prec)
125 return fmtE(buf, fmt, prec, d)
127 return fmtF(buf, prec, d)
130 eprec := prec
143 if prec > len(d.mant) {
144 prec = len(d.mant)
146 return fmtE(buf, fmt+'e'-'g', prec-1, d)
148 if prec > d.exp {
149 prec = len(d.mant)
151 return fmtF(buf, max(prec-d.exp, 0), d)
162 func fmtE(buf []byte, fmt byte, prec int, d decimal) []byte {
171 if prec > 0 {
174 m := min(len(d.mant), prec+1)
179 for ; i <= prec; i++ {
206 func fmtF(buf []byte, prec int, d decimal) []byte {
219 if prec > 0 {
221 for i := 0; i < prec; i++ {
236 // The mantissa is normalized such that is uses x.Prec() bits in binary
249 // adjust mantissa to use exactly x.prec bits
252 case w < x.prec:
253 m = nat(nil).shl(m, uint(x.prec-w))
254 case w > x.prec:
255 m = nat(nil).shr(m, uint(w-x.prec))
260 e := int64(x.exp) - int64(x.prec)
322 prec, hasPrec := s.Precision()
324 prec = 6 // default precision for 'e', 'f'
339 // TODO(gri) uncomment once (*Float).Text handles prec < 0
340 // prec = -1 // default precision for 'g', 'G'
347 buf = x.Append(buf, byte(format), prec)
