Lines Matching full:prec
26 // according to the format fmt and precision prec. It rounds the
38 // The precision prec controls the number of digits
44 func FormatFloat(f float64, fmt byte, prec, bitSize int) string {
45 return string(genericFtoa(make([]byte, 0, max(prec+4, 24)), f, fmt, prec, bitSize))
50 func AppendFloat(dst []byte, f float64, fmt byte, prec, bitSize int) []byte {
51 return genericFtoa(dst, f, fmt, prec, bitSize)
54 func genericFtoa(dst []byte, val float64, fmt byte, prec, bitSize int) []byte {
102 return bigFtoa(dst, prec, fmt, neg, mant, exp, flt)
108 shortest := prec < 0
117 return bigFtoa(dst, prec, fmt, neg, mant, exp, flt)
122 prec = max(digs.nd-1, 0)
124 prec = max(digs.nd-digs.dp, 0)
126 prec = digs.nd
130 digits := prec
135 if prec == 0 {
136 prec = 1
138 digits = prec
149 return bigFtoa(dst, prec, fmt, neg, mant, exp, flt)
151 return formatDigits(dst, shortest, neg, digs, prec, fmt)
155 func bigFtoa(dst []byte, prec int, fmt byte, neg bool, mant uint64, exp int, flt *floatInfo) []byte {
160 shortest := prec < 0
167 prec = digs.nd - 1
169 prec = max(digs.nd-digs.dp, 0)
171 prec = digs.nd
177 d.Round(prec + 1)
179 d.Round(d.dp + prec)
181 if prec == 0 {
182 prec = 1
184 d.Round(prec)
188 return formatDigits(dst, shortest, neg, digs, prec, fmt)
191 func formatDigits(dst []byte, shortest bool, neg bool, digs decimalSlice, prec int, fmt byte) []byte {
194 return fmtE(dst, neg, digs, prec, fmt)
196 return fmtF(dst, neg, digs, prec)
199 eprec := prec
211 if prec > digs.nd {
212 prec = digs.nd
214 return fmtE(dst, neg, digs, prec-1, fmt+'e'-'g')
216 if prec > digs.dp {
217 prec = digs.nd
219 return fmtF(dst, neg, digs, max(prec-digs.dp, 0))
333 func fmtE(dst []byte, neg bool, d decimalSlice, prec int, fmt byte) []byte {
347 if prec > 0 {
350 m := min(d.nd, prec+1)
355 for ; i <= prec; i++ {
388 func fmtF(dst []byte, neg bool, d decimalSlice, prec int) []byte {
406 if prec > 0 {
408 for i := 0; i < prec; i++ {
