Lines Matching refs:precision
41 // int precision = (ceil(-log10(fabs(axmax-axmin))));
42 // if ((axmin*axmax > 0) && precision > 4)
43 // sprintf(ticfmt[axis],"%%.%df", (precision>14) ? 14 : precision);
847 if (value >= half && (bits <= 32 || value > half)) { // for huge values, we can hit the precision limit and always get true here. so don't do that
1634 // Handle precision.
1637 var precision = 0;
1642 precision = getNextArg('i32');
1649 precision = precision * 10 + (precisionChr - 48);
1655 var precision = 6; // Standard default.
1761 while (argText.length < precision) {
1804 var effectivePrecision = Math.min(precision, 20);
1809 precision = precision || 1;
1811 if (precision > exponent && exponent >= -4) {
1813 precision -= exponent + 1;
1816 precision--;
1818 effectivePrecision = Math.min(precision, 20);
1842 // Zero pad until required precision.
1843 while (precision > effectivePrecision++) parts[0] += '0';
1877 if (precisionSet) argLength = Math.min(argLength, precision);
