Lines Matching refs:precision
179 int precision)
182 sprintf(number, "%.*f", precision, d);
197 /* Convert a double precision value to fixed point. */
1944 * precision, not the output precision.
2043 * with the precision of the input, not the precision of the output depth.
2046 * bit precision. This only affects those of the following limits that pertain
2145 /* This is the case where the value was calculated at 8-bit precision then
2183 /* This is the case where the value was calculated at 8-bit precision then
5282 /* For checking the code calculates double precision floating point values
5373 * 'max' and the error is initialized to the double precision epsilon value
5600 * double precision values set up by the preceding 'mod' calls:
5940 /* use-input-precision means assume that if the input has 8 bit (or less)
5942 * with 8 bit precision, not 16.
5953 /* Else calculations are at 8 bit precision, and the output actually
8391 pass = "within input precision limits:\n";
8455 /* Need either 1/255 or 1/65535 precision here; 3 or 6 decimal
8458 int precision = (outmax >= 1000 ? 6 : 3);
8502 /* sBIT has reduced the precision of the input: */
8536 pos = safecatd(msg, sizeof msg, pos, alpha, precision);
8547 pos = safecatd(msg, sizeof msg, pos, background, precision);
8549 pos = safecatd(msg, sizeof msg, pos, 1-alpha, precision);
8561 pos = safecatd(msg, sizeof msg, pos, input_sample, precision);
8563 pos = safecatd(msg, sizeof msg, pos, output, precision);
8655 * precision it is possible to calculate an exact answer given an input
8658 * output to the limits of double precision arithmetic and compare with
8684 * Because there is limited precision in the input it is arguable that
8879 * given gamma value and the sBIT chunk to the given precision.
9013 0 /*no input precision*/,
10278 else if (strcmp(*argv, "--use-input-precision") == 0)
10281 else if (strcmp(*argv, "--use-calculation-precision") == 0)
10284 else if (strcmp(*argv, "--calculations-use-input-precision") == 0)
