Searched
full:precision (Results
2576 -
2600 of
7852) sorted by null
<<101102103104105106107108109110>>
| /external/pdfium/fxjs/ |
| cjs_util.cpp | 396 enum State { BEFORE, FLAGS, WIDTH, PRECISION, SPECIFIER, AFTER };
423 state = PRECISION;
429 case PRECISION:
|
| /external/protobuf/js/binary/ |
| arith.js | 46 * top of JavaScript's number type, which has only 53 bits of precision, by
169 // precision, thus losing precision because of the 53-bit mantissa of
|
| /external/python/cpython2/Lib/test/decimaltestdata/ |
| clamp.decTest | 27 precision: 16
188 precision: 16
202 precision: 7
|
| rotate.decTest | 23 precision: 9
49 -- and |rhs| <= precision
246 precision: 5
|
| /external/python/cpython3/Lib/test/decimaltestdata/ |
| clamp.decTest | 27 precision: 16
188 precision: 16
202 precision: 7
|
| rotate.decTest | 23 precision: 9
49 -- and |rhs| <= precision
246 precision: 5
|
| /external/python/cpython3/Tools/unicode/ |
| gencodec.py | 124 def hexrepr(t, precision=4):
131 return '0x%0*X' % (precision, t)
133 return '(' + ', '.join(['0x%0*X' % (precision, item)
|
| /external/skia/src/gpu/ccpr/ |
| GrCoverageCountingPathRenderer.cpp | 28 // precision.
122 // The path is too large. We need to crop it or analytic AA can run out of fp32 precision.
259 // The path is too large. We need to crop it or analytic AA can run out of fp32 precision.
|
| /external/skia/tests/ |
| SRGBReadWritePixelsTest.cpp | 116 // This is a bit of a hack to check identity transformations that may lose precision.
266 // We allow more error on GPUs with lower precision shader variables.
269 // precision variation because the sRGB data gets converted to linear and back in
|
| /external/swiftshader/src/OpenGL/compiler/ |
| SymbolTable.h | 466 return false; // Only set default precision for int/float
474 // Searches down the precisionStack for a precision qualifier for the specified TBasicType
477 // unsigned integers use the same precision as signed
|
| /external/tcpdump/missing/ |
| snprintf.c | 133 /* given precision, ignore zero flag */
138 /* zero value with zero precision -> "" */
327 /* precision */
|
| /external/tensorflow/tensorflow/compiler/xla/tests/ |
| reduce_precision_test.cc | 73 // various precision-reduction cases.
266 // reduce-precision operation showed up in the correct place in the
270 // Insert precision-reduction after the Abs(x) operation, rounding that
|
| /external/tensorflow/tensorflow/contrib/layers/python/layers/ |
| target_column.py | 448 precision, and recall metrics. If None, defaults to [0.5].
465 # Precision for positive examples.
524 PRECISION_MEAN = "precision/positive_threshold_%f_mean"
|
| /external/tensorflow/tensorflow/tools/proto_text/ |
| gen_proto_text_functions_lib_test.cc | 96 // TODO(b/67475677): Re-enable after resolving float precision issue
103 // TODO(b/67475677): Re-enable after resolving float precision issue
112 // TODO(b/67475677): Re-enable after resolving float precision issue
|
| /external/v8/src/ |
| strtod.cc | 21 // (which has a 53bit significand) without loss of precision.
309 // We add 1 for the lost precision of error, and kDenominator for
310 // the lost precision of input.f().
|
| /frameworks/base/media/mca/filterpacks/java/android/filterpacks/imageproc/ |
| AutoFixFilter.java | 108 "precision mediump float;\n" +
201 long precision = (256l * 256l - 1l); local
205 long temp = normal_cdf[i] * precision / histDim;
|
| /frameworks/layoutlib/bridge/src/com/android/layoutlib/bridge/util/ |
| CachedPathIteratorFactory.java | 42 private static final float PRECISION = 0.002f;
91 getFlatPathLength(cubicCurve.getPathIterator(null, PRECISION));
97 getFlatPathLength(quadCurve.getPathIterator(null, PRECISION));
|
| /hardware/interfaces/neuralnetworks/1.1/ |
| types.hal | 336 * calculations with range and/or precision as low as that of the IEEE
417 * precision as low as that of the IEEE 754 16-bit floating-point format.
419 * range and precision of the IEEE 754 32-bit floating-point format.
|
| /libcore/ojluni/src/main/java/java/time/temporal/ |
| TemporalAccessor.java | 276 * query == TemporalQueries.chronology() || query == TemporalQueries.precision()) {
293 * if (query == TemporalQueries.precision()) {
311 || query == TemporalQueries.precision()) {
|
| /packages/apps/Test/connectivity/sl4n/rapidjson/doc/ |
| dom.md | 117 `kParseFullPrecisionFlag` | Parse number in full precision (slower). If this flag is not set, the normal precision (faster) is used. Normal precision has maximum 3 [ULP](http://en.wikipedia.org/wiki/Unit_in_the_last_place) error.
|
| /prebuilts/gdb/darwin-x86/lib/python2.7/test/decimaltestdata/ |
| clamp.decTest | 27 precision: 16
188 precision: 16
202 precision: 7
|
| rotate.decTest | 23 precision: 9
49 -- and |rhs| <= precision
246 precision: 5
|
| /prebuilts/gdb/linux-x86/lib/python2.7/test/decimaltestdata/ |
| clamp.decTest | 27 precision: 16
188 precision: 16
202 precision: 7
|
| rotate.decTest | 23 precision: 9
49 -- and |rhs| <= precision
246 precision: 5
|
| /prebuilts/go/darwin-x86/src/math/big/ |
| float_test.go | 42 // zero value has precision 0
92 // TODO(gri) test how precision is set for zero value results
397 // (rounding by SetPrec after SetInt64 using default precision
399 // precision)
678 // check precision
721 // check precision when set automatically
[all...] |
Completed in 1261 milliseconds
<<101102103104105106107108109110>>
