| /prebuilts/python/darwin-x86/2.7.5/lib/python2.7/test/ |
| formatfloat_testcases.txt | 1 -- 'f' code formatting, with explicit precision (>= 0). Output always
12 -- precision 0; result should never include a .
28 -- precision 1
44 -- precision 2
61 -- makes a difference when the precision is 0.
71 -- if precision is omitted it defaults to 6
92 -- 'e' code formatting with explicit precision (>= 0). Output should
103 -- precision 0. no decimal point in the output
137 -- precision 1
149 -- precision
[all...] |
| /prebuilts/python/linux-x86/2.7.5/lib/python2.7/test/ |
| formatfloat_testcases.txt | 1 -- 'f' code formatting, with explicit precision (>= 0). Output always
12 -- precision 0; result should never include a .
28 -- precision 1
44 -- precision 2
61 -- makes a difference when the precision is 0.
71 -- if precision is omitted it defaults to 6
92 -- 'e' code formatting with explicit precision (>= 0). Output should
103 -- precision 0. no decimal point in the output
137 -- precision 1
149 -- precision
[all...] |
| /external/deqp/modules/gles3/functional/ |
| es3fTransformFeedbackTests.cpp | 540 const glu::Precision precision = attrib.type.getPrecision(); local
551 switch (precision)
563 switch (precision)
575 switch (precision)
727 glu::Precision precision = attribute.type.getPrecision(); local
746 switch (precision)
[all...] |
| /external/eigen/unsupported/Eigen/src/MatrixFunctions/ |
| MatrixLogarithm.h | 28 static const int value = std::numeric_limits<RealScalar>::digits<= 24? 5: // single precision
29 std::numeric_limits<RealScalar>::digits<= 53? 7: // double precision
30 std::numeric_limits<RealScalar>::digits<= 64? 8: // extended precision
32 11; // quadruple precision
101 #if LDBL_MANT_DIG == 53 // double precision
104 #elif LDBL_MANT_DIG <= 64 // extended precision
114 #else // quadruple precision
236 const RealScalar maxNormForPade = maxPadeDegree<= 5? 5.3149729967117310e-1L: // single precision
237 maxPadeDegree<= 7? 2.6429608311114350e-1L: // double precision
238 maxPadeDegree<= 8? 2.32777776523703892094e-1L: // extended precision
[all...] |
| /prebuilts/ndk/r16/sources/third_party/shaderc/third_party/glslang/SPIRV/ |
| SpvBuilder.h | 250 Function* makeFunctionEntry(Decoration precision, Id returnType, const char* name, const std::vector<Id>& paramTypes,
304 Id createRvalueSwizzle(Decoration precision, Id typeId, Id source, const std::vector<unsigned>& channels);
312 // If both the id and precision are valid, the id
313 // gets tagged with the requested precision.
315 Id setPrecision(Id id, Decoration precision)
317 if (precision != NoPrecision && id != NoResult)
318 addDecoration(id, precision);
335 void promoteScalar(Decoration precision, Id& left, Id& right);
340 Id smearScalar(Decoration precision, Id scalarVal, Id vectorType);
363 Id createTextureCall(Decoration precision, Id resultType, bool sparse, bool fetch, bool proj, bool gather, bool noImplicit, const TexturePar (…)
[all...] |
| /prebuilts/gcc/linux-x86/host/x86_64-linux-glibc2.15-4.8/x86_64-linux/include/c++/4.8/ext/ |
| random.tcc | 473 const std::streamsize __precision = __os.precision();
477 __os.precision(std::numeric_limits<_RealType>::max_digits10);
483 __os.precision(__precision);
670 const std::streamsize __precision = __os.precision();
674 __os.precision(std::numeric_limits<_RealType>::max_digits10);
687 __os.precision(__precision);
755 const std::streamsize __precision = __os.precision();
759 __os.precision(std::numeric_limits<_RealType>::max_digits10);
767 __os.precision(__precision);
821 const std::streamsize __precision = __os.precision();
[all...] |
| /prebuilts/gcc/linux-x86/host/x86_64-w64-mingw32-4.8/x86_64-w64-mingw32/include/c++/4.8.3/ext/ |
| random.tcc | 473 const std::streamsize __precision = __os.precision();
477 __os.precision(std::numeric_limits<_RealType>::max_digits10);
483 __os.precision(__precision);
670 const std::streamsize __precision = __os.precision();
674 __os.precision(std::numeric_limits<_RealType>::max_digits10);
687 __os.precision(__precision);
755 const std::streamsize __precision = __os.precision();
759 __os.precision(std::numeric_limits<_RealType>::max_digits10);
767 __os.precision(__precision);
821 const std::streamsize __precision = __os.precision();
[all...] |
| /external/swiftshader/third_party/LLVM/lib/Target/ARM/ |
| ARMInstrVFP.td | 78 // Some single precision VFP instructions may be executed on both NEON and VFP
92 // Some single precision VFP instructions may be executed on both NEON and VFP
103 // Double Precision
131 // Single Precision
140 // Some single precision VFP instructions may be executed on both NEON and
153 // Some single precision VFP instructions may be executed on both NEON and
166 // Some single precision VFP instructions may be executed on both NEON and
209 // Some single precision VFP instructions may be executed on both NEON and
223 // Some single precision VFP instructions may be executed on both NEON and
247 // Some single precision VFP instructions may be executed on both NEON and
[all...] |
| /external/crcalc/src/com/hp/creals/ |
| CR.java | 124 * is accurate to the specified precision.
129 * later approximate it to arbitrary precision.
132 * accurate to the requested precision; no cumulative rounding errors
134 * In order to achieve this precision, the approximation function will often
135 * need to approximate subexpressions to greater precision than was originally
138 * evaluation to very high precision. This usually makes such computations
168 * If the precision request generated during any subcalculation overflows
192 * Indicates that the number of bits of precision requested by
229 * Returns value / 2 ** precision rounded to an integer.
233 * Implementations may safely assume that precision i
[all...] |
| /device/linaro/bootloader/edk2/AppPkg/Applications/Python/Python-2.7.2/Objects/stringlib/ |
| formatter.h | 140 Py_ssize_t precision;
member in struct:__anon4862
157 printf("internal format spec: precision %zd\n", format->precision);
192 format->precision = -1;
249 /* Parse field precision */
253 consumed = get_integer(&ptr, end, &format->precision);
258 /* Not having a precision after a dot is an error. */
261 "Format specifier missing precision");
433 /* not all fields of format are used. for example, precision is
705 /* if precision is specified, output no more that format.precision
933 Py_ssize_t precision = format->precision; local
1083 Py_ssize_t precision = format->precision; local
[all...] |
| /device/linaro/bootloader/edk2/AppPkg/Applications/Python/Python-2.7.2/Lib/test/decimaltestdata/ |
| fma.decTest | 23 precision: 9
144 precision: 6
152 precision: 15
154 precision: 30
156 precision: 9
202 precision: 9
208 precision: 8
211 precision: 7
215 precision: 9
221 precision: 8
[all...] |
| /external/python/cpython2/Lib/test/decimaltestdata/ |
| fma.decTest | 23 precision: 9
144 precision: 6
152 precision: 15
154 precision: 30
156 precision: 9
202 precision: 9
208 precision: 8
211 precision: 7
215 precision: 9
221 precision:
[all...] |
| /external/python/cpython3/Lib/test/decimaltestdata/ |
| fma.decTest | 23 precision: 9
144 precision: 6
152 precision: 15
154 precision: 30
156 precision: 9
202 precision: 9
208 precision: 8
211 precision: 7
215 precision: 9
221 precision: 8
[all...] |
| /prebuilts/gdb/darwin-x86/lib/python2.7/test/decimaltestdata/ |
| fma.decTest | 23 precision: 9
144 precision: 6
152 precision: 15
154 precision: 30
156 precision: 9
202 precision: 9
208 precision: 8
211 precision: 7
215 precision: 9
221 precision:
[all...] |
| /prebuilts/gdb/linux-x86/lib/python2.7/test/decimaltestdata/ |
| fma.decTest | 23 precision: 9
144 precision: 6
152 precision: 15
154 precision: 30
156 precision: 9
202 precision: 9
208 precision: 8
211 precision: 7
215 precision: 9
221 precision:
[all...] |
| /prebuilts/python/darwin-x86/2.7.5/lib/python2.7/test/decimaltestdata/ |
| fma.decTest | 23 precision: 9
144 precision: 6
152 precision: 15
154 precision: 30
156 precision: 9
202 precision: 9
208 precision: 8
211 precision: 7
215 precision: 9
221 precision:
[all...] |
| /prebuilts/python/linux-x86/2.7.5/lib/python2.7/test/decimaltestdata/ |
| fma.decTest | 23 precision: 9
144 precision: 6
152 precision: 15
154 precision: 30
156 precision: 9
202 precision: 9
208 precision: 8
211 precision: 7
215 precision: 9
221 precision:
[all...] |
| /art/compiler/utils/arm64/ |
| managed_register_arm64.h | 40 // [W..D[ double precision VFP registers (enum DRegister)
41 // [D..S[ single precision VFP registers (enum SRegister)
53 // * VFP double precision register (enum DRegister)
54 // * VFP single precision register (enum SRegister)
|
| /bionic/libc/tzcode/ |
| difftime.c | 22 ** (assuming that the larger type has more precision).
|
| /cts/tests/openglperf2/assets/fragment/ |
| blur | 14 precision mediump float;
|
| perspective | 14 precision mediump float;
|
| /cts/tests/openglperf2/assets/vertex/ |
| perspective | 14 precision mediump float;
|
| /cts/tests/tests/opengl/libopengltest/ |
| attach_shader_one.cpp | 34 "precision mediump float;\n"
|
| /device/google/contexthub/firmware/lib/builtins/ |
| CREDITS.TXT | 28 D: IEEE Quad-precision functions
|
| /device/linaro/bootloader/edk2/AppPkg/Applications/Python/Python-2.7.10/Include/ |
| complexobject.h | 36 PyComplexObject represents a complex number with double-precision
|
