| /prebuilts/gdb/linux-x86/lib/python2.7/test/decimaltestdata/ |
| minus.decTest | 26 precision: 9
72 precision: 7
75 precision: 6
78 precision: 3
83 precision: 9
113 precision: 3
118 precision: 3
151 precision: 9
159 precision: 15
|
| /prebuilts/python/darwin-x86/2.7.5/lib/python2.7/test/decimaltestdata/ |
| minus.decTest | 26 precision: 9
72 precision: 7
75 precision: 6
78 precision: 3
83 precision: 9
113 precision: 3
118 precision: 3
151 precision: 9
159 precision: 15
|
| /prebuilts/python/linux-x86/2.7.5/lib/python2.7/test/decimaltestdata/ |
| minus.decTest | 26 precision: 9
72 precision: 7
75 precision: 6
78 precision: 3
83 precision: 9
113 precision: 3
118 precision: 3
151 precision: 9
159 precision: 15
|
| /art/compiler/utils/mips/ |
| managed_register_mips.h | 66 // [R..F[ single precision FP registers (enum FRegister)
67 // [F..D[ double precision FP registers (enum DRegister)
77 // [R..F[ single precision FP registers (enum FRegister)
84 // Register), a single precision FP register (enum FRegister), a double precision
|
| /external/clang/test/Sema/ |
| ext_vector_casts.c | 105 vs = vs + i; // expected-warning {{implicit conversion loses integer precision}}
109 vi = l + vi; // expected-warning {{implicit conversion loses integer precision}}
115 vl = vl + t; // expected-warning {{implicit conversion loses integer precision}}
120 vf = d + vf; // expected-warning {{implicit conversion loses floating-point precision}}
122 vf = vf + 2.1; // expected-warning {{implicit conversion loses floating-point precision}}
|
| /external/deqp/data/gles31/shaders/es31/ |
| tessellation_negative_user_defined_io.test | 50 precision mediump float;
108 precision mediump float;
154 precision mediump float;
200 precision mediump float;
246 precision mediump float;
|
| /external/deqp/modules/gles2/scripts/ |
| gen-qualification_order.py | 36 precision mediump float;
49 precision mediump float;
68 precision mediump float;
176 DECL_PRECISION = CaseFormat("precision", ["lowp", "mediump", "mediump"])
180 PARAM_PRECISION = CaseFormat("precision", [ "lowp", "mediump", "mediump" ])
|
| /external/eigen/blas/f2c/ |
| drotm.c | 63 /* DX (input/output) DOUBLE PRECISION array, dimension N */
64 /* double precision vector with N elements */
69 /* DY (input/output) DOUBLE PRECISION array, dimension N */
70 /* double precision vector with N elements */
75 /* DPARAM (input/output) DOUBLE PRECISION array, dimension 5 */
|
| /external/python/cpython3/Include/ |
| floatobject.h | 5 PyFloatObject represents a (double precision) floating point number.
65 * 4- byte format is identical to the IEEE-754 single precision format, and
66 * the 8-byte format to the IEEE-754 double precision format, although the
71 * On non-IEEE platforms with more precision, or larger dynamic range, than
73 * precision, or smaller dynamic range, not all values can be unpacked. What
|
| /frameworks/av/media/libeffects/testlibs/ |
| AudioShelvingFilter.h | 112 // Precision for the mFrequency member.
114 // Precision for the mGain member.
122 // GAIN_PRECISION_BITS precision.
125 // FREQ_PRECISION_BITS precision.
129 // 1/Nyquist[mHz], in 42-bit precision (very small).
|
| /prebuilts/go/darwin-x86/src/math/big/ |
| example_test.go | 98 // a precision of 200 bits, and how to print the result as a decimal number.
100 // We'll do computations with 200 bits of precision in the mantissa.
118 // We use t as a temporary variable. There's no need to set its precision
119 // since big.Float values with unset (== 0) precision automatically assume
120 // the largest precision of the arguments when used as the result (receiver)
|
| /prebuilts/go/linux-x86/src/math/big/ |
| example_test.go | 98 // a precision of 200 bits, and how to print the result as a decimal number.
100 // We'll do computations with 200 bits of precision in the mantissa.
118 // We use t as a temporary variable. There's no need to set its precision
119 // since big.Float values with unset (== 0) precision automatically assume
120 // the largest precision of the arguments when used as the result (receiver)
|
| /external/icu/android_icu4j/src/main/java/android/icu/impl/number/ |
| DecimalQuantity_AbstractBCD.java | 34 * The number of digits in the BCD. For example, "1007" has BCD "0x1007" and precision 4. The
35 * maximum precision is 16 since a long can hold only 16 digits.
40 protected int precision; field in class:DecimalQuantity_AbstractBCD
121 precision = other.precision;
134 setBcdToZero(); // sets scale, precision, hasDouble, origDouble, origDelta, and BCD data
199 if (precision == 0) {
202 return scale + precision - 1;
208 if (precision != 0) {
262 int magnitude = scale + precision;
[all...] |
| /external/icu/icu4j/main/classes/core/src/com/ibm/icu/impl/number/ |
| DecimalQuantity_AbstractBCD.java | 32 * The number of digits in the BCD. For example, "1007" has BCD "0x1007" and precision 4. The
33 * maximum precision is 16 since a long can hold only 16 digits.
38 protected int precision; field in class:DecimalQuantity_AbstractBCD
119 precision = other.precision;
132 setBcdToZero(); // sets scale, precision, hasDouble, origDouble, origDelta, and BCD data
197 if (precision == 0) {
200 return scale + precision - 1;
206 if (precision != 0) {
260 int magnitude = scale + precision;
[all...] |
| /device/linaro/bootloader/edk2/AppPkg/Applications/Python/Python-2.7.10/Objects/stringlib/ |
| formatter.h | 138 Py_ssize_t precision;
member in struct:__anon4517
155 printf("internal format spec: precision %zd\n", format->precision);
191 format->precision = -1;
249 /* Parse field precision */
253 consumed = get_integer(&ptr, end, &format->precision);
258 /* Not having a precision after a dot is an error. */
261 "Format specifier missing precision");
433 /* not all fields of format are used. for example, precision is
705 /* if precision is specified, output no more that format.precision
931 Py_ssize_t precision; local
1086 Py_ssize_t precision; local
[all...] |
| /external/python/cpython2/Objects/stringlib/ |
| formatter.h | 138 Py_ssize_t precision; member in struct:__anon32921
155 printf("internal format spec: precision %zd\n", format->precision);
191 format->precision = -1;
249 /* Parse field precision */
253 consumed = get_integer(&ptr, end, &format->precision);
258 /* Not having a precision after a dot is an error. */
261 "Format specifier missing precision");
433 /* not all fields of format are used. for example, precision is
705 /* if precision is specified, output no more that format.precisio
939 Py_ssize_t precision; local
1094 Py_ssize_t precision; local
[all...] |
| /device/linaro/bootloader/edk2/EdkCompatibilityPkg/Foundation/Library/EdkIIGlueLib/Library/BasePrintLib/ |
| PrintLib.c | 104 UINTN Precision;
179 // Set the default width to zero, and the default precision to 1
182 Precision = 1;
199 Flags |= PRECISION;
218 if ((Flags & PRECISION) == 0) {
222 Precision = VA_ARG (Marker, UINTN);
226 if ((Flags & PRECISION) == 0) {
244 if ((Flags & PRECISION) == 0) {
248 Precision = Count;
255 // looking up for flag, width, precision and type.
[all...] |
| /external/apache-commons-math/src/main/java/org/apache/commons/math/dfp/ |
| DfpField.java | 74 /** High precision string representation of √2. */
77 /** High precision string representation of √2 / 2. */
80 /** High precision string representation of √3. */
83 /** High precision string representation of √3 / 3. */
86 /** High precision string representation of π. */
89 /** High precision string representation of e. */
92 /** High precision string representation of ln(2). */
95 /** High precision string representation of ln(5). */
98 /** High precision string representation of ln(10). */
185 * @param computeConstants if true, the transcendental constants for the given precision
[all...] |
| /external/tensorflow/tensorflow/contrib/coder/kernels/ |
| range_coder_ops_test.cc | 82 Status RunEncodeOp(int precision, gtl::ArraySlice<Tensor> input,
87 .Attr("precision", precision)
106 Status RunDecodeOp(int precision, gtl::ArraySlice<Tensor> input,
112 .Attr("precision", precision)
131 void TestEncodeAndDecode(int precision, const Tensor& data,
134 TF_ASSERT_OK(RunEncodeOp(precision, {data, cdf}, &encoded));
143 TF_ASSERT_OK(RunDecodeOp(precision, {encoded, shape, cdf}, &decoded));
335 .Attr("precision", 10
[all...] |
| /bionic/libm/upstream-freebsd/lib/msun/src/ |
| s_rintl.c | 74 * evaluated in long double precision. If they are evaluated in
75 * greater precision, double rounding may occur, and if they are
76 * evaluated in less precision (as on i386), results will be
|
| /external/cblas/include/ |
| cblas_f77.h | 185 /* Single Precision */
204 /* Double Precision */
224 /* Single Complex Precision */
241 /* Double Complex Precision */
262 /* Single Precision */
281 /* Double Precision */
300 /* Single Complex Precision */
320 /* Double Complex Precision */
344 /* Single Precision */
353 /* Double Precision */
[all...] |
| /external/deqp/external/vulkancts/modules/vulkan/shaderexecutor/ |
| vktShaderPackingFunctionTests.cpp | 67 static const char* getPrecisionPostfix (glu::Precision precision)
76 DE_ASSERT(de::inBounds<int>(precision, 0, DE_LENGTH_OF_ARRAY(s_postfix)));
77 return s_postfix[precision];
157 PackSnorm2x16CaseInstance (Context& context, glu::ShaderType shaderType, const ShaderSpec& spec, glu::Precision precision, const char* name)
159 , m_precision (precision)
251 const glu::Precision m_precision;
257 PackSnorm2x16Case (tcu::TestContext& testCtx, glu::ShaderType shaderType, glu::Precision precision)
[all...] |
| /external/flac/libFLAC/include/private/ |
| float.h | 83 * 'precision' roughly limits the number of iterations that are done;
84 * use (unsigned)(-1) for maximum precision.
91 FLAC__uint32 FLAC__fixedpoint_log2(FLAC__uint32 x, unsigned fracbits, unsigned precision);
|
| /external/llvm/test/CodeGen/Mips/ |
| no-odd-spreg.ll | 17 ; The intention is that if odd single precision registers are permitted, the
20 ; On the other hand, if odd single precision registers are not permitted, it
44 ; -mno-odd-sp-reg doesn't need to affect double precision values so both cases
|
| /external/llvm/test/CodeGen/X86/ |
| x86-64-double-shifts-Oz-Os-O2.ll | 6 ; precision shift instructions.
28 ; precision shift instructions.
49 ; precision shift instructions.
|
