| /prebuilts/gdb/linux-x86/lib/python2.7/ |
| decimal.py | 30 Decimal floating point has finite precision with arbitrarily large bounds.
256 longer than precision). The result is [0,qNaN].
290 on creation and either the precision exceeds the capability of the
308 operation (or sequence of operations) caused a loss of precision.
336 current precision, with the sign of the intermediate result. For
474 # uses a precision 2 greater than normal
475 return +s # Convert result to normal precision
[all...] |
| /prebuilts/python/darwin-x86/2.7.5/lib/python2.7/ |
| decimal.py | 30 Decimal floating point has finite precision with arbitrarily large bounds.
256 longer than precision). The result is [0,qNaN].
290 on creation and either the precision exceeds the capability of the
308 operation (or sequence of operations) caused a loss of precision.
336 current precision, with the sign of the intermediate result. For
474 # uses a precision 2 greater than normal
475 return +s # Convert result to normal precision
[all...] |
| /prebuilts/python/linux-x86/2.7.5/lib/python2.7/ |
| decimal.py | 30 Decimal floating point has finite precision with arbitrarily large bounds.
256 longer than precision). The result is [0,qNaN].
290 on creation and either the precision exceeds the capability of the
308 operation (or sequence of operations) caused a loss of precision.
336 current precision, with the sign of the intermediate result. For
474 # uses a precision 2 greater than normal
475 return +s # Convert result to normal precision
[all...] |
| /cts/tests/tests/renderscript/src/android/renderscript/cts/ |
| CoreMathVerifier.java | 26 /* The level of precision we expect out of the half_* functions. floats (f32) have 23 bits of
30 // The level of precision we expect out of the fast_* functions.
32 // The level of precision we expect out of the native_* functions.
258 // double-precision version later.
300 // double-precision version later.
342 // double-precision version later.
469 // Computes the cross product of two double-precision 3D vectors.
507 // Convert a double-precision radian value to degrees.
514 // Returns the distance between two points (in double-precision) in n-dimensional space.
536 // Computes the error function for a double-precision input
[all...] |
| /device/linaro/bootloader/edk2/AppPkg/Applications/Python/Python-2.7.2/Lib/ |
| decimal.py | 30 Decimal floating point has finite precision with arbitrarily large bounds.
256 longer than precision). The result is [0,qNaN].
290 on creation and either the precision exceeds the capability of the
308 operation (or sequence of operations) caused a loss of precision.
336 current precision, with the sign of the intermediate result. For
474 # uses a precision 2 greater than normal
475 return +s # Convert result to normal precision
[all...] |
| /external/tensorflow/tensorflow/python/ops/ |
| metrics_impl.py | 589 thresholds is used to compute pairs of recall and precision values. The area
592 computed using the height of the precision values by the recall.
595 computes the area under a discretized curve of precision versus recall values
627 'PR' for the Precision-Recall-curve.
1810 def precision(labels, function
[all...] |
| /external/deqp/external/vulkancts/modules/vulkan/pipeline/ |
| vktPipelineImageSamplingInstance.cpp | [all...] |
| /device/linaro/bootloader/edk2/AppPkg/Applications/Python/Python-2.7.2/Lib/test/decimaltestdata/ |
| maxmag.decTest | 26 precision: 9
95 precision: 9
196 precision: 9
216 precision: 15
337 precision: 3
365 precision: 3
370 precision: 3
|
| minmag.decTest | 26 precision: 9
84 precision: 9
185 precision: 9
205 precision: 15
326 precision: 3
355 precision: 3
360 precision: 3
|
| /external/ImageMagick/ImageMagick/script/ |
| formats.html | 213 <td>Use <a href="command-line-options.php#size">-size</a> and <a href="command-line-options.php#depth">-depth</a> to specify the image width, height, and depth. To specify a single precision floating-point format, use <code>-define quantum:format=floating-point</code>. Set the depth to 32 for single precision floats, 64 for double precision, and 16 for half-precision.</td>
220 <td>Use <a href="command-line-options.php#size">-size</a> and <a href="command-line-options.php#depth">-depth</a> to specify the image width, height, and depth. To specify a single precision floating-point format, use <code>-define quantum:format=floating-point</code>. Set the depth to 32 for single precision floats, 64 for double precision, and 16 for half-precision.</td>
403 <td>To specify a single-precision floating-point format, use <code>-define quantum:format=floating-point</code>. Set the depth to 64 for a double-precision floating-point format.</td
[all...] |
| /external/ImageMagick/www/ |
| formats.html | 217 <td>Use <a href="command-line-options.html#size">-size</a> and <a href="command-line-options.html#depth">-depth</a> to specify the image width, height, and depth. To specify a single precision floating-point format, use <code>-define quantum:format=floating-point</code>. Set the depth to 32 for single precision floats, 64 for double precision, and 16 for half-precision.</td>
224 <td>Use <a href="command-line-options.html#size">-size</a> and <a href="command-line-options.html#depth">-depth</a> to specify the image width, height, and depth. To specify a single precision floating-point format, use <code>-define quantum:format=floating-point</code>. Set the depth to 32 for single precision floats, 64 for double precision, and 16 for half-precision.</td>
407 <td>To specify a single-precision floating-point format, use <code>-define quantum:format=floating-point</code>. Set the depth to 64 for a double-precision floating-point format.</td
[all...] |
| /external/deqp/modules/gles31/functional/ |
| es31fFboTestUtil.cpp | 79 const char* precision = "highp"; local
89 src << "uniform " << precision << " " << glu::getDataTypeName(samplerTypes[samplerNdx]) << " u_sampler" << samplerNdx << ";\n";
90 src << "uniform " << precision << " vec4 u_texScale" << samplerNdx << ";\n";
91 src << "uniform " << precision << " vec4 u_texBias" << samplerNdx << ";\n";
95 src << "uniform " << precision << " vec4 u_outScale0;\n"
96 << "uniform " << precision << " vec4 u_outBias0;\n"; local
101 << " " << precision << " vec4 out0 = vec4(0.0);\n"; local
|
| /external/deqp/modules/gles31/scripts/ |
| gen-implicit-conversions.py | 45 precision highp float;
69 precision highp float;
98 precision highp float;
123 precision highp float;
154 precision highp float;
176 precision highp float;
198 precision highp float;
|
| /external/eigen/lapack/ |
| dlarfb.f | 29 * DOUBLE PRECISION C( LDC, * ), T( LDT, * ), V( LDV, * ),
99 *> V is DOUBLE PRECISION array, dimension
117 *> T is DOUBLE PRECISION array, dimension (LDT,K)
130 *> C is DOUBLE PRECISION array, dimension (LDC,N)
143 *> WORK is DOUBLE PRECISION array, dimension (LDWORK,K)
208 DOUBLE PRECISION C( LDC, * ), T( LDT, * ), V( LDV, * ),
215 DOUBLE PRECISION ONE
|
| /external/gemmlowp/doc/ |
| less-than-8-bit.md | 5 We assume familiarity with gemmlowp's low-precision uint8 computation paradigm,
6 which is described in [low-precision.md](low-precision.md).
8 This document is about the possibility of further reducing precision below 8
85 allow gemmlowp to use lower precision internally, but the input/output data
91 as specified in [low-precision.md](low-precision.md). With lower bit depths,
|
| /external/libcxx/include/ |
| ios | 79 streamsize precision() const;
80 streamsize precision(streamsize prec);
290 _LIBCPP_INLINE_VISIBILITY streamsize precision() const;
291 _LIBCPP_INLINE_VISIBILITY streamsize precision(streamsize __prec);
480 // precision
484 ios_base::precision() const
491 ios_base::precision(streamsize __prec)
|
| /external/mesa3d/docs/relnotes/ |
| 7.10.1.html | 56 <li>Fix numerous issues handling precision qualifiers in GLSL ES.</li>
178 <li>glsl: Change default value of ast_type_specifier::precision</li>
179 <li>glsl: Add semantic checks for precision qualifiers</li>
180 <li>glsl: Add support for default precision statements</li>
182 <li>glsl: Fix semantic checks on precision qualifiers</li>
279 <li>i965: Set correct values for range/precision of fragment shader types</li>
280 <li>i915: Set correct values for range/precision of fragment shader types</li>
|
| 7.9.2.html | 58 <li>Fix numerous issues handling precision qualifiers in GLSL ES.</li>
174 <li>glsl: Change default value of ast_type_specifier::precision</li>
175 <li>glsl: Add semantic checks for precision qualifiers</li>
176 <li>glsl: Add support for default precision statements</li>
178 <li>glsl: Fix semantic checks on precision qualifiers</li>
258 <li>i965: Set correct values for range/precision of fragment shader types</li>
259 <li>i915: Set correct values for range/precision of fragment shader types</li>
|
| /external/python/cpython2/Lib/test/decimaltestdata/ |
| maxmag.decTest | 26 precision: 9
95 precision: 9
196 precision: 9
216 precision: 15
337 precision: 3
365 precision: 3
370 precision: 3
|
| minmag.decTest | 26 precision: 9
84 precision: 9
185 precision: 9
205 precision: 15
326 precision: 3
355 precision: 3
360 precision: 3
|
| /external/python/cpython3/Lib/test/decimaltestdata/ |
| maxmag.decTest | 26 precision: 9
95 precision: 9
196 precision: 9
216 precision: 15
337 precision: 3
365 precision: 3
370 precision: 3
|
| minmag.decTest | 26 precision: 9
84 precision: 9
185 precision: 9
205 precision: 15
326 precision: 3
355 precision: 3
360 precision: 3
|
| /external/python/cpython3/Lib/test/ |
| test_fstring.py | 227 precision = 4
229 self.assertEqual(f'result: {value:{width}.{precision}}', 'result: 12.35')
230 self.assertEqual(f'result: {value:{width!r}.{precision}}', 'result: 12.35')
231 self.assertEqual(f'result: {value:{width:0}.{precision:1}}', 'result: 12.35')
232 self.assertEqual(f'result: {value:{1}{0:0}.{precision:1}}', 'result: 12.35')
233 self.assertEqual(f'result: {value:{ 1}{ 0:0}.{ precision:1}}', 'result: 12.35')
253 "f'result: {value:{width:{0}}.{precision:1}}'",
|
| /frameworks/compile/slang/ |
| slang_rs_pragma_handler.cpp | 402 std::string Precision = getName();
404 if (Precision == "rs_fp_imprecise") {
409 Precision = "rs_fp_relaxed";
411 // Check if we have already encountered a precision pragma already.
415 if (PreviousPrecision != Precision) {
426 mContext->addPragma(Precision, "");
427 mContext->setPrecision(Precision);
|
| /packages/apps/Settings/tests/robotests/src/com/android/settings/fuelgauge/ |
| BatteryUtilsTest.java | 105 private static final double PRECISION = 0.001;
288 assertThat(totalUsage).isWithin(PRECISION).of(BATTERY_SYSTEM_USAGE);
361 .isWithin(PRECISION).of(PERCENT_SYSTEM_USAGE);
383 assertThat(sipperNull.totalPowerMah).isWithin(PRECISION).of(BATTERY_APP_USAGE);
384 assertThat(sipperBg.totalPowerMah).isWithin(PRECISION).of(BATTERY_APP_USAGE);
385 assertThat(sipperFg.totalPowerMah).isWithin(PRECISION).of(
387 assertThat(sipperFg2.totalPowerMah).isWithin(PRECISION).of(
|
