| /device/linaro/bootloader/edk2/ArmPkg/Library/ArmSoftFloatLib/bits32/ |
| softfloat.c | 113 Returns the fraction bits of the single-precision floating-point value `a'.
125 Returns the exponent bits of the single-precision floating-point value `a'.
137 Returns the sign bit of the single-precision floating-point value `a'.
149 Normalizes the subnormal single-precision floating-point value represented
169 single-precision floating-point value, returning the result. After being
188 and significand `zSig', and returns the proper single-precision floating-
190 value is simply rounded and packed into the single-precision format, with
197 precision floating-point number.
263 and significand `zSig', and returns the proper single-precision floating-
282 Returns the least-significant 32 fraction bits of the double-precision
[all...] |
| /device/linaro/bootloader/edk2/StdLib/LibC/Softfloat/bits32/ |
| softfloat.c | 114 Returns the fraction bits of the single-precision floating-point value `a'.
126 Returns the exponent bits of the single-precision floating-point value `a'.
138 Returns the sign bit of the single-precision floating-point value `a'.
150 Normalizes the subnormal single-precision floating-point value represented
170 single-precision floating-point value, returning the result. After being
189 and significand `zSig', and returns the proper single-precision floating-
191 value is simply rounded and packed into the single-precision format, with
198 precision floating-point number.
264 and significand `zSig', and returns the proper single-precision floating-
283 Returns the least-significant 32 fraction bits of the double-precision
[all...] |
| /prebuilts/gcc/linux-x86/host/x86_64-w64-mingw32-4.8/share/info/ |
| mpfr.info | 3 This manual documents how to install and use the Multiple Precision
19 * mpfr: (mpfr). Multiple Precision Floating-Point Reliable Library.
28 This manual documents how to install and use the Multiple Precision
100 MPFR is a portable library written in C for arbitrary precision
104 most arbitrary precision floating-point software tools, are:
110 * the precision in bits can be set _exactly_ to any valid value for
111 each variable (including very small precision);
117 In particular, with a precision of 53 bits, MPFR is able to exactly
118 reproduce all computations with double-precision machine floating-point
421 precision significand (also called mantissa) with a limited precisio
[all...] |
| /external/deqp/modules/gles2/functional/ |
| es2fShaderOperatorTests.cpp | 191 PRECMASK_NA = 0, //!< Precision not applicable (booleans)
392 , precision (PRECISION_LAST)
400 Precision precision; member in struct:deqp::gles2::Functional::ShaderDataSpec
425 const char* precision = spec.precision != PRECISION_LAST ? getPrecisionName(spec.precision) : DE_NULL; local
432 // Compute precision for inputs.
438 Precision prec = isBoolVal || (isIntVal && spec.precision == PRECISION_LOWP) ? PRECISION_MEDIUMP : spec.precision
482 if (precision && !isBool) op << precision << " "; local
513 if (precision && !isBoolOut) op << precision << " "; local
[all...] |
| /external/python/cpython3/Doc/library/ |
| decimal.rst | 57 alterable precision (defaulting to 28 places) which can be as large as needed for
89 The context for arithmetic is an environment specifying precision, rounding
124 precision, rounding, or enabled traps::
132 >>> getcontext().prec = 7 # Set a new precision
178 input. Context precision and rounding only come into play during arithmetic
308 rounded (digits beyond the context precision were thrown away) and that the
373 53 or more digits of precision. For example, ``Decimal(float('1.1'))``
377 The *context* precision does not affect how many digits are stored. That is
379 ``Decimal('3.00000')`` records all five zeros even if the context precision is
772 quantize operation would be greater than precision, then a
[all...] |
| /external/deqp/data/gles31/shaders/es31/ |
| linkage_tessellation_geometry_varying_types.test | 62 precision mediump float;
132 precision mediump float;
202 precision mediump float;
272 precision mediump float;
342 precision mediump float;
412 precision mediump float;
482 precision mediump float;
552 precision mediump float;
622 precision mediump float;
692 precision mediump float
[all...] |
| /external/deqp/data/gles31/shaders/es32/ |
| linkage_tessellation_geometry_varying_types.test | 78 precision mediump float;
146 precision mediump float;
214 precision mediump float;
282 precision mediump float;
350 precision mediump float;
418 precision mediump float;
486 precision mediump float;
554 precision mediump float;
622 precision mediump float;
690 precision mediump float
[all...] |
| /external/icu/android_icu4j/src/main/tests/android/icu/impl/number/ |
| DecimalQuantity_SimpleStorage.java | 61 // the "precision" (the number of digits in the long). "primary" and "primaryScale" are the only
296 * Utility method to compute the number of digits ("precision") in a long.
299 * @return The precision of the long.
302 int precision = 0;
305 precision++;
307 return precision;
467 int precision = (primary == -1) ? precisionBigDecimal(fallback) : primaryPrecision;
468 if (precision == 0) {
471 return scale + precision - 1;
795 * Helper method to compute the precision of a BigDecimal by our definition of precision, which i
[all...] |
| /external/icu/icu4j/main/tests/core/src/com/ibm/icu/impl/number/ |
| DecimalQuantity_SimpleStorage.java | 58 // the "precision" (the number of digits in the long). "primary" and "primaryScale" are the only
293 * Utility method to compute the number of digits ("precision") in a long.
296 * @return The precision of the long.
299 int precision = 0;
302 precision++;
304 return precision;
464 int precision = (primary == -1) ? precisionBigDecimal(fallback) : primaryPrecision;
465 if (precision == 0) {
468 return scale + precision - 1;
792 * Helper method to compute the precision of a BigDecimal by our definition of precision, which i
[all...] |
| /bionic/libm/upstream-freebsd/lib/msun/src/ |
| s_fma.c | 37 * A struct dd represents a floating-point number with twice the precision
164 * canonical precision-doubling technique adapted from:
167 * Available Precision. Numer. Math. 18, 224-242 (1971).
169 * This algorithm is sensitive to the rounding precision. FPUs such
170 * as the i387 must be set in double-precision mode if variables are
|
| /external/apache-xml/src/main/java/org/apache/xpath/objects/ |
| XNumber.java | 127 // private static final int PRECISION = 16;
210 // // reduce significant digits to PRECISION if necessary
211 // if (sigDig > PRECISION)
213 // // re-scale BigDecimal in order to get significant digits = PRECISION
215 // int newScale = num.scale() - (sigDig - PRECISION);
|
| /external/eigen/blas/f2c/ |
| dspmv.c | 64 /* ALPHA - DOUBLE PRECISION. */
68 /* AP - DOUBLE PRECISION array of DIMENSION at least */
82 /* X - DOUBLE PRECISION array of dimension at least */
93 /* BETA - DOUBLE PRECISION. */
98 /* Y - DOUBLE PRECISION array of dimension at least */
|
| /external/eigen/blas/testing/ |
| dblat3.f | 19 *> Test program for the DOUBLE PRECISION Level 3 Blas.
97 DOUBLE PRECISION ZERO, ONE
104 DOUBLE PRECISION EPS, ERR, THRESH
112 DOUBLE PRECISION AA( NMAX*NMAX ), AB( NMAX, 2*NMAX ),
122 DOUBLE PRECISION DDIFF
229 * Compute EPS (the machine precision).
367 9998 FORMAT( ' RELATIVE MACHINE PRECISION IS TAKEN TO BE', 1P, D9.1 )
371 9995 FORMAT( ' TESTS OF THE DOUBLE PRECISION LEVEL 3 BLAS', //' THE F',
410 DOUBLE PRECISION ZERO
413 DOUBLE PRECISION EPS, THRES
[all...] |
| /external/gemmlowp/doc/ |
| packing.md | 15 [low-precision.md](low-precision.md) as the packing stage also has to compute
182 dimension. If this is any mysterious, read [low-precision.md](low-precision.md).
185 Second, if the BitDepthSetting requires less than 8 bit of precision, then at
|
| /external/icu/icu4c/source/i18n/ |
| precision.h | 8 * precision.h
33 * A precision manager for values to be formatted as fixed point.
99 * precision passes the exponent value. Essentially, it divides value by
242 * A precision manager for values to be expressed as scientific notation.
263 * the precision settings, the resulting mantissa may not fall
|
| /external/mesa3d/src/mesa/tnl/ |
| t_pipeline.c | 118 * START_FAST_MATH: Set x86 FPU to faster, 32-bit precision mode (and save
124 * Set the x86 FPU control word to guarentee only 32 bits of precision
132 /* Hardware default: All exceptions masked, extended double precision,
136 /* All exceptions masked, single precision, round to nearest:
228 /* Don't adjust FPU precision mode in case multiple threads are to be used.
|
| /external/mksh/src/ |
| shf.c | 794 size_t field, precision, len; local
820 field = precision = 0;
846 precision = 0;
853 precision = 0;
859 precision = (unsigned int)tmp;
892 precision = (unsigned int)tmp;
990 if (precision > len) {
991 field = precision;
995 precision = len;
1027 if (!(flags & FL_DOT) || len < precision)
[all...] |
| /external/tensorflow/tensorflow/contrib/factorization/examples/ |
| mnist.py | 102 Precision value on the dataset.
115 precision = true_count / num_examples
116 print(' Num examples: %d Num correct: %d Precision @ 1: %0.04f' %
117 (num_examples, true_count, precision))
118 return precision
|
| /external/vulkan-validation-layers/libs/glm/gtc/ |
| matrix_inverse.inl | 35 template <typename T, precision P>
49 template <typename T, precision P>
63 template <typename T, precision P>
80 template <typename T, precision P>
106 template <typename T, precision P>
|
| /external/webp/src/utils/ |
| quant_levels_dec_utils.c | 25 #define FIX 16 // fix-point precision for averaging
26 #define LFIX 2 // extra precision for look-up table
31 #define DFIX 4 // extra precision for ordered dithering
35 { 0, 8, 2, 10 }, // coefficients are in DFIX fixed-point precision
53 int scale_; // normalization factor, in FIX bits precision
|
| /frameworks/av/media/libeffects/lvm/lib/Eq/src/ |
| LVEQNB_CalcCoef.c | 42 /* Calculate double precision coefficients for a peaking filter */
70 /* 2. The double precision coefficients are only used when fc is less than fs/85, so */
73 /* lower precision maths. */
75 /* 3. The value of the B2 coefficient is only calculated as a single precision value, */
181 /* Calculate single precision coefficients for a peaking filter */
|
| /prebuilts/gcc/linux-x86/host/x86_64-linux-glibc2.15-4.8/x86_64-linux/include/c++/4.8/ |
| iomanip | 185 * @brief Manipulator for @c precision.
186 * @param __n The new precision.
188 * Sent to a stream object, this manipulator calls @c precision(__n) for
199 __is.precision(__f._M_n);
207 __os.precision(__f._M_n);
|
| /prebuilts/gcc/linux-x86/host/x86_64-w64-mingw32-4.8/x86_64-w64-mingw32/include/c++/4.8.3/ |
| iomanip | 185 * @brief Manipulator for @c precision.
186 * @param __n The new precision.
188 * Sent to a stream object, this manipulator calls @c precision(__n) for
199 __is.precision(__f._M_n);
207 __os.precision(__f._M_n);
|
| /prebuilts/ndk/r16/sources/third_party/vulkan/src/libs/glm/gtc/ |
| matrix_inverse.inl | 35 template <typename T, precision P>
49 template <typename T, precision P>
63 template <typename T, precision P>
80 template <typename T, precision P>
106 template <typename T, precision P>
|
| /toolchain/binutils/binutils-2.27/gas/doc/ |
| c-sh64.texi | 111 registers, @samp{fr0} through @samp{fr63} for single-precision floating
113 only) for double-precision floating point registers, @samp{fv0} through
114 @samp{fv60} (multiples of four only) for single-precision floating point
116 for single-precision floating point pairs, @samp{mtrx0} through
118 single-precision floating point registers, @samp{pc} for the program
|
