HomeSort by relevance Sort by last modified time
    Searched full:precision (Results 2026 - 2050 of 7852) sorted by null

<<81828384858687888990>>

  /external/deqp/framework/opengl/
gluShaderUtil.cpp 162 // Precision
164 const char* getPrecisionName (Precision precision)
174 DE_ASSERT(deInBounds32((int)precision, 0, PRECISION_LAST));
175 return s_names[(int)precision];
  /external/deqp/modules/gles3/functional/
es3fShaderMatrixTests.cpp 217 ShaderInput (InputType inputType_, DataType dataType_, Precision precision_)
220 , precision (precision_)
226 Precision precision; member in struct:deqp::gles3::Functional::MatrixCaseUtils::ShaderInput
1878 Precision precision = precisions[precNdx]; local
    [all...]
  /external/eigen/Eigen/src/Core/arch/SSE/
MathFunctions.h 248 Precision is excellent as long as x < 8192 (I did not bother to
251 the extra precision is missing).
319 /* The magic pass: "Extended precision modular arithmetic"
408 /* The magic pass: "Extended precision modular arithmetic"
449 // of Newton's method, at a cost of 1-2 bits of precision as opposed to the
  /external/eigen/blas/f2c/
dsbmv.c 70 /* ALPHA - DOUBLE PRECISION. */
74 /* A - DOUBLE PRECISION array of DIMENSION ( LDA, n ). */
119 /* X - DOUBLE PRECISION array of DIMENSION at least */
130 /* BETA - DOUBLE PRECISION. */
134 /* Y - DOUBLE PRECISION array of DIMENSION at least */
  /external/guava/guava/src/com/google/common/math/
DoubleMath.java 163 * We cannot store Long.MAX_VALUE as a double without losing precision. Instead, we store
420 * be converted to doubles, which does not cause any loss of precision for ints.
433 * be converted to doubles, which causes loss of precision for longs of magnitude over 2^53
447 * be finite. The values will be converted to doubles, which may cause loss of precision for some
461 * be finite. The values will be converted to doubles, which may cause loss of precision for some
  /external/icu/icu4c/source/i18n/
decimfmtimpl.h 26 #include "precision.h"
105 * 2. Precision attributes
143 * handles the complex interactions amoung the various precision attributes
409 // The actual precision used when formatting
414 SciFormatterOptions fOptions; // Encapsulates fixed precision options
  /external/icu/icu4c/source/i18n/unicode/
utmscale.h 121 * without losing precision to the corresponding unit.
173 * .NET <code>System.DateTime</code> values without severe loss of precision. ICU4C time addresses this by using a
176 * But they only have 53 bits of accuracy, which means that they will lose precision when
184 * fractional field). <code>BigDecimal</code> would allow for arbitrary precision and arbitrary range,
192 * This time scale is very fine grained, does not lose precision, and covers a range that
  /external/llvm/test/CodeGen/AArch64/
machine-combiner.ll 94 ; Verify that scalar single-precision multiplies are reassociated.
108 ; Verify that scalar double-precision adds are reassociated.
122 ; Verify that scalar double-precision multiplies are reassociated.
182 ; Verify that 128-bit vector single-precision multiplies are reassociated.
196 ; Verify that 128-bit vector double-precision multiplies are reassociated.
  /external/pcre/dist2/src/sljit/
sljitConfigInternal.h 40 sljit_f32 : 32 bit single precision floating point value
41 sljit_f64 : 64 bit double precision floating point value
60 a single precision floating point array by index
62 a double precision floating point array by index
397 /* Shift for double precision sized data. */
  /external/python/cpython2/Lib/test/decimaltestdata/
tointegralx.decTest 30 precision: 9
84 -- numbers around precision
85 precision: 9
167 precision: 9
243 precision: 16
  /external/python/cpython3/Doc/tutorial/
floatingpoint.rst 154 accounting applications and high-precision applications.
196 loss-of-precision during summation. It tracks "lost digits" as values are
222 almost all platforms map Python floats to IEEE-754 "double precision". 754
223 doubles contain 53 bits of precision, so on input the computer strives to
252 Therefore the best possible approximation to 1/10 in 754 double precision is::
  /external/python/cpython3/Lib/test/decimaltestdata/
tointegralx.decTest 30 precision: 9
84 -- numbers around precision
85 precision: 9
167 precision: 9
243 precision: 16
  /external/python/cpython3/Modules/_decimal/tests/
formathelper.py 237 # [[fill]align][sign][#][0][width][.precision][type]
266 # [[fill]align][sign][#][0][width][,][.precision][type]
281 # [[fill]align][sign][#][0][width][,][.precision][type]
294 # [[fill]align][sign][#][0][width][,][.precision][type]
317 # [[fill]align][sign][#][0][width][,][.precision][type]
  /external/tensorflow/tensorflow/docs_src/performance/
quantization.md 25 and these small increments typically need floating point precision to work
34 ability means that they seem to treat low-precision calculations as just another
74 ## Why Not Train in Lower Precision Directly?
171 quantizing down to a small set of values will not hurt the precision of the
226 community to explore what's possible with low-precision neural networks.
  /prebuilts/clang/host/darwin-x86/clang-3289846/lib64/clang/3.8/include/
pmmintrin.h 225 /// \brief Moves and duplicates one double-precision value to double-precision
237 /// A pointer to a double-precision value to be moved and duplicated.
242 /// \brief Moves and duplicates the double-precision value in the lower bits of
243 /// a 128-bit vector of [2 x double] to double-precision values stored in a
  /prebuilts/clang/host/darwin-x86/clang-3859424/lib64/clang/4.0/include/
pmmintrin.h 225 /// \brief Moves and duplicates one double-precision value to double-precision
237 /// A pointer to a double-precision value to be moved and duplicated.
242 /// \brief Moves and duplicates the double-precision value in the lower bits of
243 /// a 128-bit vector of [2 x double] to double-precision values stored in a
  /prebuilts/clang/host/darwin-x86/clang-4053586/lib64/clang/5.0/include/
pmmintrin.h 224 /// \brief Moves and duplicates one double-precision value to double-precision
236 /// A pointer to a double-precision value to be moved and duplicated.
241 /// \brief Moves and duplicates the double-precision value in the lower bits of
242 /// a 128-bit vector of [2 x double] to double-precision values stored in a
  /prebuilts/clang/host/darwin-x86/clang-4393122/lib64/clang/5.0.1/include/
pmmintrin.h 226 /// \brief Moves and duplicates one double-precision value to double-precision
238 /// A pointer to a double-precision value to be moved and duplicated.
243 /// \brief Moves and duplicates the double-precision value in the lower bits of
244 /// a 128-bit vector of [2 x double] to double-precision values stored in a
  /prebuilts/clang/host/darwin-x86/clang-4479392/lib64/clang/5.0.2/include/
pmmintrin.h 226 /// \brief Moves and duplicates one double-precision value to double-precision
238 /// A pointer to a double-precision value to be moved and duplicated.
243 /// \brief Moves and duplicates the double-precision value in the lower bits of
244 /// a 128-bit vector of [2 x double] to double-precision values stored in a
  /prebuilts/clang/host/darwin-x86/clang-4579689/lib64/clang/6.0.1/include/
pmmintrin.h 226 /// \brief Moves and duplicates one double-precision value to double-precision
238 /// A pointer to a double-precision value to be moved and duplicated.
243 /// \brief Moves and duplicates the double-precision value in the lower bits of
244 /// a 128-bit vector of [2 x double] to double-precision values stored in a
  /prebuilts/clang/host/darwin-x86/clang-4630689/lib64/clang/6.0.1/include/
pmmintrin.h 226 /// \brief Moves and duplicates one double-precision value to double-precision
238 /// A pointer to a double-precision value to be moved and duplicated.
243 /// \brief Moves and duplicates the double-precision value in the lower bits of
244 /// a 128-bit vector of [2 x double] to double-precision values stored in a
  /prebuilts/clang/host/darwin-x86/clang-4639204/lib64/clang/6.0.1/include/
pmmintrin.h 226 /// \brief Moves and duplicates one double-precision value to double-precision
238 /// A pointer to a double-precision value to be moved and duplicated.
243 /// \brief Moves and duplicates the double-precision value in the lower bits of
244 /// a 128-bit vector of [2 x double] to double-precision values stored in a
  /prebuilts/clang/host/darwin-x86/clang-4691093/lib64/clang/6.0.2/include/
pmmintrin.h 226 /// \brief Moves and duplicates one double-precision value to double-precision
238 /// A pointer to a double-precision value to be moved and duplicated.
243 /// \brief Moves and duplicates the double-precision value in the lower bits of
244 /// a 128-bit vector of [2 x double] to double-precision values stored in a
  /prebuilts/clang/host/linux-x86/clang-3289846/lib64/clang/3.8/include/
pmmintrin.h 225 /// \brief Moves and duplicates one double-precision value to double-precision
237 /// A pointer to a double-precision value to be moved and duplicated.
242 /// \brief Moves and duplicates the double-precision value in the lower bits of
243 /// a 128-bit vector of [2 x double] to double-precision values stored in a
  /prebuilts/clang/host/linux-x86/clang-3859424/lib64/clang/4.0/include/
pmmintrin.h 225 /// \brief Moves and duplicates one double-precision value to double-precision
237 /// A pointer to a double-precision value to be moved and duplicated.
242 /// \brief Moves and duplicates the double-precision value in the lower bits of
243 /// a 128-bit vector of [2 x double] to double-precision values stored in a

Completed in 1137 milliseconds

<<81828384858687888990>>