| /external/vulkan-validation-layers/libs/glm/detail/ |
| precision.hpp | 24 /// @file glm/core/precision.hpp
34 enum precision enum in namespace:glm
|
| /prebuilts/ndk/r16/sources/third_party/vulkan/src/libs/glm/detail/ |
| precision.hpp | 24 /// @file glm/core/precision.hpp
34 enum precision enum in namespace:glm
|
| /libcore/luni/src/test/java/libcore/java/math/ |
| OldBigDecimalScaleOperationsTest.java | 89 x1.precision();
94 x2.precision();
|
| /external/tensorflow/tensorflow/contrib/coder/kernels/ |
| range_coder_test.cc | 28 void RangeEncodeDecodeTest(int precision, random::SimplePhilox* gen) {
39 const int multiplier = (precision > 7) ? 32 : 1;
43 (multiplier << precision) - histogram.size() * (multiplier - 1);
59 ASSERT_EQ(cdf.back(), 1 << precision);
62 const double normalizer = static_cast<double>(1 << precision);
67 RangeEncoder encoder(precision);
82 RangeDecoder decoder(encoded, precision);
92 const int precision = 1 + rand.Uniform(11); local
93 RangeEncodeDecodeTest(precision, &rand);
99 for (int precision = 12; precision < 17; ++precision)
[all...] |
| /external/tensorflow/tensorflow/core/kernels/ |
| as_string_op.cc | 34 int32 precision; local
41 OP_REQUIRES_OK(ctx, ctx->GetAttr("precision", &precision));
56 OP_REQUIRES(ctx, precision < 0,
57 errors::InvalidArgument("precision not supported "
71 if (precision > -1) {
72 strings::Appendf(&format_, ".%d", precision);
|
| /libcore/harmony-tests/src/test/java/org/apache/harmony/tests/java/util/ |
| IllegalFormatPrecisionExceptionTest.java | 42 int precision = 12345; local
44 precision);
45 assertEquals(precision, illegalFormatPrecisionException.getPrecision());
52 int precision = 12345; local
54 precision);
70 assertEquals("Precision", initEx.getPrecision(), desrEx
|
| /libcore/ojluni/src/main/java/java/time/temporal/ |
| TemporalQueries.java | 111 * <li> a precision,
213 * This queries a {@code TemporalAccessor} for the time precision.
215 * date or time then this must return the smallest precision actually supported.
217 * are defined to always return ignoring the precision, thus this is the only
220 * it would return a precision of {@code MILLIS}.
241 * @return a query that can obtain the precision of a temporal, not null
243 public static TemporalQuery<TemporalUnit> precision() { method in class:TemporalQueries
244 return TemporalQueries.PRECISION;
356 static final TemporalQuery<TemporalUnit> PRECISION = (temporal) ->
357 temporal.query(TemporalQueries.PRECISION);
[all...] |
| /external/deqp/external/vulkancts/modules/vulkan/ |
| vktInfoTests.cpp | 162 int precision; member in struct:vkt::__anon18037::PrettySize
166 , precision (precision_)
203 tmp << std::fixed << std::setprecision(size.precision)
211 PrettySize<SizeType> prettySize (SizeType value, int precision = 2)
213 return PrettySize<SizeType>(value, precision);
|
| /external/icu/android_icu4j/src/main/tests/android/icu/impl/number/ |
| DecimalQuantity_64BitBCD.java | 71 assert precision + numDigits <= 16;
74 precision += numDigits;
81 precision -= numDigits;
88 precision = 0;
105 precision = 16 - i;
119 precision = 16 - i; field in class:DecimalQuantity_64BitBCD
140 for (int shift = (precision - 1); shift >= 0; shift--) {
154 precision = 0;
163 // Compute precision
164 precision = 16 - (Long.numberOfLeadingZeros(bcd) / 4)
[all...] |
| /external/icu/icu4c/source/i18n/ |
| number_decimalquantity.h | 246 * The number of digits in the BCD. For example, "1007" has BCD "0x1007" and precision 4. The
247 * maximum precision is 16 since a long can hold only 16 digits.
252 int32_t precision; member in class:number::impl::DecimalQuantity
375 * Sets the internal representation to zero. Clears any values stored in scale, precision,
402 * precision. The precision is the number of digits in the number up through the greatest nonzero
|
| /external/icu/icu4j/main/tests/core/src/com/ibm/icu/impl/number/ |
| DecimalQuantity_64BitBCD.java | 68 assert precision + numDigits <= 16;
71 precision += numDigits;
78 precision -= numDigits;
85 precision = 0;
102 precision = 16 - i;
116 precision = 16 - i; field in class:DecimalQuantity_64BitBCD
137 for (int shift = (precision - 1); shift >= 0; shift--) {
151 precision = 0;
160 // Compute precision
161 precision = 16 - (Long.numberOfLeadingZeros(bcd) / 4)
[all...] |
| /libcore/luni/src/main/java/java/math/ |
| MathContext.java | 27 * precision for the numerical operations provided by class {@link BigDecimal}.
34 * decimal precision format: 34 digit precision and
41 * precision format: 7 digit precision and {@link RoundingMode#HALF_EVEN}
48 * precision format: 16 digit precision and {@link RoundingMode#HALF_EVEN}
54 * A {@code MathContext} for unlimited precision with
61 * this precision.
63 private final int precision; field in class:MathContext
[all...] |
| /external/deqp/modules/gles2/scripts/ |
| gen-swizzles.py | 107 for (inputType, dataType, precision) in combinations.iterate():
109 print inputType, precision, dataType
131 precision mediump float;
151 precision mediump float;
152 precision mediump int;
167 def __init__(self, name, precision, dataType, swizzle, inputs, outputs):
169 self.precision = precision
250 # \todo [petri] Uses fixed precision.
253 precision = "mediump variable
[all...] |
| /external/deqp/modules/gles3/scripts/ |
| gen-swizzles.py | 107 for (inputType, dataType, precision) in combinations.iterate():
109 print inputType, precision, dataType
133 precision mediump float;
155 precision mediump float;
156 precision mediump int;
171 def __init__(self, name, precision, dataType, swizzle, inputs, outputs):
173 self.precision = precision
254 # \todo [petri] Uses fixed precision.
257 precision = "mediump variable
[all...] |
| /external/eigen/Eigen/src/Core/ |
| IO.h | 31 * - \b precision number of digits for floating point values, or one of the special constants \c StreamPrecision and \c FullPrecision.
33 * stream's own precision setting, as set for instance using \c cout.precision(3). The other special value
34 * \c FullPrecision means that the number of digits will be computed to match the full precision of each floating-point
58 rowSpacer(""), coeffSeparator(_coeffSeparator), precision(_precision), flags(_flags)
74 int precision; member in struct:Eigen::IOFormat
143 if(fmt.precision == StreamPrecision)
147 else if(fmt.precision == FullPrecision)
160 explicit_precision = fmt.precision;
164 if(explicit_precision) old_precision = s.precision(explicit_precision)
[all...] |
| /external/eigen/unsupported/Eigen/src/Skyline/ |
| SkylineInplaceLU.h | 51 * \sa precision() */
56 /** \returns the current precision.
59 RealScalar precision() const { function in class:Eigen::SkylineInplaceLU
|
| /external/llvm/lib/Support/ |
| APFloat.cpp | 10 // This file implements a class to represent arbitrary precision floating
55 unsigned int precision; member in struct:llvm::fltSemantics
639 unsigned bitsToPreserve = semantics->precision - 1;
647 unsigned QNaNBit = semantics->precision - 2;
663 // For x87 extended precision, we want to make a NaN, not a
709 semantics->precision - 1) == 0);
732 PartCount*integerPartWidth - semantics->precision + 1;
754 PartCount*integerPartWidth - semantics->precision + 1;
805 exponent = ourSemantics.precision - 1;
848 return partCountForBits(semantics->precision + 1)
[all...] |
| /external/ltp/testcases/misc/math/float/ |
| tfloat.h | 76 * precision int. value used to distinguish between rounding
79 * 2 ** precision)
88 int precision; member in struct:__anon28550
|
| /external/syslinux/dos/ |
| printf.c | 43 static char *number(char *str, long num, int base, int size, int precision,
83 if (i > precision)
84 precision = i;
85 size -= precision;
102 while (i < precision--)
125 int precision; /* min. # of digits for integers; max local
171 /* get the precision */
172 precision = -1;
176 precision = skip_atou(&fmt);
180 precision = va_arg(args, int)
[all...] |
| /frameworks/base/core/tests/coretests/src/android/view/ |
| PinchZoomAction.java | 52 float[] precision) {
58 checkNotNull(precision);
224 final float[] precision = {1.0f, 1.0f, 1.0f, 1.0f}; local
229 this.mSecondFingerEndCoords, precision);
|
| /toolchain/binutils/binutils-2.27/gas/config/ |
| atof-vax.c | 23 /* Precision in LittleNums. */
203 int precision; /* Number of 16-bit words in the format. */
213 if (what_kind_of_float (what_kind, &precision, &exponent_bits))
226 f.high = f.low + precision - 1 + GUARD;
250 int precision;
254 return_value = what_kind_of_float (format_letter, &precision, &exponent_bits);
263 memset (words, '\0', sizeof (LITTLENUM_TYPE) * precision);
333 for (; lp < words + precision; lp++)
200 int precision; \/* Number of 16-bit words in the format. *\/ local
246 int precision; local
|
| /device/google/contexthub/firmware/os/core/ |
| printf.c | 32 uint32_t precision; member in struct:PrintfData
51 if (data->precision > sizeof(buf) - 1)
52 data->precision = sizeof(buf) - 1;
92 while (data->precision > numPrinted) {
219 data.precision = 0;
247 if (data.precision)
248 len = StrVPrintf_StrLen_withMax(str,data.precision);
301 data.precision = (data.precision * 10) + c - '0';
|
| /external/compiler-rt/lib/sanitizer_common/ |
| sanitizer_printf.cc | 98 static int AppendString(char **buff, const char *buff_end, int precision,
104 if (precision >= 0 && result >= precision)
143 int precision = -1; local
146 precision = va_arg(args, int);
155 // Only %s supports precision for now
156 CHECK(!(precision >= 0 && *cur != 's'));
182 result += AppendString(&buff, buff_end, precision, va_arg(args, char*));
|
| /external/deqp/modules/gles31/functional/ |
| es31fFboTestUtil.cpp | 79 const char* precision = "highp"; local
89 src << "uniform " << precision << " " << glu::getDataTypeName(samplerTypes[samplerNdx]) << " u_sampler" << samplerNdx << ";\n";
90 src << "uniform " << precision << " vec4 u_texScale" << samplerNdx << ";\n";
91 src << "uniform " << precision << " vec4 u_texBias" << samplerNdx << ";\n";
95 src << "uniform " << precision << " vec4 u_outScale0;\n"
96 << "uniform " << precision << " vec4 u_outBias0;\n"; local
101 << " " << precision << " vec4 out0 = vec4(0.0);\n"; local
|
| /external/e2fsprogs/intl/ |
| vasnprintf.c | 278 size_t precision; local
302 precision = 6;
312 precision = (arg < 0 ? 0 : arg);
318 precision = 0;
320 precision = xsum (xtimes (precision, 10), *digitp++ - '0');
431 tmp_length = xsum (tmp_length, precision);
438 tmp_length = xsum (tmp_length, precision);
|
