| /device/linaro/bootloader/edk2/AppPkg/Applications/Python/Python-2.7.2/Lib/ |
| decimal.py | 61 >>> getcontext().prec = 18
351 return _dec_from_triple(sign, '9'*context.prec,
352 context.Emax-context.prec+1)
356 return _dec_from_triple(sign, '9'*context.prec,
357 context.Emax-context.prec+1)
472 ctx.prec += 2
485 >>> print getcontext().prec
489 ... ctx.prec += 2
490 ... print ctx.prec
494 ... print getcontext().prec
[all...] |
| /external/python/cpython2/Lib/ |
| decimal.py | 61 >>> getcontext().prec = 18
350 return _dec_from_triple(sign, '9'*context.prec,
351 context.Emax-context.prec+1)
355 return _dec_from_triple(sign, '9'*context.prec,
356 context.Emax-context.prec+1)
471 ctx.prec += 2
484 >>> print getcontext().prec
488 ... ctx.prec += 2
489 ... print ctx.prec
493 ... print getcontext().prec
[all...] |
| /prebuilts/gdb/darwin-x86/lib/python2.7/ |
| decimal.py | 61 >>> getcontext().prec = 18
351 return _dec_from_triple(sign, '9'*context.prec,
352 context.Emax-context.prec+1)
356 return _dec_from_triple(sign, '9'*context.prec,
357 context.Emax-context.prec+1)
472 ctx.prec += 2
485 >>> print getcontext().prec
489 ... ctx.prec += 2
490 ... print ctx.prec
494 ... print getcontext().prec
[all...] |
| /prebuilts/gdb/linux-x86/lib/python2.7/ |
| decimal.py | 61 >>> getcontext().prec = 18
351 return _dec_from_triple(sign, '9'*context.prec,
352 context.Emax-context.prec+1)
356 return _dec_from_triple(sign, '9'*context.prec,
357 context.Emax-context.prec+1)
472 ctx.prec += 2
485 >>> print getcontext().prec
489 ... ctx.prec += 2
490 ... print ctx.prec
494 ... print getcontext().prec
[all...] |
| /prebuilts/python/darwin-x86/2.7.5/lib/python2.7/ |
| decimal.py | 61 >>> getcontext().prec = 18
351 return _dec_from_triple(sign, '9'*context.prec,
352 context.Emax-context.prec+1)
356 return _dec_from_triple(sign, '9'*context.prec,
357 context.Emax-context.prec+1)
472 ctx.prec += 2
485 >>> print getcontext().prec
489 ... ctx.prec += 2
490 ... print ctx.prec
494 ... print getcontext().prec
[all...] |
| /prebuilts/python/linux-x86/2.7.5/lib/python2.7/ |
| decimal.py | 61 >>> getcontext().prec = 18
351 return _dec_from_triple(sign, '9'*context.prec,
352 context.Emax-context.prec+1)
356 return _dec_from_triple(sign, '9'*context.prec,
357 context.Emax-context.prec+1)
472 ctx.prec += 2
485 >>> print getcontext().prec
489 ... ctx.prec += 2
490 ... print ctx.prec
494 ... print getcontext().prec
[all...] |
| /prebuilts/go/darwin-x86/src/math/big/ |
| floatmarsh.go | 26 sz := 1 + 1 + 4 // version + mode|acc|form|neg (3+2+2+1bit) + prec
30 n = int((x.prec + (_W - 1)) / _W) // required mantissa length in words for given precision
50 binary.BigEndian.PutUint32(buf[2:], x.prec)
75 oldPrec := z.prec
83 z.prec = binary.BigEndian.Uint32(buf[2:])
|
| floatconv.go | 34 prec := z.prec
35 if prec == 0 {
36 prec = 64
65 z.prec = prec
122 z.prec = prec
139 p := new(Float).SetPrec(z.Prec() + 64) // use more bits for p -- TODO(gri) what is the right number?
200 f := new(Float).SetPrec(z.Prec() + 64).SetUint64(5
[all...] |
| float.go | 60 prec uint32
97 // Float x are stored in a nat slice long enough to hold up to x.prec bits;
101 // trailing 0 bits or x.prec is not a multiple of the Word size _W,
153 // SetPrec sets z's precision to prec and returns the (possibly) rounded
155 // cannot be represented in prec bits without loss of precision.
157 // If prec > MaxPrec, it is set to MaxPrec.
158 func (z *Float) SetPrec(prec uint) *Float {
162 if prec == 0 {
163 z.prec = 0
173 if prec > MaxPrec
[all...] |
| bits_test.go | 106 // to prec bits according to mode.
107 func (x Bits) round(prec uint, mode RoundingMode) *Float {
121 if prec >= prec0 {
124 // prec < prec0
129 r := max - int(prec)
152 f.SetMode(ToZero).SetPrec(prec)
|
| example_test.go | 101 const prec = 200
108 // iteration, we need at least log_2(prec) steps.
109 steps := int(math.Log2(prec))
112 two := new(big.Float).SetPrec(prec).SetInt64(2)
113 half := new(big.Float).SetPrec(prec).SetFloat64(0.5)
116 x := new(big.Float).SetPrec(prec).SetInt64(1)
|
| /prebuilts/go/linux-x86/src/math/big/ |
| floatmarsh.go | 26 sz := 1 + 1 + 4 // version + mode|acc|form|neg (3+2+2+1bit) + prec
30 n = int((x.prec + (_W - 1)) / _W) // required mantissa length in words for given precision
50 binary.BigEndian.PutUint32(buf[2:], x.prec)
75 oldPrec := z.prec
83 z.prec = binary.BigEndian.Uint32(buf[2:])
|
| floatconv.go | 34 prec := z.prec
35 if prec == 0 {
36 prec = 64
65 z.prec = prec
122 z.prec = prec
139 p := new(Float).SetPrec(z.Prec() + 64) // use more bits for p -- TODO(gri) what is the right number?
200 f := new(Float).SetPrec(z.Prec() + 64).SetUint64(5
[all...] |
| float.go | 60 prec uint32
97 // Float x are stored in a nat slice long enough to hold up to x.prec bits;
101 // trailing 0 bits or x.prec is not a multiple of the Word size _W,
153 // SetPrec sets z's precision to prec and returns the (possibly) rounded
155 // cannot be represented in prec bits without loss of precision.
157 // If prec > MaxPrec, it is set to MaxPrec.
158 func (z *Float) SetPrec(prec uint) *Float {
162 if prec == 0 {
163 z.prec = 0
173 if prec > MaxPrec
[all...] |
| bits_test.go | 106 // to prec bits according to mode.
107 func (x Bits) round(prec uint, mode RoundingMode) *Float {
121 if prec >= prec0 {
124 // prec < prec0
129 r := max - int(prec)
152 f.SetMode(ToZero).SetPrec(prec)
|
| example_test.go | 101 const prec = 200
108 // iteration, we need at least log_2(prec) steps.
109 steps := int(math.Log2(prec))
112 two := new(big.Float).SetPrec(prec).SetInt64(2)
113 half := new(big.Float).SetPrec(prec).SetFloat64(0.5)
116 x := new(big.Float).SetPrec(prec).SetInt64(1)
|
| /libcore/ojluni/src/main/java/sun/misc/ |
| FormattedFloatingDecimal.java | 120 private static int applyPrecision(int decExp, char[] digits, int nDigits, int prec) {
121 if (prec >= nDigits || prec < 0) {
125 if (prec == 0) {
137 int q = digits[prec];
139 int i = prec;
155 Arrays.fill(digits, prec, nDigits, '0');
|
| /external/pdfium/core/fxcodec/codec/ |
| fx_codec_jpx_opj.cpp | 60 int prec = img->comps[0].prec; local
62 if (prec > 30)
64 int offset = 1 << (prec - 1);
65 int upb = (1 << prec) - 1;
121 int prec = img->comps[0].prec; local
122 if (prec <= 0 || prec >= 32)
125 int offset = 1 << (prec - 1)
[all...] |
| /device/google/contexthub/firmware/lib/libm/ |
| kf_rem_pio2.c | 56 int __kernel_rem_pio2f(float *x, float *y, int e0, int nx, int prec, const __int32_t *ipio2)
58 int __kernel_rem_pio2f(x,y,e0,nx,prec,ipio2)
59 float x[], y[]; int e0,nx,prec; __int32_t ipio2[];
66 jk = init_jk[prec];
175 switch(prec) {
|
| /external/fdlibm/ |
| k_rem_pio2.c | 15 * __kernel_rem_pio2(x,y,e0,nx,prec,ipio2)
16 * double x[],y[]; int e0,nx,prec; int ipio2[];
64 * prec an integer indicating the precision:
164 int __kernel_rem_pio2(double *x, double *y, int e0, int nx, int prec, const int *ipio2)
166 int __kernel_rem_pio2(x,y,e0,nx,prec,ipio2)
167 double x[], y[]; int e0,nx,prec; int ipio2[];
174 jk = init_jk[prec];
282 switch(prec) {
|
| /libcore/ojluni/src/main/java/java/util/ |
| Formatter.java | [all...] |
| /external/python/cpython3/Modules/_decimal/libmpdec/ |
| io.c | 228 /* prec >= 1, clamp is 0 or 1 */
229 if (digits > (size_t)(ctx->prec-ctx->clamp))
244 if (digits > (size_t)(ctx->prec-ctx->clamp))
774 spec->prec = -1;
850 spec->prec = mpd_strtossize(cp, &cp, 10);
1268 mpd_ssize_t prec; local
[all...] |
| mpdecimal.c | 202 return ctx->emin - (ctx->prec - 1);
209 return ctx->emax - (ctx->prec - 1);
720 workctx->prec = ctx->prec;
759 _mpd_idiv_word(&len, &r, ctx->prec, MPD_RDIGITS);
767 result->digits = ctx->prec;
779 * Cut off the most significant digits so that the rest fits in ctx->prec.
788 if (result->len > 0 && result->digits > ctx->prec) {
789 _mpd_idiv_word(&len, &r, ctx->prec, MPD_RDIGITS);
809 * fits in ctx->prec - ctx->clamp. Cannot fail
815 mpd_ssize_t prec; local
4352 mpd_ssize_t prec; local
4909 mpd_ssize_t prec; local
5058 mpd_ssize_t prec; local
7708 mpd_ssize_t prec, ideal_exp; local
[all...] |
| /external/libcups/cups/ |
| debug.c | 341 prec; /* Number of characters of precision */ local
417 prec = va_arg(ap, int);
419 snprintf(tptr, sizeof(tformat) - (size_t)(tptr - tformat), "%d", prec);
424 prec = 0;
431 prec = prec * 10 + *format++ - '0';
|
| /bionic/libc/stdio/ |
| printf_common.h | 700 // string representation. If not -1, prec specifies the maximum number of
703 static char* wcsconv(wchar_t* wcsarg, int prec) {
711 if (prec < 0) {
720 if (prec < 128) {
721 nbytes = prec;
728 if (clen == 0 || clen == (size_t)-1 || nbytes + clen > (size_t)prec) break;
772 // string representation. ``prec'' specifies the maximum number of bytes
773 // to output. If ``prec'' is greater than or equal to zero, we can't assume
778 static wchar_t* mbsconv(char* mbsarg, int prec) {
786 if (prec >= 0)
[all...] |
