OpenGrok
Home
Sort by relevance
Sort by last modified time
Full Search
Definition
Symbol
File Path
History
|
|
Help
Searched
full:inexact
(Results
1 - 25
of
382
) sorted by null
1
2
3
4
5
6
7
8
9
10
11
>>
/prebuilts/python/darwin-x86/2.7.5/lib/python2.7/test/decimaltestdata/
powersqrt.decTest
26
-- x ** 0.5 (x>0) has no preferred exponent, and is
Inexact
31
-- Some exact subnormals become
inexact
and hence underflows
40
pwsx001 power 1 0.5 -> 1.00000000
Inexact
Rounded
42
pwsx003 power 1.00 0.5 -> 1.00000000
Inexact
Rounded
53
pwsx013 power 2 0.5 -> 1.41421356
Inexact
Rounded
55
pwsx015 power 2.00 0.5 -> 1.41421356
Inexact
Rounded
67
pwsx027 power 4.0 0.5 -> 2.00000000
Inexact
Rounded
68
pwsx028 power 4.00 0.5 -> 2.00000000
Inexact
Rounded
70
pwsx030 power +0.1 0.5 -> 0.316227766
Inexact
Rounded
72
pwsx032 power +0.01 0.5 -> 0.100000000
Inexact
Rounde
[
all
...]
exp.decTest
33
expx002 exp -10 -> 0.0000453999298
Inexact
Rounded
34
expx003 exp -1 -> 0.367879441
Inexact
Rounded
37
expx006 exp 1 -> 2.71828183
Inexact
Rounded
38
expx007 exp 0.693147181 -> 2.00000000
Inexact
Rounded
39
expx008 exp 10 -> 22026.4658
Inexact
Rounded
44
expx011 exp 0.1 -> 1.105171
Inexact
Rounded
45
expx012 exp 0.01 -> 1.010050
Inexact
Rounded
46
expx013 exp 0.001 -> 1.001001
Inexact
Rounded
47
expx014 exp 0.0001 -> 1.000100
Inexact
Rounded
48
expx015 exp 0.00001 -> 1.000010
Inexact
Rounde
[
all
...]
squareroot.decTest
42
sqtx013 squareroot 2 -> 1.41421356
Inexact
Rounded
44
sqtx015 squareroot 2.00 -> 1.41421356
Inexact
Rounded
59
sqtx030 squareroot +0.1 -> 0.316227766
Inexact
Rounded
63
sqtx034 squareroot +0.001 -> 0.0316227766
Inexact
Rounded
70
sqtx041 squareroot 1.1 -> 1.04880885
Inexact
Rounded
71
sqtx042 squareroot 1.10 -> 1.04880885
Inexact
Rounded
72
sqtx043 squareroot 1.100 -> 1.04880885
Inexact
Rounded
73
sqtx044 squareroot 1.110 -> 1.05356538
Inexact
Rounded
78
sqtx049 squareroot 9.9 -> 3.14642654
Inexact
Rounded
79
sqtx050 squareroot 9.90 -> 3.14642654
Inexact
Rounde
[
all
...]
log10.decTest
36
logxs003 log10 2 -> 0.301029996
Inexact
Rounded
38
logxs005 log10 70 -> 1.84509804
Inexact
Rounded
45
logx0001 log10 7E-1000 -> -999.1549019599857
Inexact
Rounded
46
logx0002 log10 1.1E-9 -> -8.958607314841775
Inexact
Rounded
47
logx0003 log10 0.0007 -> -3.154901959985743
Inexact
Rounded
48
logx0004 log10 0.11 -> -0.9586073148417750
Inexact
Rounded
49
logx0005 log10 0.7 -> -0.1549019599857432
Inexact
Rounded
51
logx0007 log10 1.5 -> 0.1760912590556812
Inexact
Rounded
52
logx0008 log10 2 -> 0.3010299956639812
Inexact
Rounded
53
logx0009 log10 2.718281828459045 -> 0.4342944819032518
Inexact
Rounde
[
all
...]
ln.decTest
32
lnxs003 ln 2.71828183 -> 1.00000000
Inexact
Rounded
33
lnxs004 ln 10 -> 2.30258509
Inexact
Rounded
40
lnx0002 ln 1E-9 -> -20.72326583694641
Inexact
Rounded
41
lnx0003 ln 0.0007 -> -7.264430222920869
Inexact
Rounded
42
lnx0004 ln 0.1 -> -2.302585092994046
Inexact
Rounded
43
lnx0005 ln 0.7 -> -0.3566749439387324
Inexact
Rounded
46
lnx0008 ln 1.5 -> 0.4054651081081644
Inexact
Rounded
47
lnx0009 ln 2 -> 0.6931471805599453
Inexact
Rounded
48
lnx0010 ln 2.718281828459045 -> 0.9999999999999999
Inexact
Rounded
49
lnx0011 ln 2.718281828459046 -> 1.000000000000000
Inexact
Rounde
[
all
...]
rounding.decTest
43
radx100 add 12345 -0.1 -> 12344
Inexact
Rounded
44
radx101 add 12345 -0.01 -> 12344
Inexact
Rounded
45
radx102 add 12345 -0.001 -> 12344
Inexact
Rounded
46
radx103 add 12345 -0.00001 -> 12344
Inexact
Rounded
47
radx104 add 12345 -0.000001 -> 12344
Inexact
Rounded
48
radx105 add 12345 -0.0000001 -> 12344
Inexact
Rounded
50
radx107 add 12345 0.0000001 -> 12345
Inexact
Rounded
51
radx108 add 12345 0.000001 -> 12345
Inexact
Rounded
52
radx109 add 12345 0.00001 -> 12345
Inexact
Rounded
53
radx110 add 12345 0.0001 -> 12345
Inexact
Rounde
[
all
...]
randomBound32.decTest
32
addx3001 add 4953734675913.065314738743322579 0218.932010396534371704930714860E+797 -> 2.189320103965343717049307148600E+799
Inexact
Rounded
34
divx3001 divide 4953734675913.065314738743322579 0218.932010396534371704930714860E+797 -> 2.262681764507965005284080800438E-787
Inexact
Rounded
36
mulx3001 multiply 4953734675913.065314738743322579 0218.932010396534371704930714860E+797 -> 1.084531091568672041923151632066E+812
Inexact
Rounded
37
powx3001 power 4953734675913.065314738743322579 2 -> 24539487239343522246155890.99495
Inexact
Rounded
39
subx3001 subtract 4953734675913.065314738743322579 0218.932010396534371704930714860E+797 -> -2.189320103965343717049307148600E+799
Inexact
Rounded
40
addx3002 add 9641.684323386955881595490347910E-844 -78864532047.12287484430980636798E+934 -> -7.886453204712287484430980636798E+944
Inexact
Rounded
42
divx3002 divide 9641.684323386955881595490347910E-844 -78864532047.12287484430980636798E+934 -> -1.222562801441069667849402782716E-1785
Inexact
Rounded
44
mulx3002 multiply 9641.684323386955881595490347910E-844 -78864532047.12287484430980636798E+934 -> -7.603869223099928141659831589905E+104
Inexact
Rounded
45
powx3002 power 9641.684323386955881595490347910E-844 -8 -> 1.338988152067180337738955757587E+6720
Inexact
Rounded
47
subx3002 subtract 9641.684323386955881595490347910E-844 -78864532047.12287484430980636798E+934 -> 7.886453204712287484430980636798E+944
Inexact
Rounde
[
all
...]
inexact.decTest
2
--
inexact
.decTest -- decimal
inexact
and rounded edge cases --
33
inx006 add 123456789 0.1 -> 123456789
Inexact
Rounded
34
inx007 add 123456789 0.01 -> 123456789
Inexact
Rounded
35
inx008 add 123456789 0.001 -> 123456789
Inexact
Rounded
36
inx009 add 123456789 0.000001 -> 123456789
Inexact
Rounded
37
inx010 add 123456789 0.000000001 -> 123456789
Inexact
Rounded
38
inx011 add 123456789 0.000000000001 -> 123456789
Inexact
Rounded
40
inx012 add 123456789 0.9 -> 123456790
Inexact
Rounded
41
inx013 add 123456789 0.09 -> 123456789
Inexact
Rounde
[
all
...]
power.decTest
60
powx027 power '2' '31' -> '2.14748365E+9'
Inexact
Rounded
62
powx028 power '2' '32' -> '4.29496730E+9'
Inexact
Rounded
128
powx093 power 101 5 -> 1.05101005E+10
Inexact
Rounded
129
powx094 power 101 6 -> 1.06152015E+12
Inexact
Rounded
130
powx095 power 101 7 -> 1.07213535E+14
Inexact
Rounded
134
powx100 power '3' '-1' -> 0.333333333
Inexact
Rounded
139
powx105 power '2' '-16' -> 0.0000152587891
Inexact
Rounded
140
powx106 power '2' '-32' -> 2.32830644E-10
Inexact
Rounded
141
powx108 power '2' '-64' -> 5.42101086E-20
Inexact
Rounded
190
powx219 power '123456789E+10' '-1.23000e+2' -> '5.54E-2226'
Inexact
Rounde
[
all
...]
randoms.decTest
30
xadd001 add 905.67402 -202896611.E-780472620 -> 905.674020
Inexact
Rounded
32
xdiv001 divide 905.67402 -202896611.E-780472620 -> -4.46372177E+780472614
Inexact
Rounded
34
xmul001 multiply 905.67402 -202896611.E-780472620 -> -1.83758189E-780472609
Inexact
Rounded
35
xpow001 power 905.67402 -2 -> 0.00000121914730
Inexact
Rounded
37
xsub001 subtract 905.67402 -202896611.E-780472620 -> 905.674020
Inexact
Rounded
38
xadd002 add 3915134.7 -597164907. -> -593249772
Inexact
Rounded
40
xdiv002 divide 3915134.7 -597164907. -> -0.00655620358
Inexact
Rounded
42
xmul002 multiply 3915134.7 -597164907. -> -2.33798105E+15
Inexact
Rounded
43
xpow002 power 3915134.7 -597164907 -> 0E-1000000007 Underflow Subnormal
Inexact
Rounded Clamped
45
xsub002 subtract 3915134.7 -597164907. -> 601080042
Inexact
Rounde
[
all
...]
ddToIntegral.decTest
28
-- ToIntegralExact (which does set
Inexact
) and the fixed-name
29
-- functions (which do not set
Inexact
).
32
-- which never sets
Inexact
; the decTest operator for that is called
44
ddintx003 tointegralx 0.1 -> 0
Inexact
Rounded
45
ddintx004 tointegralx 0.2 -> 0
Inexact
Rounded
46
ddintx005 tointegralx 0.3 -> 0
Inexact
Rounded
47
ddintx006 tointegralx 0.4 -> 0
Inexact
Rounded
48
ddintx007 tointegralx 0.5 -> 0
Inexact
Rounded
49
ddintx008 tointegralx 0.6 -> 1
Inexact
Rounded
50
ddintx009 tointegralx 0.7 -> 1
Inexact
Rounde
[
all
...]
dqToIntegral.decTest
28
-- ToIntegralExact (which does set
Inexact
) and the fixed-name
29
-- functions (which do not set
Inexact
).
32
-- which never sets
Inexact
; the decTest operator for that is called
44
dqintx003 tointegralx 0.1 -> 0
Inexact
Rounded
45
dqintx004 tointegralx 0.2 -> 0
Inexact
Rounded
46
dqintx005 tointegralx 0.3 -> 0
Inexact
Rounded
47
dqintx006 tointegralx 0.4 -> 0
Inexact
Rounded
48
dqintx007 tointegralx 0.5 -> 0
Inexact
Rounded
49
dqintx008 tointegralx 0.6 -> 1
Inexact
Rounded
50
dqintx009 tointegralx 0.7 -> 1
Inexact
Rounde
[
all
...]
tointegralx.decTest
27
-- This tests toIntegraExact, which may set
Inexact
37
intxx003 tointegralx 0.1 -> 0
Inexact
Rounded
38
intxx004 tointegralx 0.2 -> 0
Inexact
Rounded
39
intxx005 tointegralx 0.3 -> 0
Inexact
Rounded
40
intxx006 tointegralx 0.4 -> 0
Inexact
Rounded
41
intxx007 tointegralx 0.5 -> 1
Inexact
Rounded
42
intxx008 tointegralx 0.6 -> 1
Inexact
Rounded
43
intxx009 tointegralx 0.7 -> 1
Inexact
Rounded
44
intxx010 tointegralx 0.8 -> 1
Inexact
Rounded
45
intxx011 tointegralx 0.9 -> 1
Inexact
Rounde
[
all
...]
/prebuilts/python/linux-x86/2.7.5/lib/python2.7/test/decimaltestdata/
powersqrt.decTest
26
-- x ** 0.5 (x>0) has no preferred exponent, and is
Inexact
31
-- Some exact subnormals become
inexact
and hence underflows
40
pwsx001 power 1 0.5 -> 1.00000000
Inexact
Rounded
42
pwsx003 power 1.00 0.5 -> 1.00000000
Inexact
Rounded
53
pwsx013 power 2 0.5 -> 1.41421356
Inexact
Rounded
55
pwsx015 power 2.00 0.5 -> 1.41421356
Inexact
Rounded
67
pwsx027 power 4.0 0.5 -> 2.00000000
Inexact
Rounded
68
pwsx028 power 4.00 0.5 -> 2.00000000
Inexact
Rounded
70
pwsx030 power +0.1 0.5 -> 0.316227766
Inexact
Rounded
72
pwsx032 power +0.01 0.5 -> 0.100000000
Inexact
Rounde
[
all
...]
exp.decTest
33
expx002 exp -10 -> 0.0000453999298
Inexact
Rounded
34
expx003 exp -1 -> 0.367879441
Inexact
Rounded
37
expx006 exp 1 -> 2.71828183
Inexact
Rounded
38
expx007 exp 0.693147181 -> 2.00000000
Inexact
Rounded
39
expx008 exp 10 -> 22026.4658
Inexact
Rounded
44
expx011 exp 0.1 -> 1.105171
Inexact
Rounded
45
expx012 exp 0.01 -> 1.010050
Inexact
Rounded
46
expx013 exp 0.001 -> 1.001001
Inexact
Rounded
47
expx014 exp 0.0001 -> 1.000100
Inexact
Rounded
48
expx015 exp 0.00001 -> 1.000010
Inexact
Rounde
[
all
...]
squareroot.decTest
42
sqtx013 squareroot 2 -> 1.41421356
Inexact
Rounded
44
sqtx015 squareroot 2.00 -> 1.41421356
Inexact
Rounded
59
sqtx030 squareroot +0.1 -> 0.316227766
Inexact
Rounded
63
sqtx034 squareroot +0.001 -> 0.0316227766
Inexact
Rounded
70
sqtx041 squareroot 1.1 -> 1.04880885
Inexact
Rounded
71
sqtx042 squareroot 1.10 -> 1.04880885
Inexact
Rounded
72
sqtx043 squareroot 1.100 -> 1.04880885
Inexact
Rounded
73
sqtx044 squareroot 1.110 -> 1.05356538
Inexact
Rounded
78
sqtx049 squareroot 9.9 -> 3.14642654
Inexact
Rounded
79
sqtx050 squareroot 9.90 -> 3.14642654
Inexact
Rounde
[
all
...]
log10.decTest
36
logxs003 log10 2 -> 0.301029996
Inexact
Rounded
38
logxs005 log10 70 -> 1.84509804
Inexact
Rounded
45
logx0001 log10 7E-1000 -> -999.1549019599857
Inexact
Rounded
46
logx0002 log10 1.1E-9 -> -8.958607314841775
Inexact
Rounded
47
logx0003 log10 0.0007 -> -3.154901959985743
Inexact
Rounded
48
logx0004 log10 0.11 -> -0.9586073148417750
Inexact
Rounded
49
logx0005 log10 0.7 -> -0.1549019599857432
Inexact
Rounded
51
logx0007 log10 1.5 -> 0.1760912590556812
Inexact
Rounded
52
logx0008 log10 2 -> 0.3010299956639812
Inexact
Rounded
53
logx0009 log10 2.718281828459045 -> 0.4342944819032518
Inexact
Rounde
[
all
...]
ln.decTest
32
lnxs003 ln 2.71828183 -> 1.00000000
Inexact
Rounded
33
lnxs004 ln 10 -> 2.30258509
Inexact
Rounded
40
lnx0002 ln 1E-9 -> -20.72326583694641
Inexact
Rounded
41
lnx0003 ln 0.0007 -> -7.264430222920869
Inexact
Rounded
42
lnx0004 ln 0.1 -> -2.302585092994046
Inexact
Rounded
43
lnx0005 ln 0.7 -> -0.3566749439387324
Inexact
Rounded
46
lnx0008 ln 1.5 -> 0.4054651081081644
Inexact
Rounded
47
lnx0009 ln 2 -> 0.6931471805599453
Inexact
Rounded
48
lnx0010 ln 2.718281828459045 -> 0.9999999999999999
Inexact
Rounded
49
lnx0011 ln 2.718281828459046 -> 1.000000000000000
Inexact
Rounde
[
all
...]
rounding.decTest
43
radx100 add 12345 -0.1 -> 12344
Inexact
Rounded
44
radx101 add 12345 -0.01 -> 12344
Inexact
Rounded
45
radx102 add 12345 -0.001 -> 12344
Inexact
Rounded
46
radx103 add 12345 -0.00001 -> 12344
Inexact
Rounded
47
radx104 add 12345 -0.000001 -> 12344
Inexact
Rounded
48
radx105 add 12345 -0.0000001 -> 12344
Inexact
Rounded
50
radx107 add 12345 0.0000001 -> 12345
Inexact
Rounded
51
radx108 add 12345 0.000001 -> 12345
Inexact
Rounded
52
radx109 add 12345 0.00001 -> 12345
Inexact
Rounded
53
radx110 add 12345 0.0001 -> 12345
Inexact
Rounde
[
all
...]
randomBound32.decTest
32
addx3001 add 4953734675913.065314738743322579 0218.932010396534371704930714860E+797 -> 2.189320103965343717049307148600E+799
Inexact
Rounded
34
divx3001 divide 4953734675913.065314738743322579 0218.932010396534371704930714860E+797 -> 2.262681764507965005284080800438E-787
Inexact
Rounded
36
mulx3001 multiply 4953734675913.065314738743322579 0218.932010396534371704930714860E+797 -> 1.084531091568672041923151632066E+812
Inexact
Rounded
37
powx3001 power 4953734675913.065314738743322579 2 -> 24539487239343522246155890.99495
Inexact
Rounded
39
subx3001 subtract 4953734675913.065314738743322579 0218.932010396534371704930714860E+797 -> -2.189320103965343717049307148600E+799
Inexact
Rounded
40
addx3002 add 9641.684323386955881595490347910E-844 -78864532047.12287484430980636798E+934 -> -7.886453204712287484430980636798E+944
Inexact
Rounded
42
divx3002 divide 9641.684323386955881595490347910E-844 -78864532047.12287484430980636798E+934 -> -1.222562801441069667849402782716E-1785
Inexact
Rounded
44
mulx3002 multiply 9641.684323386955881595490347910E-844 -78864532047.12287484430980636798E+934 -> -7.603869223099928141659831589905E+104
Inexact
Rounded
45
powx3002 power 9641.684323386955881595490347910E-844 -8 -> 1.338988152067180337738955757587E+6720
Inexact
Rounded
47
subx3002 subtract 9641.684323386955881595490347910E-844 -78864532047.12287484430980636798E+934 -> 7.886453204712287484430980636798E+944
Inexact
Rounde
[
all
...]
inexact.decTest
2
--
inexact
.decTest -- decimal
inexact
and rounded edge cases --
33
inx006 add 123456789 0.1 -> 123456789
Inexact
Rounded
34
inx007 add 123456789 0.01 -> 123456789
Inexact
Rounded
35
inx008 add 123456789 0.001 -> 123456789
Inexact
Rounded
36
inx009 add 123456789 0.000001 -> 123456789
Inexact
Rounded
37
inx010 add 123456789 0.000000001 -> 123456789
Inexact
Rounded
38
inx011 add 123456789 0.000000000001 -> 123456789
Inexact
Rounded
40
inx012 add 123456789 0.9 -> 123456790
Inexact
Rounded
41
inx013 add 123456789 0.09 -> 123456789
Inexact
Rounde
[
all
...]
power.decTest
60
powx027 power '2' '31' -> '2.14748365E+9'
Inexact
Rounded
62
powx028 power '2' '32' -> '4.29496730E+9'
Inexact
Rounded
128
powx093 power 101 5 -> 1.05101005E+10
Inexact
Rounded
129
powx094 power 101 6 -> 1.06152015E+12
Inexact
Rounded
130
powx095 power 101 7 -> 1.07213535E+14
Inexact
Rounded
134
powx100 power '3' '-1' -> 0.333333333
Inexact
Rounded
139
powx105 power '2' '-16' -> 0.0000152587891
Inexact
Rounded
140
powx106 power '2' '-32' -> 2.32830644E-10
Inexact
Rounded
141
powx108 power '2' '-64' -> 5.42101086E-20
Inexact
Rounded
190
powx219 power '123456789E+10' '-1.23000e+2' -> '5.54E-2226'
Inexact
Rounde
[
all
...]
randoms.decTest
30
xadd001 add 905.67402 -202896611.E-780472620 -> 905.674020
Inexact
Rounded
32
xdiv001 divide 905.67402 -202896611.E-780472620 -> -4.46372177E+780472614
Inexact
Rounded
34
xmul001 multiply 905.67402 -202896611.E-780472620 -> -1.83758189E-780472609
Inexact
Rounded
35
xpow001 power 905.67402 -2 -> 0.00000121914730
Inexact
Rounded
37
xsub001 subtract 905.67402 -202896611.E-780472620 -> 905.674020
Inexact
Rounded
38
xadd002 add 3915134.7 -597164907. -> -593249772
Inexact
Rounded
40
xdiv002 divide 3915134.7 -597164907. -> -0.00655620358
Inexact
Rounded
42
xmul002 multiply 3915134.7 -597164907. -> -2.33798105E+15
Inexact
Rounded
43
xpow002 power 3915134.7 -597164907 -> 0E-1000000007 Underflow Subnormal
Inexact
Rounded Clamped
45
xsub002 subtract 3915134.7 -597164907. -> 601080042
Inexact
Rounde
[
all
...]
ddToIntegral.decTest
28
-- ToIntegralExact (which does set
Inexact
) and the fixed-name
29
-- functions (which do not set
Inexact
).
32
-- which never sets
Inexact
; the decTest operator for that is called
44
ddintx003 tointegralx 0.1 -> 0
Inexact
Rounded
45
ddintx004 tointegralx 0.2 -> 0
Inexact
Rounded
46
ddintx005 tointegralx 0.3 -> 0
Inexact
Rounded
47
ddintx006 tointegralx 0.4 -> 0
Inexact
Rounded
48
ddintx007 tointegralx 0.5 -> 0
Inexact
Rounded
49
ddintx008 tointegralx 0.6 -> 1
Inexact
Rounded
50
ddintx009 tointegralx 0.7 -> 1
Inexact
Rounde
[
all
...]
dqToIntegral.decTest
28
-- ToIntegralExact (which does set
Inexact
) and the fixed-name
29
-- functions (which do not set
Inexact
).
32
-- which never sets
Inexact
; the decTest operator for that is called
44
dqintx003 tointegralx 0.1 -> 0
Inexact
Rounded
45
dqintx004 tointegralx 0.2 -> 0
Inexact
Rounded
46
dqintx005 tointegralx 0.3 -> 0
Inexact
Rounded
47
dqintx006 tointegralx 0.4 -> 0
Inexact
Rounded
48
dqintx007 tointegralx 0.5 -> 0
Inexact
Rounded
49
dqintx008 tointegralx 0.6 -> 1
Inexact
Rounded
50
dqintx009 tointegralx 0.7 -> 1
Inexact
Rounde
[
all
...]
Completed in 1030 milliseconds
1
2
3
4
5
6
7
8
9
10
11
>>