1 # Copyright 2015 The Chromium Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 import unittest 6 7 import mock 8 9 from dashboard import ttest 10 11 12 class TTestTest(unittest.TestCase): 13 """Tests for the t-test functions.""" 14 15 def setUp(self): 16 """Sets the t-table values for the tests below.""" 17 table_patch = mock.patch.object( 18 ttest, '_TABLE', 19 [ 20 (1, [0, 6.314, 12.71, 31.82, 63.66, 318.31]), 21 (2, [0, 2.920, 4.303, 6.965, 9.925, 22.327]), 22 (3, [0, 2.353, 3.182, 4.541, 5.841, 10.215]), 23 (4, [0, 2.132, 2.776, 3.747, 4.604, 7.173]), 24 (10, [0, 1.372, 1.812, 2.228, 2.764, 3.169]), 25 (100, [0, 1.290, 1.660, 1.984, 2.364, 2.626]), 26 ]) 27 table_patch.start() 28 self.addCleanup(table_patch.stop) 29 two_tail_patch = mock.patch.object( 30 ttest, '_TWO_TAIL', 31 [1, 0.2, 0.1, 0.05, 0.02, 0.01]) 32 two_tail_patch.start() 33 self.addCleanup(two_tail_patch.stop) 34 35 def testWelchsTTest(self): 36 """Tests the t value and degrees of freedom output of Welch's t-test.""" 37 # The t-value can be checked with scipy.stats.ttest_ind(equal_var=False). 38 # However the t-value output by scipy.stats.ttest_ind is -6.32455532034. 39 # This implementation produces slightly different results. 40 result = ttest.WelchsTTest([2, 3, 2, 3, 2, 3], [4, 5, 4, 5, 4, 5]) 41 self.assertAlmostEqual(10.0, result.df) 42 self.assertAlmostEqual(-6.325, result.t, delta=1.0) 43 44 def testWelchsTTest_EmptySample_RaisesError(self): 45 """An error should be raised when an empty sample is passed in.""" 46 with self.assertRaises(RuntimeError): 47 ttest.WelchsTTest([], []) 48 with self.assertRaises(RuntimeError): 49 ttest.WelchsTTest([], [1, 2, 3]) 50 with self.assertRaises(RuntimeError): 51 ttest.WelchsTTest([1, 2, 3], []) 52 53 def testTTest_EqualSamples_PValueIsOne(self): 54 """Checks that t = 0 and p = 1 when the samples are the same.""" 55 result = ttest.WelchsTTest([1, 2, 3], [1, 2, 3]) 56 self.assertEqual(0, result.t) 57 self.assertEqual(1, result.p) 58 59 def testTTest_VeryDifferentSamples_PValueIsLow(self): 60 """Checks that p is very low when the samples are clearly different.""" 61 result = ttest.WelchsTTest([100, 101, 100, 101, 100], 62 [1, 2, 1, 2, 1, 2, 1, 2]) 63 self.assertLessEqual(250, result.t) 64 self.assertLessEqual(0.01, result.p) 65 66 def testTTest_DifferentVariance(self): 67 """Verifies that higher variance -> higher p value.""" 68 result_low_var = ttest.WelchsTTest([2, 3, 2, 3], [4, 5, 4, 5]) 69 result_high_var = ttest.WelchsTTest([1, 4, 1, 4], [3, 6, 3, 6]) 70 self.assertLess(result_low_var.p, result_high_var.p) 71 72 def testTTest_DifferentSampleSize(self): 73 """Verifies that smaller sample size -> higher p value.""" 74 result_larger_sample = ttest.WelchsTTest([2, 3, 2, 3], [4, 5, 4, 5]) 75 result_smaller_sample = ttest.WelchsTTest([2, 3, 2, 3], [4, 5]) 76 self.assertLess(result_larger_sample.p, result_smaller_sample.p) 77 78 def testTTest_DifferentMeanDifference(self): 79 """Verifies that smaller difference between means -> higher p value.""" 80 result_far_means = ttest.WelchsTTest([2, 3, 2, 3], [5, 6, 5, 6]) 81 result_near_means = ttest.WelchsTTest([2, 3, 2, 3], [3, 4, 3, 4]) 82 self.assertLess(result_far_means.p, result_near_means.p) 83 84 def testTValue(self): 85 """Tests calculation of the t-value using Welch's formula.""" 86 # Results can be verified by directly plugging variables into Welch's 87 # equation (e.g. using a calculator or the Python interpreter). 88 stats1 = ttest.SampleStats(mean=0.299, var=0.05, size=150) 89 stats2 = ttest.SampleStats(mean=0.307, var=0.08, size=165) 90 # Note that a negative t-value is obtained when the first sample has a 91 # smaller mean than the second, otherwise a positive value is returned. 92 self.assertAlmostEqual(-0.27968236, ttest._TValue(stats1, stats2)) 93 self.assertAlmostEqual(0.27968236, ttest._TValue(stats2, stats1)) 94 95 def testTValue_ConstantSamples_ResultIsInfinity(self): 96 """If there is no variation, infinity is used as the t-statistic value.""" 97 stats = ttest.SampleStats(mean=1.0, var=0, size=10) 98 self.assertEqual(float('inf'), ttest._TValue(stats, stats)) 99 100 def testDegreesOfFreedom(self): 101 """Tests calculation of estimated degrees of freedom.""" 102 # The formula used to estimate degrees of freedom for independent-samples 103 # t-test is called the Welch-Satterthwaite equation. Note that since the 104 # Welch-Satterthwaite equation gives an estimate of degrees of freedom, 105 # the result is a floating-point number and not an integer. 106 stats1 = ttest.SampleStats(mean=0.299, var=0.05, size=150) 107 stats2 = ttest.SampleStats(mean=0.307, var=0.08, size=165) 108 self.assertAlmostEqual( 109 307.19879975, ttest._DegreesOfFreedom(stats1, stats2)) 110 111 def testDegreesOfFreedom_ZeroVariance_ResultIsOne(self): 112 """The lowest possible value is returned for df if variance is zero.""" 113 stats = ttest.SampleStats(mean=1.0, var=0, size=10) 114 self.assertEqual(1.0, ttest._DegreesOfFreedom(stats, stats)) 115 116 def testDegreesOfFreedom_SmallSample_RaisesError(self): 117 """Degrees of freedom can't be calculated if sample size is too small.""" 118 size_0 = ttest.SampleStats(mean=0, var=0, size=0) 119 size_1 = ttest.SampleStats(mean=1.0, var=0, size=1) 120 size_5 = ttest.SampleStats(mean=2.0, var=0.5, size=5) 121 122 # An error is raised if the size of one of the samples is too small. 123 with self.assertRaises(RuntimeError): 124 ttest._DegreesOfFreedom(size_0, size_5) 125 with self.assertRaises(RuntimeError): 126 ttest._DegreesOfFreedom(size_1, size_5) 127 with self.assertRaises(RuntimeError): 128 ttest._DegreesOfFreedom(size_5, size_0) 129 with self.assertRaises(RuntimeError): 130 ttest._DegreesOfFreedom(size_5, size_1) 131 132 # If both of the samples have a variance of 0, no error is raised. 133 self.assertEqual(1.0, ttest._DegreesOfFreedom(size_1, size_1)) 134 135 136 class LookupPValueTest(unittest.TestCase): 137 138 def setUp(self): 139 """Sets the t-table values for the tests below.""" 140 table_patch = mock.patch.object( 141 ttest, '_TABLE', 142 [ 143 (1, [0, 6.314, 12.71, 31.82, 63.66, 318.31]), 144 (2, [0, 2.920, 4.303, 6.965, 9.925, 22.327]), 145 (3, [0, 2.353, 3.182, 4.541, 5.841, 10.215]), 146 (4, [0, 2.132, 2.776, 3.747, 4.604, 7.173]), 147 (10, [0, 1.372, 1.812, 2.228, 2.764, 3.169]), 148 (100, [0, 1.290, 1.660, 1.984, 2.364, 2.626]), 149 ]) 150 table_patch.start() 151 self.addCleanup(table_patch.stop) 152 two_tail_patch = mock.patch.object( 153 ttest, '_TWO_TAIL', 154 [1, 0.2, 0.1, 0.05, 0.02, 0.01]) 155 two_tail_patch.start() 156 self.addCleanup(two_tail_patch.stop) 157 158 def testLookupPValue_ExactMatchInTable(self): 159 """Tests looking up an entry that is in the table.""" 160 self.assertEqual(0.1, ttest._LookupPValue(3.182, 3.0)) 161 self.assertEqual(0.1, ttest._LookupPValue(-3.182, 3.0)) 162 163 def testLookupPValue_TValueBetweenTwoValues_SmallerColumnIsUsed(self): 164 # The second column is used because 3.1 is below 4.303, 165 # so the next-lowest t-value, 2.920, is used. 166 self.assertEqual(0.2, ttest._LookupPValue(3.1, 2.0)) 167 self.assertEqual(0.2, ttest._LookupPValue(-3.1, 2.0)) 168 169 def testLookup_DFBetweenTwoValues_SmallerRowIsUsed(self): 170 self.assertEqual(0.05, ttest._LookupPValue(2.228, 45.0)) 171 self.assertEqual(0.05, ttest._LookupPValue(-2.228, 45.0)) 172 173 def testLookup_DFAndTValueBetweenTwoValues_SmallerRowAndColumnIsUsed(self): 174 self.assertEqual(0.1, ttest._LookupPValue(2.0, 45.0)) 175 self.assertEqual(0.1, ttest._LookupPValue(-2.0, 45.0)) 176 177 def testLookupPValue_LargeTValue_LastColumnIsUsed(self): 178 # The smallest possible p-value will be used when t is large. 179 self.assertEqual(0.01, ttest._LookupPValue(500.0, 1.0)) 180 self.assertEqual(0.01, ttest._LookupPValue(-500.0, 1.0)) 181 182 def testLookupPValue_ZeroTValue_FirstColumnIsUsed(self): 183 # The largest possible p-value will be used when t is zero. 184 self.assertEqual(1.0, ttest._LookupPValue(0.0, 1.0)) 185 self.assertEqual(1.0, ttest._LookupPValue(0.0, 2.0)) 186 187 def testLookupPValue_SmallTValue_FirstColumnIsUsed(self): 188 # The largest possible p-value will be used when t is almost zero. 189 self.assertEqual(1.0, ttest._LookupPValue(0.1, 2.0)) 190 self.assertEqual(1.0, ttest._LookupPValue(-0.1, 2.0)) 191 192 def testLookupPValue_LargeDegreesOfFreedom_LastRowIsUsed(self): 193 # The last row of the table should be used. 194 self.assertEqual(0.02, ttest._LookupPValue(2.365, 100.0)) 195 196 197 if __name__ == '__main__': 198 unittest.main() 199
