1 # Copyright 2014 The Chromium Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 import math 6 import unittest 7 8 # Special import necessary because filename contains dash characters. 9 bisect_perf_module = __import__('bisect-perf-regression') 10 11 12 RESULTS_OUTPUT = """RESULT write_operations: write_operations= 23089 count 13 RESULT read_bytes_gpu: read_bytes_gpu= 35201 kb 14 RESULT write_bytes_gpu: write_bytes_gpu= 542 kb 15 RESULT telemetry_page_measurement_results: num_failed= 0 count 16 RESULT telemetry_page_measurement_results: num_errored= 0 count 17 *RESULT Total: Total_ref= %(value)s 18 """ 19 20 21 class BisectPerfRegressionTest(unittest.TestCase): 22 """Test case for top-level functions in the bisect-perf-regrssion module.""" 23 24 def setUp(self): 25 """Sets up the test environment before each test method.""" 26 pass 27 28 def tearDown(self): 29 """Cleans up the test environment after each test method.""" 30 pass 31 32 def testParseDEPSStringManually(self): 33 """Tests DEPS parsing.""" 34 bisect_options = bisect_perf_module.BisectOptions() 35 bisect_instance = bisect_perf_module.BisectPerformanceMetrics( 36 None, bisect_options) 37 38 deps_file_contents = """ 39 vars = { 40 'ffmpeg_hash': 41 '@ac4a9f31fe2610bd146857bbd55d7a260003a888', 42 'webkit_url': 43 'https://chromium.googlesource.com/chromium/blink.git', 44 'git_url': 45 'https://chromium.googlesource.com', 46 'webkit_rev': 47 '@e01ac0a267d1017288bc67fa3c366b10469d8a24', 48 'angle_revision': 49 '74697cf2064c0a2c0d7e1b1b28db439286766a05' 50 }""" 51 52 # Should only expect svn/git revisions to come through, and urls to be 53 # filtered out. 54 expected_vars_dict = { 55 'ffmpeg_hash': '@ac4a9f31fe2610bd146857bbd55d7a260003a888', 56 'webkit_rev': '@e01ac0a267d1017288bc67fa3c366b10469d8a24', 57 'angle_revision': '74697cf2064c0a2c0d7e1b1b28db439286766a05' 58 } 59 vars_dict = bisect_instance._ParseRevisionsFromDEPSFileManually( 60 deps_file_contents) 61 self.assertEqual(vars_dict, expected_vars_dict) 62 63 def testCalculateTruncatedMeanRaisesError(self): 64 """CalculateTrunctedMean raises an error when passed an empty list.""" 65 with self.assertRaises(TypeError): 66 bisect_perf_module.CalculateTruncatedMean([], 0) 67 68 def testCalculateMeanSingleNum(self): 69 """Tests the CalculateMean function with a single number.""" 70 self.assertEqual(3.0, bisect_perf_module.CalculateMean([3])) 71 72 def testCalculateMeanShortList(self): 73 """Tests the CalculateMean function with a short list.""" 74 self.assertEqual(0.5, bisect_perf_module.CalculateMean([-3, 0, 1, 4])) 75 76 def testCalculateMeanCompareAlternateImplementation(self): 77 """Tests CalculateMean by comparing against an alternate implementation.""" 78 def AlternateMeanFunction(values): 79 """Simple arithmetic mean function.""" 80 return sum(values) / float(len(values)) 81 test_values_lists = [[1], [5, 6.5, 1.2, 3], [-3, 0, 1, 4], 82 [-3, -1, 0.12, 0.752, 3.33, 8, 16, 32, 439]] 83 for values in test_values_lists: 84 self.assertEqual( 85 AlternateMeanFunction(values), 86 bisect_perf_module.CalculateMean(values)) 87 88 def testCalculateConfidence(self): 89 """Tests the confidence calculation.""" 90 bad_values = [[0, 1], [1, 2]] 91 good_values = [[6, 7], [7, 8]] 92 # Closest means are mean(1, 2) and mean(6, 7). 93 distance = 6.5 - 1.5 94 # Standard deviation of [n-1, n, n, n+1] is 0.8165. 95 stddev_sum = 0.8165 + 0.8165 96 # Expected confidence is an int in the range [0, 100]. 97 expected_confidence = min(100, int(100 * distance / float(stddev_sum))) 98 self.assertEqual( 99 expected_confidence, 100 bisect_perf_module.CalculateConfidence(bad_values, good_values)) 101 102 def testCalculateConfidence0(self): 103 """Tests the confidence calculation when it's expected to be 0.""" 104 bad_values = [[0, 1], [1, 2], [4, 5], [0, 2]] 105 good_values = [[4, 5], [6, 7], [7, 8]] 106 # Both groups have value lists with means of 4.5, which means distance 107 # between groups is zero, and thus confidence is zero. 108 self.assertEqual( 109 0, bisect_perf_module.CalculateConfidence(bad_values, good_values)) 110 111 def testCalculateConfidence100(self): 112 """Tests the confidence calculation when it's expected to be 100.""" 113 bad_values = [[1, 1], [1, 1]] 114 good_values = [[1.2, 1.2], [1.2, 1.2]] 115 # Standard deviation in both groups is zero, so confidence is 100. 116 self.assertEqual( 117 100, bisect_perf_module.CalculateConfidence(bad_values, good_values)) 118 119 def testCalculateRelativeChange(self): 120 """Tests the common cases for calculating relative change.""" 121 # The change is relative to the first value, regardless of which is bigger. 122 self.assertEqual(0.5, bisect_perf_module.CalculateRelativeChange(1.0, 1.5)) 123 self.assertEqual(0.5, bisect_perf_module.CalculateRelativeChange(2.0, 1.0)) 124 125 def testCalculateRelativeChangeFromZero(self): 126 """Tests what happens when relative change from zero is calculated.""" 127 # If the first number is zero, then the result is not a number. 128 self.assertEqual(0, bisect_perf_module.CalculateRelativeChange(0, 0)) 129 self.assertTrue( 130 math.isnan(bisect_perf_module.CalculateRelativeChange(0, 1))) 131 self.assertTrue( 132 math.isnan(bisect_perf_module.CalculateRelativeChange(0, -1))) 133 134 def testCalculateRelativeChangeWithNegatives(self): 135 """Tests that relative change given is always positive.""" 136 self.assertEqual(3.0, bisect_perf_module.CalculateRelativeChange(-1, 2)) 137 self.assertEqual(3.0, bisect_perf_module.CalculateRelativeChange(1, -2)) 138 self.assertEqual(1.0, bisect_perf_module.CalculateRelativeChange(-1, -2)) 139 140 def testTryParseResultValuesFromOutputWithSingleValue(self): 141 """Tests result pattern <*>RESULT <graph>: <trace>= <value>""" 142 bisect_options = bisect_perf_module.BisectOptions() 143 bisect_instance = bisect_perf_module.BisectPerformanceMetrics( 144 None, bisect_options) 145 metrics = ['Total', 'Total_ref'] 146 self.assertEqual( 147 [66.88], bisect_instance.TryParseResultValuesFromOutput( 148 metrics, RESULTS_OUTPUT % {'value': '66.88 kb'})) 149 self.assertEqual( 150 [66.88], bisect_instance.TryParseResultValuesFromOutput( 151 metrics, RESULTS_OUTPUT % {'value': '66.88kb'})) 152 self.assertEqual( 153 [66.88], bisect_instance.TryParseResultValuesFromOutput( 154 metrics, RESULTS_OUTPUT % {'value': ' 66.88 '})) 155 self.assertEqual( 156 [-66.88], bisect_instance.TryParseResultValuesFromOutput( 157 metrics, RESULTS_OUTPUT % {'value': ' -66.88 kb'})) 158 self.assertEqual( 159 [66], bisect_instance.TryParseResultValuesFromOutput( 160 metrics, RESULTS_OUTPUT % {'value': '66 kb'})) 161 self.assertEqual( 162 [.66], bisect_instance.TryParseResultValuesFromOutput( 163 metrics, RESULTS_OUTPUT % {'value': '.66 kb'})) 164 self.assertEqual( 165 [], bisect_instance.TryParseResultValuesFromOutput( 166 metrics, RESULTS_OUTPUT % {'value': '. kb'})) 167 self.assertEqual( 168 [], bisect_instance.TryParseResultValuesFromOutput( 169 metrics, RESULTS_OUTPUT % {'value': 'aaa kb'})) 170 171 def testTryParseResultValuesFromOutputWithMulitValue(self): 172 """Tests result pattern <*>RESULT <graph>: <trace>= [<value>,<value>, ..]""" 173 bisect_options = bisect_perf_module.BisectOptions() 174 bisect_instance = bisect_perf_module.BisectPerformanceMetrics( 175 None, bisect_options) 176 metrics = ['Total', 'Total_ref'] 177 self.assertEqual( 178 [66.88], bisect_instance.TryParseResultValuesFromOutput( 179 metrics, RESULTS_OUTPUT % {'value': '[66.88] kb'})) 180 self.assertEqual( 181 [66.88, 99.44], bisect_instance.TryParseResultValuesFromOutput( 182 metrics, RESULTS_OUTPUT % {'value': '[66.88, 99.44]kb'})) 183 self.assertEqual( 184 [66.88, 99.44], bisect_instance.TryParseResultValuesFromOutput( 185 metrics, RESULTS_OUTPUT % {'value': '[ 66.88, 99.44 ]'})) 186 self.assertEqual( 187 [-66.88, 99.44], bisect_instance.TryParseResultValuesFromOutput( 188 metrics, RESULTS_OUTPUT % {'value': '[-66.88,99.44] kb'})) 189 self.assertEqual( 190 [-66, 99], bisect_instance.TryParseResultValuesFromOutput( 191 metrics, RESULTS_OUTPUT % {'value': '[-66,99] kb'})) 192 self.assertEqual( 193 [-66, 99], bisect_instance.TryParseResultValuesFromOutput( 194 metrics, RESULTS_OUTPUT % {'value': '[-66,99,] kb'})) 195 self.assertEqual( 196 [.66, .99], bisect_instance.TryParseResultValuesFromOutput( 197 metrics, RESULTS_OUTPUT % {'value': '[.66,.99] kb'})) 198 self.assertEqual( 199 [], bisect_instance.TryParseResultValuesFromOutput( 200 metrics, RESULTS_OUTPUT % {'value': '[] kb'})) 201 self.assertEqual( 202 [], bisect_instance.TryParseResultValuesFromOutput( 203 metrics, RESULTS_OUTPUT % {'value': '[-66,abc] kb'})) 204 205 def testTryParseResultValuesFromOutputWithMeanStd(self): 206 """Tests result pattern <*>RESULT <graph>: <trace>= {<mean, std}""" 207 bisect_options = bisect_perf_module.BisectOptions() 208 bisect_instance = bisect_perf_module.BisectPerformanceMetrics( 209 None, bisect_options) 210 metrics = ['Total', 'Total_ref'] 211 self.assertEqual( 212 [33.22], bisect_instance.TryParseResultValuesFromOutput( 213 metrics, RESULTS_OUTPUT % {'value': '{33.22, 3.6} kb'})) 214 self.assertEqual( 215 [33.22], bisect_instance.TryParseResultValuesFromOutput( 216 metrics, RESULTS_OUTPUT % {'value': '{33.22,3.6}kb'})) 217 self.assertEqual( 218 [33.22], bisect_instance.TryParseResultValuesFromOutput( 219 metrics, RESULTS_OUTPUT % {'value': '{33.22,3.6} kb'})) 220 self.assertEqual( 221 [33.22], bisect_instance.TryParseResultValuesFromOutput( 222 metrics, RESULTS_OUTPUT % {'value': '{ 33.22,3.6 }kb'})) 223 self.assertEqual( 224 [-33.22], bisect_instance.TryParseResultValuesFromOutput( 225 metrics, RESULTS_OUTPUT % {'value': '{-33.22,3.6}kb'})) 226 self.assertEqual( 227 [22], bisect_instance.TryParseResultValuesFromOutput( 228 metrics, RESULTS_OUTPUT % {'value': '{22,6}kb'})) 229 self.assertEqual( 230 [.22], bisect_instance.TryParseResultValuesFromOutput( 231 metrics, RESULTS_OUTPUT % {'value': '{.22,6}kb'})) 232 self.assertEqual( 233 [], bisect_instance.TryParseResultValuesFromOutput( 234 metrics, RESULTS_OUTPUT % {'value': '{.22,6, 44}kb'})) 235 self.assertEqual( 236 [], bisect_instance.TryParseResultValuesFromOutput( 237 metrics, RESULTS_OUTPUT % {'value': '{}kb'})) 238 self.assertEqual( 239 [], bisect_instance.TryParseResultValuesFromOutput( 240 metrics, RESULTS_OUTPUT % {'value': '{XYZ}kb'})) 241 242 243 if __name__ == '__main__': 244 unittest.main() 245
