1 #!/usr/bin/python 2 3 """Unit tests for the perf_uploader.py module. 4 5 """ 6 7 import json, unittest 8 9 import common 10 from autotest_lib.tko import models as tko_models 11 from autotest_lib.tko.perf_upload import perf_uploader 12 13 14 class test_aggregate_iterations(unittest.TestCase): 15 """Tests for the aggregate_iterations function.""" 16 17 _PERF_ITERATION_DATA = { 18 '1': [ 19 { 20 'description': 'metric1', 21 'value': 1, 22 'stddev': 0.0, 23 'units': 'units1', 24 'higher_is_better': True, 25 'graph': None 26 }, 27 { 28 'description': 'metric2', 29 'value': 10, 30 'stddev': 0.0, 31 'units': 'units2', 32 'higher_is_better': True, 33 'graph': 'graph1', 34 }, 35 { 36 'description': 'metric2', 37 'value': 100, 38 'stddev': 1.7, 39 'units': 'units3', 40 'higher_is_better': False, 41 'graph': 'graph2', 42 } 43 ], 44 '2': [ 45 { 46 'description': 'metric1', 47 'value': 2, 48 'stddev': 0.0, 49 'units': 'units1', 50 'higher_is_better': True, 51 'graph': None, 52 }, 53 { 54 'description': 'metric2', 55 'value': 20, 56 'stddev': 0.0, 57 'units': 'units2', 58 'higher_is_better': True, 59 'graph': 'graph1', 60 }, 61 { 62 'description': 'metric2', 63 'value': 200, 64 'stddev': 21.2, 65 'units': 'units3', 66 'higher_is_better': False, 67 'graph': 'graph2', 68 } 69 ], 70 } 71 72 73 def setUp(self): 74 """Sets up for each test case.""" 75 self._perf_values = [] 76 for iter_num, iter_data in self._PERF_ITERATION_DATA.iteritems(): 77 self._perf_values.append( 78 tko_models.perf_value_iteration(iter_num, iter_data)) 79 80 81 82 class test_json_config_file_sanity(unittest.TestCase): 83 """Sanity tests for the JSON-formatted presentation config file.""" 84 85 def test_parse_json(self): 86 """Verifies _parse_config_file function.""" 87 perf_uploader._parse_config_file( 88 perf_uploader._PRESENTATION_CONFIG_FILE) 89 90 91 def test_proper_json(self): 92 """Verifies the file can be parsed as proper JSON.""" 93 try: 94 with open(perf_uploader._PRESENTATION_CONFIG_FILE, 'r') as fp: 95 json.load(fp) 96 except: 97 self.fail('Presentation config file could not be parsed as JSON.') 98 99 100 def test_unique_test_names(self): 101 """Verifies that each test name appears only once in the JSON file.""" 102 json_obj = [] 103 try: 104 with open(perf_uploader._PRESENTATION_CONFIG_FILE, 'r') as fp: 105 json_obj = json.load(fp) 106 except: 107 self.fail('Presentation config file could not be parsed as JSON.') 108 109 name_set = set([x['autotest_name'] for x in json_obj]) 110 self.assertEqual(len(name_set), len(json_obj), 111 msg='Autotest names not unique in the JSON file.') 112 113 114 def test_required_master_name(self): 115 """Verifies that master name must be specified.""" 116 json_obj = [] 117 try: 118 with open(perf_uploader._PRESENTATION_CONFIG_FILE, 'r') as fp: 119 json_obj = json.load(fp) 120 except: 121 self.fail('Presentation config file could not be parsed as JSON.') 122 123 for entry in json_obj: 124 if not 'master_name' in entry: 125 self.fail('Missing master field for test %s.' % 126 entry['autotest_name']) 127 128 129 class test_gather_presentation_info(unittest.TestCase): 130 """Tests for the gather_presentation_info function.""" 131 132 _PRESENT_INFO = { 133 'test_name': { 134 'master_name': 'new_master_name', 135 'dashboard_test_name': 'new_test_name', 136 } 137 } 138 139 _PRESENT_INFO_MISSING_MASTER = { 140 'test_name': { 141 'dashboard_test_name': 'new_test_name', 142 } 143 } 144 145 146 def test_test_name_specified(self): 147 """Verifies gathers presentation info correctly.""" 148 result = perf_uploader._gather_presentation_info( 149 self._PRESENT_INFO, 'test_name') 150 self.assertTrue( 151 all([key in result for key in 152 ['test_name', 'master_name']]), 153 msg='Unexpected keys in resulting dictionary: %s' % result) 154 self.assertEqual(result['master_name'], 'new_master_name', 155 msg='Unexpected "master_name" value: %s' % 156 result['master_name']) 157 self.assertEqual(result['test_name'], 'new_test_name', 158 msg='Unexpected "test_name" value: %s' % 159 result['test_name']) 160 161 162 def test_test_name_not_specified(self): 163 """Verifies exception raised if test is not there.""" 164 self.assertRaises( 165 perf_uploader.PerfUploadingError, 166 perf_uploader._gather_presentation_info, 167 self._PRESENT_INFO, 'other_test_name') 168 169 170 def test_master_not_specified(self): 171 """Verifies exception raised if master is not there.""" 172 self.assertRaises( 173 perf_uploader.PerfUploadingError, 174 perf_uploader._gather_presentation_info, 175 self._PRESENT_INFO_MISSING_MASTER, 'test_name') 176 177 178 class test_get_id_from_version(unittest.TestCase): 179 """Tests for the _get_id_from_version function.""" 180 181 def test_correctly_formatted_versions(self): 182 """Verifies that the expected ID is returned when input is OK.""" 183 chrome_version = '27.0.1452.2' 184 cros_version = '27.3906.0.0' 185 # 1452.2 + 3906.0.0 186 # --> 01452 + 002 + 03906 + 000 + 00 187 # --> 14520020390600000 188 self.assertEqual( 189 14520020390600000, 190 perf_uploader._get_id_from_version( 191 chrome_version, cros_version)) 192 193 chrome_version = '25.10.1000.0' 194 cros_version = '25.1200.0.0' 195 # 1000.0 + 1200.0.0 196 # --> 01000 + 000 + 01200 + 000 + 00 197 # --> 10000000120000000 198 self.assertEqual( 199 10000000120000000, 200 perf_uploader._get_id_from_version( 201 chrome_version, cros_version)) 202 203 def test_returns_none_when_given_invalid_input(self): 204 """Checks the return value when invalid input is given.""" 205 chrome_version = '27.0' 206 cros_version = '27.3906.0.0' 207 self.assertIsNone(perf_uploader._get_id_from_version( 208 chrome_version, cros_version)) 209 210 211 class test_get_version_numbers(unittest.TestCase): 212 """Tests for the _get_version_numbers function.""" 213 214 def test_with_valid_versions(self): 215 """Checks the version numbers used when data is formatted as expected.""" 216 self.assertEqual( 217 ('34.5678.9.0', '34.5.678.9'), 218 perf_uploader._get_version_numbers( 219 { 220 'CHROME_VERSION': '34.5.678.9', 221 'CHROMEOS_RELEASE_VERSION': '5678.9.0', 222 })) 223 224 def test_with_missing_version_raises_error(self): 225 """Checks that an error is raised when a version is missing.""" 226 with self.assertRaises(perf_uploader.PerfUploadingError): 227 perf_uploader._get_version_numbers( 228 { 229 'CHROMEOS_RELEASE_VERSION': '5678.9.0', 230 }) 231 232 def test_with_unexpected_version_format_raises_error(self): 233 """Checks that an error is raised when there's a rN suffix.""" 234 with self.assertRaises(perf_uploader.PerfUploadingError): 235 perf_uploader._get_version_numbers( 236 { 237 'CHROME_VERSION': '34.5.678.9', 238 'CHROMEOS_RELEASE_VERSION': '5678.9.0r1', 239 }) 240 241 def test_with_valid_release_milestone(self): 242 """Checks the version numbers used when data is formatted as expected.""" 243 self.assertEqual( 244 ('54.5678.9.0', '34.5.678.9'), 245 perf_uploader._get_version_numbers( 246 { 247 'CHROME_VERSION': '34.5.678.9', 248 'CHROMEOS_RELEASE_VERSION': '5678.9.0', 249 'CHROMEOS_RELEASE_CHROME_MILESTONE': '54', 250 })) 251 252 253 class test_format_for_upload(unittest.TestCase): 254 """Tests for the format_for_upload function.""" 255 256 _PERF_DATA = { 257 "charts": { 258 "metric1": { 259 "summary": { 260 "improvement_direction": "down", 261 "type": "scalar", 262 "units": "msec", 263 "value": 2.7, 264 } 265 }, 266 "metric2": { 267 "summary": { 268 "improvement_direction": "up", 269 "type": "scalar", 270 "units": "frames_per_sec", 271 "value": 101.35, 272 } 273 } 274 }, 275 } 276 _PRESENT_INFO = { 277 'master_name': 'new_master_name', 278 'test_name': 'new_test_name', 279 } 280 281 def setUp(self): 282 self._perf_data = self._PERF_DATA 283 284 def _verify_result_string(self, actual_result, expected_result): 285 """Verifies a JSON string matches the expected result. 286 287 This function compares JSON objects rather than strings, because of 288 possible floating-point values that need to be compared using 289 assertAlmostEqual(). 290 291 @param actual_result: The candidate JSON string. 292 @param expected_result: The reference JSON string that the candidate 293 must match. 294 295 """ 296 actual = json.loads(actual_result) 297 expected = json.loads(expected_result) 298 299 def ordered(obj): 300 if isinstance(obj, dict): 301 return sorted((k, ordered(v)) for k, v in obj.items()) 302 if isinstance(obj, list): 303 return sorted(ordered(x) for x in obj) 304 else: 305 return obj 306 fail_msg = 'Unexpected result string: %s' % actual_result 307 self.assertEqual(ordered(expected), ordered(actual), msg=fail_msg) 308 309 310 def test_format_for_upload(self): 311 """Verifies format_for_upload generates correct json data.""" 312 result = perf_uploader._format_for_upload( 313 'platform', '25.1200.0.0', '25.10.1000.0', 'WINKY E2A-F2K-Q35', 314 'i7', 'test_machine', self._perf_data, self._PRESENT_INFO, 315 '52926644-username/hostname') 316 expected_result_string = ( 317 '{"versions": {' 318 '"cros_version": "25.1200.0.0",' 319 '"chrome_version": "25.10.1000.0"' 320 '},' 321 '"point_id": 10000000120000000,' 322 '"bot": "cros-platform-i7",' 323 '"chart_data": {' 324 '"charts": {' 325 '"metric2": {' 326 '"summary": {' 327 '"units": "frames_per_sec",' 328 '"type": "scalar",' 329 '"value": 101.35,' 330 '"improvement_direction": "up"' 331 '}' 332 '},' 333 '"metric1": {' 334 '"summary": {' 335 '"units": "msec",' 336 '"type": "scalar",' 337 '"value": 2.7,' 338 '"improvement_direction": "down"}' 339 '}' 340 '}' 341 '},' 342 '"master": "new_master_name",' 343 '"supplemental": {' 344 '"hardware_identifier": "WINKY E2A-F2K-Q35",' 345 '"jobname": "52926644-username/hostname",' 346 '"hardware_hostname": "test_machine",' 347 '"default_rev": "r_cros_version",' 348 '"variant_name": "i7"}' 349 '}') 350 self._verify_result_string(result['data'], expected_result_string) 351 352 353 if __name__ == '__main__': 354 unittest.main() 355
