1 #!/usr/bin/python 2 3 """Unit tests for the perf_uploader.py module. 4 5 """ 6 7 import json, unittest 8 9 import common 10 from autotest_lib.tko import models as tko_models 11 from autotest_lib.tko.perf_upload import perf_uploader 12 13 14 class test_aggregate_iterations(unittest.TestCase): 15 """Tests for the aggregate_iterations function.""" 16 17 _PERF_ITERATION_DATA = { 18 '1': [ 19 { 20 'description': 'metric1', 21 'value': 1, 22 'stddev': 0.0, 23 'units': 'units1', 24 'higher_is_better': True, 25 'graph': None 26 }, 27 { 28 'description': 'metric2', 29 'value': 10, 30 'stddev': 0.0, 31 'units': 'units2', 32 'higher_is_better': True, 33 'graph': 'graph1', 34 }, 35 { 36 'description': 'metric2', 37 'value': 100, 38 'stddev': 1.7, 39 'units': 'units3', 40 'higher_is_better': False, 41 'graph': 'graph2', 42 } 43 ], 44 '2': [ 45 { 46 'description': 'metric1', 47 'value': 2, 48 'stddev': 0.0, 49 'units': 'units1', 50 'higher_is_better': True, 51 'graph': None, 52 }, 53 { 54 'description': 'metric2', 55 'value': 20, 56 'stddev': 0.0, 57 'units': 'units2', 58 'higher_is_better': True, 59 'graph': 'graph1', 60 }, 61 { 62 'description': 'metric2', 63 'value': 200, 64 'stddev': 21.2, 65 'units': 'units3', 66 'higher_is_better': False, 67 'graph': 'graph2', 68 } 69 ], 70 } 71 72 73 def setUp(self): 74 """Sets up for each test case.""" 75 self._perf_values = [] 76 for iter_num, iter_data in self._PERF_ITERATION_DATA.iteritems(): 77 self._perf_values.append( 78 tko_models.perf_value_iteration(iter_num, iter_data)) 79 80 81 82 class test_json_config_file_sanity(unittest.TestCase): 83 """Sanity tests for the JSON-formatted presentation config file.""" 84 85 def test_parse_json(self): 86 """Verifies _parse_config_file function.""" 87 perf_uploader._parse_config_file( 88 perf_uploader._PRESENTATION_CONFIG_FILE) 89 90 91 def test_proper_json(self): 92 """Verifies the file can be parsed as proper JSON.""" 93 try: 94 with open(perf_uploader._PRESENTATION_CONFIG_FILE, 'r') as fp: 95 json.load(fp) 96 except: 97 self.fail('Presentation config file could not be parsed as JSON.') 98 99 100 def test_unique_test_names(self): 101 """Verifies that each test name appears only once in the JSON file.""" 102 json_obj = [] 103 try: 104 with open(perf_uploader._PRESENTATION_CONFIG_FILE, 'r') as fp: 105 json_obj = json.load(fp) 106 except: 107 self.fail('Presentation config file could not be parsed as JSON.') 108 109 name_set = set([x['autotest_name'] for x in json_obj]) 110 self.assertEqual(len(name_set), len(json_obj), 111 msg='Autotest names not unique in the JSON file.') 112 113 114 def test_required_master_name(self): 115 """Verifies that master name must be specified.""" 116 json_obj = [] 117 try: 118 with open(perf_uploader._PRESENTATION_CONFIG_FILE, 'r') as fp: 119 json_obj = json.load(fp) 120 except: 121 self.fail('Presentation config file could not be parsed as JSON.') 122 123 for entry in json_obj: 124 if not 'master_name' in entry: 125 self.fail('Missing master field for test %s.' % 126 entry['autotest_name']) 127 128 129 class test_gather_presentation_info(unittest.TestCase): 130 """Tests for the gather_presentation_info function.""" 131 132 _PRESENT_INFO = { 133 'test_name': { 134 'master_name': 'new_master_name', 135 'dashboard_test_name': 'new_test_name', 136 } 137 } 138 139 _PRESENT_INFO_MISSING_MASTER = { 140 'test_name': { 141 'dashboard_test_name': 'new_test_name', 142 } 143 } 144 145 _PRESENT_INFO_REGEX = { 146 'test_name.*': { 147 'master_name': 'new_master_name', 148 'dashboard_test_name': 'new_test_name', 149 } 150 } 151 152 def test_test_name_regex_specified(self): 153 """Verifies gathers presentation info for regex search correctly""" 154 for test_name in ['test_name.arm.7.1', 'test_name.x86.7.1']: 155 result = perf_uploader._gather_presentation_info( 156 self._PRESENT_INFO, 'test_name_P') 157 self.assertTrue( 158 all([key in result for key in 159 ['test_name', 'master_name']]), 160 msg='Unexpected keys in resulting dictionary: %s' % result) 161 self.assertEqual(result['master_name'], 'new_master_name', 162 msg='Unexpected "master_name" value: %s' % 163 result['master_name']) 164 self.assertEqual(result['test_name'], 'new_test_name', 165 msg='Unexpected "test_name" value: %s' % 166 result['test_name']) 167 168 def test_test_name_specified(self): 169 """Verifies gathers presentation info correctly.""" 170 result = perf_uploader._gather_presentation_info( 171 self._PRESENT_INFO, 'test_name') 172 self.assertTrue( 173 all([key in result for key in 174 ['test_name', 'master_name']]), 175 msg='Unexpected keys in resulting dictionary: %s' % result) 176 self.assertEqual(result['master_name'], 'new_master_name', 177 msg='Unexpected "master_name" value: %s' % 178 result['master_name']) 179 self.assertEqual(result['test_name'], 'new_test_name', 180 msg='Unexpected "test_name" value: %s' % 181 result['test_name']) 182 183 184 def test_test_name_not_specified(self): 185 """Verifies exception raised if test is not there.""" 186 self.assertRaises( 187 perf_uploader.PerfUploadingError, 188 perf_uploader._gather_presentation_info, 189 self._PRESENT_INFO, 'other_test_name') 190 191 192 def test_master_not_specified(self): 193 """Verifies exception raised if master is not there.""" 194 self.assertRaises( 195 perf_uploader.PerfUploadingError, 196 perf_uploader._gather_presentation_info, 197 self._PRESENT_INFO_MISSING_MASTER, 'test_name') 198 199 200 class test_get_id_from_version(unittest.TestCase): 201 """Tests for the _get_id_from_version function.""" 202 203 def test_correctly_formatted_versions(self): 204 """Verifies that the expected ID is returned when input is OK.""" 205 chrome_version = '27.0.1452.2' 206 cros_version = '27.3906.0.0' 207 # 1452.2 + 3906.0.0 208 # --> 01452 + 002 + 03906 + 000 + 00 209 # --> 14520020390600000 210 self.assertEqual( 211 14520020390600000, 212 perf_uploader._get_id_from_version( 213 chrome_version, cros_version)) 214 215 chrome_version = '25.10.1000.0' 216 cros_version = '25.1200.0.0' 217 # 1000.0 + 1200.0.0 218 # --> 01000 + 000 + 01200 + 000 + 00 219 # --> 10000000120000000 220 self.assertEqual( 221 10000000120000000, 222 perf_uploader._get_id_from_version( 223 chrome_version, cros_version)) 224 225 def test_returns_none_when_given_invalid_input(self): 226 """Checks the return value when invalid input is given.""" 227 chrome_version = '27.0' 228 cros_version = '27.3906.0.0' 229 self.assertIsNone(perf_uploader._get_id_from_version( 230 chrome_version, cros_version)) 231 232 233 class test_get_version_numbers(unittest.TestCase): 234 """Tests for the _get_version_numbers function.""" 235 236 def test_with_valid_versions(self): 237 """Checks the version numbers used when data is formatted as expected.""" 238 self.assertEqual( 239 ('34.5678.9.0', '34.5.678.9'), 240 perf_uploader._get_version_numbers( 241 { 242 'CHROME_VERSION': '34.5.678.9', 243 'CHROMEOS_RELEASE_VERSION': '5678.9.0', 244 })) 245 246 def test_with_missing_version_raises_error(self): 247 """Checks that an error is raised when a version is missing.""" 248 with self.assertRaises(perf_uploader.PerfUploadingError): 249 perf_uploader._get_version_numbers( 250 { 251 'CHROMEOS_RELEASE_VERSION': '5678.9.0', 252 }) 253 254 def test_with_unexpected_version_format_raises_error(self): 255 """Checks that an error is raised when there's a rN suffix.""" 256 with self.assertRaises(perf_uploader.PerfUploadingError): 257 perf_uploader._get_version_numbers( 258 { 259 'CHROME_VERSION': '34.5.678.9', 260 'CHROMEOS_RELEASE_VERSION': '5678.9.0r1', 261 }) 262 263 def test_with_valid_release_milestone(self): 264 """Checks the version numbers used when data is formatted as expected.""" 265 self.assertEqual( 266 ('54.5678.9.0', '34.5.678.9'), 267 perf_uploader._get_version_numbers( 268 { 269 'CHROME_VERSION': '34.5.678.9', 270 'CHROMEOS_RELEASE_VERSION': '5678.9.0', 271 'CHROMEOS_RELEASE_CHROME_MILESTONE': '54', 272 })) 273 274 275 class test_format_for_upload(unittest.TestCase): 276 """Tests for the format_for_upload function.""" 277 278 _PERF_DATA = { 279 "charts": { 280 "metric1": { 281 "summary": { 282 "improvement_direction": "down", 283 "type": "scalar", 284 "units": "msec", 285 "value": 2.7, 286 } 287 }, 288 "metric2": { 289 "summary": { 290 "improvement_direction": "up", 291 "type": "scalar", 292 "units": "frames_per_sec", 293 "value": 101.35, 294 } 295 } 296 }, 297 } 298 _PRESENT_INFO = { 299 'master_name': 'new_master_name', 300 'test_name': 'new_test_name', 301 } 302 303 def setUp(self): 304 self._perf_data = self._PERF_DATA 305 306 def _verify_result_string(self, actual_result, expected_result): 307 """Verifies a JSON string matches the expected result. 308 309 This function compares JSON objects rather than strings, because of 310 possible floating-point values that need to be compared using 311 assertAlmostEqual(). 312 313 @param actual_result: The candidate JSON string. 314 @param expected_result: The reference JSON string that the candidate 315 must match. 316 317 """ 318 actual = json.loads(actual_result) 319 expected = json.loads(expected_result) 320 321 def ordered(obj): 322 if isinstance(obj, dict): 323 return sorted((k, ordered(v)) for k, v in obj.items()) 324 if isinstance(obj, list): 325 return sorted(ordered(x) for x in obj) 326 else: 327 return obj 328 fail_msg = 'Unexpected result string: %s' % actual_result 329 self.assertEqual(ordered(expected), ordered(actual), msg=fail_msg) 330 331 332 def test_format_for_upload(self): 333 """Verifies format_for_upload generates correct json data.""" 334 result = perf_uploader._format_for_upload( 335 'platform', '25.1200.0.0', '25.10.1000.0', 'WINKY E2A-F2K-Q35', 336 'i7', 'test_machine', self._perf_data, self._PRESENT_INFO, 337 '52926644-username/hostname') 338 expected_result_string = ( 339 '{"versions": {' 340 '"cros_version": "25.1200.0.0",' 341 '"chrome_version": "25.10.1000.0"' 342 '},' 343 '"point_id": 10000000120000000,' 344 '"bot": "cros-platform-i7",' 345 '"chart_data": {' 346 '"charts": {' 347 '"metric2": {' 348 '"summary": {' 349 '"units": "frames_per_sec",' 350 '"type": "scalar",' 351 '"value": 101.35,' 352 '"improvement_direction": "up"' 353 '}' 354 '},' 355 '"metric1": {' 356 '"summary": {' 357 '"units": "msec",' 358 '"type": "scalar",' 359 '"value": 2.7,' 360 '"improvement_direction": "down"}' 361 '}' 362 '}' 363 '},' 364 '"master": "new_master_name",' 365 '"supplemental": {' 366 '"hardware_identifier": "WINKY E2A-F2K-Q35",' 367 '"jobname": "52926644-username/hostname",' 368 '"hardware_hostname": "test_machine",' 369 '"default_rev": "r_cros_version",' 370 '"variant_name": "i7"}' 371 '}') 372 self._verify_result_string(result['data'], expected_result_string) 373 374 375 if __name__ == '__main__': 376 unittest.main() 377
