1 # Copyright (C) 2010 Google Inc. All rights reserved. 2 # 3 # Redistribution and use in source and binary forms, with or without 4 # modification, are permitted provided that the following conditions are 5 # met: 6 # 7 # * Redistributions of source code must retain the above copyright 8 # notice, this list of conditions and the following disclaimer. 9 # * Redistributions in binary form must reproduce the above 10 # copyright notice, this list of conditions and the following disclaimer 11 # in the documentation and/or other materials provided with the 12 # distribution. 13 # * Neither the name of Google Inc. nor the names of its 14 # contributors may be used to endorse or promote products derived from 15 # this software without specific prior written permission. 16 # 17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 import json 30 import logging 31 import re 32 import sys 33 import traceback 34 35 from testfile import TestFile 36 37 JSON_RESULTS_FILE = "results.json" 38 JSON_RESULTS_FILE_SMALL = "results-small.json" 39 JSON_RESULTS_PREFIX = "ADD_RESULTS(" 40 JSON_RESULTS_SUFFIX = ");" 41 42 JSON_RESULTS_MIN_TIME = 3 43 JSON_RESULTS_HIERARCHICAL_VERSION = 4 44 JSON_RESULTS_MAX_BUILDS = 500 45 JSON_RESULTS_MAX_BUILDS_SMALL = 100 46 47 ACTUAL_KEY = "actual" 48 BUG_KEY = "bugs" 49 BUILD_NUMBERS_KEY = "buildNumbers" 50 BUILDER_NAME_KEY = "builder_name" 51 EXPECTED_KEY = "expected" 52 FAILURE_MAP_KEY = "failure_map" 53 FAILURES_BY_TYPE_KEY = "num_failures_by_type" 54 FIXABLE_COUNTS_KEY = "fixableCounts" 55 RESULTS_KEY = "results" 56 TESTS_KEY = "tests" 57 TIME_KEY = "time" 58 TIMES_KEY = "times" 59 VERSIONS_KEY = "version" 60 61 AUDIO = "A" 62 CRASH = "C" 63 FAIL = "Q" 64 # This is only output by gtests. 65 FLAKY = "L" 66 IMAGE = "I" 67 IMAGE_PLUS_TEXT = "Z" 68 MISSING = "O" 69 NO_DATA = "N" 70 NOTRUN = "Y" 71 PASS = "P" 72 SKIP = "X" 73 TEXT = "F" 74 TIMEOUT = "T" 75 76 AUDIO_STRING = "AUDIO" 77 CRASH_STRING = "CRASH" 78 IMAGE_PLUS_TEXT_STRING = "IMAGE+TEXT" 79 IMAGE_STRING = "IMAGE" 80 FAIL_STRING = "FAIL" 81 FLAKY_STRING = "FLAKY" 82 MISSING_STRING = "MISSING" 83 NO_DATA_STRING = "NO DATA" 84 NOTRUN_STRING = "NOTRUN" 85 PASS_STRING = "PASS" 86 SKIP_STRING = "SKIP" 87 TEXT_STRING = "TEXT" 88 TIMEOUT_STRING = "TIMEOUT" 89 90 FAILURE_TO_CHAR = { 91 AUDIO_STRING: AUDIO, 92 CRASH_STRING: CRASH, 93 IMAGE_PLUS_TEXT_STRING: IMAGE_PLUS_TEXT, 94 IMAGE_STRING: IMAGE, 95 FLAKY_STRING: FLAKY, 96 FAIL_STRING: FAIL, 97 MISSING_STRING: MISSING, 98 NO_DATA_STRING: NO_DATA, 99 NOTRUN_STRING: NOTRUN, 100 PASS_STRING: PASS, 101 SKIP_STRING: SKIP, 102 TEXT_STRING: TEXT, 103 TIMEOUT_STRING: TIMEOUT, 104 } 105 106 # FIXME: Use dict comprehensions once we update the server to python 2.7. 107 CHAR_TO_FAILURE = dict((value, key) for key, value in FAILURE_TO_CHAR.items()) 108 109 def _is_directory(subtree): 110 return RESULTS_KEY not in subtree 111 112 113 class JsonResults(object): 114 @classmethod 115 def _strip_prefix_suffix(cls, data): 116 if data.startswith(JSON_RESULTS_PREFIX) and data.endswith(JSON_RESULTS_SUFFIX): 117 return data[len(JSON_RESULTS_PREFIX):len(data) - len(JSON_RESULTS_SUFFIX)] 118 return data 119 120 @classmethod 121 def _generate_file_data(cls, jsonObject, sort_keys=False): 122 return json.dumps(jsonObject, separators=(',', ':'), sort_keys=sort_keys) 123 124 @classmethod 125 def _load_json(cls, file_data): 126 json_results_str = cls._strip_prefix_suffix(file_data) 127 if not json_results_str: 128 logging.warning("No json results data.") 129 return None 130 131 try: 132 return json.loads(json_results_str) 133 except: 134 logging.debug(json_results_str) 135 logging.error("Failed to load json results: %s", traceback.print_exception(*sys.exc_info())) 136 return None 137 138 @classmethod 139 def _merge_json(cls, aggregated_json, incremental_json, num_runs): 140 # We have to delete expected entries because the incremental json may not have any 141 # entry for every test in the aggregated json. But, the incremental json will have 142 # all the correct expected entries for that run. 143 cls._delete_expected_entries(aggregated_json[TESTS_KEY]) 144 cls._merge_non_test_data(aggregated_json, incremental_json, num_runs) 145 incremental_tests = incremental_json[TESTS_KEY] 146 if incremental_tests: 147 aggregated_tests = aggregated_json[TESTS_KEY] 148 cls._merge_tests(aggregated_tests, incremental_tests, num_runs) 149 150 @classmethod 151 def _delete_expected_entries(cls, aggregated_json): 152 for key in aggregated_json: 153 item = aggregated_json[key] 154 if _is_directory(item): 155 cls._delete_expected_entries(item) 156 else: 157 if EXPECTED_KEY in item: 158 del item[EXPECTED_KEY] 159 if BUG_KEY in item: 160 del item[BUG_KEY] 161 162 @classmethod 163 def _merge_non_test_data(cls, aggregated_json, incremental_json, num_runs): 164 incremental_builds = incremental_json[BUILD_NUMBERS_KEY] 165 aggregated_builds = aggregated_json[BUILD_NUMBERS_KEY] 166 aggregated_build_number = int(aggregated_builds[0]) 167 168 # FIXME: It's no longer possible to have multiple runs worth of data in the incremental_json, 169 # So we can get rid of this for-loop and the associated index. 170 for index in reversed(range(len(incremental_builds))): 171 build_number = int(incremental_builds[index]) 172 logging.debug("Merging build %s, incremental json index: %d.", build_number, index) 173 174 # Merge this build into aggreagated results. 175 cls._merge_one_build(aggregated_json, incremental_json, index, num_runs) 176 177 @classmethod 178 def _merge_one_build(cls, aggregated_json, incremental_json, incremental_index, num_runs): 179 for key in incremental_json.keys(): 180 # Merge json results except "tests" properties (results, times etc). 181 # "tests" properties will be handled separately. 182 if key == TESTS_KEY or key == FAILURE_MAP_KEY: 183 continue 184 185 if key in aggregated_json: 186 if key == FAILURES_BY_TYPE_KEY: 187 cls._merge_one_build(aggregated_json[key], incremental_json[key], incremental_index, num_runs=num_runs) 188 else: 189 aggregated_json[key].insert(0, incremental_json[key][incremental_index]) 190 aggregated_json[key] = aggregated_json[key][:num_runs] 191 else: 192 aggregated_json[key] = incremental_json[key] 193 194 @classmethod 195 def _merge_tests(cls, aggregated_json, incremental_json, num_runs): 196 # FIXME: Some data got corrupted and has results/times at the directory level. 197 # Once the data is fixe, this should assert that the directory level does not have 198 # results or times and just return "RESULTS_KEY not in subtree". 199 if RESULTS_KEY in aggregated_json: 200 del aggregated_json[RESULTS_KEY] 201 if TIMES_KEY in aggregated_json: 202 del aggregated_json[TIMES_KEY] 203 204 all_tests = set(aggregated_json.iterkeys()) 205 if incremental_json: 206 all_tests |= set(incremental_json.iterkeys()) 207 208 for test_name in all_tests: 209 if test_name not in aggregated_json: 210 aggregated_json[test_name] = incremental_json[test_name] 211 continue 212 213 incremental_sub_result = incremental_json[test_name] if incremental_json and test_name in incremental_json else None 214 if _is_directory(aggregated_json[test_name]): 215 cls._merge_tests(aggregated_json[test_name], incremental_sub_result, num_runs) 216 continue 217 218 aggregated_test = aggregated_json[test_name] 219 220 if incremental_sub_result: 221 results = incremental_sub_result[RESULTS_KEY] 222 times = incremental_sub_result[TIMES_KEY] 223 if EXPECTED_KEY in incremental_sub_result and incremental_sub_result[EXPECTED_KEY] != PASS_STRING: 224 aggregated_test[EXPECTED_KEY] = incremental_sub_result[EXPECTED_KEY] 225 if BUG_KEY in incremental_sub_result: 226 aggregated_test[BUG_KEY] = incremental_sub_result[BUG_KEY] 227 else: 228 results = [[1, NO_DATA]] 229 times = [[1, 0]] 230 231 cls._insert_item_run_length_encoded(results, aggregated_test[RESULTS_KEY], num_runs) 232 cls._insert_item_run_length_encoded(times, aggregated_test[TIMES_KEY], num_runs) 233 234 @classmethod 235 def _insert_item_run_length_encoded(cls, incremental_item, aggregated_item, num_runs): 236 for item in incremental_item: 237 if len(aggregated_item) and item[1] == aggregated_item[0][1]: 238 aggregated_item[0][0] = min(aggregated_item[0][0] + item[0], num_runs) 239 else: 240 aggregated_item.insert(0, item) 241 242 @classmethod 243 def _normalize_results(cls, aggregated_json, num_runs, run_time_pruning_threshold): 244 names_to_delete = [] 245 for test_name in aggregated_json: 246 if _is_directory(aggregated_json[test_name]): 247 cls._normalize_results(aggregated_json[test_name], num_runs, run_time_pruning_threshold) 248 # If normalizing deletes all the children of this directory, also delete the directory. 249 if not aggregated_json[test_name]: 250 names_to_delete.append(test_name) 251 else: 252 leaf = aggregated_json[test_name] 253 leaf[RESULTS_KEY] = cls._remove_items_over_max_number_of_builds(leaf[RESULTS_KEY], num_runs) 254 leaf[TIMES_KEY] = cls._remove_items_over_max_number_of_builds(leaf[TIMES_KEY], num_runs) 255 if cls._should_delete_leaf(leaf, run_time_pruning_threshold): 256 names_to_delete.append(test_name) 257 258 for test_name in names_to_delete: 259 del aggregated_json[test_name] 260 261 @classmethod 262 def _should_delete_leaf(cls, leaf, run_time_pruning_threshold): 263 if leaf.get(EXPECTED_KEY, PASS_STRING) != PASS_STRING: 264 return False 265 266 if BUG_KEY in leaf: 267 return False 268 269 deletable_types = set((PASS, NO_DATA, NOTRUN)) 270 for result in leaf[RESULTS_KEY]: 271 if result[1] not in deletable_types: 272 return False 273 274 for time in leaf[TIMES_KEY]: 275 if time[1] >= run_time_pruning_threshold: 276 return False 277 278 return True 279 280 @classmethod 281 def _remove_items_over_max_number_of_builds(cls, encoded_list, num_runs): 282 num_builds = 0 283 index = 0 284 for result in encoded_list: 285 num_builds = num_builds + result[0] 286 index = index + 1 287 if num_builds >= num_runs: 288 return encoded_list[:index] 289 290 return encoded_list 291 292 @classmethod 293 def _convert_gtest_json_to_aggregate_results_format(cls, json): 294 # FIXME: Change gtests over to uploading the full results format like layout-tests 295 # so we don't have to do this normalizing. 296 # http://crbug.com/247192. 297 298 if FAILURES_BY_TYPE_KEY in json: 299 # This is already in the right format. 300 return 301 302 failures_by_type = {} 303 for fixableCount in json[FIXABLE_COUNTS_KEY]: 304 for failure_type, count in fixableCount.items(): 305 failure_string = CHAR_TO_FAILURE[failure_type] 306 if failure_string not in failures_by_type: 307 failures_by_type[failure_string] = [] 308 failures_by_type[failure_string].append(count) 309 json[FAILURES_BY_TYPE_KEY] = failures_by_type 310 311 @classmethod 312 def _check_json(cls, builder, json): 313 version = json[VERSIONS_KEY] 314 if version > JSON_RESULTS_HIERARCHICAL_VERSION: 315 return "Results JSON version '%s' is not supported." % version 316 317 if not builder in json: 318 return "Builder '%s' is not in json results." % builder 319 320 results_for_builder = json[builder] 321 if not BUILD_NUMBERS_KEY in results_for_builder: 322 return "Missing build number in json results." 323 324 cls._convert_gtest_json_to_aggregate_results_format(json[builder]) 325 326 # FIXME: Remove this once all the bots have cycled with this code. 327 # The failure map was moved from the top-level to being below the builder 328 # like everything else. 329 if FAILURE_MAP_KEY in json: 330 del json[FAILURE_MAP_KEY] 331 332 # FIXME: Remove this code once the gtests switch over to uploading the full_results.json format. 333 # Once the bots have cycled with this code, we can move this loop into _convert_gtest_json_to_aggregate_results_format. 334 KEYS_TO_DELETE = ["fixableCount", "fixableCounts", "allFixableCount"] 335 for key in KEYS_TO_DELETE: 336 if key in json[builder]: 337 del json[builder][key] 338 339 return "" 340 341 @classmethod 342 def _populate_tests_from_full_results(cls, full_results, new_results): 343 if EXPECTED_KEY in full_results: 344 expected = full_results[EXPECTED_KEY] 345 if expected != PASS_STRING and expected != NOTRUN_STRING: 346 new_results[EXPECTED_KEY] = expected 347 time = int(round(full_results[TIME_KEY])) if TIME_KEY in full_results else 0 348 new_results[TIMES_KEY] = [[1, time]] 349 350 actual_failures = full_results[ACTUAL_KEY] 351 # Treat unexpected skips like NOTRUNs to avoid exploding the results JSON files 352 # when a bot exits early (e.g. due to too many crashes/timeouts). 353 if expected != SKIP_STRING and actual_failures == SKIP_STRING: 354 expected = first_actual_failure = NOTRUN_STRING 355 elif expected == NOTRUN_STRING: 356 first_actual_failure = expected 357 else: 358 # FIXME: Include the retry result as well and find a nice way to display it in the flakiness dashboard. 359 first_actual_failure = actual_failures.split(' ')[0] 360 new_results[RESULTS_KEY] = [[1, FAILURE_TO_CHAR[first_actual_failure]]] 361 362 if BUG_KEY in full_results: 363 new_results[BUG_KEY] = full_results[BUG_KEY] 364 return 365 366 for key in full_results: 367 new_results[key] = {} 368 cls._populate_tests_from_full_results(full_results[key], new_results[key]) 369 370 @classmethod 371 def _convert_full_results_format_to_aggregate(cls, full_results_format): 372 num_total_tests = 0 373 num_failing_tests = 0 374 failures_by_type = full_results_format[FAILURES_BY_TYPE_KEY] 375 376 tests = {} 377 cls._populate_tests_from_full_results(full_results_format[TESTS_KEY], tests) 378 379 aggregate_results_format = { 380 VERSIONS_KEY: JSON_RESULTS_HIERARCHICAL_VERSION, 381 full_results_format[BUILDER_NAME_KEY]: { 382 # FIXME: Use dict comprehensions once we update the server to python 2.7. 383 FAILURES_BY_TYPE_KEY: dict((key, [value]) for key, value in failures_by_type.items()), 384 TESTS_KEY: tests, 385 # FIXME: Have all the consumers of this switch over to the full_results_format keys 386 # so we don't have to do this silly conversion. Or switch the full_results_format keys 387 # to be camel-case. 388 BUILD_NUMBERS_KEY: [full_results_format['build_number']], 389 'chromeRevision': [full_results_format['chromium_revision']], 390 'blinkRevision': [full_results_format['blink_revision']], 391 'secondsSinceEpoch': [full_results_format['seconds_since_epoch']], 392 } 393 } 394 return aggregate_results_format 395 396 @classmethod 397 def _get_incremental_json(cls, builder, incremental_string, is_full_results_format): 398 if not incremental_string: 399 return "No incremental JSON data to merge.", 403 400 401 logging.info("Loading incremental json.") 402 incremental_json = cls._load_json(incremental_string) 403 if not incremental_json: 404 return "Incremental JSON data is not valid JSON.", 403 405 406 if is_full_results_format: 407 logging.info("Converting full results format to aggregate.") 408 incremental_json = cls._convert_full_results_format_to_aggregate(incremental_json) 409 410 logging.info("Checking incremental json.") 411 check_json_error_string = cls._check_json(builder, incremental_json) 412 if check_json_error_string: 413 return check_json_error_string, 403 414 return incremental_json, 200 415 416 @classmethod 417 def _get_aggregated_json(cls, builder, aggregated_string): 418 logging.info("Loading existing aggregated json.") 419 aggregated_json = cls._load_json(aggregated_string) 420 if not aggregated_json: 421 return None, 200 422 423 logging.info("Checking existing aggregated json.") 424 check_json_error_string = cls._check_json(builder, aggregated_json) 425 if check_json_error_string: 426 return check_json_error_string, 500 427 428 return aggregated_json, 200 429 430 @classmethod 431 def merge(cls, builder, aggregated_string, incremental_json, num_runs, sort_keys=False): 432 aggregated_json, status_code = cls._get_aggregated_json(builder, aggregated_string) 433 if not aggregated_json: 434 aggregated_json = incremental_json 435 elif status_code != 200: 436 return aggregated_json, status_code 437 else: 438 if aggregated_json[builder][BUILD_NUMBERS_KEY][0] == incremental_json[builder][BUILD_NUMBERS_KEY][0]: 439 status_string = "Incremental JSON's build number %s is the latest build number in the aggregated JSON." % str(aggregated_json[builder][BUILD_NUMBERS_KEY][0]) 440 return status_string, 409 441 442 logging.info("Merging json results.") 443 try: 444 cls._merge_json(aggregated_json[builder], incremental_json[builder], num_runs) 445 except: 446 return "Failed to merge json results: %s", traceback.print_exception(*sys.exc_info()), 500 447 448 aggregated_json[VERSIONS_KEY] = JSON_RESULTS_HIERARCHICAL_VERSION 449 aggregated_json[builder][FAILURE_MAP_KEY] = CHAR_TO_FAILURE 450 451 is_debug_builder = re.search(r"(Debug|Dbg)", builder, re.I) 452 run_time_pruning_threshold = 3 * JSON_RESULTS_MIN_TIME if is_debug_builder else JSON_RESULTS_MIN_TIME 453 cls._normalize_results(aggregated_json[builder][TESTS_KEY], num_runs, run_time_pruning_threshold) 454 return cls._generate_file_data(aggregated_json, sort_keys), 200 455 456 @classmethod 457 def _get_file(cls, master, builder, test_type, filename): 458 files = TestFile.get_files(master, builder, test_type, filename) 459 if files: 460 return files[0] 461 462 file = TestFile() 463 file.master = master 464 file.builder = builder 465 file.test_type = test_type 466 file.name = filename 467 file.data = "" 468 return file 469 470 @classmethod 471 def update(cls, master, builder, test_type, incremental_string, is_full_results_format): 472 logging.info("Updating %s and %s." % (JSON_RESULTS_FILE_SMALL, JSON_RESULTS_FILE)) 473 small_file = cls._get_file(master, builder, test_type, JSON_RESULTS_FILE_SMALL) 474 large_file = cls._get_file(master, builder, test_type, JSON_RESULTS_FILE) 475 return cls.update_files(builder, incremental_string, small_file, large_file, is_full_results_format) 476 477 @classmethod 478 def update_files(cls, builder, incremental_string, small_file, large_file, is_full_results_format): 479 incremental_json, status_code = cls._get_incremental_json(builder, incremental_string, is_full_results_format) 480 if status_code != 200: 481 return incremental_json, status_code 482 483 status_string, status_code = cls.update_file(builder, small_file, incremental_json, JSON_RESULTS_MAX_BUILDS_SMALL) 484 if status_code != 200: 485 return status_string, status_code 486 487 return cls.update_file(builder, large_file, incremental_json, JSON_RESULTS_MAX_BUILDS) 488 489 @classmethod 490 def update_file(cls, builder, file, incremental_json, num_runs): 491 new_results, status_code = cls.merge(builder, file.data, incremental_json, num_runs) 492 if status_code != 200: 493 return new_results, status_code 494 return TestFile.save_file(file, new_results) 495 496 @classmethod 497 def _delete_results_and_times(cls, tests): 498 for key in tests.keys(): 499 if key in (RESULTS_KEY, TIMES_KEY): 500 del tests[key] 501 else: 502 cls._delete_results_and_times(tests[key]) 503 504 @classmethod 505 def get_test_list(cls, builder, json_file_data): 506 logging.debug("Loading test results json...") 507 json = cls._load_json(json_file_data) 508 if not json: 509 return None 510 511 logging.debug("Checking test results json...") 512 513 check_json_error_string = cls._check_json(builder, json) 514 if check_json_error_string: 515 return None 516 517 test_list_json = {} 518 tests = json[builder][TESTS_KEY] 519 cls._delete_results_and_times(tests) 520 test_list_json[builder] = {TESTS_KEY: tests} 521 return cls._generate_file_data(test_list_json) 522
