1 # Copyright (C) 2010 Google Inc. All rights reserved. 2 # 3 # Redistribution and use in source and binary forms, with or without 4 # modification, are permitted provided that the following conditions are 5 # met: 6 # 7 # * Redistributions of source code must retain the above copyright 8 # notice, this list of conditions and the following disclaimer. 9 # * Redistributions in binary form must reproduce the above 10 # copyright notice, this list of conditions and the following disclaimer 11 # in the documentation and/or other materials provided with the 12 # distribution. 13 # * Neither the name of Google Inc. nor the names of its 14 # contributors may be used to endorse or promote products derived from 15 # this software without specific prior written permission. 16 # 17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 import json 30 import logging 31 import subprocess 32 import sys 33 import time 34 import urllib2 35 import xml.dom.minidom 36 37 from webkitpy.common.checkout.scm.detection import SCMDetector 38 from webkitpy.common.net.file_uploader import FileUploader 39 40 # FIXME: This is the left-overs from when we used to generate JSON here. 41 # What's still used by webkitpy is just a group of functions used by a 42 # hodge-podge of different classes. Those functions should be move to where they are 43 # used and this file should just go away entirely. 44 # 45 # Unfortunately, a big chunk of this file is used by 46 # chromium/src/build/android/pylib/utils/flakiness_dashboard_results_uploader.py 47 # so we can't delete it until that code is migrated over. 48 # See crbug.com/242206 49 50 _log = logging.getLogger(__name__) 51 52 _JSON_PREFIX = "ADD_RESULTS(" 53 _JSON_SUFFIX = ");" 54 55 56 def has_json_wrapper(string): 57 return string.startswith(_JSON_PREFIX) and string.endswith(_JSON_SUFFIX) 58 59 60 def strip_json_wrapper(json_content): 61 # FIXME: Kill this code once the server returns json instead of jsonp. 62 if has_json_wrapper(json_content): 63 return json_content[len(_JSON_PREFIX):len(json_content) - len(_JSON_SUFFIX)] 64 return json_content 65 66 67 def load_json(filesystem, file_path): 68 content = filesystem.read_text_file(file_path) 69 content = strip_json_wrapper(content) 70 return json.loads(content) 71 72 73 def write_json(filesystem, json_object, file_path, callback=None): 74 # Specify separators in order to get compact encoding. 75 json_string = json.dumps(json_object, separators=(',', ':')) 76 if callback: 77 json_string = callback + "(" + json_string + ");" 78 filesystem.write_text_file(file_path, json_string) 79 80 81 def convert_trie_to_flat_paths(trie, prefix=None): 82 """Converts the directory structure in the given trie to flat paths, prepending a prefix to each.""" 83 result = {} 84 for name, data in trie.iteritems(): 85 if prefix: 86 name = prefix + "/" + name 87 88 if len(data) and not "results" in data: 89 result.update(convert_trie_to_flat_paths(data, name)) 90 else: 91 result[name] = data 92 93 return result 94 95 96 def add_path_to_trie(path, value, trie): 97 """Inserts a single flat directory path and associated value into a directory trie structure.""" 98 if not "/" in path: 99 trie[path] = value 100 return 101 102 directory, slash, rest = path.partition("/") 103 if not directory in trie: 104 trie[directory] = {} 105 add_path_to_trie(rest, value, trie[directory]) 106 107 def test_timings_trie(port, individual_test_timings): 108 """Breaks a test name into chunks by directory and puts the test time as a value in the lowest part, e.g. 109 foo/bar/baz.html: 1ms 110 foo/bar/baz1.html: 3ms 111 112 becomes 113 foo: { 114 bar: { 115 baz.html: 1, 116 baz1.html: 3 117 } 118 } 119 """ 120 trie = {} 121 for test_result in individual_test_timings: 122 test = test_result.test_name 123 124 add_path_to_trie(test, int(1000 * test_result.test_run_time), trie) 125 126 return trie 127 128 # FIXME: We already have a TestResult class in test_results.py 129 class TestResult(object): 130 """A simple class that represents a single test result.""" 131 132 # Test modifier constants. 133 (NONE, FAILS, FLAKY, DISABLED) = range(4) 134 135 def __init__(self, test, failed=False, elapsed_time=0): 136 self.test_name = test 137 self.failed = failed 138 self.test_run_time = elapsed_time 139 140 test_name = test 141 try: 142 test_name = test.split('.')[1] 143 except IndexError: 144 _log.warn("Invalid test name: %s.", test) 145 pass 146 147 if test_name.startswith('FAILS_'): 148 self.modifier = self.FAILS 149 elif test_name.startswith('FLAKY_'): 150 self.modifier = self.FLAKY 151 elif test_name.startswith('DISABLED_'): 152 self.modifier = self.DISABLED 153 else: 154 self.modifier = self.NONE 155 156 def fixable(self): 157 return self.failed or self.modifier == self.DISABLED 158 159 160 class JSONResultsGeneratorBase(object): 161 """A JSON results generator for generic tests.""" 162 163 MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG = 750 164 # Min time (seconds) that will be added to the JSON. 165 MIN_TIME = 1 166 167 # Note that in non-chromium tests those chars are used to indicate 168 # test modifiers (FAILS, FLAKY, etc) but not actual test results. 169 PASS_RESULT = "P" 170 SKIP_RESULT = "X" 171 FAIL_RESULT = "F" 172 FLAKY_RESULT = "L" 173 NO_DATA_RESULT = "N" 174 175 MODIFIER_TO_CHAR = {TestResult.NONE: PASS_RESULT, 176 TestResult.DISABLED: SKIP_RESULT, 177 TestResult.FAILS: FAIL_RESULT, 178 TestResult.FLAKY: FLAKY_RESULT} 179 180 VERSION = 4 181 VERSION_KEY = "version" 182 RESULTS = "results" 183 TIMES = "times" 184 BUILD_NUMBERS = "buildNumbers" 185 TIME = "secondsSinceEpoch" 186 TESTS = "tests" 187 188 FIXABLE_COUNT = "fixableCount" 189 FIXABLE = "fixableCounts" 190 ALL_FIXABLE_COUNT = "allFixableCount" 191 192 RESULTS_FILENAME = "results.json" 193 TIMES_MS_FILENAME = "times_ms.json" 194 INCREMENTAL_RESULTS_FILENAME = "incremental_results.json" 195 196 URL_FOR_TEST_LIST_JSON = "http://%s/testfile?builder=%s&name=%s&testlistjson=1&testtype=%s&master=%s" 197 198 # FIXME: Remove generate_incremental_results once the reference to it in 199 # http://src.chromium.org/viewvc/chrome/trunk/tools/build/scripts/slave/gtest_slave_utils.py 200 # has been removed. 201 def __init__(self, port, builder_name, build_name, build_number, 202 results_file_base_path, builder_base_url, 203 test_results_map, svn_repositories=None, 204 test_results_server=None, 205 test_type="", 206 master_name="", 207 generate_incremental_results=None): 208 """Modifies the results.json file. Grabs it off the archive directory 209 if it is not found locally. 210 211 Args 212 port: port-specific wrapper 213 builder_name: the builder name (e.g. Webkit). 214 build_name: the build name (e.g. webkit-rel). 215 build_number: the build number. 216 results_file_base_path: Absolute path to the directory containing the 217 results json file. 218 builder_base_url: the URL where we have the archived test results. 219 If this is None no archived results will be retrieved. 220 test_results_map: A dictionary that maps test_name to TestResult. 221 svn_repositories: A (json_field_name, svn_path) pair for SVN 222 repositories that tests rely on. The SVN revision will be 223 included in the JSON with the given json_field_name. 224 test_results_server: server that hosts test results json. 225 test_type: test type string (e.g. 'layout-tests'). 226 master_name: the name of the buildbot master. 227 """ 228 self._port = port 229 self._filesystem = port._filesystem 230 self._executive = port._executive 231 self._builder_name = builder_name 232 self._build_name = build_name 233 self._build_number = build_number 234 self._builder_base_url = builder_base_url 235 self._results_directory = results_file_base_path 236 237 self._test_results_map = test_results_map 238 self._test_results = test_results_map.values() 239 240 self._svn_repositories = svn_repositories 241 if not self._svn_repositories: 242 self._svn_repositories = {} 243 244 self._test_results_server = test_results_server 245 self._test_type = test_type 246 self._master_name = master_name 247 248 self._archived_results = None 249 250 def generate_json_output(self): 251 json_object = self.get_json() 252 if json_object: 253 file_path = self._filesystem.join(self._results_directory, self.INCREMENTAL_RESULTS_FILENAME) 254 write_json(self._filesystem, json_object, file_path) 255 256 def generate_times_ms_file(self): 257 # FIXME: rename to generate_times_ms_file. This needs to be coordinated with 258 # changing the calls to this on the chromium build slaves. 259 times = test_timings_trie(self._port, self._test_results_map.values()) 260 file_path = self._filesystem.join(self._results_directory, self.TIMES_MS_FILENAME) 261 write_json(self._filesystem, times, file_path) 262 263 def get_json(self): 264 """Gets the results for the results.json file.""" 265 results_json = {} 266 267 if not results_json: 268 results_json, error = self._get_archived_json_results() 269 if error: 270 # If there was an error don't write a results.json 271 # file at all as it would lose all the information on the 272 # bot. 273 _log.error("Archive directory is inaccessible. Not " 274 "modifying or clobbering the results.json " 275 "file: " + str(error)) 276 return None 277 278 builder_name = self._builder_name 279 if results_json and builder_name not in results_json: 280 _log.debug("Builder name (%s) is not in the results.json file." 281 % builder_name) 282 283 self._convert_json_to_current_version(results_json) 284 285 if builder_name not in results_json: 286 results_json[builder_name] = ( 287 self._create_results_for_builder_json()) 288 289 results_for_builder = results_json[builder_name] 290 291 if builder_name: 292 self._insert_generic_metadata(results_for_builder) 293 294 self._insert_failure_summaries(results_for_builder) 295 296 # Update the all failing tests with result type and time. 297 tests = results_for_builder[self.TESTS] 298 all_failing_tests = self._get_failed_test_names() 299 all_failing_tests.update(convert_trie_to_flat_paths(tests)) 300 301 for test in all_failing_tests: 302 self._insert_test_time_and_result(test, tests) 303 304 return results_json 305 306 def set_archived_results(self, archived_results): 307 self._archived_results = archived_results 308 309 def upload_json_files(self, json_files): 310 """Uploads the given json_files to the test_results_server (if the 311 test_results_server is given).""" 312 if not self._test_results_server: 313 return 314 315 if not self._master_name: 316 _log.error("--test-results-server was set, but --master-name was not. Not uploading JSON files.") 317 return 318 319 _log.info("Uploading JSON files for builder: %s", self._builder_name) 320 attrs = [("builder", self._builder_name), 321 ("testtype", self._test_type), 322 ("master", self._master_name)] 323 324 files = [(file, self._filesystem.join(self._results_directory, file)) 325 for file in json_files] 326 327 url = "http://%s/testfile/upload" % self._test_results_server 328 # Set uploading timeout in case appengine server is having problems. 329 # 120 seconds are more than enough to upload test results. 330 uploader = FileUploader(url, 120) 331 try: 332 response = uploader.upload_as_multipart_form_data(self._filesystem, files, attrs) 333 if response: 334 if response.code == 200: 335 _log.info("JSON uploaded.") 336 else: 337 _log.debug("JSON upload failed, %d: '%s'" % (response.code, response.read())) 338 else: 339 _log.error("JSON upload failed; no response returned") 340 except Exception, err: 341 _log.error("Upload failed: %s" % err) 342 return 343 344 345 def _get_test_timing(self, test_name): 346 """Returns test timing data (elapsed time) in second 347 for the given test_name.""" 348 if test_name in self._test_results_map: 349 # Floor for now to get time in seconds. 350 return int(self._test_results_map[test_name].test_run_time) 351 return 0 352 353 def _get_failed_test_names(self): 354 """Returns a set of failed test names.""" 355 return set([r.test_name for r in self._test_results if r.failed]) 356 357 def _get_modifier_char(self, test_name): 358 """Returns a single char (e.g. SKIP_RESULT, FAIL_RESULT, 359 PASS_RESULT, NO_DATA_RESULT, etc) that indicates the test modifier 360 for the given test_name. 361 """ 362 if test_name not in self._test_results_map: 363 return self.__class__.NO_DATA_RESULT 364 365 test_result = self._test_results_map[test_name] 366 if test_result.modifier in self.MODIFIER_TO_CHAR.keys(): 367 return self.MODIFIER_TO_CHAR[test_result.modifier] 368 369 return self.__class__.PASS_RESULT 370 371 def _get_result_char(self, test_name): 372 """Returns a single char (e.g. SKIP_RESULT, FAIL_RESULT, 373 PASS_RESULT, NO_DATA_RESULT, etc) that indicates the test result 374 for the given test_name. 375 """ 376 if test_name not in self._test_results_map: 377 return self.__class__.NO_DATA_RESULT 378 379 test_result = self._test_results_map[test_name] 380 if test_result.modifier == TestResult.DISABLED: 381 return self.__class__.SKIP_RESULT 382 383 if test_result.failed: 384 return self.__class__.FAIL_RESULT 385 386 return self.__class__.PASS_RESULT 387 388 def _get_svn_revision(self, in_directory): 389 """Returns the svn revision for the given directory. 390 391 Args: 392 in_directory: The directory where svn is to be run. 393 """ 394 395 # FIXME: We initialize this here in order to engage the stupid windows hacks :). 396 # We can't reuse an existing scm object because the specific directories may 397 # be part of other checkouts. 398 self._port.host.initialize_scm() 399 scm = SCMDetector(self._filesystem, self._executive).detect_scm_system(in_directory) 400 if scm: 401 return scm.svn_revision(in_directory) 402 return "" 403 404 def _get_archived_json_results(self): 405 """Download JSON file that only contains test 406 name list from test-results server. This is for generating incremental 407 JSON so the file generated has info for tests that failed before but 408 pass or are skipped from current run. 409 410 Returns (archived_results, error) tuple where error is None if results 411 were successfully read. 412 """ 413 results_json = {} 414 old_results = None 415 error = None 416 417 if not self._test_results_server: 418 return {}, None 419 420 results_file_url = (self.URL_FOR_TEST_LIST_JSON % 421 (urllib2.quote(self._test_results_server), 422 urllib2.quote(self._builder_name), 423 self.RESULTS_FILENAME, 424 urllib2.quote(self._test_type), 425 urllib2.quote(self._master_name))) 426 427 try: 428 # FIXME: We should talk to the network via a Host object. 429 results_file = urllib2.urlopen(results_file_url) 430 info = results_file.info() 431 old_results = results_file.read() 432 except urllib2.HTTPError, http_error: 433 # A non-4xx status code means the bot is hosed for some reason 434 # and we can't grab the results.json file off of it. 435 if (http_error.code < 400 and http_error.code >= 500): 436 error = http_error 437 except urllib2.URLError, url_error: 438 error = url_error 439 440 if old_results: 441 # Strip the prefix and suffix so we can get the actual JSON object. 442 old_results = strip_json_wrapper(old_results) 443 444 try: 445 results_json = json.loads(old_results) 446 except: 447 _log.debug("results.json was not valid JSON. Clobbering.") 448 # The JSON file is not valid JSON. Just clobber the results. 449 results_json = {} 450 else: 451 _log.debug('Old JSON results do not exist. Starting fresh.') 452 results_json = {} 453 454 return results_json, error 455 456 def _insert_failure_summaries(self, results_for_builder): 457 """Inserts aggregate pass/failure statistics into the JSON. 458 This method reads self._test_results and generates 459 FIXABLE, FIXABLE_COUNT and ALL_FIXABLE_COUNT entries. 460 461 Args: 462 results_for_builder: Dictionary containing the test results for a 463 single builder. 464 """ 465 # Insert the number of tests that failed or skipped. 466 fixable_count = len([r for r in self._test_results if r.fixable()]) 467 self._insert_item_into_raw_list(results_for_builder, 468 fixable_count, self.FIXABLE_COUNT) 469 470 # Create a test modifiers (FAILS, FLAKY etc) summary dictionary. 471 entry = {} 472 for test_name in self._test_results_map.iterkeys(): 473 result_char = self._get_modifier_char(test_name) 474 entry[result_char] = entry.get(result_char, 0) + 1 475 476 # Insert the pass/skip/failure summary dictionary. 477 self._insert_item_into_raw_list(results_for_builder, entry, 478 self.FIXABLE) 479 480 # Insert the number of all the tests that are supposed to pass. 481 all_test_count = len(self._test_results) 482 self._insert_item_into_raw_list(results_for_builder, 483 all_test_count, self.ALL_FIXABLE_COUNT) 484 485 def _insert_item_into_raw_list(self, results_for_builder, item, key): 486 """Inserts the item into the list with the given key in the results for 487 this builder. Creates the list if no such list exists. 488 489 Args: 490 results_for_builder: Dictionary containing the test results for a 491 single builder. 492 item: Number or string to insert into the list. 493 key: Key in results_for_builder for the list to insert into. 494 """ 495 if key in results_for_builder: 496 raw_list = results_for_builder[key] 497 else: 498 raw_list = [] 499 500 raw_list.insert(0, item) 501 raw_list = raw_list[:self.MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG] 502 results_for_builder[key] = raw_list 503 504 def _insert_item_run_length_encoded(self, item, encoded_results): 505 """Inserts the item into the run-length encoded results. 506 507 Args: 508 item: String or number to insert. 509 encoded_results: run-length encoded results. An array of arrays, e.g. 510 [[3,'A'],[1,'Q']] encodes AAAQ. 511 """ 512 if len(encoded_results) and item == encoded_results[0][1]: 513 num_results = encoded_results[0][0] 514 if num_results <= self.MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG: 515 encoded_results[0][0] = num_results + 1 516 else: 517 # Use a list instead of a class for the run-length encoding since 518 # we want the serialized form to be concise. 519 encoded_results.insert(0, [1, item]) 520 521 def _insert_generic_metadata(self, results_for_builder): 522 """ Inserts generic metadata (such as version number, current time etc) 523 into the JSON. 524 525 Args: 526 results_for_builder: Dictionary containing the test results for 527 a single builder. 528 """ 529 self._insert_item_into_raw_list(results_for_builder, 530 self._build_number, self.BUILD_NUMBERS) 531 532 # Include SVN revisions for the given repositories. 533 for (name, path) in self._svn_repositories: 534 # Note: for JSON file's backward-compatibility we use 'chrome' rather 535 # than 'chromium' here. 536 lowercase_name = name.lower() 537 if lowercase_name == 'chromium': 538 lowercase_name = 'chrome' 539 self._insert_item_into_raw_list(results_for_builder, 540 self._get_svn_revision(path), 541 lowercase_name + 'Revision') 542 543 self._insert_item_into_raw_list(results_for_builder, 544 int(time.time()), 545 self.TIME) 546 547 def _insert_test_time_and_result(self, test_name, tests): 548 """ Insert a test item with its results to the given tests dictionary. 549 550 Args: 551 tests: Dictionary containing test result entries. 552 """ 553 554 result = self._get_result_char(test_name) 555 time = self._get_test_timing(test_name) 556 557 this_test = tests 558 for segment in test_name.split("/"): 559 if segment not in this_test: 560 this_test[segment] = {} 561 this_test = this_test[segment] 562 563 if not len(this_test): 564 self._populate_results_and_times_json(this_test) 565 566 if self.RESULTS in this_test: 567 self._insert_item_run_length_encoded(result, this_test[self.RESULTS]) 568 else: 569 this_test[self.RESULTS] = [[1, result]] 570 571 if self.TIMES in this_test: 572 self._insert_item_run_length_encoded(time, this_test[self.TIMES]) 573 else: 574 this_test[self.TIMES] = [[1, time]] 575 576 def _convert_json_to_current_version(self, results_json): 577 """If the JSON does not match the current version, converts it to the 578 current version and adds in the new version number. 579 """ 580 if self.VERSION_KEY in results_json: 581 archive_version = results_json[self.VERSION_KEY] 582 if archive_version == self.VERSION: 583 return 584 else: 585 archive_version = 3 586 587 # version 3->4 588 if archive_version == 3: 589 num_results = len(results_json.values()) 590 for builder, results in results_json.iteritems(): 591 self._convert_tests_to_trie(results) 592 593 results_json[self.VERSION_KEY] = self.VERSION 594 595 def _convert_tests_to_trie(self, results): 596 if not self.TESTS in results: 597 return 598 599 test_results = results[self.TESTS] 600 test_results_trie = {} 601 for test in test_results.iterkeys(): 602 single_test_result = test_results[test] 603 add_path_to_trie(test, single_test_result, test_results_trie) 604 605 results[self.TESTS] = test_results_trie 606 607 def _populate_results_and_times_json(self, results_and_times): 608 results_and_times[self.RESULTS] = [] 609 results_and_times[self.TIMES] = [] 610 return results_and_times 611 612 def _create_results_for_builder_json(self): 613 results_for_builder = {} 614 results_for_builder[self.TESTS] = {} 615 return results_for_builder 616 617 def _remove_items_over_max_number_of_builds(self, encoded_list): 618 """Removes items from the run-length encoded list after the final 619 item that exceeds the max number of builds to track. 620 621 Args: 622 encoded_results: run-length encoded results. An array of arrays, e.g. 623 [[3,'A'],[1,'Q']] encodes AAAQ. 624 """ 625 num_builds = 0 626 index = 0 627 for result in encoded_list: 628 num_builds = num_builds + result[0] 629 index = index + 1 630 if num_builds > self.MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG: 631 return encoded_list[:index] 632 return encoded_list 633 634 def _normalize_results_json(self, test, test_name, tests): 635 """ Prune tests where all runs pass or tests that no longer exist and 636 truncate all results to maxNumberOfBuilds. 637 638 Args: 639 test: ResultsAndTimes object for this test. 640 test_name: Name of the test. 641 tests: The JSON object with all the test results for this builder. 642 """ 643 test[self.RESULTS] = self._remove_items_over_max_number_of_builds( 644 test[self.RESULTS]) 645 test[self.TIMES] = self._remove_items_over_max_number_of_builds( 646 test[self.TIMES]) 647 648 is_all_pass = self._is_results_all_of_type(test[self.RESULTS], 649 self.PASS_RESULT) 650 is_all_no_data = self._is_results_all_of_type(test[self.RESULTS], 651 self.NO_DATA_RESULT) 652 max_time = max([time[1] for time in test[self.TIMES]]) 653 654 # Remove all passes/no-data from the results to reduce noise and 655 # filesize. If a test passes every run, but takes > MIN_TIME to run, 656 # don't throw away the data. 657 if is_all_no_data or (is_all_pass and max_time <= self.MIN_TIME): 658 del tests[test_name] 659 660 def _is_results_all_of_type(self, results, type): 661 """Returns whether all the results are of the given type 662 (e.g. all passes).""" 663 return len(results) == 1 and results[0][1] == type 664 665 666 # Left here not to break anything. 667 class JSONResultsGenerator(JSONResultsGeneratorBase): 668 pass 669
