1 #!/usr/bin/env python 2 # Copyright (C) 2010 Google Inc. All rights reserved. 3 # 4 # Redistribution and use in source and binary forms, with or without 5 # modification, are permitted provided that the following conditions are 6 # met: 7 # 8 # * Redistributions of source code must retain the above copyright 9 # notice, this list of conditions and the following disclaimer. 10 # * Redistributions in binary form must reproduce the above 11 # copyright notice, this list of conditions and the following disclaimer 12 # in the documentation and/or other materials provided with the 13 # distribution. 14 # * Neither the name of Google Inc. nor the names of its 15 # contributors may be used to endorse or promote products derived from 16 # this software without specific prior written permission. 17 # 18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 30 """Run layout tests using the test_shell. 31 32 This is a port of the existing webkit test script run-webkit-tests. 33 34 The TestRunner class runs a series of tests (TestType interface) against a set 35 of test files. If a test file fails a TestType, it returns a list TestFailure 36 objects to the TestRunner. The TestRunner then aggregates the TestFailures to 37 create a final report. 38 39 This script reads several files, if they exist in the test_lists subdirectory 40 next to this script itself. Each should contain a list of paths to individual 41 tests or entire subdirectories of tests, relative to the outermost test 42 directory. Entire lines starting with '//' (comments) will be ignored. 43 44 For details of the files' contents and purposes, see test_lists/README. 45 """ 46 47 import errno 48 import glob 49 import logging 50 import math 51 import optparse 52 import os 53 import Queue 54 import random 55 import re 56 import shutil 57 import sys 58 import time 59 import traceback 60 61 import simplejson 62 63 from layout_package import test_expectations 64 from layout_package import json_layout_results_generator 65 from layout_package import metered_stream 66 from layout_package import test_failures 67 from layout_package import test_shell_thread 68 from layout_package import test_files 69 from test_types import fuzzy_image_diff 70 from test_types import image_diff 71 from test_types import test_type_base 72 from test_types import text_diff 73 74 import port 75 76 # Indicates that we want detailed progress updates in the output (prints 77 # directory-by-directory feedback). 78 LOG_DETAILED_PROGRESS = 'detailed-progress' 79 80 # Log any unexpected results while running (instead of just at the end). 81 LOG_UNEXPECTED = 'unexpected' 82 83 # Builder base URL where we have the archived test results. 84 BUILDER_BASE_URL = "http://build.chromium.org/buildbot/layout_test_results/" 85 86 TestExpectationsFile = test_expectations.TestExpectationsFile 87 88 89 class TestInfo: 90 """Groups information about a test for easy passing of data.""" 91 92 def __init__(self, port, filename, timeout): 93 """Generates the URI and stores the filename and timeout for this test. 94 Args: 95 filename: Full path to the test. 96 timeout: Timeout for running the test in TestShell. 97 """ 98 self.filename = filename 99 self.uri = port.filename_to_uri(filename) 100 self.timeout = timeout 101 expected_hash_file = port.expected_filename(filename, '.checksum') 102 try: 103 self.image_hash = open(expected_hash_file, "r").read() 104 except IOError, e: 105 if errno.ENOENT != e.errno: 106 raise 107 self.image_hash = None 108 109 110 class ResultSummary(object): 111 """A class for partitioning the test results we get into buckets. 112 113 This class is basically a glorified struct and it's private to this file 114 so we don't bother with any information hiding.""" 115 116 def __init__(self, expectations, test_files): 117 self.total = len(test_files) 118 self.remaining = self.total 119 self.expectations = expectations 120 self.expected = 0 121 self.unexpected = 0 122 self.tests_by_expectation = {} 123 self.tests_by_timeline = {} 124 self.results = {} 125 self.unexpected_results = {} 126 self.failures = {} 127 self.tests_by_expectation[test_expectations.SKIP] = set() 128 for expectation in TestExpectationsFile.EXPECTATIONS.values(): 129 self.tests_by_expectation[expectation] = set() 130 for timeline in TestExpectationsFile.TIMELINES.values(): 131 self.tests_by_timeline[timeline] = ( 132 expectations.get_tests_with_timeline(timeline)) 133 134 def add(self, test, failures, result, expected): 135 """Add a result into the appropriate bin. 136 137 Args: 138 test: test file name 139 failures: list of failure objects from test execution 140 result: result of test (PASS, IMAGE, etc.). 141 expected: whether the result was what we expected it to be. 142 """ 143 144 self.tests_by_expectation[result].add(test) 145 self.results[test] = result 146 self.remaining -= 1 147 if len(failures): 148 self.failures[test] = failures 149 if expected: 150 self.expected += 1 151 else: 152 self.unexpected_results[test] = result 153 self.unexpected += 1 154 155 156 class TestRunner: 157 """A class for managing running a series of tests on a series of layout 158 test files.""" 159 160 HTTP_SUBDIR = os.sep.join(['', 'http', '']) 161 WEBSOCKET_SUBDIR = os.sep.join(['', 'websocket', '']) 162 163 # The per-test timeout in milliseconds, if no --time-out-ms option was 164 # given to run_webkit_tests. This should correspond to the default timeout 165 # in test_shell.exe. 166 DEFAULT_TEST_TIMEOUT_MS = 6 * 1000 167 168 NUM_RETRY_ON_UNEXPECTED_FAILURE = 1 169 170 def __init__(self, port, options, meter): 171 """Initialize test runner data structures. 172 173 Args: 174 port: an object implementing port-specific 175 options: a dictionary of command line options 176 meter: a MeteredStream object to record updates to. 177 """ 178 self._port = port 179 self._options = options 180 self._meter = meter 181 182 # disable wss server. need to install pyOpenSSL on buildbots. 183 # self._websocket_secure_server = websocket_server.PyWebSocket( 184 # options.results_directory, use_tls=True, port=9323) 185 186 # a list of TestType objects 187 self._test_types = [] 188 189 # a set of test files, and the same tests as a list 190 self._test_files = set() 191 self._test_files_list = None 192 self._result_queue = Queue.Queue() 193 194 # These are used for --log detailed-progress to track status by 195 # directory. 196 self._current_dir = None 197 self._current_progress_str = "" 198 self._current_test_number = 0 199 200 def __del__(self): 201 logging.debug("flushing stdout") 202 sys.stdout.flush() 203 logging.debug("flushing stderr") 204 sys.stderr.flush() 205 logging.debug("stopping http server") 206 self._port.stop_http_server() 207 logging.debug("stopping websocket server") 208 self._port.stop_websocket_server() 209 210 def gather_file_paths(self, paths): 211 """Find all the files to test. 212 213 Args: 214 paths: a list of globs to use instead of the defaults.""" 215 self._test_files = test_files.gather_test_files(self._port, paths) 216 217 def parse_expectations(self, test_platform_name, is_debug_mode): 218 """Parse the expectations from the test_list files and return a data 219 structure holding them. Throws an error if the test_list files have 220 invalid syntax.""" 221 if self._options.lint_test_files: 222 test_files = None 223 else: 224 test_files = self._test_files 225 226 try: 227 expectations_str = self._port.test_expectations() 228 self._expectations = test_expectations.TestExpectations( 229 self._port, test_files, expectations_str, test_platform_name, 230 is_debug_mode, self._options.lint_test_files) 231 return self._expectations 232 except Exception, err: 233 if self._options.lint_test_files: 234 print str(err) 235 else: 236 raise err 237 238 def prepare_lists_and_print_output(self, write): 239 """Create appropriate subsets of test lists and returns a 240 ResultSummary object. Also prints expected test counts. 241 242 Args: 243 write: A callback to write info to (e.g., a LoggingWriter) or 244 sys.stdout.write. 245 """ 246 247 # Remove skipped - both fixable and ignored - files from the 248 # top-level list of files to test. 249 num_all_test_files = len(self._test_files) 250 write("Found: %d tests" % (len(self._test_files))) 251 skipped = set() 252 if num_all_test_files > 1 and not self._options.force: 253 skipped = self._expectations.get_tests_with_result_type( 254 test_expectations.SKIP) 255 self._test_files -= skipped 256 257 # Create a sorted list of test files so the subset chunk, 258 # if used, contains alphabetically consecutive tests. 259 self._test_files_list = list(self._test_files) 260 if self._options.randomize_order: 261 random.shuffle(self._test_files_list) 262 else: 263 self._test_files_list.sort() 264 265 # If the user specifies they just want to run a subset of the tests, 266 # just grab a subset of the non-skipped tests. 267 if self._options.run_chunk or self._options.run_part: 268 chunk_value = self._options.run_chunk or self._options.run_part 269 test_files = self._test_files_list 270 try: 271 (chunk_num, chunk_len) = chunk_value.split(":") 272 chunk_num = int(chunk_num) 273 assert(chunk_num >= 0) 274 test_size = int(chunk_len) 275 assert(test_size > 0) 276 except: 277 logging.critical("invalid chunk '%s'" % chunk_value) 278 sys.exit(1) 279 280 # Get the number of tests 281 num_tests = len(test_files) 282 283 # Get the start offset of the slice. 284 if self._options.run_chunk: 285 chunk_len = test_size 286 # In this case chunk_num can be really large. We need 287 # to make the slave fit in the current number of tests. 288 slice_start = (chunk_num * chunk_len) % num_tests 289 else: 290 # Validate the data. 291 assert(test_size <= num_tests) 292 assert(chunk_num <= test_size) 293 294 # To count the chunk_len, and make sure we don't skip 295 # some tests, we round to the next value that fits exactly 296 # all the parts. 297 rounded_tests = num_tests 298 if rounded_tests % test_size != 0: 299 rounded_tests = (num_tests + test_size - 300 (num_tests % test_size)) 301 302 chunk_len = rounded_tests / test_size 303 slice_start = chunk_len * (chunk_num - 1) 304 # It does not mind if we go over test_size. 305 306 # Get the end offset of the slice. 307 slice_end = min(num_tests, slice_start + chunk_len) 308 309 files = test_files[slice_start:slice_end] 310 311 tests_run_msg = 'Running: %d tests (chunk slice [%d:%d] of %d)' % ( 312 (slice_end - slice_start), slice_start, slice_end, num_tests) 313 write(tests_run_msg) 314 315 # If we reached the end and we don't have enough tests, we run some 316 # from the beginning. 317 if (self._options.run_chunk and 318 (slice_end - slice_start < chunk_len)): 319 extra = 1 + chunk_len - (slice_end - slice_start) 320 extra_msg = (' last chunk is partial, appending [0:%d]' % 321 extra) 322 write(extra_msg) 323 tests_run_msg += "\n" + extra_msg 324 files.extend(test_files[0:extra]) 325 tests_run_filename = os.path.join(self._options.results_directory, 326 "tests_run.txt") 327 tests_run_file = open(tests_run_filename, "w") 328 tests_run_file.write(tests_run_msg + "\n") 329 tests_run_file.close() 330 331 len_skip_chunk = int(len(files) * len(skipped) / 332 float(len(self._test_files))) 333 skip_chunk_list = list(skipped)[0:len_skip_chunk] 334 skip_chunk = set(skip_chunk_list) 335 336 # Update expectations so that the stats are calculated correctly. 337 # We need to pass a list that includes the right # of skipped files 338 # to ParseExpectations so that ResultSummary() will get the correct 339 # stats. So, we add in the subset of skipped files, and then 340 # subtract them back out. 341 self._test_files_list = files + skip_chunk_list 342 self._test_files = set(self._test_files_list) 343 344 self._expectations = self.parse_expectations( 345 self._port.test_platform_name(), 346 self._options.target == 'Debug') 347 348 self._test_files = set(files) 349 self._test_files_list = files 350 else: 351 skip_chunk = skipped 352 353 result_summary = ResultSummary(self._expectations, 354 self._test_files | skip_chunk) 355 self._print_expected_results_of_type(write, result_summary, 356 test_expectations.PASS, "passes") 357 self._print_expected_results_of_type(write, result_summary, 358 test_expectations.FAIL, "failures") 359 self._print_expected_results_of_type(write, result_summary, 360 test_expectations.FLAKY, "flaky") 361 self._print_expected_results_of_type(write, result_summary, 362 test_expectations.SKIP, "skipped") 363 364 365 if self._options.force: 366 write('Running all tests, including skips (--force)') 367 else: 368 # Note that we don't actually run the skipped tests (they were 369 # subtracted out of self._test_files, above), but we stub out the 370 # results here so the statistics can remain accurate. 371 for test in skip_chunk: 372 result_summary.add(test, [], test_expectations.SKIP, 373 expected=True) 374 write("") 375 376 return result_summary 377 378 def add_test_type(self, test_type): 379 """Add a TestType to the TestRunner.""" 380 self._test_types.append(test_type) 381 382 def _get_dir_for_test_file(self, test_file): 383 """Returns the highest-level directory by which to shard the given 384 test file.""" 385 index = test_file.rfind(os.sep + 'LayoutTests' + os.sep) 386 387 test_file = test_file[index + len('LayoutTests/'):] 388 test_file_parts = test_file.split(os.sep, 1) 389 directory = test_file_parts[0] 390 test_file = test_file_parts[1] 391 392 # The http tests are very stable on mac/linux. 393 # TODO(ojan): Make the http server on Windows be apache so we can 394 # turn shard the http tests there as well. Switching to apache is 395 # what made them stable on linux/mac. 396 return_value = directory 397 while ((directory != 'http' or sys.platform in ('darwin', 'linux2')) 398 and test_file.find(os.sep) >= 0): 399 test_file_parts = test_file.split(os.sep, 1) 400 directory = test_file_parts[0] 401 return_value = os.path.join(return_value, directory) 402 test_file = test_file_parts[1] 403 404 return return_value 405 406 def _get_test_info_for_file(self, test_file): 407 """Returns the appropriate TestInfo object for the file. Mostly this 408 is used for looking up the timeout value (in ms) to use for the given 409 test.""" 410 if self._expectations.has_modifier(test_file, test_expectations.SLOW): 411 return TestInfo(self._port, test_file, 412 self._options.slow_time_out_ms) 413 return TestInfo(self._port, test_file, self._options.time_out_ms) 414 415 def _get_test_file_queue(self, test_files): 416 """Create the thread safe queue of lists of (test filenames, test URIs) 417 tuples. Each TestShellThread pulls a list from this queue and runs 418 those tests in order before grabbing the next available list. 419 420 Shard the lists by directory. This helps ensure that tests that depend 421 on each other (aka bad tests!) continue to run together as most 422 cross-tests dependencies tend to occur within the same directory. 423 424 Return: 425 The Queue of lists of TestInfo objects. 426 """ 427 428 if (self._options.experimental_fully_parallel or 429 self._is_single_threaded()): 430 filename_queue = Queue.Queue() 431 for test_file in test_files: 432 filename_queue.put( 433 ('.', [self._get_test_info_for_file(test_file)])) 434 return filename_queue 435 436 tests_by_dir = {} 437 for test_file in test_files: 438 directory = self._get_dir_for_test_file(test_file) 439 tests_by_dir.setdefault(directory, []) 440 tests_by_dir[directory].append( 441 self._get_test_info_for_file(test_file)) 442 443 # Sort by the number of tests in the dir so that the ones with the 444 # most tests get run first in order to maximize parallelization. 445 # Number of tests is a good enough, but not perfect, approximation 446 # of how long that set of tests will take to run. We can't just use 447 # a PriorityQueue until we move # to Python 2.6. 448 test_lists = [] 449 http_tests = None 450 for directory in tests_by_dir: 451 test_list = tests_by_dir[directory] 452 # Keep the tests in alphabetical order. 453 # TODO: Remove once tests are fixed so they can be run in any 454 # order. 455 test_list.reverse() 456 test_list_tuple = (directory, test_list) 457 if directory == 'LayoutTests' + os.sep + 'http': 458 http_tests = test_list_tuple 459 else: 460 test_lists.append(test_list_tuple) 461 test_lists.sort(lambda a, b: cmp(len(b[1]), len(a[1]))) 462 463 # Put the http tests first. There are only a couple hundred of them, 464 # but each http test takes a very long time to run, so sorting by the 465 # number of tests doesn't accurately capture how long they take to run. 466 if http_tests: 467 test_lists.insert(0, http_tests) 468 469 filename_queue = Queue.Queue() 470 for item in test_lists: 471 filename_queue.put(item) 472 return filename_queue 473 474 def _get_test_shell_args(self, index): 475 """Returns the tuple of arguments for tests and for test_shell.""" 476 shell_args = [] 477 test_args = test_type_base.TestArguments() 478 png_path = None 479 if not self._options.no_pixel_tests: 480 png_path = os.path.join(self._options.results_directory, 481 "png_result%s.png" % index) 482 shell_args.append("--pixel-tests=" + png_path) 483 test_args.png_path = png_path 484 485 test_args.new_baseline = self._options.new_baseline 486 487 test_args.show_sources = self._options.sources 488 489 if self._options.startup_dialog: 490 shell_args.append('--testshell-startup-dialog') 491 492 if self._options.gp_fault_error_box: 493 shell_args.append('--gp-fault-error-box') 494 495 return test_args, png_path, shell_args 496 497 def _contains_tests(self, subdir): 498 for test_file in self._test_files_list: 499 if test_file.find(subdir) >= 0: 500 return True 501 return False 502 503 def _instantiate_test_shell_threads(self, test_files, result_summary): 504 """Instantitates and starts the TestShellThread(s). 505 506 Return: 507 The list of threads. 508 """ 509 filename_queue = self._get_test_file_queue(test_files) 510 511 # Instantiate TestShellThreads and start them. 512 threads = [] 513 for i in xrange(int(self._options.num_test_shells)): 514 # Create separate TestTypes instances for each thread. 515 test_types = [] 516 for t in self._test_types: 517 test_types.append(t(self._port, self._options.platform, 518 self._options.results_directory)) 519 520 test_args, png_path, shell_args = self._get_test_shell_args(i) 521 thread = test_shell_thread.TestShellThread(self._port, 522 filename_queue, 523 self._result_queue, 524 test_types, 525 test_args, 526 png_path, 527 shell_args, 528 self._options) 529 if self._is_single_threaded(): 530 thread.run_in_main_thread(self, result_summary) 531 else: 532 thread.start() 533 threads.append(thread) 534 535 return threads 536 537 def _is_single_threaded(self): 538 """Returns whether we should run all the tests in the main thread.""" 539 return int(self._options.num_test_shells) == 1 540 541 def _run_tests(self, file_list, result_summary): 542 """Runs the tests in the file_list. 543 544 Return: A tuple (failures, thread_timings, test_timings, 545 individual_test_timings) 546 failures is a map from test to list of failure types 547 thread_timings is a list of dicts with the total runtime 548 of each thread with 'name', 'num_tests', 'total_time' properties 549 test_timings is a list of timings for each sharded subdirectory 550 of the form [time, directory_name, num_tests] 551 individual_test_timings is a list of run times for each test 552 in the form {filename:filename, test_run_time:test_run_time} 553 result_summary: summary object to populate with the results 554 """ 555 threads = self._instantiate_test_shell_threads(file_list, 556 result_summary) 557 558 # Wait for the threads to finish and collect test failures. 559 failures = {} 560 test_timings = {} 561 individual_test_timings = [] 562 thread_timings = [] 563 try: 564 for thread in threads: 565 while thread.isAlive(): 566 # Let it timeout occasionally so it can notice a 567 # KeyboardInterrupt. Actually, the timeout doesn't 568 # really matter: apparently it suffices to not use 569 # an indefinite blocking join for it to 570 # be interruptible by KeyboardInterrupt. 571 thread.join(0.1) 572 self.update_summary(result_summary) 573 thread_timings.append({'name': thread.getName(), 574 'num_tests': thread.get_num_tests(), 575 'total_time': thread.get_total_time()}) 576 test_timings.update(thread.get_directory_timing_stats()) 577 individual_test_timings.extend( 578 thread.get_individual_test_stats()) 579 except KeyboardInterrupt: 580 for thread in threads: 581 thread.cancel() 582 self._port.stop_helper() 583 raise 584 for thread in threads: 585 # Check whether a TestShellThread died before normal completion. 586 exception_info = thread.get_exception_info() 587 if exception_info is not None: 588 # Re-raise the thread's exception here to make it clear that 589 # testing was aborted. Otherwise, the tests that did not run 590 # would be assumed to have passed. 591 raise exception_info[0], exception_info[1], exception_info[2] 592 593 # Make sure we pick up any remaining tests. 594 self.update_summary(result_summary) 595 return (thread_timings, test_timings, individual_test_timings) 596 597 def run(self, result_summary): 598 """Run all our tests on all our test files. 599 600 For each test file, we run each test type. If there are any failures, 601 we collect them for reporting. 602 603 Args: 604 result_summary: a summary object tracking the test results. 605 606 Return: 607 We return nonzero if there are regressions compared to the last run. 608 """ 609 if not self._test_files: 610 return 0 611 start_time = time.time() 612 613 # Start up any helper needed 614 if not self._options.no_pixel_tests: 615 self._port.start_helper() 616 617 if self._contains_tests(self.HTTP_SUBDIR): 618 self._port.start_http_server() 619 620 if self._contains_tests(self.WEBSOCKET_SUBDIR): 621 self._port.start_websocket_server() 622 # self._websocket_secure_server.Start() 623 624 thread_timings, test_timings, individual_test_timings = ( 625 self._run_tests(self._test_files_list, result_summary)) 626 627 # We exclude the crashes from the list of results to retry, because 628 # we want to treat even a potentially flaky crash as an error. 629 failures = self._get_failures(result_summary, include_crashes=False) 630 retries = 0 631 retry_summary = result_summary 632 while (retries < self.NUM_RETRY_ON_UNEXPECTED_FAILURE and 633 len(failures)): 634 logging.debug("Retrying %d unexpected failure(s)" % len(failures)) 635 retries += 1 636 retry_summary = ResultSummary(self._expectations, failures.keys()) 637 self._run_tests(failures.keys(), retry_summary) 638 failures = self._get_failures(retry_summary, include_crashes=True) 639 640 self._port.stop_helper() 641 end_time = time.time() 642 643 write = create_logging_writer(self._options, 'timing') 644 self._print_timing_statistics(write, end_time - start_time, 645 thread_timings, test_timings, 646 individual_test_timings, 647 result_summary) 648 649 self._meter.update("") 650 651 if self._options.verbose: 652 # We write this block to stdout for compatibility with the 653 # buildbot log parser, which only looks at stdout, not stderr :( 654 write = lambda s: sys.stdout.write("%s\n" % s) 655 else: 656 write = create_logging_writer(self._options, 'actual') 657 658 self._print_result_summary(write, result_summary) 659 660 sys.stdout.flush() 661 sys.stderr.flush() 662 663 if (LOG_DETAILED_PROGRESS in self._options.log or 664 (LOG_UNEXPECTED in self._options.log and 665 result_summary.total != result_summary.expected)): 666 print 667 668 # This summary data gets written to stdout regardless of log level 669 self._print_one_line_summary(result_summary.total, 670 result_summary.expected) 671 672 unexpected_results = self._summarize_unexpected_results(result_summary, 673 retry_summary) 674 self._print_unexpected_results(unexpected_results) 675 676 # Write the same data to log files. 677 self._write_json_files(unexpected_results, result_summary, 678 individual_test_timings) 679 680 # Write the summary to disk (results.html) and maybe open the 681 # test_shell to this file. 682 wrote_results = self._write_results_html_file(result_summary) 683 if not self._options.noshow_results and wrote_results: 684 self._show_results_html_file() 685 686 # Ignore flaky failures and unexpected passes so we don't turn the 687 # bot red for those. 688 return unexpected_results['num_regressions'] 689 690 def update_summary(self, result_summary): 691 """Update the summary while running tests.""" 692 while True: 693 try: 694 (test, fail_list) = self._result_queue.get_nowait() 695 result = test_failures.determine_result_type(fail_list) 696 expected = self._expectations.matches_an_expected_result(test, 697 result) 698 result_summary.add(test, fail_list, result, expected) 699 if (LOG_DETAILED_PROGRESS in self._options.log and 700 (self._options.experimental_fully_parallel or 701 self._is_single_threaded())): 702 self._display_detailed_progress(result_summary) 703 else: 704 if not expected and LOG_UNEXPECTED in self._options.log: 705 self._print_unexpected_test_result(test, result) 706 self._display_one_line_progress(result_summary) 707 except Queue.Empty: 708 return 709 710 def _display_one_line_progress(self, result_summary): 711 """Displays the progress through the test run.""" 712 self._meter.update("Testing: %d ran as expected, %d didn't, %d left" % 713 (result_summary.expected, result_summary.unexpected, 714 result_summary.remaining)) 715 716 def _display_detailed_progress(self, result_summary): 717 """Display detailed progress output where we print the directory name 718 and one dot for each completed test. This is triggered by 719 "--log detailed-progress".""" 720 if self._current_test_number == len(self._test_files_list): 721 return 722 723 next_test = self._test_files_list[self._current_test_number] 724 next_dir = os.path.dirname( 725 self._port.relative_test_filename(next_test)) 726 if self._current_progress_str == "": 727 self._current_progress_str = "%s: " % (next_dir) 728 self._current_dir = next_dir 729 730 while next_test in result_summary.results: 731 if next_dir != self._current_dir: 732 self._meter.write("%s\n" % (self._current_progress_str)) 733 self._current_progress_str = "%s: ." % (next_dir) 734 self._current_dir = next_dir 735 else: 736 self._current_progress_str += "." 737 738 if (next_test in result_summary.unexpected_results and 739 LOG_UNEXPECTED in self._options.log): 740 result = result_summary.unexpected_results[next_test] 741 self._meter.write("%s\n" % self._current_progress_str) 742 self._print_unexpected_test_result(next_test, result) 743 self._current_progress_str = "%s: " % self._current_dir 744 745 self._current_test_number += 1 746 if self._current_test_number == len(self._test_files_list): 747 break 748 749 next_test = self._test_files_list[self._current_test_number] 750 next_dir = os.path.dirname( 751 self._port.relative_test_filename(next_test)) 752 753 if result_summary.remaining: 754 remain_str = " (%d)" % (result_summary.remaining) 755 self._meter.update("%s%s" % 756 (self._current_progress_str, remain_str)) 757 else: 758 self._meter.write("%s\n" % (self._current_progress_str)) 759 760 def _get_failures(self, result_summary, include_crashes): 761 """Filters a dict of results and returns only the failures. 762 763 Args: 764 result_summary: the results of the test run 765 include_crashes: whether crashes are included in the output. 766 We use False when finding the list of failures to retry 767 to see if the results were flaky. Although the crashes may also be 768 flaky, we treat them as if they aren't so that they're not ignored. 769 Returns: 770 a dict of files -> results 771 """ 772 failed_results = {} 773 for test, result in result_summary.unexpected_results.iteritems(): 774 if (result == test_expectations.PASS or 775 result == test_expectations.CRASH and not include_crashes): 776 continue 777 failed_results[test] = result 778 779 return failed_results 780 781 def _summarize_unexpected_results(self, result_summary, retry_summary): 782 """Summarize any unexpected results as a dict. 783 784 TODO(dpranke): split this data structure into a separate class? 785 786 Args: 787 result_summary: summary object from initial test runs 788 retry_summary: summary object from final test run of retried tests 789 Returns: 790 A dictionary containing a summary of the unexpected results from the 791 run, with the following fields: 792 'version': a version indicator (1 in this version) 793 'fixable': # of fixable tests (NOW - PASS) 794 'skipped': # of skipped tests (NOW & SKIPPED) 795 'num_regressions': # of non-flaky failures 796 'num_flaky': # of flaky failures 797 'num_passes': # of unexpected passes 798 'tests': a dict of tests -> {'expected': '...', 'actual': '...'} 799 """ 800 results = {} 801 results['version'] = 1 802 803 tbe = result_summary.tests_by_expectation 804 tbt = result_summary.tests_by_timeline 805 results['fixable'] = len(tbt[test_expectations.NOW] - 806 tbe[test_expectations.PASS]) 807 results['skipped'] = len(tbt[test_expectations.NOW] & 808 tbe[test_expectations.SKIP]) 809 810 num_passes = 0 811 num_flaky = 0 812 num_regressions = 0 813 keywords = {} 814 for k, v in TestExpectationsFile.EXPECTATIONS.iteritems(): 815 keywords[v] = k.upper() 816 817 tests = {} 818 for filename, result in result_summary.unexpected_results.iteritems(): 819 # Note that if a test crashed in the original run, we ignore 820 # whether or not it crashed when we retried it (if we retried it), 821 # and always consider the result not flaky. 822 test = self._port.relative_test_filename(filename) 823 expected = self._expectations.get_expectations_string(filename) 824 actual = [keywords[result]] 825 826 if result == test_expectations.PASS: 827 num_passes += 1 828 elif result == test_expectations.CRASH: 829 num_regressions += 1 830 else: 831 if filename not in retry_summary.unexpected_results: 832 actual.extend( 833 self._expectations.get_expectations_string( 834 filename).split(" ")) 835 num_flaky += 1 836 else: 837 retry_result = retry_summary.unexpected_results[filename] 838 if result != retry_result: 839 actual.append(keywords[retry_result]) 840 num_flaky += 1 841 else: 842 num_regressions += 1 843 844 tests[test] = {} 845 tests[test]['expected'] = expected 846 tests[test]['actual'] = " ".join(actual) 847 848 results['tests'] = tests 849 results['num_passes'] = num_passes 850 results['num_flaky'] = num_flaky 851 results['num_regressions'] = num_regressions 852 853 return results 854 855 def _write_json_files(self, unexpected_results, result_summary, 856 individual_test_timings): 857 """Writes the results of the test run as JSON files into the results 858 dir. 859 860 There are three different files written into the results dir: 861 unexpected_results.json: A short list of any unexpected results. 862 This is used by the buildbots to display results. 863 expectations.json: This is used by the flakiness dashboard. 864 results.json: A full list of the results - used by the flakiness 865 dashboard and the aggregate results dashboard. 866 867 Args: 868 unexpected_results: dict of unexpected results 869 result_summary: full summary object 870 individual_test_timings: list of test times (used by the flakiness 871 dashboard). 872 """ 873 logging.debug("Writing JSON files in %s." % 874 self._options.results_directory) 875 unexpected_file = open(os.path.join(self._options.results_directory, 876 "unexpected_results.json"), "w") 877 unexpected_file.write(simplejson.dumps(unexpected_results, 878 sort_keys=True, indent=2)) 879 unexpected_file.close() 880 881 # Write a json file of the test_expectations.txt file for the layout 882 # tests dashboard. 883 expectations_file = open(os.path.join(self._options.results_directory, 884 "expectations.json"), "w") 885 expectations_json = \ 886 self._expectations.get_expectations_json_for_all_platforms() 887 expectations_file.write("ADD_EXPECTATIONS(" + expectations_json + ");") 888 expectations_file.close() 889 890 json_layout_results_generator.JSONLayoutResultsGenerator( 891 self._port, self._options.builder_name, self._options.build_name, 892 self._options.build_number, self._options.results_directory, 893 BUILDER_BASE_URL, individual_test_timings, 894 self._expectations, result_summary, self._test_files_list) 895 896 logging.debug("Finished writing JSON files.") 897 898 def _print_expected_results_of_type(self, write, result_summary, 899 result_type, result_type_str): 900 """Print the number of the tests in a given result class. 901 902 Args: 903 write: A callback to write info to (e.g., a LoggingWriter) or 904 sys.stdout.write. 905 result_summary - the object containing all the results to report on 906 result_type - the particular result type to report in the summary. 907 result_type_str - a string description of the result_type. 908 """ 909 tests = self._expectations.get_tests_with_result_type(result_type) 910 now = result_summary.tests_by_timeline[test_expectations.NOW] 911 wontfix = result_summary.tests_by_timeline[test_expectations.WONTFIX] 912 defer = result_summary.tests_by_timeline[test_expectations.DEFER] 913 914 # We use a fancy format string in order to print the data out in a 915 # nicely-aligned table. 916 fmtstr = ("Expect: %%5d %%-8s (%%%dd now, %%%dd defer, %%%dd wontfix)" 917 % (self._num_digits(now), self._num_digits(defer), 918 self._num_digits(wontfix))) 919 write(fmtstr % (len(tests), result_type_str, len(tests & now), 920 len(tests & defer), len(tests & wontfix))) 921 922 def _num_digits(self, num): 923 """Returns the number of digits needed to represent the length of a 924 sequence.""" 925 ndigits = 1 926 if len(num): 927 ndigits = int(math.log10(len(num))) + 1 928 return ndigits 929 930 def _print_timing_statistics(self, write, total_time, thread_timings, 931 directory_test_timings, individual_test_timings, 932 result_summary): 933 """Record timing-specific information for the test run. 934 935 Args: 936 write: A callback to write info to (e.g., a LoggingWriter) or 937 sys.stdout.write. 938 total_time: total elapsed time (in seconds) for the test run 939 thread_timings: wall clock time each thread ran for 940 directory_test_timings: timing by directory 941 individual_test_timings: timing by file 942 result_summary: summary object for the test run 943 """ 944 write("Test timing:") 945 write(" %6.2f total testing time" % total_time) 946 write("") 947 write("Thread timing:") 948 cuml_time = 0 949 for t in thread_timings: 950 write(" %10s: %5d tests, %6.2f secs" % 951 (t['name'], t['num_tests'], t['total_time'])) 952 cuml_time += t['total_time'] 953 write(" %6.2f cumulative, %6.2f optimal" % 954 (cuml_time, cuml_time / int(self._options.num_test_shells))) 955 write("") 956 957 self._print_aggregate_test_statistics(write, individual_test_timings) 958 self._print_individual_test_times(write, individual_test_timings, 959 result_summary) 960 self._print_directory_timings(write, directory_test_timings) 961 962 def _print_aggregate_test_statistics(self, write, individual_test_timings): 963 """Prints aggregate statistics (e.g. median, mean, etc.) for all tests. 964 Args: 965 write: A callback to write info to (e.g., a LoggingWriter) or 966 sys.stdout.write. 967 individual_test_timings: List of test_shell_thread.TestStats for all 968 tests. 969 """ 970 test_types = individual_test_timings[0].time_for_diffs.keys() 971 times_for_test_shell = [] 972 times_for_diff_processing = [] 973 times_per_test_type = {} 974 for test_type in test_types: 975 times_per_test_type[test_type] = [] 976 977 for test_stats in individual_test_timings: 978 times_for_test_shell.append(test_stats.test_run_time) 979 times_for_diff_processing.append( 980 test_stats.total_time_for_all_diffs) 981 time_for_diffs = test_stats.time_for_diffs 982 for test_type in test_types: 983 times_per_test_type[test_type].append( 984 time_for_diffs[test_type]) 985 986 self._print_statistics_for_test_timings(write, 987 "PER TEST TIME IN TESTSHELL (seconds):", times_for_test_shell) 988 self._print_statistics_for_test_timings(write, 989 "PER TEST DIFF PROCESSING TIMES (seconds):", 990 times_for_diff_processing) 991 for test_type in test_types: 992 self._print_statistics_for_test_timings(write, 993 "PER TEST TIMES BY TEST TYPE: %s" % test_type, 994 times_per_test_type[test_type]) 995 996 def _print_individual_test_times(self, write, individual_test_timings, 997 result_summary): 998 """Prints the run times for slow, timeout and crash tests. 999 Args: 1000 write: A callback to write info to (e.g., a LoggingWriter) or 1001 sys.stdout.write. 1002 individual_test_timings: List of test_shell_thread.TestStats for all 1003 tests. 1004 result_summary: summary object for test run 1005 """ 1006 # Reverse-sort by the time spent in test_shell. 1007 individual_test_timings.sort(lambda a, b: 1008 cmp(b.test_run_time, a.test_run_time)) 1009 1010 num_printed = 0 1011 slow_tests = [] 1012 timeout_or_crash_tests = [] 1013 unexpected_slow_tests = [] 1014 for test_tuple in individual_test_timings: 1015 filename = test_tuple.filename 1016 is_timeout_crash_or_slow = False 1017 if self._expectations.has_modifier(filename, 1018 test_expectations.SLOW): 1019 is_timeout_crash_or_slow = True 1020 slow_tests.append(test_tuple) 1021 1022 if filename in result_summary.failures: 1023 result = result_summary.results[filename] 1024 if (result == test_expectations.TIMEOUT or 1025 result == test_expectations.CRASH): 1026 is_timeout_crash_or_slow = True 1027 timeout_or_crash_tests.append(test_tuple) 1028 1029 if (not is_timeout_crash_or_slow and 1030 num_printed < self._options.num_slow_tests_to_log): 1031 num_printed = num_printed + 1 1032 unexpected_slow_tests.append(test_tuple) 1033 1034 write("") 1035 self._print_test_list_timing(write, "%s slowest tests that are not " 1036 "marked as SLOW and did not timeout/crash:" % 1037 self._options.num_slow_tests_to_log, unexpected_slow_tests) 1038 write("") 1039 self._print_test_list_timing(write, "Tests marked as SLOW:", 1040 slow_tests) 1041 write("") 1042 self._print_test_list_timing(write, "Tests that timed out or crashed:", 1043 timeout_or_crash_tests) 1044 write("") 1045 1046 def _print_test_list_timing(self, write, title, test_list): 1047 """Print timing info for each test. 1048 1049 Args: 1050 write: A callback to write info to (e.g., a LoggingWriter) or 1051 sys.stdout.write. 1052 title: section heading 1053 test_list: tests that fall in this section 1054 """ 1055 write(title) 1056 for test_tuple in test_list: 1057 filename = test_tuple.filename[len( 1058 self._port.layout_tests_dir()) + 1:] 1059 filename = filename.replace('\\', '/') 1060 test_run_time = round(test_tuple.test_run_time, 1) 1061 write(" %s took %s seconds" % (filename, test_run_time)) 1062 1063 def _print_directory_timings(self, write, directory_test_timings): 1064 """Print timing info by directory for any directories that 1065 take > 10 seconds to run. 1066 1067 Args: 1068 write: A callback to write info to (e.g., a LoggingWriter) or 1069 sys.stdout.write. 1070 directory_test_timing: time info for each directory 1071 """ 1072 timings = [] 1073 for directory in directory_test_timings: 1074 num_tests, time_for_directory = directory_test_timings[directory] 1075 timings.append((round(time_for_directory, 1), directory, 1076 num_tests)) 1077 timings.sort() 1078 1079 write("Time to process slowest subdirectories:") 1080 min_seconds_to_print = 10 1081 for timing in timings: 1082 if timing[0] > min_seconds_to_print: 1083 write(" %s took %s seconds to run %s tests." % (timing[1], 1084 timing[0], timing[2])) 1085 write("") 1086 1087 def _print_statistics_for_test_timings(self, write, title, timings): 1088 """Prints the median, mean and standard deviation of the values in 1089 timings. 1090 1091 Args: 1092 write: A callback to write info to (e.g., a LoggingWriter) or 1093 sys.stdout.write. 1094 title: Title for these timings. 1095 timings: A list of floats representing times. 1096 """ 1097 write(title) 1098 timings.sort() 1099 1100 num_tests = len(timings) 1101 percentile90 = timings[int(.9 * num_tests)] 1102 percentile99 = timings[int(.99 * num_tests)] 1103 1104 if num_tests % 2 == 1: 1105 median = timings[((num_tests - 1) / 2) - 1] 1106 else: 1107 lower = timings[num_tests / 2 - 1] 1108 upper = timings[num_tests / 2] 1109 median = (float(lower + upper)) / 2 1110 1111 mean = sum(timings) / num_tests 1112 1113 for time in timings: 1114 sum_of_deviations = math.pow(time - mean, 2) 1115 1116 std_deviation = math.sqrt(sum_of_deviations / num_tests) 1117 write(" Median: %6.3f" % median) 1118 write(" Mean: %6.3f" % mean) 1119 write(" 90th percentile: %6.3f" % percentile90) 1120 write(" 99th percentile: %6.3f" % percentile99) 1121 write(" Standard dev: %6.3f" % std_deviation) 1122 write("") 1123 1124 def _print_result_summary(self, write, result_summary): 1125 """Print a short summary about how many tests passed. 1126 1127 Args: 1128 write: A callback to write info to (e.g., a LoggingWriter) or 1129 sys.stdout.write. 1130 result_summary: information to log 1131 """ 1132 failed = len(result_summary.failures) 1133 skipped = len( 1134 result_summary.tests_by_expectation[test_expectations.SKIP]) 1135 total = result_summary.total 1136 passed = total - failed - skipped 1137 pct_passed = 0.0 1138 if total > 0: 1139 pct_passed = float(passed) * 100 / total 1140 1141 write("") 1142 write("=> Results: %d/%d tests passed (%.1f%%)" % 1143 (passed, total, pct_passed)) 1144 write("") 1145 self._print_result_summary_entry(write, result_summary, 1146 test_expectations.NOW, "Tests to be fixed for the current release") 1147 1148 write("") 1149 self._print_result_summary_entry(write, result_summary, 1150 test_expectations.DEFER, 1151 "Tests we'll fix in the future if they fail (DEFER)") 1152 1153 write("") 1154 self._print_result_summary_entry(write, result_summary, 1155 test_expectations.WONTFIX, 1156 "Tests that will only be fixed if they crash (WONTFIX)") 1157 1158 def _print_result_summary_entry(self, write, result_summary, timeline, 1159 heading): 1160 """Print a summary block of results for a particular timeline of test. 1161 1162 Args: 1163 write: A callback to write info to (e.g., a LoggingWriter) or 1164 sys.stdout.write. 1165 result_summary: summary to print results for 1166 timeline: the timeline to print results for (NOT, WONTFIX, etc.) 1167 heading: a textual description of the timeline 1168 """ 1169 total = len(result_summary.tests_by_timeline[timeline]) 1170 not_passing = (total - 1171 len(result_summary.tests_by_expectation[test_expectations.PASS] & 1172 result_summary.tests_by_timeline[timeline])) 1173 write("=> %s (%d):" % (heading, not_passing)) 1174 1175 for result in TestExpectationsFile.EXPECTATION_ORDER: 1176 if result == test_expectations.PASS: 1177 continue 1178 results = (result_summary.tests_by_expectation[result] & 1179 result_summary.tests_by_timeline[timeline]) 1180 desc = TestExpectationsFile.EXPECTATION_DESCRIPTIONS[result] 1181 if not_passing and len(results): 1182 pct = len(results) * 100.0 / not_passing 1183 write(" %5d %-24s (%4.1f%%)" % (len(results), 1184 desc[len(results) != 1], pct)) 1185 1186 def _print_one_line_summary(self, total, expected): 1187 """Print a one-line summary of the test run to stdout. 1188 1189 Args: 1190 total: total number of tests run 1191 expected: number of expected results 1192 """ 1193 unexpected = total - expected 1194 if unexpected == 0: 1195 print "All %d tests ran as expected." % expected 1196 elif expected == 1: 1197 print "1 test ran as expected, %d didn't:" % unexpected 1198 else: 1199 print "%d tests ran as expected, %d didn't:" % (expected, 1200 unexpected) 1201 1202 def _print_unexpected_results(self, unexpected_results): 1203 """Prints any unexpected results in a human-readable form to stdout.""" 1204 passes = {} 1205 flaky = {} 1206 regressions = {} 1207 1208 if len(unexpected_results['tests']): 1209 print "" 1210 1211 for test, results in unexpected_results['tests'].iteritems(): 1212 actual = results['actual'].split(" ") 1213 expected = results['expected'].split(" ") 1214 if actual == ['PASS']: 1215 if 'CRASH' in expected: 1216 _add_to_dict_of_lists(passes, 1217 'Expected to crash, but passed', 1218 test) 1219 elif 'TIMEOUT' in expected: 1220 _add_to_dict_of_lists(passes, 1221 'Expected to timeout, but passed', 1222 test) 1223 else: 1224 _add_to_dict_of_lists(passes, 1225 'Expected to fail, but passed', 1226 test) 1227 elif len(actual) > 1: 1228 # We group flaky tests by the first actual result we got. 1229 _add_to_dict_of_lists(flaky, actual[0], test) 1230 else: 1231 _add_to_dict_of_lists(regressions, results['actual'], test) 1232 1233 if len(passes): 1234 for key, tests in passes.iteritems(): 1235 print "%s: (%d)" % (key, len(tests)) 1236 tests.sort() 1237 for test in tests: 1238 print " %s" % test 1239 print 1240 1241 if len(flaky): 1242 descriptions = TestExpectationsFile.EXPECTATION_DESCRIPTIONS 1243 for key, tests in flaky.iteritems(): 1244 result = TestExpectationsFile.EXPECTATIONS[key.lower()] 1245 print "Unexpected flakiness: %s (%d)" % ( 1246 descriptions[result][1], len(tests)) 1247 tests.sort() 1248 1249 for test in tests: 1250 result = unexpected_results['tests'][test] 1251 actual = result['actual'].split(" ") 1252 expected = result['expected'].split(" ") 1253 result = TestExpectationsFile.EXPECTATIONS[key.lower()] 1254 new_expectations_list = list(set(actual) | set(expected)) 1255 print " %s = %s" % (test, " ".join(new_expectations_list)) 1256 print 1257 1258 if len(regressions): 1259 descriptions = TestExpectationsFile.EXPECTATION_DESCRIPTIONS 1260 for key, tests in regressions.iteritems(): 1261 result = TestExpectationsFile.EXPECTATIONS[key.lower()] 1262 print "Regressions: Unexpected %s : (%d)" % ( 1263 descriptions[result][1], len(tests)) 1264 tests.sort() 1265 for test in tests: 1266 print " %s = %s" % (test, key) 1267 print 1268 1269 if len(unexpected_results['tests']) and self._options.verbose: 1270 print "-" * 78 1271 1272 def _print_unexpected_test_result(self, test, result): 1273 """Prints one unexpected test result line.""" 1274 desc = TestExpectationsFile.EXPECTATION_DESCRIPTIONS[result][0] 1275 self._meter.write(" %s -> unexpected %s\n" % 1276 (self._port.relative_test_filename(test), desc)) 1277 1278 def _write_results_html_file(self, result_summary): 1279 """Write results.html which is a summary of tests that failed. 1280 1281 Args: 1282 result_summary: a summary of the results :) 1283 1284 Returns: 1285 True if any results were written (since expected failures may be 1286 omitted) 1287 """ 1288 # test failures 1289 if self._options.full_results_html: 1290 test_files = result_summary.failures.keys() 1291 else: 1292 unexpected_failures = self._get_failures(result_summary, 1293 include_crashes=True) 1294 test_files = unexpected_failures.keys() 1295 if not len(test_files): 1296 return False 1297 1298 out_filename = os.path.join(self._options.results_directory, 1299 "results.html") 1300 out_file = open(out_filename, 'w') 1301 # header 1302 if self._options.full_results_html: 1303 h2 = "Test Failures" 1304 else: 1305 h2 = "Unexpected Test Failures" 1306 out_file.write("<html><head><title>Layout Test Results (%(time)s)" 1307 "</title></head><body><h2>%(h2)s (%(time)s)</h2>\n" 1308 % {'h2': h2, 'time': time.asctime()}) 1309 1310 test_files.sort() 1311 for test_file in test_files: 1312 test_failures = result_summary.failures.get(test_file, []) 1313 out_file.write("<p><a href='%s'>%s</a><br />\n" 1314 % (self._port.filename_to_uri(test_file), 1315 self._port.relative_test_filename(test_file))) 1316 for failure in test_failures: 1317 out_file.write(" %s<br/>" 1318 % failure.result_html_output( 1319 self._port.relative_test_filename(test_file))) 1320 out_file.write("</p>\n") 1321 1322 # footer 1323 out_file.write("</body></html>\n") 1324 return True 1325 1326 def _show_results_html_file(self): 1327 """Launches the test shell open to the results.html page.""" 1328 results_filename = os.path.join(self._options.results_directory, 1329 "results.html") 1330 self._port.show_results_html_file(results_filename) 1331 1332 1333 def _add_to_dict_of_lists(dict, key, value): 1334 dict.setdefault(key, []).append(value) 1335 1336 1337 def read_test_files(files): 1338 tests = [] 1339 for file in files: 1340 for line in open(file): 1341 line = test_expectations.strip_comments(line) 1342 if line: 1343 tests.append(line) 1344 return tests 1345 1346 1347 def create_logging_writer(options, log_option): 1348 """Returns a write() function that will write the string to logging.info() 1349 if comp was specified in --log or if --verbose is true. Otherwise the 1350 message is dropped. 1351 1352 Args: 1353 options: list of command line options from optparse 1354 log_option: option to match in options.log in order for the messages 1355 to be logged (e.g., 'actual' or 'expected') 1356 """ 1357 if options.verbose or log_option in options.log.split(","): 1358 return logging.info 1359 return lambda str: 1 1360 1361 1362 def main(options, args): 1363 """Run the tests. Will call sys.exit when complete. 1364 1365 Args: 1366 options: a dictionary of command line options 1367 args: a list of sub directories or files to test 1368 """ 1369 1370 if options.sources: 1371 options.verbose = True 1372 1373 # Set up our logging format. 1374 meter = metered_stream.MeteredStream(options.verbose, sys.stderr) 1375 log_fmt = '%(message)s' 1376 log_datefmt = '%y%m%d %H:%M:%S' 1377 log_level = logging.INFO 1378 if options.verbose: 1379 log_fmt = ('%(asctime)s %(filename)s:%(lineno)-4d %(levelname)s ' 1380 '%(message)s') 1381 log_level = logging.DEBUG 1382 logging.basicConfig(level=log_level, format=log_fmt, datefmt=log_datefmt, 1383 stream=meter) 1384 1385 if not options.target: 1386 if options.debug: 1387 options.target = "Debug" 1388 else: 1389 options.target = "Release" 1390 1391 port_obj = port.get(options.platform, options) 1392 1393 if not options.use_apache: 1394 options.use_apache = sys.platform in ('darwin', 'linux2') 1395 1396 if options.results_directory.startswith("/"): 1397 # Assume it's an absolute path and normalize. 1398 options.results_directory = port_obj.get_absolute_path( 1399 options.results_directory) 1400 else: 1401 # If it's a relative path, make the output directory relative to 1402 # Debug or Release. 1403 options.results_directory = port_obj.results_directory() 1404 1405 if options.clobber_old_results: 1406 # Just clobber the actual test results directories since the other 1407 # files in the results directory are explicitly used for cross-run 1408 # tracking. 1409 path = os.path.join(options.results_directory, 'LayoutTests') 1410 if os.path.exists(path): 1411 shutil.rmtree(path) 1412 1413 if not options.num_test_shells: 1414 # TODO(ojan): Investigate perf/flakiness impact of using numcores + 1. 1415 options.num_test_shells = port_obj.num_cores() 1416 1417 write = create_logging_writer(options, 'config') 1418 write("Running %s test_shells in parallel" % options.num_test_shells) 1419 1420 if not options.time_out_ms: 1421 if options.target == "Debug": 1422 options.time_out_ms = str(2 * TestRunner.DEFAULT_TEST_TIMEOUT_MS) 1423 else: 1424 options.time_out_ms = str(TestRunner.DEFAULT_TEST_TIMEOUT_MS) 1425 1426 options.slow_time_out_ms = str(5 * int(options.time_out_ms)) 1427 write("Regular timeout: %s, slow test timeout: %s" % 1428 (options.time_out_ms, options.slow_time_out_ms)) 1429 1430 # Include all tests if none are specified. 1431 new_args = [] 1432 for arg in args: 1433 if arg and arg != '': 1434 new_args.append(arg) 1435 1436 paths = new_args 1437 if not paths: 1438 paths = [] 1439 if options.test_list: 1440 paths += read_test_files(options.test_list) 1441 1442 # Create the output directory if it doesn't already exist. 1443 port_obj.maybe_make_directory(options.results_directory) 1444 meter.update("Gathering files ...") 1445 1446 test_runner = TestRunner(port_obj, options, meter) 1447 test_runner.gather_file_paths(paths) 1448 1449 if options.lint_test_files: 1450 # Creating the expecations for each platform/target pair does all the 1451 # test list parsing and ensures it's correct syntax (e.g. no dupes). 1452 for platform in port_obj.test_platform_names(): 1453 test_runner.parse_expectations(platform, is_debug_mode=True) 1454 test_runner.parse_expectations(platform, is_debug_mode=False) 1455 print ("If there are no fail messages, errors or exceptions, then the " 1456 "lint succeeded.") 1457 sys.exit(0) 1458 1459 # Check that the system dependencies (themes, fonts, ...) are correct. 1460 if not options.nocheck_sys_deps: 1461 if not port_obj.check_sys_deps(): 1462 sys.exit(1) 1463 1464 write = create_logging_writer(options, "config") 1465 write("Using port '%s'" % port_obj.name()) 1466 write("Placing test results in %s" % options.results_directory) 1467 if options.new_baseline: 1468 write("Placing new baselines in %s" % port_obj.baseline_path()) 1469 write("Using %s build" % options.target) 1470 if options.no_pixel_tests: 1471 write("Not running pixel tests") 1472 write("") 1473 1474 meter.update("Parsing expectations ...") 1475 test_runner.parse_expectations(port_obj.test_platform_name(), 1476 options.target == 'Debug') 1477 1478 meter.update("Preparing tests ...") 1479 write = create_logging_writer(options, "expected") 1480 result_summary = test_runner.prepare_lists_and_print_output(write) 1481 1482 port_obj.setup_test_run() 1483 1484 test_runner.add_test_type(text_diff.TestTextDiff) 1485 if not options.no_pixel_tests: 1486 test_runner.add_test_type(image_diff.ImageDiff) 1487 if options.fuzzy_pixel_tests: 1488 test_runner.add_test_type(fuzzy_image_diff.FuzzyImageDiff) 1489 1490 meter.update("Starting ...") 1491 has_new_failures = test_runner.run(result_summary) 1492 1493 logging.debug("Exit status: %d" % has_new_failures) 1494 sys.exit(has_new_failures) 1495 1496 1497 def parse_args(args=None): 1498 """Provides a default set of command line args. 1499 1500 Returns a tuple of options, args from optparse""" 1501 option_parser = optparse.OptionParser() 1502 option_parser.add_option("", "--no-pixel-tests", action="store_true", 1503 default=False, 1504 help="disable pixel-to-pixel PNG comparisons") 1505 option_parser.add_option("", "--fuzzy-pixel-tests", action="store_true", 1506 default=False, 1507 help="Also use fuzzy matching to compare pixel " 1508 "test outputs.") 1509 option_parser.add_option("", "--results-directory", 1510 default="layout-test-results", 1511 help="Output results directory source dir," 1512 " relative to Debug or Release") 1513 option_parser.add_option("", "--new-baseline", action="store_true", 1514 default=False, 1515 help="save all generated results as new baselines" 1516 " into the platform directory, overwriting " 1517 "whatever's already there.") 1518 option_parser.add_option("", "--noshow-results", action="store_true", 1519 default=False, help="don't launch the test_shell" 1520 " with results after the tests are done") 1521 option_parser.add_option("", "--full-results-html", action="store_true", 1522 default=False, help="show all failures in " 1523 "results.html, rather than only regressions") 1524 option_parser.add_option("", "--clobber-old-results", action="store_true", 1525 default=False, help="Clobbers test results from " 1526 "previous runs.") 1527 option_parser.add_option("", "--lint-test-files", action="store_true", 1528 default=False, help="Makes sure the test files " 1529 "parse for all configurations. Does not run any " 1530 "tests.") 1531 option_parser.add_option("", "--force", action="store_true", 1532 default=False, 1533 help="Run all tests, even those marked SKIP " 1534 "in the test list") 1535 option_parser.add_option("", "--num-test-shells", 1536 help="Number of testshells to run in parallel.") 1537 option_parser.add_option("", "--use-apache", action="store_true", 1538 default=False, 1539 help="Whether to use apache instead of lighttpd.") 1540 option_parser.add_option("", "--time-out-ms", default=None, 1541 help="Set the timeout for each test") 1542 option_parser.add_option("", "--run-singly", action="store_true", 1543 default=False, 1544 help="run a separate test_shell for each test") 1545 option_parser.add_option("", "--debug", action="store_true", default=False, 1546 help="use the debug binary instead of the release" 1547 " binary") 1548 option_parser.add_option("", "--num-slow-tests-to-log", default=50, 1549 help="Number of slow tests whose timings " 1550 "to print.") 1551 option_parser.add_option("", "--platform", 1552 help="Override the platform for expected results") 1553 option_parser.add_option("", "--target", default="", 1554 help="Set the build target configuration " 1555 "(overrides --debug)") 1556 option_parser.add_option("", "--log", action="store", 1557 default="detailed-progress,unexpected", 1558 help="log various types of data. The param should" 1559 " be a comma-separated list of values from: " 1560 "actual,config," + LOG_DETAILED_PROGRESS + 1561 ",expected,timing," + LOG_UNEXPECTED + " " 1562 "(defaults to " + 1563 "--log detailed-progress,unexpected)") 1564 option_parser.add_option("-v", "--verbose", action="store_true", 1565 default=False, help="include debug-level logging") 1566 option_parser.add_option("", "--sources", action="store_true", 1567 help="show expected result file path for each " 1568 "test (implies --verbose)") 1569 option_parser.add_option("", "--startup-dialog", action="store_true", 1570 default=False, 1571 help="create a dialog on test_shell.exe startup") 1572 option_parser.add_option("", "--gp-fault-error-box", action="store_true", 1573 default=False, 1574 help="enable Windows GP fault error box") 1575 option_parser.add_option("", "--wrapper", 1576 help="wrapper command to insert before " 1577 "invocations of test_shell; option is split " 1578 "on whitespace before running. (Example: " 1579 "--wrapper='valgrind --smc-check=all')") 1580 option_parser.add_option("", "--test-list", action="append", 1581 help="read list of tests to run from file", 1582 metavar="FILE") 1583 option_parser.add_option("", "--nocheck-sys-deps", action="store_true", 1584 default=False, 1585 help="Don't check the system dependencies " 1586 "(themes)") 1587 option_parser.add_option("", "--randomize-order", action="store_true", 1588 default=False, 1589 help=("Run tests in random order (useful for " 1590 "tracking down corruption)")) 1591 option_parser.add_option("", "--run-chunk", 1592 default=None, 1593 help=("Run a specified chunk (n:l), the " 1594 "nth of len l, of the layout tests")) 1595 option_parser.add_option("", "--run-part", 1596 default=None, 1597 help=("Run a specified part (n:m), the nth of m" 1598 " parts, of the layout tests")) 1599 option_parser.add_option("", "--batch-size", 1600 default=None, 1601 help=("Run a the tests in batches (n), after " 1602 "every n tests, the test shell is " 1603 "relaunched.")) 1604 option_parser.add_option("", "--builder-name", 1605 default="DUMMY_BUILDER_NAME", 1606 help=("The name of the builder shown on the " 1607 "waterfall running this script e.g. " 1608 "WebKit.")) 1609 option_parser.add_option("", "--build-name", 1610 default="DUMMY_BUILD_NAME", 1611 help=("The name of the builder used in its path, " 1612 "e.g. webkit-rel.")) 1613 option_parser.add_option("", "--build-number", 1614 default="DUMMY_BUILD_NUMBER", 1615 help=("The build number of the builder running" 1616 "this script.")) 1617 option_parser.add_option("", "--experimental-fully-parallel", 1618 action="store_true", default=False, 1619 help="run all tests in parallel") 1620 return option_parser.parse_args(args) 1621 1622 if '__main__' == __name__: 1623 options, args = parse_args() 1624 main(options, args) 1625