1 # Copyright (C) 2010 Google Inc. All rights reserved. 2 # 3 # Redistribution and use in source and binary forms, with or without 4 # modification, are permitted provided that the following conditions are 5 # met: 6 # 7 # * Redistributions of source code must retain the above copyright 8 # notice, this list of conditions and the following disclaimer. 9 # * Redistributions in binary form must reproduce the above 10 # copyright notice, this list of conditions and the following disclaimer 11 # in the documentation and/or other materials provided with the 12 # distribution. 13 # * Neither the Google name nor the names of its 14 # contributors may be used to endorse or promote products derived from 15 # this software without specific prior written permission. 16 # 17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 """Abstract base class of Port-specific entry points for the layout tests 30 test infrastructure (the Port and Driver classes).""" 31 32 import cgi 33 import difflib 34 import errno 35 import itertools 36 import json 37 import logging 38 import os 39 import operator 40 import optparse 41 import re 42 import sys 43 44 try: 45 from collections import OrderedDict 46 except ImportError: 47 # Needed for Python < 2.7 48 from webkitpy.thirdparty.ordered_dict import OrderedDict 49 50 51 from webkitpy.common import find_files 52 from webkitpy.common import read_checksum_from_png 53 from webkitpy.common.memoized import memoized 54 from webkitpy.common.system import path 55 from webkitpy.common.system.executive import ScriptError 56 from webkitpy.common.system.path import cygpath 57 from webkitpy.common.system.systemhost import SystemHost 58 from webkitpy.common.webkit_finder import WebKitFinder 59 from webkitpy.layout_tests.layout_package.bot_test_expectations import BotTestExpectationsFactory 60 from webkitpy.layout_tests.models import test_run_results 61 from webkitpy.layout_tests.models.test_configuration import TestConfiguration 62 from webkitpy.layout_tests.port import config as port_config 63 from webkitpy.layout_tests.port import driver 64 from webkitpy.layout_tests.port import server_process 65 from webkitpy.layout_tests.port.factory import PortFactory 66 from webkitpy.layout_tests.servers import apache_http 67 from webkitpy.layout_tests.servers import pywebsocket 68 69 _log = logging.getLogger(__name__) 70 71 72 # FIXME: This class should merge with WebKitPort now that Chromium behaves mostly like other webkit ports. 73 class Port(object): 74 """Abstract class for Port-specific hooks for the layout_test package.""" 75 76 # Subclasses override this. This should indicate the basic implementation 77 # part of the port name, e.g., 'mac', 'win', 'gtk'; there is probably (?) 78 # one unique value per class. 79 80 # FIXME: We should probably rename this to something like 'implementation_name'. 81 port_name = None 82 83 # Test names resemble unix relative paths, and use '/' as a directory separator. 84 TEST_PATH_SEPARATOR = '/' 85 86 ALL_BUILD_TYPES = ('debug', 'release') 87 88 CONTENT_SHELL_NAME = 'content_shell' 89 90 # True if the port as aac and mp3 codecs built in. 91 PORT_HAS_AUDIO_CODECS_BUILT_IN = False 92 93 ALL_SYSTEMS = ( 94 ('snowleopard', 'x86'), 95 ('lion', 'x86'), 96 97 # FIXME: We treat Retina (High-DPI) devices as if they are running 98 # a different operating system version. This isn't accurate, but will work until 99 # we need to test and support baselines across multiple O/S versions. 100 ('retina', 'x86'), 101 102 ('mountainlion', 'x86'), 103 ('mavericks', 'x86'), 104 ('xp', 'x86'), 105 ('win7', 'x86'), 106 ('lucid', 'x86'), 107 ('lucid', 'x86_64'), 108 # FIXME: Technically this should be 'arm', but adding a third architecture type breaks TestConfigurationConverter. 109 # If we need this to be 'arm' in the future, then we first have to fix TestConfigurationConverter. 110 ('icecreamsandwich', 'x86'), 111 ) 112 113 ALL_BASELINE_VARIANTS = [ 114 'mac-mavericks', 'mac-mountainlion', 'mac-retina', 'mac-lion', 'mac-snowleopard', 115 'win-win7', 'win-xp', 116 'linux-x86_64', 'linux-x86', 117 ] 118 119 CONFIGURATION_SPECIFIER_MACROS = { 120 'mac': ['snowleopard', 'lion', 'retina', 'mountainlion', 'mavericks'], 121 'win': ['xp', 'win7'], 122 'linux': ['lucid'], 123 'android': ['icecreamsandwich'], 124 } 125 126 DEFAULT_BUILD_DIRECTORIES = ('out',) 127 128 # overridden in subclasses. 129 FALLBACK_PATHS = {} 130 131 SUPPORTED_VERSIONS = [] 132 133 # URL to the build requirements page. 134 BUILD_REQUIREMENTS_URL = '' 135 136 @classmethod 137 def latest_platform_fallback_path(cls): 138 return cls.FALLBACK_PATHS[cls.SUPPORTED_VERSIONS[-1]] 139 140 @classmethod 141 def _static_build_path(cls, filesystem, build_directory, chromium_base, configuration, comps): 142 if build_directory: 143 return filesystem.join(build_directory, configuration, *comps) 144 145 hits = [] 146 for directory in cls.DEFAULT_BUILD_DIRECTORIES: 147 base_dir = filesystem.join(chromium_base, directory, configuration) 148 path = filesystem.join(base_dir, *comps) 149 if filesystem.exists(path): 150 hits.append((filesystem.mtime(path), path)) 151 152 if hits: 153 hits.sort(reverse=True) 154 return hits[0][1] # Return the newest file found. 155 156 # We have to default to something, so pick the last one. 157 return filesystem.join(base_dir, *comps) 158 159 @classmethod 160 def determine_full_port_name(cls, host, options, port_name): 161 """Return a fully-specified port name that can be used to construct objects.""" 162 # Subclasses will usually override this. 163 assert port_name.startswith(cls.port_name) 164 return port_name 165 166 def __init__(self, host, port_name, options=None, **kwargs): 167 168 # This value may be different from cls.port_name by having version modifiers 169 # and other fields appended to it (for example, 'qt-arm' or 'mac-wk2'). 170 self._name = port_name 171 172 # These are default values that should be overridden in a subclasses. 173 self._version = '' 174 self._architecture = 'x86' 175 176 # FIXME: Ideally we'd have a package-wide way to get a 177 # well-formed options object that had all of the necessary 178 # options defined on it. 179 self._options = options or optparse.Values() 180 181 self.host = host 182 self._executive = host.executive 183 self._filesystem = host.filesystem 184 self._webkit_finder = WebKitFinder(host.filesystem) 185 self._config = port_config.Config(self._executive, self._filesystem, self.port_name) 186 187 self._helper = None 188 self._http_server = None 189 self._websocket_server = None 190 self._image_differ = None 191 self._server_process_constructor = server_process.ServerProcess # overridable for testing 192 self._http_lock = None # FIXME: Why does this live on the port object? 193 self._dump_reader = None 194 195 # Python's Popen has a bug that causes any pipes opened to a 196 # process that can't be executed to be leaked. Since this 197 # code is specifically designed to tolerate exec failures 198 # to gracefully handle cases where wdiff is not installed, 199 # the bug results in a massive file descriptor leak. As a 200 # workaround, if an exec failure is ever experienced for 201 # wdiff, assume it's not available. This will leak one 202 # file descriptor but that's better than leaking each time 203 # wdiff would be run. 204 # 205 # http://mail.python.org/pipermail/python-list/ 206 # 2008-August/505753.html 207 # http://bugs.python.org/issue3210 208 self._wdiff_available = None 209 210 # FIXME: prettypatch.py knows this path, why is it copied here? 211 self._pretty_patch_path = self.path_from_webkit_base("Tools", "Scripts", "webkitruby", "PrettyPatch", "prettify.rb") 212 self._pretty_patch_available = None 213 214 if not hasattr(options, 'configuration') or not options.configuration: 215 self.set_option_default('configuration', self.default_configuration()) 216 self._test_configuration = None 217 self._reftest_list = {} 218 self._results_directory = None 219 self._virtual_test_suites = None 220 221 def buildbot_archives_baselines(self): 222 return True 223 224 def additional_drt_flag(self): 225 if self.driver_name() == self.CONTENT_SHELL_NAME: 226 return ['--dump-render-tree'] 227 return [] 228 229 def supports_per_test_timeout(self): 230 return False 231 232 def default_pixel_tests(self): 233 return True 234 235 def default_smoke_test_only(self): 236 return False 237 238 def default_timeout_ms(self): 239 timeout_ms = 6 * 1000 240 if self.get_option('configuration') == 'Debug': 241 # Debug is usually 2x-3x slower than Release. 242 return 3 * timeout_ms 243 return timeout_ms 244 245 def driver_stop_timeout(self): 246 """ Returns the amount of time in seconds to wait before killing the process in driver.stop().""" 247 # We want to wait for at least 3 seconds, but if we are really slow, we want to be slow on cleanup as 248 # well (for things like ASAN, Valgrind, etc.) 249 return 3.0 * float(self.get_option('time_out_ms', '0')) / self.default_timeout_ms() 250 251 def wdiff_available(self): 252 if self._wdiff_available is None: 253 self._wdiff_available = self.check_wdiff(logging=False) 254 return self._wdiff_available 255 256 def pretty_patch_available(self): 257 if self._pretty_patch_available is None: 258 self._pretty_patch_available = self.check_pretty_patch(logging=False) 259 return self._pretty_patch_available 260 261 def default_child_processes(self): 262 """Return the number of drivers to use for this port.""" 263 return self._executive.cpu_count() 264 265 def default_max_locked_shards(self): 266 """Return the number of "locked" shards to run in parallel (like the http tests).""" 267 max_locked_shards = int(self.default_child_processes()) / 4 268 if not max_locked_shards: 269 return 1 270 return max_locked_shards 271 272 def baseline_path(self): 273 """Return the absolute path to the directory to store new baselines in for this port.""" 274 # FIXME: remove once all callers are calling either baseline_version_dir() or baseline_platform_dir() 275 return self.baseline_version_dir() 276 277 def baseline_platform_dir(self): 278 """Return the absolute path to the default (version-independent) platform-specific results.""" 279 return self._filesystem.join(self.layout_tests_dir(), 'platform', self.port_name) 280 281 def baseline_version_dir(self): 282 """Return the absolute path to the platform-and-version-specific results.""" 283 baseline_search_paths = self.baseline_search_path() 284 return baseline_search_paths[0] 285 286 def virtual_baseline_search_path(self, test_name): 287 suite = self.lookup_virtual_suite(test_name) 288 if not suite: 289 return None 290 return [self._filesystem.join(path, suite.name) for path in self.default_baseline_search_path()] 291 292 def baseline_search_path(self): 293 return self.get_option('additional_platform_directory', []) + self._compare_baseline() + self.default_baseline_search_path() 294 295 def default_baseline_search_path(self): 296 """Return a list of absolute paths to directories to search under for 297 baselines. The directories are searched in order.""" 298 return map(self._webkit_baseline_path, self.FALLBACK_PATHS[self.version()]) 299 300 @memoized 301 def _compare_baseline(self): 302 factory = PortFactory(self.host) 303 target_port = self.get_option('compare_port') 304 if target_port: 305 return factory.get(target_port).default_baseline_search_path() 306 return [] 307 308 def _check_file_exists(self, path_to_file, file_description, 309 override_step=None, logging=True): 310 """Verify the file is present where expected or log an error. 311 312 Args: 313 file_name: The (human friendly) name or description of the file 314 you're looking for (e.g., "HTTP Server"). Used for error logging. 315 override_step: An optional string to be logged if the check fails. 316 logging: Whether or not log the error messages.""" 317 if not self._filesystem.exists(path_to_file): 318 if logging: 319 _log.error('Unable to find %s' % file_description) 320 _log.error(' at %s' % path_to_file) 321 if override_step: 322 _log.error(' %s' % override_step) 323 _log.error('') 324 return False 325 return True 326 327 def check_build(self, needs_http, printer): 328 result = True 329 330 dump_render_tree_binary_path = self._path_to_driver() 331 result = self._check_file_exists(dump_render_tree_binary_path, 332 'test driver') and result 333 if not result and self.get_option('build'): 334 result = self._check_driver_build_up_to_date( 335 self.get_option('configuration')) 336 else: 337 _log.error('') 338 339 helper_path = self._path_to_helper() 340 if helper_path: 341 result = self._check_file_exists(helper_path, 342 'layout test helper') and result 343 344 if self.get_option('pixel_tests'): 345 result = self.check_image_diff( 346 'To override, invoke with --no-pixel-tests') and result 347 348 # It's okay if pretty patch and wdiff aren't available, but we will at least log messages. 349 self._pretty_patch_available = self.check_pretty_patch() 350 self._wdiff_available = self.check_wdiff() 351 352 if self._dump_reader: 353 result = self._dump_reader.check_is_functional() and result 354 355 if needs_http: 356 result = self.check_httpd() and result 357 358 return test_run_results.OK_EXIT_STATUS if result else test_run_results.UNEXPECTED_ERROR_EXIT_STATUS 359 360 def _check_driver(self): 361 driver_path = self._path_to_driver() 362 if not self._filesystem.exists(driver_path): 363 _log.error("%s was not found at %s" % (self.driver_name(), driver_path)) 364 return False 365 return True 366 367 def _check_port_build(self): 368 # Ports can override this method to do additional checks. 369 return True 370 371 def check_sys_deps(self, needs_http): 372 """If the port needs to do some runtime checks to ensure that the 373 tests can be run successfully, it should override this routine. 374 This step can be skipped with --nocheck-sys-deps. 375 376 Returns whether the system is properly configured.""" 377 cmd = [self._path_to_driver(), '--check-layout-test-sys-deps'] 378 379 local_error = ScriptError() 380 381 def error_handler(script_error): 382 local_error.exit_code = script_error.exit_code 383 384 output = self._executive.run_command(cmd, error_handler=error_handler) 385 if local_error.exit_code: 386 _log.error('System dependencies check failed.') 387 _log.error('To override, invoke with --nocheck-sys-deps') 388 _log.error('') 389 _log.error(output) 390 if self.BUILD_REQUIREMENTS_URL is not '': 391 _log.error('') 392 _log.error('For complete build requirements, please see:') 393 _log.error(self.BUILD_REQUIREMENTS_URL) 394 return test_run_results.SYS_DEPS_EXIT_STATUS 395 return test_run_results.OK_EXIT_STATUS 396 397 def check_image_diff(self, override_step=None, logging=True): 398 """This routine is used to check whether image_diff binary exists.""" 399 image_diff_path = self._path_to_image_diff() 400 if not self._filesystem.exists(image_diff_path): 401 _log.error("image_diff was not found at %s" % image_diff_path) 402 return False 403 return True 404 405 def check_pretty_patch(self, logging=True): 406 """Checks whether we can use the PrettyPatch ruby script.""" 407 try: 408 _ = self._executive.run_command(['ruby', '--version']) 409 except OSError, e: 410 if e.errno in [errno.ENOENT, errno.EACCES, errno.ECHILD]: 411 if logging: 412 _log.warning("Ruby is not installed; can't generate pretty patches.") 413 _log.warning('') 414 return False 415 416 if not self._filesystem.exists(self._pretty_patch_path): 417 if logging: 418 _log.warning("Unable to find %s; can't generate pretty patches." % self._pretty_patch_path) 419 _log.warning('') 420 return False 421 422 return True 423 424 def check_wdiff(self, logging=True): 425 if not self._path_to_wdiff(): 426 # Don't need to log here since this is the port choosing not to use wdiff. 427 return False 428 429 try: 430 _ = self._executive.run_command([self._path_to_wdiff(), '--help']) 431 except OSError: 432 if logging: 433 message = self._wdiff_missing_message() 434 if message: 435 for line in message.splitlines(): 436 _log.warning(' ' + line) 437 _log.warning('') 438 return False 439 440 return True 441 442 def _wdiff_missing_message(self): 443 return 'wdiff is not installed; please install it to generate word-by-word diffs.' 444 445 def check_httpd(self): 446 httpd_path = self.path_to_apache() 447 try: 448 server_name = self._filesystem.basename(httpd_path) 449 env = self.setup_environ_for_server(server_name) 450 if self._executive.run_command([httpd_path, "-v"], env=env, return_exit_code=True) != 0: 451 _log.error("httpd seems broken. Cannot run http tests.") 452 return False 453 return True 454 except OSError: 455 _log.error("No httpd found. Cannot run http tests.") 456 return False 457 458 def do_text_results_differ(self, expected_text, actual_text): 459 return expected_text != actual_text 460 461 def do_audio_results_differ(self, expected_audio, actual_audio): 462 return expected_audio != actual_audio 463 464 def diff_image(self, expected_contents, actual_contents): 465 """Compare two images and return a tuple of an image diff, and an error string. 466 467 If an error occurs (like image_diff isn't found, or crashes, we log an error and return True (for a diff). 468 """ 469 # If only one of them exists, return that one. 470 if not actual_contents and not expected_contents: 471 return (None, None) 472 if not actual_contents: 473 return (expected_contents, None) 474 if not expected_contents: 475 return (actual_contents, None) 476 477 tempdir = self._filesystem.mkdtemp() 478 479 expected_filename = self._filesystem.join(str(tempdir), "expected.png") 480 self._filesystem.write_binary_file(expected_filename, expected_contents) 481 482 actual_filename = self._filesystem.join(str(tempdir), "actual.png") 483 self._filesystem.write_binary_file(actual_filename, actual_contents) 484 485 diff_filename = self._filesystem.join(str(tempdir), "diff.png") 486 487 # image_diff needs native win paths as arguments, so we need to convert them if running under cygwin. 488 native_expected_filename = self._convert_path(expected_filename) 489 native_actual_filename = self._convert_path(actual_filename) 490 native_diff_filename = self._convert_path(diff_filename) 491 492 executable = self._path_to_image_diff() 493 # Note that although we are handed 'old', 'new', image_diff wants 'new', 'old'. 494 comand = [executable, '--diff', native_actual_filename, native_expected_filename, native_diff_filename] 495 496 result = None 497 err_str = None 498 try: 499 exit_code = self._executive.run_command(comand, return_exit_code=True) 500 if exit_code == 0: 501 # The images are the same. 502 result = None 503 elif exit_code == 1: 504 result = self._filesystem.read_binary_file(native_diff_filename) 505 else: 506 err_str = "Image diff returned an exit code of %s. See http://crbug.com/278596" % exit_code 507 except OSError, e: 508 err_str = 'error running image diff: %s' % str(e) 509 finally: 510 self._filesystem.rmtree(str(tempdir)) 511 512 return (result, err_str or None) 513 514 def diff_text(self, expected_text, actual_text, expected_filename, actual_filename): 515 """Returns a string containing the diff of the two text strings 516 in 'unified diff' format.""" 517 518 # The filenames show up in the diff output, make sure they're 519 # raw bytes and not unicode, so that they don't trigger join() 520 # trying to decode the input. 521 def to_raw_bytes(string_value): 522 if isinstance(string_value, unicode): 523 return string_value.encode('utf-8') 524 return string_value 525 expected_filename = to_raw_bytes(expected_filename) 526 actual_filename = to_raw_bytes(actual_filename) 527 diff = difflib.unified_diff(expected_text.splitlines(True), 528 actual_text.splitlines(True), 529 expected_filename, 530 actual_filename) 531 532 # The diff generated by the difflib is incorrect if one of the files 533 # does not have a newline at the end of the file and it is present in 534 # the diff. Relevant Python issue: http://bugs.python.org/issue2142 535 def diff_fixup(diff): 536 for line in diff: 537 yield line 538 if not line.endswith('\n'): 539 yield '\n\ No newline at end of file\n' 540 541 return ''.join(diff_fixup(diff)) 542 543 def driver_name(self): 544 if self.get_option('driver_name'): 545 return self.get_option('driver_name') 546 return self.CONTENT_SHELL_NAME 547 548 def expected_baselines_by_extension(self, test_name): 549 """Returns a dict mapping baseline suffix to relative path for each baseline in 550 a test. For reftests, it returns ".==" or ".!=" instead of the suffix.""" 551 # FIXME: The name similarity between this and expected_baselines() below, is unfortunate. 552 # We should probably rename them both. 553 baseline_dict = {} 554 reference_files = self.reference_files(test_name) 555 if reference_files: 556 # FIXME: How should this handle more than one type of reftest? 557 baseline_dict['.' + reference_files[0][0]] = self.relative_test_filename(reference_files[0][1]) 558 559 for extension in self.baseline_extensions(): 560 path = self.expected_filename(test_name, extension, return_default=False) 561 baseline_dict[extension] = self.relative_test_filename(path) if path else path 562 563 return baseline_dict 564 565 def baseline_extensions(self): 566 """Returns a tuple of all of the non-reftest baseline extensions we use. The extensions include the leading '.'.""" 567 return ('.wav', '.txt', '.png') 568 569 def expected_baselines(self, test_name, suffix, all_baselines=False): 570 """Given a test name, finds where the baseline results are located. 571 572 Args: 573 test_name: name of test file (usually a relative path under LayoutTests/) 574 suffix: file suffix of the expected results, including dot; e.g. 575 '.txt' or '.png'. This should not be None, but may be an empty 576 string. 577 all_baselines: If True, return an ordered list of all baseline paths 578 for the given platform. If False, return only the first one. 579 Returns 580 a list of ( platform_dir, results_filename ), where 581 platform_dir - abs path to the top of the results tree (or test 582 tree) 583 results_filename - relative path from top of tree to the results 584 file 585 (port.join() of the two gives you the full path to the file, 586 unless None was returned.) 587 Return values will be in the format appropriate for the current 588 platform (e.g., "\\" for path separators on Windows). If the results 589 file is not found, then None will be returned for the directory, 590 but the expected relative pathname will still be returned. 591 592 This routine is generic but lives here since it is used in 593 conjunction with the other baseline and filename routines that are 594 platform specific. 595 """ 596 baseline_filename = self._filesystem.splitext(test_name)[0] + '-expected' + suffix 597 baseline_search_path = self.baseline_search_path() 598 599 baselines = [] 600 for platform_dir in baseline_search_path: 601 if self._filesystem.exists(self._filesystem.join(platform_dir, baseline_filename)): 602 baselines.append((platform_dir, baseline_filename)) 603 604 if not all_baselines and baselines: 605 return baselines 606 607 # If it wasn't found in a platform directory, return the expected 608 # result in the test directory, even if no such file actually exists. 609 platform_dir = self.layout_tests_dir() 610 if self._filesystem.exists(self._filesystem.join(platform_dir, baseline_filename)): 611 baselines.append((platform_dir, baseline_filename)) 612 613 if baselines: 614 return baselines 615 616 return [(None, baseline_filename)] 617 618 def expected_filename(self, test_name, suffix, return_default=True): 619 """Given a test name, returns an absolute path to its expected results. 620 621 If no expected results are found in any of the searched directories, 622 the directory in which the test itself is located will be returned. 623 The return value is in the format appropriate for the platform 624 (e.g., "\\" for path separators on windows). 625 626 Args: 627 test_name: name of test file (usually a relative path under LayoutTests/) 628 suffix: file suffix of the expected results, including dot; e.g. '.txt' 629 or '.png'. This should not be None, but may be an empty string. 630 platform: the most-specific directory name to use to build the 631 search list of directories, e.g., 'win', or 632 'chromium-cg-mac-leopard' (we follow the WebKit format) 633 return_default: if True, returns the path to the generic expectation if nothing 634 else is found; if False, returns None. 635 636 This routine is generic but is implemented here to live alongside 637 the other baseline and filename manipulation routines. 638 """ 639 # FIXME: The [0] here is very mysterious, as is the destructured return. 640 platform_dir, baseline_filename = self.expected_baselines(test_name, suffix)[0] 641 if platform_dir: 642 return self._filesystem.join(platform_dir, baseline_filename) 643 644 actual_test_name = self.lookup_virtual_test_base(test_name) 645 if actual_test_name: 646 return self.expected_filename(actual_test_name, suffix) 647 648 if return_default: 649 return self._filesystem.join(self.layout_tests_dir(), baseline_filename) 650 return None 651 652 def expected_checksum(self, test_name): 653 """Returns the checksum of the image we expect the test to produce, or None if it is a text-only test.""" 654 png_path = self.expected_filename(test_name, '.png') 655 656 if self._filesystem.exists(png_path): 657 with self._filesystem.open_binary_file_for_reading(png_path) as filehandle: 658 return read_checksum_from_png.read_checksum(filehandle) 659 660 return None 661 662 def expected_image(self, test_name): 663 """Returns the image we expect the test to produce.""" 664 baseline_path = self.expected_filename(test_name, '.png') 665 if not self._filesystem.exists(baseline_path): 666 return None 667 return self._filesystem.read_binary_file(baseline_path) 668 669 def expected_audio(self, test_name): 670 baseline_path = self.expected_filename(test_name, '.wav') 671 if not self._filesystem.exists(baseline_path): 672 return None 673 return self._filesystem.read_binary_file(baseline_path) 674 675 def expected_text(self, test_name): 676 """Returns the text output we expect the test to produce, or None 677 if we don't expect there to be any text output. 678 End-of-line characters are normalized to '\n'.""" 679 # FIXME: DRT output is actually utf-8, but since we don't decode the 680 # output from DRT (instead treating it as a binary string), we read the 681 # baselines as a binary string, too. 682 baseline_path = self.expected_filename(test_name, '.txt') 683 if not self._filesystem.exists(baseline_path): 684 return None 685 text = self._filesystem.read_binary_file(baseline_path) 686 return text.replace("\r\n", "\n") 687 688 def _get_reftest_list(self, test_name): 689 dirname = self._filesystem.join(self.layout_tests_dir(), self._filesystem.dirname(test_name)) 690 if dirname not in self._reftest_list: 691 self._reftest_list[dirname] = Port._parse_reftest_list(self._filesystem, dirname) 692 return self._reftest_list[dirname] 693 694 @staticmethod 695 def _parse_reftest_list(filesystem, test_dirpath): 696 reftest_list_path = filesystem.join(test_dirpath, 'reftest.list') 697 if not filesystem.isfile(reftest_list_path): 698 return None 699 reftest_list_file = filesystem.read_text_file(reftest_list_path) 700 701 parsed_list = {} 702 for line in reftest_list_file.split('\n'): 703 line = re.sub('#.+$', '', line) 704 split_line = line.split() 705 if len(split_line) == 4: 706 # FIXME: Probably one of mozilla's extensions in the reftest.list format. Do we need to support this? 707 _log.warning("unsupported reftest.list line '%s' in %s" % (line, reftest_list_path)) 708 continue 709 if len(split_line) < 3: 710 continue 711 expectation_type, test_file, ref_file = split_line 712 parsed_list.setdefault(filesystem.join(test_dirpath, test_file), []).append((expectation_type, filesystem.join(test_dirpath, ref_file))) 713 return parsed_list 714 715 def reference_files(self, test_name): 716 """Return a list of expectation (== or !=) and filename pairs""" 717 718 reftest_list = self._get_reftest_list(test_name) 719 if not reftest_list: 720 reftest_list = [] 721 for expectation, prefix in (('==', ''), ('!=', '-mismatch')): 722 for extention in Port._supported_file_extensions: 723 path = self.expected_filename(test_name, prefix + extention) 724 if self._filesystem.exists(path): 725 reftest_list.append((expectation, path)) 726 return reftest_list 727 728 return reftest_list.get(self._filesystem.join(self.layout_tests_dir(), test_name), []) # pylint: disable=E1103 729 730 def tests(self, paths): 731 """Return the list of tests found matching paths.""" 732 tests = self._real_tests(paths) 733 734 suites = self.virtual_test_suites() 735 if paths: 736 tests.extend(self._virtual_tests_matching_paths(paths, suites)) 737 else: 738 tests.extend(self._all_virtual_tests(suites)) 739 return tests 740 741 def _real_tests(self, paths): 742 # When collecting test cases, skip these directories 743 skipped_directories = set(['.svn', '_svn', 'platform', 'resources', 'support', 'script-tests', 'reference', 'reftest']) 744 files = find_files.find(self._filesystem, self.layout_tests_dir(), paths, skipped_directories, Port.is_test_file, self.test_key) 745 return [self.relative_test_filename(f) for f in files] 746 747 # When collecting test cases, we include any file with these extensions. 748 _supported_file_extensions = set(['.html', '.xml', '.xhtml', '.xht', '.pl', 749 '.htm', '.php', '.svg', '.mht', '.pdf']) 750 751 @staticmethod 752 # If any changes are made here be sure to update the isUsedInReftest method in old-run-webkit-tests as well. 753 def is_reference_html_file(filesystem, dirname, filename): 754 if filename.startswith('ref-') or filename.startswith('notref-'): 755 return True 756 filename_wihout_ext, unused = filesystem.splitext(filename) 757 for suffix in ['-expected', '-expected-mismatch', '-ref', '-notref']: 758 if filename_wihout_ext.endswith(suffix): 759 return True 760 return False 761 762 @staticmethod 763 def _has_supported_extension(filesystem, filename): 764 """Return true if filename is one of the file extensions we want to run a test on.""" 765 extension = filesystem.splitext(filename)[1] 766 return extension in Port._supported_file_extensions 767 768 @staticmethod 769 def is_test_file(filesystem, dirname, filename): 770 return Port._has_supported_extension(filesystem, filename) and not Port.is_reference_html_file(filesystem, dirname, filename) 771 772 ALL_TEST_TYPES = ['audio', 'harness', 'pixel', 'ref', 'text', 'unknown'] 773 774 def test_type(self, test_name): 775 fs = self._filesystem 776 if fs.exists(self.expected_filename(test_name, '.png')): 777 return 'pixel' 778 if fs.exists(self.expected_filename(test_name, '.wav')): 779 return 'audio' 780 if self.reference_files(test_name): 781 return 'ref' 782 txt = self.expected_text(test_name) 783 if txt: 784 if 'layer at (0,0) size 800x600' in txt: 785 return 'pixel' 786 for line in txt.splitlines(): 787 if line.startswith('FAIL') or line.startswith('TIMEOUT') or line.startswith('PASS'): 788 return 'harness' 789 return 'text' 790 return 'unknown' 791 792 def test_key(self, test_name): 793 """Turns a test name into a list with two sublists, the natural key of the 794 dirname, and the natural key of the basename. 795 796 This can be used when sorting paths so that files in a directory. 797 directory are kept together rather than being mixed in with files in 798 subdirectories.""" 799 dirname, basename = self.split_test(test_name) 800 return (self._natural_sort_key(dirname + self.TEST_PATH_SEPARATOR), self._natural_sort_key(basename)) 801 802 def _natural_sort_key(self, string_to_split): 803 """ Turns a string into a list of string and number chunks, i.e. "z23a" -> ["z", 23, "a"] 804 805 This can be used to implement "natural sort" order. See: 806 http://www.codinghorror.com/blog/2007/12/sorting-for-humans-natural-sort-order.html 807 http://nedbatchelder.com/blog/200712.html#e20071211T054956 808 """ 809 def tryint(val): 810 try: 811 return int(val) 812 except ValueError: 813 return val 814 815 return [tryint(chunk) for chunk in re.split('(\d+)', string_to_split)] 816 817 def test_dirs(self): 818 """Returns the list of top-level test directories.""" 819 layout_tests_dir = self.layout_tests_dir() 820 return filter(lambda x: self._filesystem.isdir(self._filesystem.join(layout_tests_dir, x)), 821 self._filesystem.listdir(layout_tests_dir)) 822 823 @memoized 824 def test_isfile(self, test_name): 825 """Return True if the test name refers to a directory of tests.""" 826 # Used by test_expectations.py to apply rules to whole directories. 827 if self._filesystem.isfile(self.abspath_for_test(test_name)): 828 return True 829 base = self.lookup_virtual_test_base(test_name) 830 return base and self._filesystem.isfile(self.abspath_for_test(base)) 831 832 @memoized 833 def test_isdir(self, test_name): 834 """Return True if the test name refers to a directory of tests.""" 835 # Used by test_expectations.py to apply rules to whole directories. 836 if self._filesystem.isdir(self.abspath_for_test(test_name)): 837 return True 838 base = self.lookup_virtual_test_base(test_name) 839 return base and self._filesystem.isdir(self.abspath_for_test(base)) 840 841 @memoized 842 def test_exists(self, test_name): 843 """Return True if the test name refers to an existing test or baseline.""" 844 # Used by test_expectations.py to determine if an entry refers to a 845 # valid test and by printing.py to determine if baselines exist. 846 return self.test_isfile(test_name) or self.test_isdir(test_name) 847 848 def split_test(self, test_name): 849 """Splits a test name into the 'directory' part and the 'basename' part.""" 850 index = test_name.rfind(self.TEST_PATH_SEPARATOR) 851 if index < 1: 852 return ('', test_name) 853 return (test_name[0:index], test_name[index:]) 854 855 def normalize_test_name(self, test_name): 856 """Returns a normalized version of the test name or test directory.""" 857 if test_name.endswith('/'): 858 return test_name 859 if self.test_isdir(test_name): 860 return test_name + '/' 861 return test_name 862 863 def driver_cmd_line(self): 864 """Prints the DRT command line that will be used.""" 865 driver = self.create_driver(0) 866 return driver.cmd_line(self.get_option('pixel_tests'), []) 867 868 def update_baseline(self, baseline_path, data): 869 """Updates the baseline for a test. 870 871 Args: 872 baseline_path: the actual path to use for baseline, not the path to 873 the test. This function is used to update either generic or 874 platform-specific baselines, but we can't infer which here. 875 data: contents of the baseline. 876 """ 877 self._filesystem.write_binary_file(baseline_path, data) 878 879 # FIXME: update callers to create a finder and call it instead of these next five routines (which should be protected). 880 def webkit_base(self): 881 return self._webkit_finder.webkit_base() 882 883 def path_from_webkit_base(self, *comps): 884 return self._webkit_finder.path_from_webkit_base(*comps) 885 886 def path_from_chromium_base(self, *comps): 887 return self._webkit_finder.path_from_chromium_base(*comps) 888 889 def path_to_script(self, script_name): 890 return self._webkit_finder.path_to_script(script_name) 891 892 def layout_tests_dir(self): 893 return self._webkit_finder.layout_tests_dir() 894 895 def perf_tests_dir(self): 896 return self._webkit_finder.perf_tests_dir() 897 898 def skipped_layout_tests(self, test_list): 899 """Returns tests skipped outside of the TestExpectations files.""" 900 tests = set(self._skipped_tests_for_unsupported_features(test_list)) 901 902 # We explicitly skip any tests in LayoutTests/w3c if need be to avoid running any tests 903 # left over from the old DEPS-pulled repos. 904 # We also will warn at the end of the test run if these directories still exist. 905 # 906 # TODO(dpranke): Remove this check after 1/1/2015 and let people deal with the warnings. 907 # Remove the check in controllers/manager.py as well. 908 if self._filesystem.isdir(self._filesystem.join(self.layout_tests_dir(), 'w3c')): 909 tests.add('w3c') 910 911 return tests 912 913 def _tests_from_skipped_file_contents(self, skipped_file_contents): 914 tests_to_skip = [] 915 for line in skipped_file_contents.split('\n'): 916 line = line.strip() 917 line = line.rstrip('/') # Best to normalize directory names to not include the trailing slash. 918 if line.startswith('#') or not len(line): 919 continue 920 tests_to_skip.append(line) 921 return tests_to_skip 922 923 def _expectations_from_skipped_files(self, skipped_file_paths): 924 tests_to_skip = [] 925 for search_path in skipped_file_paths: 926 filename = self._filesystem.join(self._webkit_baseline_path(search_path), "Skipped") 927 if not self._filesystem.exists(filename): 928 _log.debug("Skipped does not exist: %s" % filename) 929 continue 930 _log.debug("Using Skipped file: %s" % filename) 931 skipped_file_contents = self._filesystem.read_text_file(filename) 932 tests_to_skip.extend(self._tests_from_skipped_file_contents(skipped_file_contents)) 933 return tests_to_skip 934 935 @memoized 936 def skipped_perf_tests(self): 937 return self._expectations_from_skipped_files([self.perf_tests_dir()]) 938 939 def skips_perf_test(self, test_name): 940 for test_or_category in self.skipped_perf_tests(): 941 if test_or_category == test_name: 942 return True 943 category = self._filesystem.join(self.perf_tests_dir(), test_or_category) 944 if self._filesystem.isdir(category) and test_name.startswith(test_or_category): 945 return True 946 return False 947 948 def is_chromium(self): 949 return True 950 951 def name(self): 952 """Returns a name that uniquely identifies this particular type of port 953 (e.g., "mac-snowleopard" or "linux-x86_x64" and can be passed 954 to factory.get() to instantiate the port.""" 955 return self._name 956 957 def operating_system(self): 958 # Subclasses should override this default implementation. 959 return 'mac' 960 961 def version(self): 962 """Returns a string indicating the version of a given platform, e.g. 963 'leopard' or 'xp'. 964 965 This is used to help identify the exact port when parsing test 966 expectations, determining search paths, and logging information.""" 967 return self._version 968 969 def architecture(self): 970 return self._architecture 971 972 def get_option(self, name, default_value=None): 973 return getattr(self._options, name, default_value) 974 975 def set_option_default(self, name, default_value): 976 return self._options.ensure_value(name, default_value) 977 978 @memoized 979 def path_to_generic_test_expectations_file(self): 980 return self._filesystem.join(self.layout_tests_dir(), 'TestExpectations') 981 982 def relative_test_filename(self, filename): 983 """Returns a test_name a relative unix-style path for a filename under the LayoutTests 984 directory. Ports may legitimately return abspaths here if no relpath makes sense.""" 985 # Ports that run on windows need to override this method to deal with 986 # filenames with backslashes in them. 987 if filename.startswith(self.layout_tests_dir()): 988 return self.host.filesystem.relpath(filename, self.layout_tests_dir()) 989 else: 990 return self.host.filesystem.abspath(filename) 991 992 @memoized 993 def abspath_for_test(self, test_name): 994 """Returns the full path to the file for a given test name. This is the 995 inverse of relative_test_filename().""" 996 return self._filesystem.join(self.layout_tests_dir(), test_name) 997 998 def results_directory(self): 999 """Absolute path to the place to store the test results (uses --results-directory).""" 1000 if not self._results_directory: 1001 option_val = self.get_option('results_directory') or self.default_results_directory() 1002 self._results_directory = self._filesystem.abspath(option_val) 1003 return self._results_directory 1004 1005 def perf_results_directory(self): 1006 return self._build_path() 1007 1008 def default_results_directory(self): 1009 """Absolute path to the default place to store the test results.""" 1010 try: 1011 return self.path_from_chromium_base('webkit', self.get_option('configuration'), 'layout-test-results') 1012 except AssertionError: 1013 return self._build_path('layout-test-results') 1014 1015 def setup_test_run(self): 1016 """Perform port-specific work at the beginning of a test run.""" 1017 # Delete the disk cache if any to ensure a clean test run. 1018 dump_render_tree_binary_path = self._path_to_driver() 1019 cachedir = self._filesystem.dirname(dump_render_tree_binary_path) 1020 cachedir = self._filesystem.join(cachedir, "cache") 1021 if self._filesystem.exists(cachedir): 1022 self._filesystem.rmtree(cachedir) 1023 1024 if self._dump_reader: 1025 self._filesystem.maybe_make_directory(self._dump_reader.crash_dumps_directory()) 1026 1027 def num_workers(self, requested_num_workers): 1028 """Returns the number of available workers (possibly less than the number requested).""" 1029 return requested_num_workers 1030 1031 def clean_up_test_run(self): 1032 """Perform port-specific work at the end of a test run.""" 1033 if self._image_differ: 1034 self._image_differ.stop() 1035 self._image_differ = None 1036 1037 # FIXME: os.environ access should be moved to onto a common/system class to be more easily mockable. 1038 def _value_or_default_from_environ(self, name, default=None): 1039 if name in os.environ: 1040 return os.environ[name] 1041 return default 1042 1043 def _copy_value_from_environ_if_set(self, clean_env, name): 1044 if name in os.environ: 1045 clean_env[name] = os.environ[name] 1046 1047 def setup_environ_for_server(self, server_name=None): 1048 # We intentionally copy only a subset of os.environ when 1049 # launching subprocesses to ensure consistent test results. 1050 clean_env = { 1051 'LOCAL_RESOURCE_ROOT': self.layout_tests_dir(), # FIXME: Is this used? 1052 } 1053 variables_to_copy = [ 1054 'WEBKIT_TESTFONTS', # FIXME: Is this still used? 1055 'WEBKITOUTPUTDIR', # FIXME: Is this still used? 1056 'CHROME_DEVEL_SANDBOX', 1057 'CHROME_IPC_LOGGING', 1058 'ASAN_OPTIONS', 1059 'VALGRIND_LIB', 1060 'VALGRIND_LIB_INNER', 1061 ] 1062 if self.host.platform.is_linux() or self.host.platform.is_freebsd(): 1063 variables_to_copy += [ 1064 'XAUTHORITY', 1065 'HOME', 1066 'LANG', 1067 'LD_LIBRARY_PATH', 1068 'DBUS_SESSION_BUS_ADDRESS', 1069 'XDG_DATA_DIRS', 1070 ] 1071 clean_env['DISPLAY'] = self._value_or_default_from_environ('DISPLAY', ':1') 1072 if self.host.platform.is_mac(): 1073 clean_env['DYLD_LIBRARY_PATH'] = self._build_path() 1074 clean_env['DYLD_FRAMEWORK_PATH'] = self._build_path() 1075 variables_to_copy += [ 1076 'HOME', 1077 ] 1078 if self.host.platform.is_win(): 1079 variables_to_copy += [ 1080 'PATH', 1081 'GYP_DEFINES', # Required to locate win sdk. 1082 ] 1083 if self.host.platform.is_cygwin(): 1084 variables_to_copy += [ 1085 'HOMEDRIVE', 1086 'HOMEPATH', 1087 '_NT_SYMBOL_PATH', 1088 ] 1089 1090 for variable in variables_to_copy: 1091 self._copy_value_from_environ_if_set(clean_env, variable) 1092 1093 for string_variable in self.get_option('additional_env_var', []): 1094 [name, value] = string_variable.split('=', 1) 1095 clean_env[name] = value 1096 1097 return clean_env 1098 1099 def show_results_html_file(self, results_filename): 1100 """This routine should display the HTML file pointed at by 1101 results_filename in a users' browser.""" 1102 return self.host.user.open_url(path.abspath_to_uri(self.host.platform, results_filename)) 1103 1104 def create_driver(self, worker_number, no_timeout=False): 1105 """Return a newly created Driver subclass for starting/stopping the test driver.""" 1106 return self._driver_class()(self, worker_number, pixel_tests=self.get_option('pixel_tests'), no_timeout=no_timeout) 1107 1108 def start_helper(self): 1109 """If a port needs to reconfigure graphics settings or do other 1110 things to ensure a known test configuration, it should override this 1111 method.""" 1112 helper_path = self._path_to_helper() 1113 if helper_path: 1114 _log.debug("Starting layout helper %s" % helper_path) 1115 # Note: Not thread safe: http://bugs.python.org/issue2320 1116 self._helper = self._executive.popen([helper_path], 1117 stdin=self._executive.PIPE, stdout=self._executive.PIPE, stderr=None) 1118 is_ready = self._helper.stdout.readline() 1119 if not is_ready.startswith('ready'): 1120 _log.error("layout_test_helper failed to be ready") 1121 1122 def requires_http_server(self): 1123 """Does the port require an HTTP server for running tests? This could 1124 be the case when the tests aren't run on the host platform.""" 1125 return False 1126 1127 def start_http_server(self, additional_dirs, number_of_drivers): 1128 """Start a web server. Raise an error if it can't start or is already running. 1129 1130 Ports can stub this out if they don't need a web server to be running.""" 1131 assert not self._http_server, 'Already running an http server.' 1132 1133 server = apache_http.ApacheHTTP(self, self.results_directory(), 1134 additional_dirs=additional_dirs, 1135 number_of_servers=(number_of_drivers * 4)) 1136 server.start() 1137 self._http_server = server 1138 1139 def start_websocket_server(self): 1140 """Start a web server. Raise an error if it can't start or is already running. 1141 1142 Ports can stub this out if they don't need a websocket server to be running.""" 1143 assert not self._websocket_server, 'Already running a websocket server.' 1144 1145 server = pywebsocket.PyWebSocket(self, self.results_directory()) 1146 server.start() 1147 self._websocket_server = server 1148 1149 def http_server_supports_ipv6(self): 1150 # Apache < 2.4 on win32 does not support IPv6, nor does cygwin apache. 1151 if self.host.platform.is_cygwin() or self.host.platform.is_win(): 1152 return False 1153 return True 1154 1155 def stop_helper(self): 1156 """Shut down the test helper if it is running. Do nothing if 1157 it isn't, or it isn't available. If a port overrides start_helper() 1158 it must override this routine as well.""" 1159 if self._helper: 1160 _log.debug("Stopping layout test helper") 1161 try: 1162 self._helper.stdin.write("x\n") 1163 self._helper.stdin.close() 1164 self._helper.wait() 1165 except IOError, e: 1166 pass 1167 finally: 1168 self._helper = None 1169 1170 def stop_http_server(self): 1171 """Shut down the http server if it is running. Do nothing if it isn't.""" 1172 if self._http_server: 1173 self._http_server.stop() 1174 self._http_server = None 1175 1176 def stop_websocket_server(self): 1177 """Shut down the websocket server if it is running. Do nothing if it isn't.""" 1178 if self._websocket_server: 1179 self._websocket_server.stop() 1180 self._websocket_server = None 1181 1182 # 1183 # TEST EXPECTATION-RELATED METHODS 1184 # 1185 1186 def test_configuration(self): 1187 """Returns the current TestConfiguration for the port.""" 1188 if not self._test_configuration: 1189 self._test_configuration = TestConfiguration(self._version, self._architecture, self._options.configuration.lower()) 1190 return self._test_configuration 1191 1192 # FIXME: Belongs on a Platform object. 1193 @memoized 1194 def all_test_configurations(self): 1195 """Returns a list of TestConfiguration instances, representing all available 1196 test configurations for this port.""" 1197 return self._generate_all_test_configurations() 1198 1199 # FIXME: Belongs on a Platform object. 1200 def configuration_specifier_macros(self): 1201 """Ports may provide a way to abbreviate configuration specifiers to conveniently 1202 refer to them as one term or alias specific values to more generic ones. For example: 1203 1204 (xp, vista, win7) -> win # Abbreviate all Windows versions into one namesake. 1205 (lucid) -> linux # Change specific name of the Linux distro to a more generic term. 1206 1207 Returns a dictionary, each key representing a macro term ('win', for example), 1208 and value being a list of valid configuration specifiers (such as ['xp', 'vista', 'win7']).""" 1209 return self.CONFIGURATION_SPECIFIER_MACROS 1210 1211 def all_baseline_variants(self): 1212 """Returns a list of platform names sufficient to cover all the baselines. 1213 1214 The list should be sorted so that a later platform will reuse 1215 an earlier platform's baselines if they are the same (e.g., 1216 'snowleopard' should precede 'leopard').""" 1217 return self.ALL_BASELINE_VARIANTS 1218 1219 def _generate_all_test_configurations(self): 1220 """Returns a sequence of the TestConfigurations the port supports.""" 1221 # By default, we assume we want to test every graphics type in 1222 # every configuration on every system. 1223 test_configurations = [] 1224 for version, architecture in self.ALL_SYSTEMS: 1225 for build_type in self.ALL_BUILD_TYPES: 1226 test_configurations.append(TestConfiguration(version, architecture, build_type)) 1227 return test_configurations 1228 1229 try_builder_names = frozenset([ 1230 'linux_layout', 1231 'mac_layout', 1232 'win_layout', 1233 'linux_layout_rel', 1234 'mac_layout_rel', 1235 'win_layout_rel', 1236 ]) 1237 1238 def warn_if_bug_missing_in_test_expectations(self): 1239 return True 1240 1241 def _port_specific_expectations_files(self): 1242 paths = [] 1243 paths.append(self.path_from_chromium_base('skia', 'skia_test_expectations.txt')) 1244 paths.append(self._filesystem.join(self.layout_tests_dir(), 'NeverFixTests')) 1245 paths.append(self._filesystem.join(self.layout_tests_dir(), 'StaleTestExpectations')) 1246 paths.append(self._filesystem.join(self.layout_tests_dir(), 'SlowTests')) 1247 paths.append(self._filesystem.join(self.layout_tests_dir(), 'FlakyTests')) 1248 1249 return paths 1250 1251 def expectations_dict(self): 1252 """Returns an OrderedDict of name -> expectations strings. 1253 The names are expected to be (but not required to be) paths in the filesystem. 1254 If the name is a path, the file can be considered updatable for things like rebaselining, 1255 so don't use names that are paths if they're not paths. 1256 Generally speaking the ordering should be files in the filesystem in cascade order 1257 (TestExpectations followed by Skipped, if the port honors both formats), 1258 then any built-in expectations (e.g., from compile-time exclusions), then --additional-expectations options.""" 1259 # FIXME: rename this to test_expectations() once all the callers are updated to know about the ordered dict. 1260 expectations = OrderedDict() 1261 1262 for path in self.expectations_files(): 1263 if self._filesystem.exists(path): 1264 expectations[path] = self._filesystem.read_text_file(path) 1265 1266 for path in self.get_option('additional_expectations', []): 1267 expanded_path = self._filesystem.expanduser(path) 1268 if self._filesystem.exists(expanded_path): 1269 _log.debug("reading additional_expectations from path '%s'" % path) 1270 expectations[path] = self._filesystem.read_text_file(expanded_path) 1271 else: 1272 _log.warning("additional_expectations path '%s' does not exist" % path) 1273 return expectations 1274 1275 def bot_expectations(self): 1276 if not self.get_option('ignore_flaky_tests'): 1277 return {} 1278 1279 full_port_name = self.determine_full_port_name(self.host, self._options, self.port_name) 1280 builder_category = self.get_option('ignore_builder_category', 'layout') 1281 factory = BotTestExpectationsFactory() 1282 # FIXME: This only grabs release builder's flakiness data. If we're running debug, 1283 # when we should grab the debug builder's data. 1284 expectations = factory.expectations_for_port(full_port_name, builder_category) 1285 1286 if not expectations: 1287 return {} 1288 1289 ignore_mode = self.get_option('ignore_flaky_tests') 1290 if ignore_mode == 'very-flaky' or ignore_mode == 'maybe-flaky': 1291 return expectations.flakes_by_path(ignore_mode == 'very-flaky') 1292 if ignore_mode == 'unexpected': 1293 return expectations.unexpected_results_by_path() 1294 _log.warning("Unexpected ignore mode: '%s'." % ignore_mode) 1295 return {} 1296 1297 def expectations_files(self): 1298 return [self.path_to_generic_test_expectations_file()] + self._port_specific_expectations_files() 1299 1300 def repository_paths(self): 1301 """Returns a list of (repository_name, repository_path) tuples of its depending code base.""" 1302 return [('blink', self.layout_tests_dir()), 1303 ('chromium', self.path_from_chromium_base('build'))] 1304 1305 _WDIFF_DEL = '##WDIFF_DEL##' 1306 _WDIFF_ADD = '##WDIFF_ADD##' 1307 _WDIFF_END = '##WDIFF_END##' 1308 1309 def _format_wdiff_output_as_html(self, wdiff): 1310 wdiff = cgi.escape(wdiff) 1311 wdiff = wdiff.replace(self._WDIFF_DEL, "<span class=del>") 1312 wdiff = wdiff.replace(self._WDIFF_ADD, "<span class=add>") 1313 wdiff = wdiff.replace(self._WDIFF_END, "</span>") 1314 html = "<head><style>.del { background: #faa; } " 1315 html += ".add { background: #afa; }</style></head>" 1316 html += "<pre>%s</pre>" % wdiff 1317 return html 1318 1319 def _wdiff_command(self, actual_filename, expected_filename): 1320 executable = self._path_to_wdiff() 1321 return [executable, 1322 "--start-delete=%s" % self._WDIFF_DEL, 1323 "--end-delete=%s" % self._WDIFF_END, 1324 "--start-insert=%s" % self._WDIFF_ADD, 1325 "--end-insert=%s" % self._WDIFF_END, 1326 actual_filename, 1327 expected_filename] 1328 1329 @staticmethod 1330 def _handle_wdiff_error(script_error): 1331 # Exit 1 means the files differed, any other exit code is an error. 1332 if script_error.exit_code != 1: 1333 raise script_error 1334 1335 def _run_wdiff(self, actual_filename, expected_filename): 1336 """Runs wdiff and may throw exceptions. 1337 This is mostly a hook for unit testing.""" 1338 # Diffs are treated as binary as they may include multiple files 1339 # with conflicting encodings. Thus we do not decode the output. 1340 command = self._wdiff_command(actual_filename, expected_filename) 1341 wdiff = self._executive.run_command(command, decode_output=False, 1342 error_handler=self._handle_wdiff_error) 1343 return self._format_wdiff_output_as_html(wdiff) 1344 1345 _wdiff_error_html = "Failed to run wdiff, see error log." 1346 1347 def wdiff_text(self, actual_filename, expected_filename): 1348 """Returns a string of HTML indicating the word-level diff of the 1349 contents of the two filenames. Returns an empty string if word-level 1350 diffing isn't available.""" 1351 if not self.wdiff_available(): 1352 return "" 1353 try: 1354 # It's possible to raise a ScriptError we pass wdiff invalid paths. 1355 return self._run_wdiff(actual_filename, expected_filename) 1356 except OSError as e: 1357 if e.errno in [errno.ENOENT, errno.EACCES, errno.ECHILD]: 1358 # Silently ignore cases where wdiff is missing. 1359 self._wdiff_available = False 1360 return "" 1361 raise 1362 except ScriptError as e: 1363 _log.error("Failed to run wdiff: %s" % e) 1364 self._wdiff_available = False 1365 return self._wdiff_error_html 1366 1367 # This is a class variable so we can test error output easily. 1368 _pretty_patch_error_html = "Failed to run PrettyPatch, see error log." 1369 1370 def pretty_patch_text(self, diff_path): 1371 if self._pretty_patch_available is None: 1372 self._pretty_patch_available = self.check_pretty_patch(logging=False) 1373 if not self._pretty_patch_available: 1374 return self._pretty_patch_error_html 1375 command = ("ruby", "-I", self._filesystem.dirname(self._pretty_patch_path), 1376 self._pretty_patch_path, diff_path) 1377 try: 1378 # Diffs are treated as binary (we pass decode_output=False) as they 1379 # may contain multiple files of conflicting encodings. 1380 return self._executive.run_command(command, decode_output=False) 1381 except OSError, e: 1382 # If the system is missing ruby log the error and stop trying. 1383 self._pretty_patch_available = False 1384 _log.error("Failed to run PrettyPatch (%s): %s" % (command, e)) 1385 return self._pretty_patch_error_html 1386 except ScriptError, e: 1387 # If ruby failed to run for some reason, log the command 1388 # output and stop trying. 1389 self._pretty_patch_available = False 1390 _log.error("Failed to run PrettyPatch (%s):\n%s" % (command, e.message_with_output())) 1391 return self._pretty_patch_error_html 1392 1393 def default_configuration(self): 1394 return self._config.default_configuration() 1395 1396 def clobber_old_port_specific_results(self): 1397 pass 1398 1399 # FIXME: This does not belong on the port object. 1400 @memoized 1401 def path_to_apache(self): 1402 """Returns the full path to the apache binary. 1403 1404 This is needed only by ports that use the apache_http_server module.""" 1405 raise NotImplementedError('Port.path_to_apache') 1406 1407 def path_to_apache_config_file(self): 1408 """Returns the full path to the apache configuration file. 1409 1410 If the WEBKIT_HTTP_SERVER_CONF_PATH environment variable is set, its 1411 contents will be used instead. 1412 1413 This is needed only by ports that use the apache_http_server module.""" 1414 config_file_from_env = os.environ.get('WEBKIT_HTTP_SERVER_CONF_PATH') 1415 if config_file_from_env: 1416 if not self._filesystem.exists(config_file_from_env): 1417 raise IOError('%s was not found on the system' % config_file_from_env) 1418 return config_file_from_env 1419 1420 config_file_name = self._apache_config_file_name_for_platform(sys.platform) 1421 return self._filesystem.join(self.layout_tests_dir(), 'http', 'conf', config_file_name) 1422 1423 # 1424 # PROTECTED ROUTINES 1425 # 1426 # The routines below should only be called by routines in this class 1427 # or any of its subclasses. 1428 # 1429 1430 # FIXME: This belongs on some platform abstraction instead of Port. 1431 def _is_redhat_based(self): 1432 return self._filesystem.exists('/etc/redhat-release') 1433 1434 def _is_debian_based(self): 1435 return self._filesystem.exists('/etc/debian_version') 1436 1437 def _apache_version(self): 1438 config = self._executive.run_command([self.path_to_apache(), '-v']) 1439 return re.sub(r'(?:.|\n)*Server version: Apache/(\d+\.\d+)(?:.|\n)*', r'\1', config) 1440 1441 # We pass sys_platform into this method to make it easy to unit test. 1442 def _apache_config_file_name_for_platform(self, sys_platform): 1443 if sys_platform == 'cygwin': 1444 return 'cygwin-httpd.conf' # CYGWIN is the only platform to still use Apache 1.3. 1445 if sys_platform.startswith('linux'): 1446 if self._is_redhat_based(): 1447 return 'fedora-httpd-' + self._apache_version() + '.conf' 1448 if self._is_debian_based(): 1449 return 'debian-httpd-' + self._apache_version() + '.conf' 1450 # All platforms use apache2 except for CYGWIN (and Mac OS X Tiger and prior, which we no longer support). 1451 return "apache2-httpd.conf" 1452 1453 def _path_to_driver(self, configuration=None): 1454 """Returns the full path to the test driver.""" 1455 return self._build_path(self.driver_name()) 1456 1457 def _path_to_webcore_library(self): 1458 """Returns the full path to a built copy of WebCore.""" 1459 return None 1460 1461 def _path_to_helper(self): 1462 """Returns the full path to the layout_test_helper binary, which 1463 is used to help configure the system for the test run, or None 1464 if no helper is needed. 1465 1466 This is likely only used by start/stop_helper().""" 1467 return None 1468 1469 def _path_to_image_diff(self): 1470 """Returns the full path to the image_diff binary, or None if it is not available. 1471 1472 This is likely used only by diff_image()""" 1473 return self._build_path('image_diff') 1474 1475 @memoized 1476 def _path_to_wdiff(self): 1477 """Returns the full path to the wdiff binary, or None if it is not available. 1478 1479 This is likely used only by wdiff_text()""" 1480 for path in ("/usr/bin/wdiff", "/usr/bin/dwdiff"): 1481 if self._filesystem.exists(path): 1482 return path 1483 return None 1484 1485 def _webkit_baseline_path(self, platform): 1486 """Return the full path to the top of the baseline tree for a 1487 given platform.""" 1488 return self._filesystem.join(self.layout_tests_dir(), 'platform', platform) 1489 1490 def _driver_class(self): 1491 """Returns the port's driver implementation.""" 1492 return driver.Driver 1493 1494 def _output_contains_sanitizer_messages(self, output): 1495 if not output: 1496 return None 1497 if 'AddressSanitizer' in output: 1498 return 'AddressSanitizer' 1499 if 'MemorySanitizer' in output: 1500 return 'MemorySanitizer' 1501 return None 1502 1503 def _get_crash_log(self, name, pid, stdout, stderr, newer_than): 1504 if self._output_contains_sanitizer_messages(stderr): 1505 # Running the symbolizer script can take a lot of memory, so we need to 1506 # serialize access to it across all the concurrently running drivers. 1507 1508 llvm_symbolizer_path = self.path_from_chromium_base('third_party', 'llvm-build', 'Release+Asserts', 'bin', 'llvm-symbolizer') 1509 if self._filesystem.exists(llvm_symbolizer_path): 1510 env = os.environ.copy() 1511 env['LLVM_SYMBOLIZER_PATH'] = llvm_symbolizer_path 1512 else: 1513 env = None 1514 sanitizer_filter_path = self.path_from_chromium_base('tools', 'valgrind', 'asan', 'asan_symbolize.py') 1515 sanitizer_strip_path_prefix = 'Release/../../' 1516 if self._filesystem.exists(sanitizer_filter_path): 1517 stderr = self._executive.run_command(['flock', sys.executable, sanitizer_filter_path, sanitizer_strip_path_prefix], input=stderr, decode_output=False, env=env) 1518 1519 name_str = name or '<unknown process name>' 1520 pid_str = str(pid or '<unknown>') 1521 stdout_lines = (stdout or '<empty>').decode('utf8', 'replace').splitlines() 1522 stderr_lines = (stderr or '<empty>').decode('utf8', 'replace').splitlines() 1523 return (stderr, 'crash log for %s (pid %s):\n%s\n%s\n' % (name_str, pid_str, 1524 '\n'.join(('STDOUT: ' + l) for l in stdout_lines), 1525 '\n'.join(('STDERR: ' + l) for l in stderr_lines))) 1526 1527 def look_for_new_crash_logs(self, crashed_processes, start_time): 1528 pass 1529 1530 def look_for_new_samples(self, unresponsive_processes, start_time): 1531 pass 1532 1533 def sample_process(self, name, pid): 1534 pass 1535 1536 def physical_test_suites(self): 1537 return [ 1538 # For example, to turn on force-compositing-mode in the svg/ directory: 1539 # PhysicalTestSuite('svg', 1540 # ['--force-compositing-mode']), 1541 ] 1542 1543 def virtual_test_suites(self): 1544 if self._virtual_test_suites is None: 1545 path_to_virtual_test_suites = self._filesystem.join(self.layout_tests_dir(), 'VirtualTestSuites') 1546 assert self._filesystem.exists(path_to_virtual_test_suites), 'LayoutTests/VirtualTestSuites not found' 1547 try: 1548 test_suite_json = json.loads(self._filesystem.read_text_file(path_to_virtual_test_suites)) 1549 self._virtual_test_suites = [VirtualTestSuite(**d) for d in test_suite_json] 1550 except ValueError as e: 1551 raise ValueError("LayoutTests/VirtualTestSuites is not a valid JSON file: %s" % str(e)) 1552 return self._virtual_test_suites 1553 1554 def _all_virtual_tests(self, suites): 1555 tests = [] 1556 for suite in suites: 1557 self._populate_virtual_suite(suite) 1558 tests.extend(suite.tests.keys()) 1559 return tests 1560 1561 def _virtual_tests_matching_paths(self, paths, suites): 1562 tests = [] 1563 for suite in suites: 1564 if any(p.startswith(suite.name) for p in paths): 1565 self._populate_virtual_suite(suite) 1566 for test in suite.tests: 1567 if any(test.startswith(p) for p in paths): 1568 tests.append(test) 1569 return tests 1570 1571 def _populate_virtual_suite(self, suite): 1572 if not suite.tests: 1573 base_tests = self._real_tests([suite.base]) 1574 suite.tests = {} 1575 for test in base_tests: 1576 suite.tests[test.replace(suite.base, suite.name, 1)] = test 1577 1578 def is_virtual_test(self, test_name): 1579 return bool(self.lookup_virtual_suite(test_name)) 1580 1581 def lookup_virtual_suite(self, test_name): 1582 for suite in self.virtual_test_suites(): 1583 if test_name.startswith(suite.name): 1584 return suite 1585 return None 1586 1587 def lookup_virtual_test_base(self, test_name): 1588 suite = self.lookup_virtual_suite(test_name) 1589 if not suite: 1590 return None 1591 return test_name.replace(suite.name, suite.base, 1) 1592 1593 def lookup_virtual_test_args(self, test_name): 1594 for suite in self.virtual_test_suites(): 1595 if test_name.startswith(suite.name): 1596 return suite.args 1597 return [] 1598 1599 def lookup_physical_test_args(self, test_name): 1600 for suite in self.physical_test_suites(): 1601 if test_name.startswith(suite.name): 1602 return suite.args 1603 return [] 1604 1605 def should_run_as_pixel_test(self, test_input): 1606 if not self._options.pixel_tests: 1607 return False 1608 if self._options.pixel_test_directories: 1609 return any(test_input.test_name.startswith(directory) for directory in self._options.pixel_test_directories) 1610 return True 1611 1612 def _modules_to_search_for_symbols(self): 1613 path = self._path_to_webcore_library() 1614 if path: 1615 return [path] 1616 return [] 1617 1618 def _symbols_string(self): 1619 symbols = '' 1620 for path_to_module in self._modules_to_search_for_symbols(): 1621 try: 1622 symbols += self._executive.run_command(['nm', path_to_module], error_handler=self._executive.ignore_error) 1623 except OSError, e: 1624 _log.warn("Failed to run nm: %s. Can't determine supported features correctly." % e) 1625 return symbols 1626 1627 # Ports which use compile-time feature detection should define this method and return 1628 # a dictionary mapping from symbol substrings to possibly disabled test directories. 1629 # When the symbol substrings are not matched, the directories will be skipped. 1630 # If ports don't ever enable certain features, then those directories can just be 1631 # in the Skipped list instead of compile-time-checked here. 1632 def _missing_symbol_to_skipped_tests(self): 1633 if self.PORT_HAS_AUDIO_CODECS_BUILT_IN: 1634 return {} 1635 else: 1636 return { 1637 "ff_mp3_decoder": ["webaudio/codec-tests/mp3"], 1638 "ff_aac_decoder": ["webaudio/codec-tests/aac"], 1639 } 1640 1641 def _has_test_in_directories(self, directory_lists, test_list): 1642 if not test_list: 1643 return False 1644 1645 directories = itertools.chain.from_iterable(directory_lists) 1646 for directory, test in itertools.product(directories, test_list): 1647 if test.startswith(directory): 1648 return True 1649 return False 1650 1651 def _skipped_tests_for_unsupported_features(self, test_list): 1652 # Only check the symbols of there are tests in the test_list that might get skipped. 1653 # This is a performance optimization to avoid the calling nm. 1654 # Runtime feature detection not supported, fallback to static detection: 1655 # Disable any tests for symbols missing from the executable or libraries. 1656 if self._has_test_in_directories(self._missing_symbol_to_skipped_tests().values(), test_list): 1657 symbols_string = self._symbols_string() 1658 if symbols_string is not None: 1659 return reduce(operator.add, [directories for symbol_substring, directories in self._missing_symbol_to_skipped_tests().items() if symbol_substring not in symbols_string], []) 1660 return [] 1661 1662 def _convert_path(self, path): 1663 """Handles filename conversion for subprocess command line args.""" 1664 # See note above in diff_image() for why we need this. 1665 if sys.platform == 'cygwin': 1666 return cygpath(path) 1667 return path 1668 1669 def _build_path(self, *comps): 1670 return self._build_path_with_configuration(None, *comps) 1671 1672 def _build_path_with_configuration(self, configuration, *comps): 1673 # Note that we don't do the option caching that the 1674 # base class does, because finding the right directory is relatively 1675 # fast. 1676 configuration = configuration or self.get_option('configuration') 1677 return self._static_build_path(self._filesystem, self.get_option('build_directory'), 1678 self.path_from_chromium_base(), configuration, comps) 1679 1680 def _check_driver_build_up_to_date(self, configuration): 1681 if configuration in ('Debug', 'Release'): 1682 try: 1683 debug_path = self._path_to_driver('Debug') 1684 release_path = self._path_to_driver('Release') 1685 1686 debug_mtime = self._filesystem.mtime(debug_path) 1687 release_mtime = self._filesystem.mtime(release_path) 1688 1689 if (debug_mtime > release_mtime and configuration == 'Release' or 1690 release_mtime > debug_mtime and configuration == 'Debug'): 1691 most_recent_binary = 'Release' if configuration == 'Debug' else 'Debug' 1692 _log.warning('You are running the %s binary. However the %s binary appears to be more recent. ' 1693 'Please pass --%s.', configuration, most_recent_binary, most_recent_binary.lower()) 1694 _log.warning('') 1695 # This will fail if we don't have both a debug and release binary. 1696 # That's fine because, in this case, we must already be running the 1697 # most up-to-date one. 1698 except OSError: 1699 pass 1700 return True 1701 1702 def _chromium_baseline_path(self, platform): 1703 if platform is None: 1704 platform = self.name() 1705 return self.path_from_webkit_base('LayoutTests', 'platform', platform) 1706 1707 class VirtualTestSuite(object): 1708 def __init__(self, prefix=None, base=None, args=None): 1709 assert base 1710 assert args 1711 assert prefix.find('/') == -1, "Virtual test suites prefixes cannot contain /'s: %s" % prefix 1712 self.name = 'virtual/' + prefix + '/' + base 1713 self.base = base 1714 self.args = args 1715 self.tests = {} 1716 1717 def __repr__(self): 1718 return "VirtualTestSuite('%s', '%s', %s)" % (self.name, self.base, self.args) 1719 1720 1721 class PhysicalTestSuite(object): 1722 def __init__(self, base, args): 1723 self.name = base 1724 self.base = base 1725 self.args = args 1726 self.tests = set() 1727 1728 def __repr__(self): 1729 return "PhysicalTestSuite('%s', '%s', %s)" % (self.name, self.base, self.args) 1730