1 # Copyright (C) 2010 Google Inc. All rights reserved. 2 # 3 # Redistribution and use in source and binary forms, with or without 4 # modification, are permitted provided that the following conditions are 5 # met: 6 # 7 # * Redistributions of source code must retain the above copyright 8 # notice, this list of conditions and the following disclaimer. 9 # * Redistributions in binary form must reproduce the above 10 # copyright notice, this list of conditions and the following disclaimer 11 # in the documentation and/or other materials provided with the 12 # distribution. 13 # * Neither the Google name nor the names of its 14 # contributors may be used to endorse or promote products derived from 15 # this software without specific prior written permission. 16 # 17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 """Abstract base class of Port-specific entry points for the layout tests 30 test infrastructure (the Port and Driver classes).""" 31 32 import cgi 33 import difflib 34 import errno 35 import itertools 36 import logging 37 import os 38 import operator 39 import optparse 40 import re 41 import sys 42 43 try: 44 from collections import OrderedDict 45 except ImportError: 46 # Needed for Python < 2.7 47 from webkitpy.thirdparty.ordered_dict import OrderedDict 48 49 50 from webkitpy.common import find_files 51 from webkitpy.common import read_checksum_from_png 52 from webkitpy.common.memoized import memoized 53 from webkitpy.common.system import path 54 from webkitpy.common.system.executive import ScriptError 55 from webkitpy.common.system.path import cygpath 56 from webkitpy.common.system.systemhost import SystemHost 57 from webkitpy.common.webkit_finder import WebKitFinder 58 from webkitpy.layout_tests.layout_package.bot_test_expectations import BotTestExpectationsFactory 59 from webkitpy.layout_tests.models.test_configuration import TestConfiguration 60 from webkitpy.layout_tests.port import config as port_config 61 from webkitpy.layout_tests.port import driver 62 from webkitpy.layout_tests.port import server_process 63 from webkitpy.layout_tests.port.factory import PortFactory 64 from webkitpy.layout_tests.servers import apache_http_server 65 from webkitpy.layout_tests.servers import http_server 66 from webkitpy.layout_tests.servers import websocket_server 67 68 _log = logging.getLogger(__name__) 69 70 71 # FIXME: This class should merge with WebKitPort now that Chromium behaves mostly like other webkit ports. 72 class Port(object): 73 """Abstract class for Port-specific hooks for the layout_test package.""" 74 75 # Subclasses override this. This should indicate the basic implementation 76 # part of the port name, e.g., 'mac', 'win', 'gtk'; there is probably (?) 77 # one unique value per class. 78 79 # FIXME: We should probably rename this to something like 'implementation_name'. 80 port_name = None 81 82 # Test names resemble unix relative paths, and use '/' as a directory separator. 83 TEST_PATH_SEPARATOR = '/' 84 85 ALL_BUILD_TYPES = ('debug', 'release') 86 87 CONTENT_SHELL_NAME = 'content_shell' 88 89 # True if the port as aac and mp3 codecs built in. 90 PORT_HAS_AUDIO_CODECS_BUILT_IN = False 91 92 @classmethod 93 def determine_full_port_name(cls, host, options, port_name): 94 """Return a fully-specified port name that can be used to construct objects.""" 95 # Subclasses will usually override this. 96 assert port_name.startswith(cls.port_name) 97 return port_name 98 99 def __init__(self, host, port_name, options=None, **kwargs): 100 101 # This value may be different from cls.port_name by having version modifiers 102 # and other fields appended to it (for example, 'qt-arm' or 'mac-wk2'). 103 self._name = port_name 104 105 # These are default values that should be overridden in a subclasses. 106 self._version = '' 107 self._architecture = 'x86' 108 109 # FIXME: Ideally we'd have a package-wide way to get a 110 # well-formed options object that had all of the necessary 111 # options defined on it. 112 self._options = options or optparse.Values() 113 114 self.host = host 115 self._executive = host.executive 116 self._filesystem = host.filesystem 117 self._webkit_finder = WebKitFinder(host.filesystem) 118 self._config = port_config.Config(self._executive, self._filesystem, self.port_name) 119 120 self._helper = None 121 self._http_server = None 122 self._websocket_server = None 123 self._image_differ = None 124 self._server_process_constructor = server_process.ServerProcess # overridable for testing 125 self._http_lock = None # FIXME: Why does this live on the port object? 126 127 # Python's Popen has a bug that causes any pipes opened to a 128 # process that can't be executed to be leaked. Since this 129 # code is specifically designed to tolerate exec failures 130 # to gracefully handle cases where wdiff is not installed, 131 # the bug results in a massive file descriptor leak. As a 132 # workaround, if an exec failure is ever experienced for 133 # wdiff, assume it's not available. This will leak one 134 # file descriptor but that's better than leaking each time 135 # wdiff would be run. 136 # 137 # http://mail.python.org/pipermail/python-list/ 138 # 2008-August/505753.html 139 # http://bugs.python.org/issue3210 140 self._wdiff_available = None 141 142 # FIXME: prettypatch.py knows this path, why is it copied here? 143 self._pretty_patch_path = self.path_from_webkit_base("Tools", "Scripts", "webkitruby", "PrettyPatch", "prettify.rb") 144 self._pretty_patch_available = None 145 146 if not hasattr(options, 'configuration') or not options.configuration: 147 self.set_option_default('configuration', self.default_configuration()) 148 self._test_configuration = None 149 self._reftest_list = {} 150 self._results_directory = None 151 152 def buildbot_archives_baselines(self): 153 return True 154 155 def additional_drt_flag(self): 156 if self.driver_name() == self.CONTENT_SHELL_NAME: 157 return ['--dump-render-tree'] 158 return [] 159 160 def supports_per_test_timeout(self): 161 return False 162 163 def default_pixel_tests(self): 164 # FIXME: Disable until they are run by default on build.webkit.org. 165 return False 166 167 def default_timeout_ms(self): 168 timeout_ms = 6 * 1000 169 if self.get_option('configuration') == 'Debug': 170 # Debug is 6x slower than Release 171 # FIXME: It should be closer to 2x. See crbug.com/254188 172 return 6 * timeout_ms 173 return timeout_ms 174 175 def driver_stop_timeout(self): 176 """ Returns the amount of time in seconds to wait before killing the process in driver.stop().""" 177 # We want to wait for at least 3 seconds, but if we are really slow, we want to be slow on cleanup as 178 # well (for things like ASAN, Valgrind, etc.) 179 return 3.0 * float(self.get_option('time_out_ms', '0')) / self.default_timeout_ms() 180 181 def wdiff_available(self): 182 if self._wdiff_available is None: 183 self._wdiff_available = self.check_wdiff(logging=False) 184 return self._wdiff_available 185 186 def pretty_patch_available(self): 187 if self._pretty_patch_available is None: 188 self._pretty_patch_available = self.check_pretty_patch(logging=False) 189 return self._pretty_patch_available 190 191 def default_child_processes(self): 192 """Return the number of drivers to use for this port.""" 193 return self._executive.cpu_count() 194 195 def default_max_locked_shards(self): 196 """Return the number of "locked" shards to run in parallel (like the http tests).""" 197 return 1 198 199 def worker_startup_delay_secs(self): 200 # FIXME: If we start workers up too quickly, DumpRenderTree appears 201 # to thrash on something and time out its first few tests. Until 202 # we can figure out what's going on, sleep a bit in between 203 # workers. See https://bugs.webkit.org/show_bug.cgi?id=79147 . 204 return 0.1 205 206 def baseline_path(self): 207 """Return the absolute path to the directory to store new baselines in for this port.""" 208 # FIXME: remove once all callers are calling either baseline_version_dir() or baseline_platform_dir() 209 return self.baseline_version_dir() 210 211 def baseline_platform_dir(self): 212 """Return the absolute path to the default (version-independent) platform-specific results.""" 213 return self._filesystem.join(self.layout_tests_dir(), 'platform', self.port_name) 214 215 def baseline_version_dir(self): 216 """Return the absolute path to the platform-and-version-specific results.""" 217 baseline_search_paths = self.baseline_search_path() 218 return baseline_search_paths[0] 219 220 def virtual_baseline_search_path(self, test_name): 221 suite = self.lookup_virtual_suite(test_name) 222 if not suite: 223 return None 224 return [self._filesystem.join(path, suite.name) for path in self.default_baseline_search_path()] 225 226 def baseline_search_path(self): 227 return self.get_option('additional_platform_directory', []) + self._compare_baseline() + self.default_baseline_search_path() 228 229 def default_baseline_search_path(self): 230 """Return a list of absolute paths to directories to search under for 231 baselines. The directories are searched in order.""" 232 search_paths = [] 233 search_paths.append(self.name()) 234 if self.name() != self.port_name: 235 search_paths.append(self.port_name) 236 return map(self._webkit_baseline_path, search_paths) 237 238 @memoized 239 def _compare_baseline(self): 240 factory = PortFactory(self.host) 241 target_port = self.get_option('compare_port') 242 if target_port: 243 return factory.get(target_port).default_baseline_search_path() 244 return [] 245 246 def check_build(self, needs_http): 247 """This routine is used to ensure that the build is up to date 248 and all the needed binaries are present.""" 249 if self.get_option('build'): 250 return False 251 if not self._check_driver(): 252 return False 253 if self.get_option('pixel_tests'): 254 if not self.check_image_diff(): 255 return False 256 if not self._check_port_build(): 257 return False 258 return True 259 260 def _check_driver(self): 261 driver_path = self._path_to_driver() 262 if not self._filesystem.exists(driver_path): 263 _log.error("%s was not found at %s" % (self.driver_name(), driver_path)) 264 return False 265 return True 266 267 def _check_port_build(self): 268 # Ports can override this method to do additional checks. 269 return True 270 271 def check_sys_deps(self, needs_http): 272 """If the port needs to do some runtime checks to ensure that the 273 tests can be run successfully, it should override this routine. 274 This step can be skipped with --nocheck-sys-deps. 275 276 Returns whether the system is properly configured.""" 277 if needs_http: 278 return self.check_httpd() 279 return True 280 281 def check_image_diff(self, override_step=None, logging=True): 282 """This routine is used to check whether image_diff binary exists.""" 283 image_diff_path = self._path_to_image_diff() 284 if not self._filesystem.exists(image_diff_path): 285 _log.error("image_diff was not found at %s" % image_diff_path) 286 return False 287 return True 288 289 def check_pretty_patch(self, logging=True): 290 """Checks whether we can use the PrettyPatch ruby script.""" 291 try: 292 _ = self._executive.run_command(['ruby', '--version']) 293 except OSError, e: 294 if e.errno in [errno.ENOENT, errno.EACCES, errno.ECHILD]: 295 if logging: 296 _log.warning("Ruby is not installed; can't generate pretty patches.") 297 _log.warning('') 298 return False 299 300 if not self._filesystem.exists(self._pretty_patch_path): 301 if logging: 302 _log.warning("Unable to find %s; can't generate pretty patches." % self._pretty_patch_path) 303 _log.warning('') 304 return False 305 306 return True 307 308 def check_wdiff(self, logging=True): 309 if not self._path_to_wdiff(): 310 # Don't need to log here since this is the port choosing not to use wdiff. 311 return False 312 313 try: 314 _ = self._executive.run_command([self._path_to_wdiff(), '--help']) 315 except OSError: 316 if logging: 317 message = self._wdiff_missing_message() 318 if message: 319 for line in message.splitlines(): 320 _log.warning(' ' + line) 321 _log.warning('') 322 return False 323 324 return True 325 326 def _wdiff_missing_message(self): 327 return 'wdiff is not installed; please install it to generate word-by-word diffs.' 328 329 def check_httpd(self): 330 if self._uses_apache(): 331 httpd_path = self._path_to_apache() 332 else: 333 httpd_path = self._path_to_lighttpd() 334 335 try: 336 server_name = self._filesystem.basename(httpd_path) 337 env = self.setup_environ_for_server(server_name) 338 if self._executive.run_command([httpd_path, "-v"], env=env, return_exit_code=True) != 0: 339 _log.error("httpd seems broken. Cannot run http tests.") 340 return False 341 return True 342 except OSError: 343 _log.error("No httpd found. Cannot run http tests.") 344 return False 345 346 def do_text_results_differ(self, expected_text, actual_text): 347 return expected_text != actual_text 348 349 def do_audio_results_differ(self, expected_audio, actual_audio): 350 return expected_audio != actual_audio 351 352 def diff_image(self, expected_contents, actual_contents): 353 """Compare two images and return a tuple of an image diff, and an error string. 354 355 If an error occurs (like image_diff isn't found, or crashes, we log an error and return True (for a diff). 356 """ 357 # If only one of them exists, return that one. 358 if not actual_contents and not expected_contents: 359 return (None, None) 360 if not actual_contents: 361 return (expected_contents, None) 362 if not expected_contents: 363 return (actual_contents, None) 364 365 tempdir = self._filesystem.mkdtemp() 366 367 expected_filename = self._filesystem.join(str(tempdir), "expected.png") 368 self._filesystem.write_binary_file(expected_filename, expected_contents) 369 370 actual_filename = self._filesystem.join(str(tempdir), "actual.png") 371 self._filesystem.write_binary_file(actual_filename, actual_contents) 372 373 diff_filename = self._filesystem.join(str(tempdir), "diff.png") 374 375 # image_diff needs native win paths as arguments, so we need to convert them if running under cygwin. 376 native_expected_filename = self._convert_path(expected_filename) 377 native_actual_filename = self._convert_path(actual_filename) 378 native_diff_filename = self._convert_path(diff_filename) 379 380 executable = self._path_to_image_diff() 381 # Note that although we are handed 'old', 'new', image_diff wants 'new', 'old'. 382 comand = [executable, '--diff', native_actual_filename, native_expected_filename, native_diff_filename] 383 384 result = None 385 err_str = None 386 try: 387 exit_code = self._executive.run_command(comand, return_exit_code=True) 388 if exit_code == 0: 389 # The images are the same. 390 result = None 391 elif exit_code == 1: 392 result = self._filesystem.read_binary_file(native_diff_filename) 393 else: 394 err_str = "image diff returned an exit code of %s" % exit_code 395 except OSError, e: 396 err_str = 'error running image diff: %s' % str(e) 397 finally: 398 self._filesystem.rmtree(str(tempdir)) 399 400 return (result, err_str or None) 401 402 def diff_text(self, expected_text, actual_text, expected_filename, actual_filename): 403 """Returns a string containing the diff of the two text strings 404 in 'unified diff' format.""" 405 406 # The filenames show up in the diff output, make sure they're 407 # raw bytes and not unicode, so that they don't trigger join() 408 # trying to decode the input. 409 def to_raw_bytes(string_value): 410 if isinstance(string_value, unicode): 411 return string_value.encode('utf-8') 412 return string_value 413 expected_filename = to_raw_bytes(expected_filename) 414 actual_filename = to_raw_bytes(actual_filename) 415 diff = difflib.unified_diff(expected_text.splitlines(True), 416 actual_text.splitlines(True), 417 expected_filename, 418 actual_filename) 419 return ''.join(diff) 420 421 def driver_name(self): 422 if self.get_option('driver_name'): 423 return self.get_option('driver_name') 424 return self.CONTENT_SHELL_NAME 425 426 def expected_baselines_by_extension(self, test_name): 427 """Returns a dict mapping baseline suffix to relative path for each baseline in 428 a test. For reftests, it returns ".==" or ".!=" instead of the suffix.""" 429 # FIXME: The name similarity between this and expected_baselines() below, is unfortunate. 430 # We should probably rename them both. 431 baseline_dict = {} 432 reference_files = self.reference_files(test_name) 433 if reference_files: 434 # FIXME: How should this handle more than one type of reftest? 435 baseline_dict['.' + reference_files[0][0]] = self.relative_test_filename(reference_files[0][1]) 436 437 for extension in self.baseline_extensions(): 438 path = self.expected_filename(test_name, extension, return_default=False) 439 baseline_dict[extension] = self.relative_test_filename(path) if path else path 440 441 return baseline_dict 442 443 def baseline_extensions(self): 444 """Returns a tuple of all of the non-reftest baseline extensions we use. The extensions include the leading '.'.""" 445 return ('.wav', '.txt', '.png') 446 447 def expected_baselines(self, test_name, suffix, all_baselines=False): 448 """Given a test name, finds where the baseline results are located. 449 450 Args: 451 test_name: name of test file (usually a relative path under LayoutTests/) 452 suffix: file suffix of the expected results, including dot; e.g. 453 '.txt' or '.png'. This should not be None, but may be an empty 454 string. 455 all_baselines: If True, return an ordered list of all baseline paths 456 for the given platform. If False, return only the first one. 457 Returns 458 a list of ( platform_dir, results_filename ), where 459 platform_dir - abs path to the top of the results tree (or test 460 tree) 461 results_filename - relative path from top of tree to the results 462 file 463 (port.join() of the two gives you the full path to the file, 464 unless None was returned.) 465 Return values will be in the format appropriate for the current 466 platform (e.g., "\\" for path separators on Windows). If the results 467 file is not found, then None will be returned for the directory, 468 but the expected relative pathname will still be returned. 469 470 This routine is generic but lives here since it is used in 471 conjunction with the other baseline and filename routines that are 472 platform specific. 473 """ 474 baseline_filename = self._filesystem.splitext(test_name)[0] + '-expected' + suffix 475 baseline_search_path = self.baseline_search_path() 476 477 baselines = [] 478 for platform_dir in baseline_search_path: 479 if self._filesystem.exists(self._filesystem.join(platform_dir, baseline_filename)): 480 baselines.append((platform_dir, baseline_filename)) 481 482 if not all_baselines and baselines: 483 return baselines 484 485 # If it wasn't found in a platform directory, return the expected 486 # result in the test directory, even if no such file actually exists. 487 platform_dir = self.layout_tests_dir() 488 if self._filesystem.exists(self._filesystem.join(platform_dir, baseline_filename)): 489 baselines.append((platform_dir, baseline_filename)) 490 491 if baselines: 492 return baselines 493 494 return [(None, baseline_filename)] 495 496 def expected_filename(self, test_name, suffix, return_default=True): 497 """Given a test name, returns an absolute path to its expected results. 498 499 If no expected results are found in any of the searched directories, 500 the directory in which the test itself is located will be returned. 501 The return value is in the format appropriate for the platform 502 (e.g., "\\" for path separators on windows). 503 504 Args: 505 test_name: name of test file (usually a relative path under LayoutTests/) 506 suffix: file suffix of the expected results, including dot; e.g. '.txt' 507 or '.png'. This should not be None, but may be an empty string. 508 platform: the most-specific directory name to use to build the 509 search list of directories, e.g., 'win', or 510 'chromium-cg-mac-leopard' (we follow the WebKit format) 511 return_default: if True, returns the path to the generic expectation if nothing 512 else is found; if False, returns None. 513 514 This routine is generic but is implemented here to live alongside 515 the other baseline and filename manipulation routines. 516 """ 517 # FIXME: The [0] here is very mysterious, as is the destructured return. 518 platform_dir, baseline_filename = self.expected_baselines(test_name, suffix)[0] 519 if platform_dir: 520 return self._filesystem.join(platform_dir, baseline_filename) 521 522 actual_test_name = self.lookup_virtual_test_base(test_name) 523 if actual_test_name: 524 return self.expected_filename(actual_test_name, suffix) 525 526 if return_default: 527 return self._filesystem.join(self.layout_tests_dir(), baseline_filename) 528 return None 529 530 def expected_checksum(self, test_name): 531 """Returns the checksum of the image we expect the test to produce, or None if it is a text-only test.""" 532 png_path = self.expected_filename(test_name, '.png') 533 534 if self._filesystem.exists(png_path): 535 with self._filesystem.open_binary_file_for_reading(png_path) as filehandle: 536 return read_checksum_from_png.read_checksum(filehandle) 537 538 return None 539 540 def expected_image(self, test_name): 541 """Returns the image we expect the test to produce.""" 542 baseline_path = self.expected_filename(test_name, '.png') 543 if not self._filesystem.exists(baseline_path): 544 return None 545 return self._filesystem.read_binary_file(baseline_path) 546 547 def expected_audio(self, test_name): 548 baseline_path = self.expected_filename(test_name, '.wav') 549 if not self._filesystem.exists(baseline_path): 550 return None 551 return self._filesystem.read_binary_file(baseline_path) 552 553 def expected_text(self, test_name): 554 """Returns the text output we expect the test to produce, or None 555 if we don't expect there to be any text output. 556 End-of-line characters are normalized to '\n'.""" 557 # FIXME: DRT output is actually utf-8, but since we don't decode the 558 # output from DRT (instead treating it as a binary string), we read the 559 # baselines as a binary string, too. 560 baseline_path = self.expected_filename(test_name, '.txt') 561 if not self._filesystem.exists(baseline_path): 562 return None 563 text = self._filesystem.read_binary_file(baseline_path) 564 return text.replace("\r\n", "\n") 565 566 def _get_reftest_list(self, test_name): 567 dirname = self._filesystem.join(self.layout_tests_dir(), self._filesystem.dirname(test_name)) 568 if dirname not in self._reftest_list: 569 self._reftest_list[dirname] = Port._parse_reftest_list(self._filesystem, dirname) 570 return self._reftest_list[dirname] 571 572 @staticmethod 573 def _parse_reftest_list(filesystem, test_dirpath): 574 reftest_list_path = filesystem.join(test_dirpath, 'reftest.list') 575 if not filesystem.isfile(reftest_list_path): 576 return None 577 reftest_list_file = filesystem.read_text_file(reftest_list_path) 578 579 parsed_list = {} 580 for line in reftest_list_file.split('\n'): 581 line = re.sub('#.+$', '', line) 582 split_line = line.split() 583 if len(split_line) < 3: 584 continue 585 expectation_type, test_file, ref_file = split_line 586 parsed_list.setdefault(filesystem.join(test_dirpath, test_file), []).append((expectation_type, filesystem.join(test_dirpath, ref_file))) 587 return parsed_list 588 589 def reference_files(self, test_name): 590 """Return a list of expectation (== or !=) and filename pairs""" 591 592 reftest_list = self._get_reftest_list(test_name) 593 if not reftest_list: 594 reftest_list = [] 595 for expectation, prefix in (('==', ''), ('!=', '-mismatch')): 596 for extention in Port._supported_file_extensions: 597 path = self.expected_filename(test_name, prefix + extention) 598 if self._filesystem.exists(path): 599 reftest_list.append((expectation, path)) 600 return reftest_list 601 602 return reftest_list.get(self._filesystem.join(self.layout_tests_dir(), test_name), []) # pylint: disable=E1103 603 604 def tests(self, paths): 605 """Return the list of tests found matching paths.""" 606 tests = self._real_tests(paths) 607 tests.extend(self._virtual_tests(paths, self.populated_virtual_test_suites())) 608 return tests 609 610 def _real_tests(self, paths): 611 # When collecting test cases, skip these directories 612 skipped_directories = set(['.svn', '_svn', 'platform', 'resources', 'script-tests', 'reference', 'reftest']) 613 files = find_files.find(self._filesystem, self.layout_tests_dir(), paths, skipped_directories, Port.is_test_file, self.test_key) 614 return [self.relative_test_filename(f) for f in files] 615 616 # When collecting test cases, we include any file with these extensions. 617 _supported_file_extensions = set(['.html', '.xml', '.xhtml', '.xht', '.pl', 618 '.htm', '.php', '.svg', '.mht']) 619 620 @staticmethod 621 # If any changes are made here be sure to update the isUsedInReftest method in old-run-webkit-tests as well. 622 def is_reference_html_file(filesystem, dirname, filename): 623 if filename.startswith('ref-') or filename.startswith('notref-'): 624 return True 625 filename_wihout_ext, unused = filesystem.splitext(filename) 626 for suffix in ['-expected', '-expected-mismatch', '-ref', '-notref']: 627 if filename_wihout_ext.endswith(suffix): 628 return True 629 return False 630 631 @staticmethod 632 def _has_supported_extension(filesystem, filename): 633 """Return true if filename is one of the file extensions we want to run a test on.""" 634 extension = filesystem.splitext(filename)[1] 635 return extension in Port._supported_file_extensions 636 637 @staticmethod 638 def is_test_file(filesystem, dirname, filename): 639 return Port._has_supported_extension(filesystem, filename) and not Port.is_reference_html_file(filesystem, dirname, filename) 640 641 def test_key(self, test_name): 642 """Turns a test name into a list with two sublists, the natural key of the 643 dirname, and the natural key of the basename. 644 645 This can be used when sorting paths so that files in a directory. 646 directory are kept together rather than being mixed in with files in 647 subdirectories.""" 648 dirname, basename = self.split_test(test_name) 649 return (self._natural_sort_key(dirname + self.TEST_PATH_SEPARATOR), self._natural_sort_key(basename)) 650 651 def _natural_sort_key(self, string_to_split): 652 """ Turns a string into a list of string and number chunks, i.e. "z23a" -> ["z", 23, "a"] 653 654 This can be used to implement "natural sort" order. See: 655 http://www.codinghorror.com/blog/2007/12/sorting-for-humans-natural-sort-order.html 656 http://nedbatchelder.com/blog/200712.html#e20071211T054956 657 """ 658 def tryint(val): 659 try: 660 return int(val) 661 except ValueError: 662 return val 663 664 return [tryint(chunk) for chunk in re.split('(\d+)', string_to_split)] 665 666 def test_dirs(self): 667 """Returns the list of top-level test directories.""" 668 layout_tests_dir = self.layout_tests_dir() 669 return filter(lambda x: self._filesystem.isdir(self._filesystem.join(layout_tests_dir, x)), 670 self._filesystem.listdir(layout_tests_dir)) 671 672 @memoized 673 def test_isfile(self, test_name): 674 """Return True if the test name refers to a directory of tests.""" 675 # Used by test_expectations.py to apply rules to whole directories. 676 if self._filesystem.isfile(self.abspath_for_test(test_name)): 677 return True 678 base = self.lookup_virtual_test_base(test_name) 679 return base and self._filesystem.isfile(self.abspath_for_test(base)) 680 681 @memoized 682 def test_isdir(self, test_name): 683 """Return True if the test name refers to a directory of tests.""" 684 # Used by test_expectations.py to apply rules to whole directories. 685 if self._filesystem.isdir(self.abspath_for_test(test_name)): 686 return True 687 base = self.lookup_virtual_test_base(test_name) 688 return base and self._filesystem.isdir(self.abspath_for_test(base)) 689 690 @memoized 691 def test_exists(self, test_name): 692 """Return True if the test name refers to an existing test or baseline.""" 693 # Used by test_expectations.py to determine if an entry refers to a 694 # valid test and by printing.py to determine if baselines exist. 695 return self.test_isfile(test_name) or self.test_isdir(test_name) 696 697 def split_test(self, test_name): 698 """Splits a test name into the 'directory' part and the 'basename' part.""" 699 index = test_name.rfind(self.TEST_PATH_SEPARATOR) 700 if index < 1: 701 return ('', test_name) 702 return (test_name[0:index], test_name[index:]) 703 704 def normalize_test_name(self, test_name): 705 """Returns a normalized version of the test name or test directory.""" 706 if test_name.endswith('/'): 707 return test_name 708 if self.test_isdir(test_name): 709 return test_name + '/' 710 return test_name 711 712 def driver_cmd_line(self): 713 """Prints the DRT command line that will be used.""" 714 driver = self.create_driver(0) 715 return driver.cmd_line(self.get_option('pixel_tests'), []) 716 717 def update_baseline(self, baseline_path, data): 718 """Updates the baseline for a test. 719 720 Args: 721 baseline_path: the actual path to use for baseline, not the path to 722 the test. This function is used to update either generic or 723 platform-specific baselines, but we can't infer which here. 724 data: contents of the baseline. 725 """ 726 self._filesystem.write_binary_file(baseline_path, data) 727 728 # FIXME: update callers to create a finder and call it instead of these next five routines (which should be protected). 729 def webkit_base(self): 730 return self._webkit_finder.webkit_base() 731 732 def path_from_webkit_base(self, *comps): 733 return self._webkit_finder.path_from_webkit_base(*comps) 734 735 def path_to_script(self, script_name): 736 return self._webkit_finder.path_to_script(script_name) 737 738 def layout_tests_dir(self): 739 return self._webkit_finder.layout_tests_dir() 740 741 def perf_tests_dir(self): 742 return self._webkit_finder.perf_tests_dir() 743 744 def skipped_layout_tests(self, test_list): 745 """Returns tests skipped outside of the TestExpectations files.""" 746 return set(self._skipped_tests_for_unsupported_features(test_list)) 747 748 def _tests_from_skipped_file_contents(self, skipped_file_contents): 749 tests_to_skip = [] 750 for line in skipped_file_contents.split('\n'): 751 line = line.strip() 752 line = line.rstrip('/') # Best to normalize directory names to not include the trailing slash. 753 if line.startswith('#') or not len(line): 754 continue 755 tests_to_skip.append(line) 756 return tests_to_skip 757 758 def _expectations_from_skipped_files(self, skipped_file_paths): 759 tests_to_skip = [] 760 for search_path in skipped_file_paths: 761 filename = self._filesystem.join(self._webkit_baseline_path(search_path), "Skipped") 762 if not self._filesystem.exists(filename): 763 _log.debug("Skipped does not exist: %s" % filename) 764 continue 765 _log.debug("Using Skipped file: %s" % filename) 766 skipped_file_contents = self._filesystem.read_text_file(filename) 767 tests_to_skip.extend(self._tests_from_skipped_file_contents(skipped_file_contents)) 768 return tests_to_skip 769 770 @memoized 771 def skipped_perf_tests(self): 772 return self._expectations_from_skipped_files([self.perf_tests_dir()]) 773 774 def skips_perf_test(self, test_name): 775 for test_or_category in self.skipped_perf_tests(): 776 if test_or_category == test_name: 777 return True 778 category = self._filesystem.join(self.perf_tests_dir(), test_or_category) 779 if self._filesystem.isdir(category) and test_name.startswith(test_or_category): 780 return True 781 return False 782 783 def is_chromium(self): 784 return False 785 786 def name(self): 787 """Returns a name that uniquely identifies this particular type of port 788 (e.g., "mac-snowleopard" or "linux-x86_x64" and can be passed 789 to factory.get() to instantiate the port.""" 790 return self._name 791 792 def operating_system(self): 793 # Subclasses should override this default implementation. 794 return 'mac' 795 796 def version(self): 797 """Returns a string indicating the version of a given platform, e.g. 798 'leopard' or 'xp'. 799 800 This is used to help identify the exact port when parsing test 801 expectations, determining search paths, and logging information.""" 802 return self._version 803 804 def architecture(self): 805 return self._architecture 806 807 def get_option(self, name, default_value=None): 808 return getattr(self._options, name, default_value) 809 810 def set_option_default(self, name, default_value): 811 return self._options.ensure_value(name, default_value) 812 813 @memoized 814 def path_to_generic_test_expectations_file(self): 815 return self._filesystem.join(self.layout_tests_dir(), 'TestExpectations') 816 817 def relative_test_filename(self, filename): 818 """Returns a test_name a relative unix-style path for a filename under the LayoutTests 819 directory. Ports may legitimately return abspaths here if no relpath makes sense.""" 820 # Ports that run on windows need to override this method to deal with 821 # filenames with backslashes in them. 822 if filename.startswith(self.layout_tests_dir()): 823 return self.host.filesystem.relpath(filename, self.layout_tests_dir()) 824 else: 825 return self.host.filesystem.abspath(filename) 826 827 @memoized 828 def abspath_for_test(self, test_name): 829 """Returns the full path to the file for a given test name. This is the 830 inverse of relative_test_filename().""" 831 return self._filesystem.join(self.layout_tests_dir(), test_name) 832 833 def results_directory(self): 834 """Absolute path to the place to store the test results (uses --results-directory).""" 835 if not self._results_directory: 836 option_val = self.get_option('results_directory') or self.default_results_directory() 837 self._results_directory = self._filesystem.abspath(option_val) 838 return self._results_directory 839 840 def perf_results_directory(self): 841 return self._build_path() 842 843 def default_results_directory(self): 844 """Absolute path to the default place to store the test results.""" 845 # Results are store relative to the built products to make it easy 846 # to have multiple copies of webkit checked out and built. 847 return self._build_path('layout-test-results') 848 849 def setup_test_run(self): 850 """Perform port-specific work at the beginning of a test run.""" 851 pass 852 853 def clean_up_test_run(self): 854 """Perform port-specific work at the end of a test run.""" 855 if self._image_differ: 856 self._image_differ.stop() 857 self._image_differ = None 858 859 # FIXME: os.environ access should be moved to onto a common/system class to be more easily mockable. 860 def _value_or_default_from_environ(self, name, default=None): 861 if name in os.environ: 862 return os.environ[name] 863 return default 864 865 def _copy_value_from_environ_if_set(self, clean_env, name): 866 if name in os.environ: 867 clean_env[name] = os.environ[name] 868 869 def setup_environ_for_server(self, server_name=None): 870 # We intentionally copy only a subset of os.environ when 871 # launching subprocesses to ensure consistent test results. 872 clean_env = {} 873 variables_to_copy = [ 874 # For Linux: 875 'XAUTHORITY', 876 'HOME', 877 'LANG', 878 'LD_LIBRARY_PATH', 879 'DBUS_SESSION_BUS_ADDRESS', 880 'XDG_DATA_DIRS', 881 882 # Darwin: 883 'DYLD_LIBRARY_PATH', 884 'HOME', 885 886 # CYGWIN: 887 'HOMEDRIVE', 888 'HOMEPATH', 889 '_NT_SYMBOL_PATH', 890 891 # Windows: 892 'PATH', 893 894 # Most ports (?): 895 'WEBKIT_TESTFONTS', 896 'WEBKITOUTPUTDIR', 897 898 # Chromium: 899 'CHROME_DEVEL_SANDBOX', 900 'CHROME_IPC_LOGGING', 901 'ASAN_OPTIONS', 902 903 ] 904 for variable in variables_to_copy: 905 self._copy_value_from_environ_if_set(clean_env, variable) 906 907 # For Linux: 908 clean_env['DISPLAY'] = self._value_or_default_from_environ('DISPLAY', ':1') 909 910 for string_variable in self.get_option('additional_env_var', []): 911 [name, value] = string_variable.split('=', 1) 912 clean_env[name] = value 913 914 return clean_env 915 916 def show_results_html_file(self, results_filename): 917 """This routine should display the HTML file pointed at by 918 results_filename in a users' browser.""" 919 return self.host.user.open_url(path.abspath_to_uri(self.host.platform, results_filename)) 920 921 def create_driver(self, worker_number, no_timeout=False): 922 """Return a newly created Driver subclass for starting/stopping the test driver.""" 923 return self._driver_class()(self, worker_number, pixel_tests=self.get_option('pixel_tests'), no_timeout=no_timeout) 924 925 def start_helper(self): 926 """If a port needs to reconfigure graphics settings or do other 927 things to ensure a known test configuration, it should override this 928 method.""" 929 pass 930 931 def requires_http_server(self): 932 """Does the port require an HTTP server for running tests? This could 933 be the case when the tests aren't run on the host platform.""" 934 return False 935 936 def start_http_server(self, additional_dirs=None, number_of_servers=None): 937 """Start a web server. Raise an error if it can't start or is already running. 938 939 Ports can stub this out if they don't need a web server to be running.""" 940 assert not self._http_server, 'Already running an http server.' 941 942 if self._uses_apache(): 943 server = apache_http_server.LayoutTestApacheHttpd(self, self.results_directory(), additional_dirs=additional_dirs, number_of_servers=number_of_servers) 944 else: 945 server = http_server.Lighttpd(self, self.results_directory(), additional_dirs=additional_dirs, number_of_servers=number_of_servers) 946 947 server.start() 948 self._http_server = server 949 950 def start_websocket_server(self): 951 """Start a web server. Raise an error if it can't start or is already running. 952 953 Ports can stub this out if they don't need a websocket server to be running.""" 954 assert not self._websocket_server, 'Already running a websocket server.' 955 956 server = websocket_server.PyWebSocket(self, self.results_directory()) 957 server.start() 958 self._websocket_server = server 959 960 def http_server_supports_ipv6(self): 961 # Cygwin is the only platform to still use Apache 1.3, which only supports IPV4. 962 # Once it moves to Apache 2, we can drop this method altogether. 963 if self.host.platform.is_cygwin(): 964 return False 965 return True 966 967 def stop_helper(self): 968 """Shut down the test helper if it is running. Do nothing if 969 it isn't, or it isn't available. If a port overrides start_helper() 970 it must override this routine as well.""" 971 pass 972 973 def stop_http_server(self): 974 """Shut down the http server if it is running. Do nothing if it isn't.""" 975 if self._http_server: 976 self._http_server.stop() 977 self._http_server = None 978 979 def stop_websocket_server(self): 980 """Shut down the websocket server if it is running. Do nothing if it isn't.""" 981 if self._websocket_server: 982 self._websocket_server.stop() 983 self._websocket_server = None 984 985 # 986 # TEST EXPECTATION-RELATED METHODS 987 # 988 989 def test_configuration(self): 990 """Returns the current TestConfiguration for the port.""" 991 if not self._test_configuration: 992 self._test_configuration = TestConfiguration(self._version, self._architecture, self._options.configuration.lower()) 993 return self._test_configuration 994 995 # FIXME: Belongs on a Platform object. 996 @memoized 997 def all_test_configurations(self): 998 """Returns a list of TestConfiguration instances, representing all available 999 test configurations for this port.""" 1000 return self._generate_all_test_configurations() 1001 1002 # FIXME: Belongs on a Platform object. 1003 def configuration_specifier_macros(self): 1004 """Ports may provide a way to abbreviate configuration specifiers to conveniently 1005 refer to them as one term or alias specific values to more generic ones. For example: 1006 1007 (xp, vista, win7) -> win # Abbreviate all Windows versions into one namesake. 1008 (lucid) -> linux # Change specific name of the Linux distro to a more generic term. 1009 1010 Returns a dictionary, each key representing a macro term ('win', for example), 1011 and value being a list of valid configuration specifiers (such as ['xp', 'vista', 'win7']).""" 1012 return {} 1013 1014 def all_baseline_variants(self): 1015 """Returns a list of platform names sufficient to cover all the baselines. 1016 1017 The list should be sorted so that a later platform will reuse 1018 an earlier platform's baselines if they are the same (e.g., 1019 'snowleopard' should precede 'leopard').""" 1020 raise NotImplementedError 1021 1022 def expectations_dict(self): 1023 """Returns an OrderedDict of name -> expectations strings. 1024 The names are expected to be (but not required to be) paths in the filesystem. 1025 If the name is a path, the file can be considered updatable for things like rebaselining, 1026 so don't use names that are paths if they're not paths. 1027 Generally speaking the ordering should be files in the filesystem in cascade order 1028 (TestExpectations followed by Skipped, if the port honors both formats), 1029 then any built-in expectations (e.g., from compile-time exclusions), then --additional-expectations options.""" 1030 # FIXME: rename this to test_expectations() once all the callers are updated to know about the ordered dict. 1031 expectations = OrderedDict() 1032 1033 for path in self.expectations_files(): 1034 if self._filesystem.exists(path): 1035 expectations[path] = self._filesystem.read_text_file(path) 1036 1037 for path in self.get_option('additional_expectations', []): 1038 expanded_path = self._filesystem.expanduser(path) 1039 if self._filesystem.exists(expanded_path): 1040 _log.debug("reading additional_expectations from path '%s'" % path) 1041 expectations[path] = self._filesystem.read_text_file(expanded_path) 1042 else: 1043 _log.warning("additional_expectations path '%s' does not exist" % path) 1044 return expectations 1045 1046 def bot_expectations(self): 1047 if not self.get_option('ignore_flaky_tests'): 1048 return {} 1049 1050 full_port_name = self.determine_full_port_name(self.host, self._options, self.port_name) 1051 builder_category = self.get_option('ignore_builder_category', 'layout') 1052 factory = BotTestExpectationsFactory() 1053 expectations = factory.expectations_for_port(full_port_name, builder_category) 1054 1055 if not expectations: 1056 return {} 1057 1058 ignore_mode = self.get_option('ignore_flaky_tests') 1059 if ignore_mode == 'very-flaky' or ignore_mode == 'maybe-flaky': 1060 return expectations.flakes_by_path(ignore_mode == 'very-flaky') 1061 if ignore_mode == 'unexpected': 1062 return expectations.unexpected_results_by_path() 1063 _log.warning("Unexpected ignore mode: '%s'." % ignore_mode) 1064 return {} 1065 1066 def _port_specific_expectations_files(self): 1067 # Unlike baseline_search_path, we only want to search [WK2-PORT, PORT-VERSION, PORT] and any directories 1068 # included via --additional-platform-directory, not the full casade. 1069 search_paths = [self.port_name] 1070 1071 non_wk2_name = self.name().replace('-wk2', '') 1072 if non_wk2_name != self.port_name: 1073 search_paths.append(non_wk2_name) 1074 1075 search_paths.extend(self.get_option("additional_platform_directory", [])) 1076 1077 return [self._filesystem.join(self._webkit_baseline_path(d), 'TestExpectations') for d in search_paths] 1078 1079 def expectations_files(self): 1080 return [self.path_to_generic_test_expectations_file()] + self._port_specific_expectations_files() 1081 1082 def repository_paths(self): 1083 """Returns a list of (repository_name, repository_path) tuples of its depending code base. 1084 By default it returns a list that only contains a ('WebKit', <webkitRepositoryPath>) tuple.""" 1085 1086 # We use LayoutTest directory here because webkit_base isn't a part of WebKit repository in Chromium port 1087 # where turnk isn't checked out as a whole. 1088 return [('blink', self.layout_tests_dir())] 1089 1090 _WDIFF_DEL = '##WDIFF_DEL##' 1091 _WDIFF_ADD = '##WDIFF_ADD##' 1092 _WDIFF_END = '##WDIFF_END##' 1093 1094 def _format_wdiff_output_as_html(self, wdiff): 1095 wdiff = cgi.escape(wdiff) 1096 wdiff = wdiff.replace(self._WDIFF_DEL, "<span class=del>") 1097 wdiff = wdiff.replace(self._WDIFF_ADD, "<span class=add>") 1098 wdiff = wdiff.replace(self._WDIFF_END, "</span>") 1099 html = "<head><style>.del { background: #faa; } " 1100 html += ".add { background: #afa; }</style></head>" 1101 html += "<pre>%s</pre>" % wdiff 1102 return html 1103 1104 def _wdiff_command(self, actual_filename, expected_filename): 1105 executable = self._path_to_wdiff() 1106 return [executable, 1107 "--start-delete=%s" % self._WDIFF_DEL, 1108 "--end-delete=%s" % self._WDIFF_END, 1109 "--start-insert=%s" % self._WDIFF_ADD, 1110 "--end-insert=%s" % self._WDIFF_END, 1111 actual_filename, 1112 expected_filename] 1113 1114 @staticmethod 1115 def _handle_wdiff_error(script_error): 1116 # Exit 1 means the files differed, any other exit code is an error. 1117 if script_error.exit_code != 1: 1118 raise script_error 1119 1120 def _run_wdiff(self, actual_filename, expected_filename): 1121 """Runs wdiff and may throw exceptions. 1122 This is mostly a hook for unit testing.""" 1123 # Diffs are treated as binary as they may include multiple files 1124 # with conflicting encodings. Thus we do not decode the output. 1125 command = self._wdiff_command(actual_filename, expected_filename) 1126 wdiff = self._executive.run_command(command, decode_output=False, 1127 error_handler=self._handle_wdiff_error) 1128 return self._format_wdiff_output_as_html(wdiff) 1129 1130 _wdiff_error_html = "Failed to run wdiff, see error log." 1131 1132 def wdiff_text(self, actual_filename, expected_filename): 1133 """Returns a string of HTML indicating the word-level diff of the 1134 contents of the two filenames. Returns an empty string if word-level 1135 diffing isn't available.""" 1136 if not self.wdiff_available(): 1137 return "" 1138 try: 1139 # It's possible to raise a ScriptError we pass wdiff invalid paths. 1140 return self._run_wdiff(actual_filename, expected_filename) 1141 except OSError as e: 1142 if e.errno in [errno.ENOENT, errno.EACCES, errno.ECHILD]: 1143 # Silently ignore cases where wdiff is missing. 1144 self._wdiff_available = False 1145 return "" 1146 raise 1147 except ScriptError as e: 1148 _log.error("Failed to run wdiff: %s" % e) 1149 self._wdiff_available = False 1150 return self._wdiff_error_html 1151 1152 # This is a class variable so we can test error output easily. 1153 _pretty_patch_error_html = "Failed to run PrettyPatch, see error log." 1154 1155 def pretty_patch_text(self, diff_path): 1156 if self._pretty_patch_available is None: 1157 self._pretty_patch_available = self.check_pretty_patch(logging=False) 1158 if not self._pretty_patch_available: 1159 return self._pretty_patch_error_html 1160 command = ("ruby", "-I", self._filesystem.dirname(self._pretty_patch_path), 1161 self._pretty_patch_path, diff_path) 1162 try: 1163 # Diffs are treated as binary (we pass decode_output=False) as they 1164 # may contain multiple files of conflicting encodings. 1165 return self._executive.run_command(command, decode_output=False) 1166 except OSError, e: 1167 # If the system is missing ruby log the error and stop trying. 1168 self._pretty_patch_available = False 1169 _log.error("Failed to run PrettyPatch (%s): %s" % (command, e)) 1170 return self._pretty_patch_error_html 1171 except ScriptError, e: 1172 # If ruby failed to run for some reason, log the command 1173 # output and stop trying. 1174 self._pretty_patch_available = False 1175 _log.error("Failed to run PrettyPatch (%s):\n%s" % (command, e.message_with_output())) 1176 return self._pretty_patch_error_html 1177 1178 def default_configuration(self): 1179 return self._config.default_configuration() 1180 1181 # 1182 # PROTECTED ROUTINES 1183 # 1184 # The routines below should only be called by routines in this class 1185 # or any of its subclasses. 1186 # 1187 1188 def _uses_apache(self): 1189 return True 1190 1191 # FIXME: This does not belong on the port object. 1192 @memoized 1193 def _path_to_apache(self): 1194 """Returns the full path to the apache binary. 1195 1196 This is needed only by ports that use the apache_http_server module.""" 1197 raise NotImplementedError('Port._path_to_apache') 1198 1199 # FIXME: This belongs on some platform abstraction instead of Port. 1200 def _is_redhat_based(self): 1201 return self._filesystem.exists('/etc/redhat-release') 1202 1203 def _is_debian_based(self): 1204 return self._filesystem.exists('/etc/debian_version') 1205 1206 def _apache_version(self): 1207 config = self._executive.run_command([self._path_to_apache(), '-v']) 1208 return re.sub(r'(?:.|\n)*Server version: Apache/(\d+\.\d+)(?:.|\n)*', r'\1', config) 1209 1210 # We pass sys_platform into this method to make it easy to unit test. 1211 def _apache_config_file_name_for_platform(self, sys_platform): 1212 if sys_platform == 'cygwin': 1213 return 'cygwin-httpd.conf' # CYGWIN is the only platform to still use Apache 1.3. 1214 if sys_platform.startswith('linux'): 1215 if self._is_redhat_based(): 1216 return 'fedora-httpd-' + self._apache_version() + '.conf' 1217 if self._is_debian_based(): 1218 return 'apache2-debian-httpd.conf' 1219 # All platforms use apache2 except for CYGWIN (and Mac OS X Tiger and prior, which we no longer support). 1220 return "apache2-httpd.conf" 1221 1222 def _path_to_apache_config_file(self): 1223 """Returns the full path to the apache configuration file. 1224 1225 If the WEBKIT_HTTP_SERVER_CONF_PATH environment variable is set, its 1226 contents will be used instead. 1227 1228 This is needed only by ports that use the apache_http_server module.""" 1229 config_file_from_env = os.environ.get('WEBKIT_HTTP_SERVER_CONF_PATH') 1230 if config_file_from_env: 1231 if not self._filesystem.exists(config_file_from_env): 1232 raise IOError('%s was not found on the system' % config_file_from_env) 1233 return config_file_from_env 1234 1235 config_file_name = self._apache_config_file_name_for_platform(sys.platform) 1236 return self._filesystem.join(self.layout_tests_dir(), 'http', 'conf', config_file_name) 1237 1238 def _build_path(self, *comps): 1239 build_directory = self.get_option('build_directory') 1240 if build_directory: 1241 root_directory = self._filesystem.join(build_directory, self.get_option('configuration')) 1242 else: 1243 root_directory = self._config.build_directory(self.get_option('configuration')) 1244 return self._filesystem.join(self._filesystem.abspath(root_directory), *comps) 1245 1246 def _path_to_driver(self, configuration=None): 1247 """Returns the full path to the test driver.""" 1248 return self._build_path(self.driver_name()) 1249 1250 def _path_to_webcore_library(self): 1251 """Returns the full path to a built copy of WebCore.""" 1252 return None 1253 1254 def _path_to_helper(self): 1255 """Returns the full path to the layout_test_helper binary, which 1256 is used to help configure the system for the test run, or None 1257 if no helper is needed. 1258 1259 This is likely only used by start/stop_helper().""" 1260 return None 1261 1262 def _path_to_image_diff(self): 1263 """Returns the full path to the image_diff binary, or None if it is not available. 1264 1265 This is likely used only by diff_image()""" 1266 return self._build_path('image_diff') 1267 1268 def _path_to_lighttpd(self): 1269 """Returns the path to the LigHTTPd binary. 1270 1271 This is needed only by ports that use the http_server.py module.""" 1272 raise NotImplementedError('Port._path_to_lighttpd') 1273 1274 def _path_to_lighttpd_modules(self): 1275 """Returns the path to the LigHTTPd modules directory. 1276 1277 This is needed only by ports that use the http_server.py module.""" 1278 raise NotImplementedError('Port._path_to_lighttpd_modules') 1279 1280 def _path_to_lighttpd_php(self): 1281 """Returns the path to the LigHTTPd PHP executable. 1282 1283 This is needed only by ports that use the http_server.py module.""" 1284 raise NotImplementedError('Port._path_to_lighttpd_php') 1285 1286 @memoized 1287 def _path_to_wdiff(self): 1288 """Returns the full path to the wdiff binary, or None if it is not available. 1289 1290 This is likely used only by wdiff_text()""" 1291 for path in ("/usr/bin/wdiff", "/usr/bin/dwdiff"): 1292 if self._filesystem.exists(path): 1293 return path 1294 return None 1295 1296 def _webkit_baseline_path(self, platform): 1297 """Return the full path to the top of the baseline tree for a 1298 given platform.""" 1299 return self._filesystem.join(self.layout_tests_dir(), 'platform', platform) 1300 1301 # FIXME: Belongs on a Platform object. 1302 def _generate_all_test_configurations(self): 1303 """Generates a list of TestConfiguration instances, representing configurations 1304 for a platform across all OSes, architectures, build and graphics types.""" 1305 raise NotImplementedError('Port._generate_all_test_configurations') 1306 1307 def _driver_class(self): 1308 """Returns the port's driver implementation.""" 1309 return driver.Driver 1310 1311 def _get_crash_log(self, name, pid, stdout, stderr, newer_than): 1312 name_str = name or '<unknown process name>' 1313 pid_str = str(pid or '<unknown>') 1314 stdout_lines = (stdout or '<empty>').decode('utf8', 'replace').splitlines() 1315 stderr_lines = (stderr or '<empty>').decode('utf8', 'replace').splitlines() 1316 return (stderr, 'crash log for %s (pid %s):\n%s\n%s\n' % (name_str, pid_str, 1317 '\n'.join(('STDOUT: ' + l) for l in stdout_lines), 1318 '\n'.join(('STDERR: ' + l) for l in stderr_lines))) 1319 1320 def look_for_new_crash_logs(self, crashed_processes, start_time): 1321 pass 1322 1323 def look_for_new_samples(self, unresponsive_processes, start_time): 1324 pass 1325 1326 def sample_process(self, name, pid): 1327 pass 1328 1329 def virtual_test_suites(self): 1330 return [] 1331 1332 @memoized 1333 def populated_virtual_test_suites(self): 1334 suites = self.virtual_test_suites() 1335 1336 # Sanity-check the suites to make sure they don't point to other suites. 1337 suite_dirs = [suite.name for suite in suites] 1338 for suite in suites: 1339 assert suite.base not in suite_dirs 1340 1341 for suite in suites: 1342 base_tests = self._real_tests([suite.base]) 1343 suite.tests = {} 1344 for test in base_tests: 1345 suite.tests[test.replace(suite.base, suite.name, 1)] = test 1346 return suites 1347 1348 def _virtual_tests(self, paths, suites): 1349 virtual_tests = list() 1350 for suite in suites: 1351 if paths: 1352 for test in suite.tests: 1353 if any(test.startswith(p) for p in paths): 1354 virtual_tests.append(test) 1355 else: 1356 virtual_tests.extend(suite.tests.keys()) 1357 return virtual_tests 1358 1359 def is_virtual_test(self, test_name): 1360 return bool(self.lookup_virtual_suite(test_name)) 1361 1362 def lookup_virtual_suite(self, test_name): 1363 for suite in self.populated_virtual_test_suites(): 1364 if test_name.startswith(suite.name): 1365 return suite 1366 return None 1367 1368 def lookup_virtual_test_base(self, test_name): 1369 suite = self.lookup_virtual_suite(test_name) 1370 if not suite: 1371 return None 1372 return test_name.replace(suite.name, suite.base, 1) 1373 1374 def lookup_virtual_test_args(self, test_name): 1375 for suite in self.populated_virtual_test_suites(): 1376 if test_name.startswith(suite.name): 1377 return suite.args 1378 return [] 1379 1380 def should_run_as_pixel_test(self, test_input): 1381 if not self._options.pixel_tests: 1382 return False 1383 if self._options.pixel_test_directories: 1384 return any(test_input.test_name.startswith(directory) for directory in self._options.pixel_test_directories) 1385 return True 1386 1387 def _modules_to_search_for_symbols(self): 1388 path = self._path_to_webcore_library() 1389 if path: 1390 return [path] 1391 return [] 1392 1393 def _symbols_string(self): 1394 symbols = '' 1395 for path_to_module in self._modules_to_search_for_symbols(): 1396 try: 1397 symbols += self._executive.run_command(['nm', path_to_module], error_handler=self._executive.ignore_error) 1398 except OSError, e: 1399 _log.warn("Failed to run nm: %s. Can't determine supported features correctly." % e) 1400 return symbols 1401 1402 # Ports which use compile-time feature detection should define this method and return 1403 # a dictionary mapping from symbol substrings to possibly disabled test directories. 1404 # When the symbol substrings are not matched, the directories will be skipped. 1405 # If ports don't ever enable certain features, then those directories can just be 1406 # in the Skipped list instead of compile-time-checked here. 1407 def _missing_symbol_to_skipped_tests(self): 1408 if self.PORT_HAS_AUDIO_CODECS_BUILT_IN: 1409 return {} 1410 else: 1411 return { 1412 "ff_mp3_decoder": ["webaudio/codec-tests/mp3"], 1413 "ff_aac_decoder": ["webaudio/codec-tests/aac"], 1414 } 1415 1416 def _has_test_in_directories(self, directory_lists, test_list): 1417 if not test_list: 1418 return False 1419 1420 directories = itertools.chain.from_iterable(directory_lists) 1421 for directory, test in itertools.product(directories, test_list): 1422 if test.startswith(directory): 1423 return True 1424 return False 1425 1426 def _skipped_tests_for_unsupported_features(self, test_list): 1427 # Only check the symbols of there are tests in the test_list that might get skipped. 1428 # This is a performance optimization to avoid the calling nm. 1429 # Runtime feature detection not supported, fallback to static detection: 1430 # Disable any tests for symbols missing from the executable or libraries. 1431 if self._has_test_in_directories(self._missing_symbol_to_skipped_tests().values(), test_list): 1432 symbols_string = self._symbols_string() 1433 if symbols_string is not None: 1434 return reduce(operator.add, [directories for symbol_substring, directories in self._missing_symbol_to_skipped_tests().items() if symbol_substring not in symbols_string], []) 1435 return [] 1436 1437 def _convert_path(self, path): 1438 """Handles filename conversion for subprocess command line args.""" 1439 # See note above in diff_image() for why we need this. 1440 if sys.platform == 'cygwin': 1441 return cygpath(path) 1442 return path 1443 1444 1445 class VirtualTestSuite(object): 1446 def __init__(self, name, base, args, tests=None): 1447 self.name = name 1448 self.base = base 1449 self.args = args 1450 self.tests = tests or set() 1451 1452 def __repr__(self): 1453 return "VirtualTestSuite('%s', '%s', %s)" % (self.name, self.base, self.args) 1454
