Home | History | Annotate | Download | only in port
      1 # Copyright (C) 2010 Google Inc. All rights reserved.
      2 #
      3 # Redistribution and use in source and binary forms, with or without
      4 # modification, are permitted provided that the following conditions are
      5 # met:
      6 #
      7 #     * Redistributions of source code must retain the above copyright
      8 # notice, this list of conditions and the following disclaimer.
      9 #     * Redistributions in binary form must reproduce the above
     10 # copyright notice, this list of conditions and the following disclaimer
     11 # in the documentation and/or other materials provided with the
     12 # distribution.
     13 #     * Neither the Google name nor the names of its
     14 # contributors may be used to endorse or promote products derived from
     15 # this software without specific prior written permission.
     16 #
     17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
     18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
     19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
     20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
     21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
     22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
     23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
     24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
     25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
     27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     28 
     29 """Abstract base class of Port-specific entry points for the layout tests
     30 test infrastructure (the Port and Driver classes)."""
     31 
     32 import cgi
     33 import difflib
     34 import errno
     35 import itertools
     36 import logging
     37 import os
     38 import operator
     39 import optparse
     40 import re
     41 import sys
     42 
     43 try:
     44     from collections import OrderedDict
     45 except ImportError:
     46     # Needed for Python < 2.7
     47     from webkitpy.thirdparty.ordered_dict import OrderedDict
     48 
     49 
     50 from webkitpy.common import find_files
     51 from webkitpy.common import read_checksum_from_png
     52 from webkitpy.common.memoized import memoized
     53 from webkitpy.common.system import path
     54 from webkitpy.common.system.executive import ScriptError
     55 from webkitpy.common.system.path import cygpath
     56 from webkitpy.common.system.systemhost import SystemHost
     57 from webkitpy.common.webkit_finder import WebKitFinder
     58 from webkitpy.layout_tests.layout_package.bot_test_expectations import BotTestExpectationsFactory
     59 from webkitpy.layout_tests.models import test_run_results
     60 from webkitpy.layout_tests.models.test_configuration import TestConfiguration
     61 from webkitpy.layout_tests.port import config as port_config
     62 from webkitpy.layout_tests.port import driver
     63 from webkitpy.layout_tests.port import server_process
     64 from webkitpy.layout_tests.port.factory import PortFactory
     65 from webkitpy.layout_tests.servers import apache_http
     66 from webkitpy.layout_tests.servers import lighttpd
     67 from webkitpy.layout_tests.servers import pywebsocket
     68 
     69 _log = logging.getLogger(__name__)
     70 
     71 
     72 # FIXME: This class should merge with WebKitPort now that Chromium behaves mostly like other webkit ports.
     73 class Port(object):
     74     """Abstract class for Port-specific hooks for the layout_test package."""
     75 
     76     # Subclasses override this. This should indicate the basic implementation
     77     # part of the port name, e.g., 'mac', 'win', 'gtk'; there is probably (?)
     78     # one unique value per class.
     79 
     80     # FIXME: We should probably rename this to something like 'implementation_name'.
     81     port_name = None
     82 
     83     # Test names resemble unix relative paths, and use '/' as a directory separator.
     84     TEST_PATH_SEPARATOR = '/'
     85 
     86     ALL_BUILD_TYPES = ('debug', 'release')
     87 
     88     CONTENT_SHELL_NAME = 'content_shell'
     89 
     90     # True if the port as aac and mp3 codecs built in.
     91     PORT_HAS_AUDIO_CODECS_BUILT_IN = False
     92 
     93     ALL_SYSTEMS = (
     94         ('snowleopard', 'x86'),
     95         ('lion', 'x86'),
     96 
     97         # FIXME: We treat Retina (High-DPI) devices as if they are running
     98         # a different operating system version. This isn't accurate, but will work until
     99         # we need to test and support baselines across multiple O/S versions.
    100         ('retina', 'x86'),
    101 
    102         ('mountainlion', 'x86'),
    103         ('mavericks', 'x86'),
    104         ('xp', 'x86'),
    105         ('win7', 'x86'),
    106         ('lucid', 'x86'),
    107         ('lucid', 'x86_64'),
    108         # FIXME: Technically this should be 'arm', but adding a third architecture type breaks TestConfigurationConverter.
    109         # If we need this to be 'arm' in the future, then we first have to fix TestConfigurationConverter.
    110         ('icecreamsandwich', 'x86'),
    111         )
    112 
    113     ALL_BASELINE_VARIANTS = [
    114         'mac-mavericks', 'mac-mountainlion', 'mac-retina', 'mac-lion', 'mac-snowleopard',
    115         'win-win7', 'win-xp',
    116         'linux-x86_64', 'linux-x86',
    117     ]
    118 
    119     CONFIGURATION_SPECIFIER_MACROS = {
    120         'mac': ['snowleopard', 'lion', 'retina', 'mountainlion', 'mavericks'],
    121         'win': ['xp', 'win7'],
    122         'linux': ['lucid'],
    123         'android': ['icecreamsandwich'],
    124     }
    125 
    126     DEFAULT_BUILD_DIRECTORIES = ('out',)
    127 
    128     # overridden in subclasses.
    129     FALLBACK_PATHS = {}
    130 
    131     SUPPORTED_VERSIONS = []
    132 
    133     # URL to the build requirements page.
    134     BUILD_REQUIREMENTS_URL = ''
    135 
    136     @classmethod
    137     def latest_platform_fallback_path(cls):
    138         return cls.FALLBACK_PATHS[cls.SUPPORTED_VERSIONS[-1]]
    139 
    140     @classmethod
    141     def _static_build_path(cls, filesystem, build_directory, chromium_base, configuration, comps):
    142         if build_directory:
    143             return filesystem.join(build_directory, configuration, *comps)
    144 
    145         hits = []
    146         for directory in cls.DEFAULT_BUILD_DIRECTORIES:
    147             base_dir = filesystem.join(chromium_base, directory, configuration)
    148             path = filesystem.join(base_dir, *comps)
    149             if filesystem.exists(path):
    150                 hits.append((filesystem.mtime(path), path))
    151 
    152         if hits:
    153             hits.sort(reverse=True)
    154             return hits[0][1]  # Return the newest file found.
    155 
    156         # We have to default to something, so pick the last one.
    157         return filesystem.join(base_dir, *comps)
    158 
    159     @classmethod
    160     def determine_full_port_name(cls, host, options, port_name):
    161         """Return a fully-specified port name that can be used to construct objects."""
    162         # Subclasses will usually override this.
    163         assert port_name.startswith(cls.port_name)
    164         return port_name
    165 
    166     def __init__(self, host, port_name, options=None, **kwargs):
    167 
    168         # This value may be different from cls.port_name by having version modifiers
    169         # and other fields appended to it (for example, 'qt-arm' or 'mac-wk2').
    170         self._name = port_name
    171 
    172         # These are default values that should be overridden in a subclasses.
    173         self._version = ''
    174         self._architecture = 'x86'
    175 
    176         # FIXME: Ideally we'd have a package-wide way to get a
    177         # well-formed options object that had all of the necessary
    178         # options defined on it.
    179         self._options = options or optparse.Values()
    180 
    181         self.host = host
    182         self._executive = host.executive
    183         self._filesystem = host.filesystem
    184         self._webkit_finder = WebKitFinder(host.filesystem)
    185         self._config = port_config.Config(self._executive, self._filesystem, self.port_name)
    186 
    187         self._helper = None
    188         self._http_server = None
    189         self._websocket_server = None
    190         self._image_differ = None
    191         self._server_process_constructor = server_process.ServerProcess  # overridable for testing
    192         self._http_lock = None  # FIXME: Why does this live on the port object?
    193         self._dump_reader = None
    194 
    195         # Python's Popen has a bug that causes any pipes opened to a
    196         # process that can't be executed to be leaked.  Since this
    197         # code is specifically designed to tolerate exec failures
    198         # to gracefully handle cases where wdiff is not installed,
    199         # the bug results in a massive file descriptor leak. As a
    200         # workaround, if an exec failure is ever experienced for
    201         # wdiff, assume it's not available.  This will leak one
    202         # file descriptor but that's better than leaking each time
    203         # wdiff would be run.
    204         #
    205         # http://mail.python.org/pipermail/python-list/
    206         #    2008-August/505753.html
    207         # http://bugs.python.org/issue3210
    208         self._wdiff_available = None
    209 
    210         # FIXME: prettypatch.py knows this path, why is it copied here?
    211         self._pretty_patch_path = self.path_from_webkit_base("Tools", "Scripts", "webkitruby", "PrettyPatch", "prettify.rb")
    212         self._pretty_patch_available = None
    213 
    214         if not hasattr(options, 'configuration') or not options.configuration:
    215             self.set_option_default('configuration', self.default_configuration())
    216         self._test_configuration = None
    217         self._reftest_list = {}
    218         self._results_directory = None
    219 
    220     def buildbot_archives_baselines(self):
    221         return True
    222 
    223     def additional_drt_flag(self):
    224         if self.driver_name() == self.CONTENT_SHELL_NAME:
    225             return ['--dump-render-tree']
    226         return []
    227 
    228     def supports_per_test_timeout(self):
    229         return False
    230 
    231     def default_pixel_tests(self):
    232         return True
    233 
    234     def default_smoke_test_only(self):
    235         return False
    236 
    237     def default_timeout_ms(self):
    238         timeout_ms = 6 * 1000
    239         if self.get_option('configuration') == 'Debug':
    240             # Debug is usually 2x-3x slower than Release.
    241             return 3 * timeout_ms
    242         return timeout_ms
    243 
    244     def driver_stop_timeout(self):
    245         """ Returns the amount of time in seconds to wait before killing the process in driver.stop()."""
    246         # We want to wait for at least 3 seconds, but if we are really slow, we want to be slow on cleanup as
    247         # well (for things like ASAN, Valgrind, etc.)
    248         return 3.0 * float(self.get_option('time_out_ms', '0')) / self.default_timeout_ms()
    249 
    250     def wdiff_available(self):
    251         if self._wdiff_available is None:
    252             self._wdiff_available = self.check_wdiff(logging=False)
    253         return self._wdiff_available
    254 
    255     def pretty_patch_available(self):
    256         if self._pretty_patch_available is None:
    257             self._pretty_patch_available = self.check_pretty_patch(logging=False)
    258         return self._pretty_patch_available
    259 
    260     def default_child_processes(self):
    261         """Return the number of drivers to use for this port."""
    262         return self._executive.cpu_count()
    263 
    264     def default_max_locked_shards(self):
    265         """Return the number of "locked" shards to run in parallel (like the http tests)."""
    266         max_locked_shards = int(self.default_child_processes()) / 4
    267         if not max_locked_shards:
    268             return 1
    269         return max_locked_shards
    270 
    271     def baseline_path(self):
    272         """Return the absolute path to the directory to store new baselines in for this port."""
    273         # FIXME: remove once all callers are calling either baseline_version_dir() or baseline_platform_dir()
    274         return self.baseline_version_dir()
    275 
    276     def baseline_platform_dir(self):
    277         """Return the absolute path to the default (version-independent) platform-specific results."""
    278         return self._filesystem.join(self.layout_tests_dir(), 'platform', self.port_name)
    279 
    280     def baseline_version_dir(self):
    281         """Return the absolute path to the platform-and-version-specific results."""
    282         baseline_search_paths = self.baseline_search_path()
    283         return baseline_search_paths[0]
    284 
    285     def virtual_baseline_search_path(self, test_name):
    286         suite = self.lookup_virtual_suite(test_name)
    287         if not suite:
    288             return None
    289         return [self._filesystem.join(path, suite.name) for path in self.default_baseline_search_path()]
    290 
    291     def baseline_search_path(self):
    292         return self.get_option('additional_platform_directory', []) + self._compare_baseline() + self.default_baseline_search_path()
    293 
    294     def default_baseline_search_path(self):
    295         """Return a list of absolute paths to directories to search under for
    296         baselines. The directories are searched in order."""
    297         return map(self._webkit_baseline_path, self.FALLBACK_PATHS[self.version()])
    298 
    299     @memoized
    300     def _compare_baseline(self):
    301         factory = PortFactory(self.host)
    302         target_port = self.get_option('compare_port')
    303         if target_port:
    304             return factory.get(target_port).default_baseline_search_path()
    305         return []
    306 
    307     def _check_file_exists(self, path_to_file, file_description,
    308                            override_step=None, logging=True):
    309         """Verify the file is present where expected or log an error.
    310 
    311         Args:
    312             file_name: The (human friendly) name or description of the file
    313                 you're looking for (e.g., "HTTP Server"). Used for error logging.
    314             override_step: An optional string to be logged if the check fails.
    315             logging: Whether or not log the error messages."""
    316         if not self._filesystem.exists(path_to_file):
    317             if logging:
    318                 _log.error('Unable to find %s' % file_description)
    319                 _log.error('    at %s' % path_to_file)
    320                 if override_step:
    321                     _log.error('    %s' % override_step)
    322                     _log.error('')
    323             return False
    324         return True
    325 
    326     def check_build(self, needs_http, printer):
    327         result = True
    328 
    329         dump_render_tree_binary_path = self._path_to_driver()
    330         result = self._check_file_exists(dump_render_tree_binary_path,
    331                                          'test driver') and result
    332         if not result and self.get_option('build'):
    333             result = self._check_driver_build_up_to_date(
    334                 self.get_option('configuration'))
    335         else:
    336             _log.error('')
    337 
    338         helper_path = self._path_to_helper()
    339         if helper_path:
    340             result = self._check_file_exists(helper_path,
    341                                              'layout test helper') and result
    342 
    343         if self.get_option('pixel_tests'):
    344             result = self.check_image_diff(
    345                 'To override, invoke with --no-pixel-tests') and result
    346 
    347         # It's okay if pretty patch and wdiff aren't available, but we will at least log messages.
    348         self._pretty_patch_available = self.check_pretty_patch()
    349         self._wdiff_available = self.check_wdiff()
    350 
    351         if self._dump_reader:
    352             result = self._dump_reader.check_is_functional() and result
    353 
    354         if needs_http:
    355             result = self.check_httpd() and result
    356 
    357         return test_run_results.OK_EXIT_STATUS if result else test_run_results.UNEXPECTED_ERROR_EXIT_STATUS
    358 
    359     def _check_driver(self):
    360         driver_path = self._path_to_driver()
    361         if not self._filesystem.exists(driver_path):
    362             _log.error("%s was not found at %s" % (self.driver_name(), driver_path))
    363             return False
    364         return True
    365 
    366     def _check_port_build(self):
    367         # Ports can override this method to do additional checks.
    368         return True
    369 
    370     def check_sys_deps(self, needs_http):
    371         """If the port needs to do some runtime checks to ensure that the
    372         tests can be run successfully, it should override this routine.
    373         This step can be skipped with --nocheck-sys-deps.
    374 
    375         Returns whether the system is properly configured."""
    376         cmd = [self._path_to_driver(), '--check-layout-test-sys-deps']
    377 
    378         local_error = ScriptError()
    379 
    380         def error_handler(script_error):
    381             local_error.exit_code = script_error.exit_code
    382 
    383         output = self._executive.run_command(cmd, error_handler=error_handler)
    384         if local_error.exit_code:
    385             _log.error('System dependencies check failed.')
    386             _log.error('To override, invoke with --nocheck-sys-deps')
    387             _log.error('')
    388             _log.error(output)
    389             if self.BUILD_REQUIREMENTS_URL is not '':
    390                 _log.error('')
    391                 _log.error('For complete build requirements, please see:')
    392                 _log.error(self.BUILD_REQUIREMENTS_URL)
    393             return test_run_results.SYS_DEPS_EXIT_STATUS
    394         return test_run_results.OK_EXIT_STATUS
    395 
    396     def check_image_diff(self, override_step=None, logging=True):
    397         """This routine is used to check whether image_diff binary exists."""
    398         image_diff_path = self._path_to_image_diff()
    399         if not self._filesystem.exists(image_diff_path):
    400             _log.error("image_diff was not found at %s" % image_diff_path)
    401             return False
    402         return True
    403 
    404     def check_pretty_patch(self, logging=True):
    405         """Checks whether we can use the PrettyPatch ruby script."""
    406         try:
    407             _ = self._executive.run_command(['ruby', '--version'])
    408         except OSError, e:
    409             if e.errno in [errno.ENOENT, errno.EACCES, errno.ECHILD]:
    410                 if logging:
    411                     _log.warning("Ruby is not installed; can't generate pretty patches.")
    412                     _log.warning('')
    413                 return False
    414 
    415         if not self._filesystem.exists(self._pretty_patch_path):
    416             if logging:
    417                 _log.warning("Unable to find %s; can't generate pretty patches." % self._pretty_patch_path)
    418                 _log.warning('')
    419             return False
    420 
    421         return True
    422 
    423     def check_wdiff(self, logging=True):
    424         if not self._path_to_wdiff():
    425             # Don't need to log here since this is the port choosing not to use wdiff.
    426             return False
    427 
    428         try:
    429             _ = self._executive.run_command([self._path_to_wdiff(), '--help'])
    430         except OSError:
    431             if logging:
    432                 message = self._wdiff_missing_message()
    433                 if message:
    434                     for line in message.splitlines():
    435                         _log.warning('    ' + line)
    436                         _log.warning('')
    437             return False
    438 
    439         return True
    440 
    441     def _wdiff_missing_message(self):
    442         return 'wdiff is not installed; please install it to generate word-by-word diffs.'
    443 
    444     def check_httpd(self):
    445         if self.uses_apache():
    446             httpd_path = self.path_to_apache()
    447         else:
    448             httpd_path = self.path_to_lighttpd()
    449 
    450         try:
    451             server_name = self._filesystem.basename(httpd_path)
    452             env = self.setup_environ_for_server(server_name)
    453             if self._executive.run_command([httpd_path, "-v"], env=env, return_exit_code=True) != 0:
    454                 _log.error("httpd seems broken. Cannot run http tests.")
    455                 return False
    456             return True
    457         except OSError:
    458             _log.error("No httpd found. Cannot run http tests.")
    459             return False
    460 
    461     def do_text_results_differ(self, expected_text, actual_text):
    462         return expected_text != actual_text
    463 
    464     def do_audio_results_differ(self, expected_audio, actual_audio):
    465         return expected_audio != actual_audio
    466 
    467     def diff_image(self, expected_contents, actual_contents):
    468         """Compare two images and return a tuple of an image diff, and an error string.
    469 
    470         If an error occurs (like image_diff isn't found, or crashes, we log an error and return True (for a diff).
    471         """
    472         # If only one of them exists, return that one.
    473         if not actual_contents and not expected_contents:
    474             return (None, None)
    475         if not actual_contents:
    476             return (expected_contents, None)
    477         if not expected_contents:
    478             return (actual_contents, None)
    479 
    480         tempdir = self._filesystem.mkdtemp()
    481 
    482         expected_filename = self._filesystem.join(str(tempdir), "expected.png")
    483         self._filesystem.write_binary_file(expected_filename, expected_contents)
    484 
    485         actual_filename = self._filesystem.join(str(tempdir), "actual.png")
    486         self._filesystem.write_binary_file(actual_filename, actual_contents)
    487 
    488         diff_filename = self._filesystem.join(str(tempdir), "diff.png")
    489 
    490         # image_diff needs native win paths as arguments, so we need to convert them if running under cygwin.
    491         native_expected_filename = self._convert_path(expected_filename)
    492         native_actual_filename = self._convert_path(actual_filename)
    493         native_diff_filename = self._convert_path(diff_filename)
    494 
    495         executable = self._path_to_image_diff()
    496         # Note that although we are handed 'old', 'new', image_diff wants 'new', 'old'.
    497         comand = [executable, '--diff', native_actual_filename, native_expected_filename, native_diff_filename]
    498 
    499         result = None
    500         err_str = None
    501         try:
    502             exit_code = self._executive.run_command(comand, return_exit_code=True)
    503             if exit_code == 0:
    504                 # The images are the same.
    505                 result = None
    506             elif exit_code == 1:
    507                 result = self._filesystem.read_binary_file(native_diff_filename)
    508             else:
    509                 err_str = "Image diff returned an exit code of %s. See http://crbug.com/278596" % exit_code
    510         except OSError, e:
    511             err_str = 'error running image diff: %s' % str(e)
    512         finally:
    513             self._filesystem.rmtree(str(tempdir))
    514 
    515         return (result, err_str or None)
    516 
    517     def diff_text(self, expected_text, actual_text, expected_filename, actual_filename):
    518         """Returns a string containing the diff of the two text strings
    519         in 'unified diff' format."""
    520 
    521         # The filenames show up in the diff output, make sure they're
    522         # raw bytes and not unicode, so that they don't trigger join()
    523         # trying to decode the input.
    524         def to_raw_bytes(string_value):
    525             if isinstance(string_value, unicode):
    526                 return string_value.encode('utf-8')
    527             return string_value
    528         expected_filename = to_raw_bytes(expected_filename)
    529         actual_filename = to_raw_bytes(actual_filename)
    530         diff = difflib.unified_diff(expected_text.splitlines(True),
    531                                     actual_text.splitlines(True),
    532                                     expected_filename,
    533                                     actual_filename)
    534         return ''.join(diff)
    535 
    536     def driver_name(self):
    537         if self.get_option('driver_name'):
    538             return self.get_option('driver_name')
    539         return self.CONTENT_SHELL_NAME
    540 
    541     def expected_baselines_by_extension(self, test_name):
    542         """Returns a dict mapping baseline suffix to relative path for each baseline in
    543         a test. For reftests, it returns ".==" or ".!=" instead of the suffix."""
    544         # FIXME: The name similarity between this and expected_baselines() below, is unfortunate.
    545         # We should probably rename them both.
    546         baseline_dict = {}
    547         reference_files = self.reference_files(test_name)
    548         if reference_files:
    549             # FIXME: How should this handle more than one type of reftest?
    550             baseline_dict['.' + reference_files[0][0]] = self.relative_test_filename(reference_files[0][1])
    551 
    552         for extension in self.baseline_extensions():
    553             path = self.expected_filename(test_name, extension, return_default=False)
    554             baseline_dict[extension] = self.relative_test_filename(path) if path else path
    555 
    556         return baseline_dict
    557 
    558     def baseline_extensions(self):
    559         """Returns a tuple of all of the non-reftest baseline extensions we use. The extensions include the leading '.'."""
    560         return ('.wav', '.txt', '.png')
    561 
    562     def expected_baselines(self, test_name, suffix, all_baselines=False):
    563         """Given a test name, finds where the baseline results are located.
    564 
    565         Args:
    566         test_name: name of test file (usually a relative path under LayoutTests/)
    567         suffix: file suffix of the expected results, including dot; e.g.
    568             '.txt' or '.png'.  This should not be None, but may be an empty
    569             string.
    570         all_baselines: If True, return an ordered list of all baseline paths
    571             for the given platform. If False, return only the first one.
    572         Returns
    573         a list of ( platform_dir, results_filename ), where
    574             platform_dir - abs path to the top of the results tree (or test
    575                 tree)
    576             results_filename - relative path from top of tree to the results
    577                 file
    578             (port.join() of the two gives you the full path to the file,
    579                 unless None was returned.)
    580         Return values will be in the format appropriate for the current
    581         platform (e.g., "\\" for path separators on Windows). If the results
    582         file is not found, then None will be returned for the directory,
    583         but the expected relative pathname will still be returned.
    584 
    585         This routine is generic but lives here since it is used in
    586         conjunction with the other baseline and filename routines that are
    587         platform specific.
    588         """
    589         baseline_filename = self._filesystem.splitext(test_name)[0] + '-expected' + suffix
    590         baseline_search_path = self.baseline_search_path()
    591 
    592         baselines = []
    593         for platform_dir in baseline_search_path:
    594             if self._filesystem.exists(self._filesystem.join(platform_dir, baseline_filename)):
    595                 baselines.append((platform_dir, baseline_filename))
    596 
    597             if not all_baselines and baselines:
    598                 return baselines
    599 
    600         # If it wasn't found in a platform directory, return the expected
    601         # result in the test directory, even if no such file actually exists.
    602         platform_dir = self.layout_tests_dir()
    603         if self._filesystem.exists(self._filesystem.join(platform_dir, baseline_filename)):
    604             baselines.append((platform_dir, baseline_filename))
    605 
    606         if baselines:
    607             return baselines
    608 
    609         return [(None, baseline_filename)]
    610 
    611     def expected_filename(self, test_name, suffix, return_default=True):
    612         """Given a test name, returns an absolute path to its expected results.
    613 
    614         If no expected results are found in any of the searched directories,
    615         the directory in which the test itself is located will be returned.
    616         The return value is in the format appropriate for the platform
    617         (e.g., "\\" for path separators on windows).
    618 
    619         Args:
    620         test_name: name of test file (usually a relative path under LayoutTests/)
    621         suffix: file suffix of the expected results, including dot; e.g. '.txt'
    622             or '.png'.  This should not be None, but may be an empty string.
    623         platform: the most-specific directory name to use to build the
    624             search list of directories, e.g., 'win', or
    625             'chromium-cg-mac-leopard' (we follow the WebKit format)
    626         return_default: if True, returns the path to the generic expectation if nothing
    627             else is found; if False, returns None.
    628 
    629         This routine is generic but is implemented here to live alongside
    630         the other baseline and filename manipulation routines.
    631         """
    632         # FIXME: The [0] here is very mysterious, as is the destructured return.
    633         platform_dir, baseline_filename = self.expected_baselines(test_name, suffix)[0]
    634         if platform_dir:
    635             return self._filesystem.join(platform_dir, baseline_filename)
    636 
    637         actual_test_name = self.lookup_virtual_test_base(test_name)
    638         if actual_test_name:
    639             return self.expected_filename(actual_test_name, suffix)
    640 
    641         if return_default:
    642             return self._filesystem.join(self.layout_tests_dir(), baseline_filename)
    643         return None
    644 
    645     def expected_checksum(self, test_name):
    646         """Returns the checksum of the image we expect the test to produce, or None if it is a text-only test."""
    647         png_path = self.expected_filename(test_name, '.png')
    648 
    649         if self._filesystem.exists(png_path):
    650             with self._filesystem.open_binary_file_for_reading(png_path) as filehandle:
    651                 return read_checksum_from_png.read_checksum(filehandle)
    652 
    653         return None
    654 
    655     def expected_image(self, test_name):
    656         """Returns the image we expect the test to produce."""
    657         baseline_path = self.expected_filename(test_name, '.png')
    658         if not self._filesystem.exists(baseline_path):
    659             return None
    660         return self._filesystem.read_binary_file(baseline_path)
    661 
    662     def expected_audio(self, test_name):
    663         baseline_path = self.expected_filename(test_name, '.wav')
    664         if not self._filesystem.exists(baseline_path):
    665             return None
    666         return self._filesystem.read_binary_file(baseline_path)
    667 
    668     def expected_text(self, test_name):
    669         """Returns the text output we expect the test to produce, or None
    670         if we don't expect there to be any text output.
    671         End-of-line characters are normalized to '\n'."""
    672         # FIXME: DRT output is actually utf-8, but since we don't decode the
    673         # output from DRT (instead treating it as a binary string), we read the
    674         # baselines as a binary string, too.
    675         baseline_path = self.expected_filename(test_name, '.txt')
    676         if not self._filesystem.exists(baseline_path):
    677             return None
    678         text = self._filesystem.read_binary_file(baseline_path)
    679         return text.replace("\r\n", "\n")
    680 
    681     def _get_reftest_list(self, test_name):
    682         dirname = self._filesystem.join(self.layout_tests_dir(), self._filesystem.dirname(test_name))
    683         if dirname not in self._reftest_list:
    684             self._reftest_list[dirname] = Port._parse_reftest_list(self._filesystem, dirname)
    685         return self._reftest_list[dirname]
    686 
    687     @staticmethod
    688     def _parse_reftest_list(filesystem, test_dirpath):
    689         reftest_list_path = filesystem.join(test_dirpath, 'reftest.list')
    690         if not filesystem.isfile(reftest_list_path):
    691             return None
    692         reftest_list_file = filesystem.read_text_file(reftest_list_path)
    693 
    694         parsed_list = {}
    695         for line in reftest_list_file.split('\n'):
    696             line = re.sub('#.+$', '', line)
    697             split_line = line.split()
    698             if len(split_line) == 4:
    699                 # FIXME: Probably one of mozilla's extensions in the reftest.list format. Do we need to support this?
    700                 _log.warning("unsupported reftest.list line '%s' in %s" % (line, reftest_list_path))
    701                 continue
    702             if len(split_line) < 3:
    703                 continue
    704             expectation_type, test_file, ref_file = split_line
    705             parsed_list.setdefault(filesystem.join(test_dirpath, test_file), []).append((expectation_type, filesystem.join(test_dirpath, ref_file)))
    706         return parsed_list
    707 
    708     def reference_files(self, test_name):
    709         """Return a list of expectation (== or !=) and filename pairs"""
    710 
    711         reftest_list = self._get_reftest_list(test_name)
    712         if not reftest_list:
    713             reftest_list = []
    714             for expectation, prefix in (('==', ''), ('!=', '-mismatch')):
    715                 for extention in Port._supported_file_extensions:
    716                     path = self.expected_filename(test_name, prefix + extention)
    717                     if self._filesystem.exists(path):
    718                         reftest_list.append((expectation, path))
    719             return reftest_list
    720 
    721         return reftest_list.get(self._filesystem.join(self.layout_tests_dir(), test_name), [])  # pylint: disable=E1103
    722 
    723     def tests(self, paths):
    724         """Return the list of tests found matching paths."""
    725         tests = self._real_tests(paths)
    726         tests.extend(self._virtual_tests(paths, self.populated_virtual_test_suites()))
    727         return tests
    728 
    729     def _real_tests(self, paths):
    730         # When collecting test cases, skip these directories
    731         skipped_directories = set(['.svn', '_svn', 'platform', 'resources', 'script-tests', 'reference', 'reftest'])
    732         files = find_files.find(self._filesystem, self.layout_tests_dir(), paths, skipped_directories, Port.is_test_file, self.test_key)
    733         return [self.relative_test_filename(f) for f in files]
    734 
    735     # When collecting test cases, we include any file with these extensions.
    736     _supported_file_extensions = set(['.html', '.xml', '.xhtml', '.xht', '.pl',
    737                                       '.htm', '.php', '.svg', '.mht'])
    738 
    739     @staticmethod
    740     # If any changes are made here be sure to update the isUsedInReftest method in old-run-webkit-tests as well.
    741     def is_reference_html_file(filesystem, dirname, filename):
    742         if filename.startswith('ref-') or filename.startswith('notref-'):
    743             return True
    744         filename_wihout_ext, unused = filesystem.splitext(filename)
    745         for suffix in ['-expected', '-expected-mismatch', '-ref', '-notref']:
    746             if filename_wihout_ext.endswith(suffix):
    747                 return True
    748         return False
    749 
    750     @staticmethod
    751     def _has_supported_extension(filesystem, filename):
    752         """Return true if filename is one of the file extensions we want to run a test on."""
    753         extension = filesystem.splitext(filename)[1]
    754         return extension in Port._supported_file_extensions
    755 
    756     @staticmethod
    757     def is_test_file(filesystem, dirname, filename):
    758         return Port._has_supported_extension(filesystem, filename) and not Port.is_reference_html_file(filesystem, dirname, filename)
    759 
    760     ALL_TEST_TYPES = ['audio', 'harness', 'pixel', 'ref', 'text', 'unknown']
    761 
    762     def test_type(self, test_name):
    763         fs = self._filesystem
    764         if fs.exists(self.expected_filename(test_name, '.png')):
    765             return 'pixel'
    766         if fs.exists(self.expected_filename(test_name, '.wav')):
    767             return 'audio'
    768         if self.reference_files(test_name):
    769             return 'ref'
    770         txt = self.expected_text(test_name)
    771         if txt:
    772             if 'layer at (0,0) size 800x600' in txt:
    773                 return 'pixel'
    774             for line in txt.splitlines():
    775                 if line.startswith('FAIL') or line.startswith('TIMEOUT') or line.startswith('PASS'):
    776                     return 'harness'
    777             return 'text'
    778         return 'unknown'
    779 
    780     def test_key(self, test_name):
    781         """Turns a test name into a list with two sublists, the natural key of the
    782         dirname, and the natural key of the basename.
    783 
    784         This can be used when sorting paths so that files in a directory.
    785         directory are kept together rather than being mixed in with files in
    786         subdirectories."""
    787         dirname, basename = self.split_test(test_name)
    788         return (self._natural_sort_key(dirname + self.TEST_PATH_SEPARATOR), self._natural_sort_key(basename))
    789 
    790     def _natural_sort_key(self, string_to_split):
    791         """ Turns a string into a list of string and number chunks, i.e. "z23a" -> ["z", 23, "a"]
    792 
    793         This can be used to implement "natural sort" order. See:
    794         http://www.codinghorror.com/blog/2007/12/sorting-for-humans-natural-sort-order.html
    795         http://nedbatchelder.com/blog/200712.html#e20071211T054956
    796         """
    797         def tryint(val):
    798             try:
    799                 return int(val)
    800             except ValueError:
    801                 return val
    802 
    803         return [tryint(chunk) for chunk in re.split('(\d+)', string_to_split)]
    804 
    805     def test_dirs(self):
    806         """Returns the list of top-level test directories."""
    807         layout_tests_dir = self.layout_tests_dir()
    808         return filter(lambda x: self._filesystem.isdir(self._filesystem.join(layout_tests_dir, x)),
    809                       self._filesystem.listdir(layout_tests_dir))
    810 
    811     @memoized
    812     def test_isfile(self, test_name):
    813         """Return True if the test name refers to a directory of tests."""
    814         # Used by test_expectations.py to apply rules to whole directories.
    815         if self._filesystem.isfile(self.abspath_for_test(test_name)):
    816             return True
    817         base = self.lookup_virtual_test_base(test_name)
    818         return base and self._filesystem.isfile(self.abspath_for_test(base))
    819 
    820     @memoized
    821     def test_isdir(self, test_name):
    822         """Return True if the test name refers to a directory of tests."""
    823         # Used by test_expectations.py to apply rules to whole directories.
    824         if self._filesystem.isdir(self.abspath_for_test(test_name)):
    825             return True
    826         base = self.lookup_virtual_test_base(test_name)
    827         return base and self._filesystem.isdir(self.abspath_for_test(base))
    828 
    829     @memoized
    830     def test_exists(self, test_name):
    831         """Return True if the test name refers to an existing test or baseline."""
    832         # Used by test_expectations.py to determine if an entry refers to a
    833         # valid test and by printing.py to determine if baselines exist.
    834         return self.test_isfile(test_name) or self.test_isdir(test_name)
    835 
    836     def split_test(self, test_name):
    837         """Splits a test name into the 'directory' part and the 'basename' part."""
    838         index = test_name.rfind(self.TEST_PATH_SEPARATOR)
    839         if index < 1:
    840             return ('', test_name)
    841         return (test_name[0:index], test_name[index:])
    842 
    843     def normalize_test_name(self, test_name):
    844         """Returns a normalized version of the test name or test directory."""
    845         if test_name.endswith('/'):
    846             return test_name
    847         if self.test_isdir(test_name):
    848             return test_name + '/'
    849         return test_name
    850 
    851     def driver_cmd_line(self):
    852         """Prints the DRT command line that will be used."""
    853         driver = self.create_driver(0)
    854         return driver.cmd_line(self.get_option('pixel_tests'), [])
    855 
    856     def update_baseline(self, baseline_path, data):
    857         """Updates the baseline for a test.
    858 
    859         Args:
    860             baseline_path: the actual path to use for baseline, not the path to
    861               the test. This function is used to update either generic or
    862               platform-specific baselines, but we can't infer which here.
    863             data: contents of the baseline.
    864         """
    865         self._filesystem.write_binary_file(baseline_path, data)
    866 
    867     # FIXME: update callers to create a finder and call it instead of these next five routines (which should be protected).
    868     def webkit_base(self):
    869         return self._webkit_finder.webkit_base()
    870 
    871     def path_from_webkit_base(self, *comps):
    872         return self._webkit_finder.path_from_webkit_base(*comps)
    873 
    874     def path_from_chromium_base(self, *comps):
    875         return self._webkit_finder.path_from_chromium_base(*comps)
    876 
    877     def path_to_script(self, script_name):
    878         return self._webkit_finder.path_to_script(script_name)
    879 
    880     def layout_tests_dir(self):
    881         return self._webkit_finder.layout_tests_dir()
    882 
    883     def perf_tests_dir(self):
    884         return self._webkit_finder.perf_tests_dir()
    885 
    886     def skipped_layout_tests(self, test_list):
    887         """Returns tests skipped outside of the TestExpectations files."""
    888         return set(self._skipped_tests_for_unsupported_features(test_list))
    889 
    890     def _tests_from_skipped_file_contents(self, skipped_file_contents):
    891         tests_to_skip = []
    892         for line in skipped_file_contents.split('\n'):
    893             line = line.strip()
    894             line = line.rstrip('/')  # Best to normalize directory names to not include the trailing slash.
    895             if line.startswith('#') or not len(line):
    896                 continue
    897             tests_to_skip.append(line)
    898         return tests_to_skip
    899 
    900     def _expectations_from_skipped_files(self, skipped_file_paths):
    901         tests_to_skip = []
    902         for search_path in skipped_file_paths:
    903             filename = self._filesystem.join(self._webkit_baseline_path(search_path), "Skipped")
    904             if not self._filesystem.exists(filename):
    905                 _log.debug("Skipped does not exist: %s" % filename)
    906                 continue
    907             _log.debug("Using Skipped file: %s" % filename)
    908             skipped_file_contents = self._filesystem.read_text_file(filename)
    909             tests_to_skip.extend(self._tests_from_skipped_file_contents(skipped_file_contents))
    910         return tests_to_skip
    911 
    912     @memoized
    913     def skipped_perf_tests(self):
    914         return self._expectations_from_skipped_files([self.perf_tests_dir()])
    915 
    916     def skips_perf_test(self, test_name):
    917         for test_or_category in self.skipped_perf_tests():
    918             if test_or_category == test_name:
    919                 return True
    920             category = self._filesystem.join(self.perf_tests_dir(), test_or_category)
    921             if self._filesystem.isdir(category) and test_name.startswith(test_or_category):
    922                 return True
    923         return False
    924 
    925     def is_chromium(self):
    926         return True
    927 
    928     def name(self):
    929         """Returns a name that uniquely identifies this particular type of port
    930         (e.g., "mac-snowleopard" or "linux-x86_x64" and can be passed
    931         to factory.get() to instantiate the port."""
    932         return self._name
    933 
    934     def operating_system(self):
    935         # Subclasses should override this default implementation.
    936         return 'mac'
    937 
    938     def version(self):
    939         """Returns a string indicating the version of a given platform, e.g.
    940         'leopard' or 'xp'.
    941 
    942         This is used to help identify the exact port when parsing test
    943         expectations, determining search paths, and logging information."""
    944         return self._version
    945 
    946     def architecture(self):
    947         return self._architecture
    948 
    949     def get_option(self, name, default_value=None):
    950         return getattr(self._options, name, default_value)
    951 
    952     def set_option_default(self, name, default_value):
    953         return self._options.ensure_value(name, default_value)
    954 
    955     @memoized
    956     def path_to_generic_test_expectations_file(self):
    957         return self._filesystem.join(self.layout_tests_dir(), 'TestExpectations')
    958 
    959     def relative_test_filename(self, filename):
    960         """Returns a test_name a relative unix-style path for a filename under the LayoutTests
    961         directory. Ports may legitimately return abspaths here if no relpath makes sense."""
    962         # Ports that run on windows need to override this method to deal with
    963         # filenames with backslashes in them.
    964         if filename.startswith(self.layout_tests_dir()):
    965             return self.host.filesystem.relpath(filename, self.layout_tests_dir())
    966         else:
    967             return self.host.filesystem.abspath(filename)
    968 
    969     @memoized
    970     def abspath_for_test(self, test_name):
    971         """Returns the full path to the file for a given test name. This is the
    972         inverse of relative_test_filename()."""
    973         return self._filesystem.join(self.layout_tests_dir(), test_name)
    974 
    975     def results_directory(self):
    976         """Absolute path to the place to store the test results (uses --results-directory)."""
    977         if not self._results_directory:
    978             option_val = self.get_option('results_directory') or self.default_results_directory()
    979             self._results_directory = self._filesystem.abspath(option_val)
    980         return self._results_directory
    981 
    982     def perf_results_directory(self):
    983         return self._build_path()
    984 
    985     def default_results_directory(self):
    986         """Absolute path to the default place to store the test results."""
    987         try:
    988             return self.path_from_chromium_base('webkit', self.get_option('configuration'), 'layout-test-results')
    989         except AssertionError:
    990             return self._build_path('layout-test-results')
    991 
    992     def setup_test_run(self):
    993         """Perform port-specific work at the beginning of a test run."""
    994         # Delete the disk cache if any to ensure a clean test run.
    995         dump_render_tree_binary_path = self._path_to_driver()
    996         cachedir = self._filesystem.dirname(dump_render_tree_binary_path)
    997         cachedir = self._filesystem.join(cachedir, "cache")
    998         if self._filesystem.exists(cachedir):
    999             self._filesystem.rmtree(cachedir)
   1000 
   1001         if self._dump_reader:
   1002             self._filesystem.maybe_make_directory(self._dump_reader.crash_dumps_directory())
   1003 
   1004     def num_workers(self, requested_num_workers):
   1005         """Returns the number of available workers (possibly less than the number requested)."""
   1006         return requested_num_workers
   1007 
   1008     def clean_up_test_run(self):
   1009         """Perform port-specific work at the end of a test run."""
   1010         if self._image_differ:
   1011             self._image_differ.stop()
   1012             self._image_differ = None
   1013 
   1014     # FIXME: os.environ access should be moved to onto a common/system class to be more easily mockable.
   1015     def _value_or_default_from_environ(self, name, default=None):
   1016         if name in os.environ:
   1017             return os.environ[name]
   1018         return default
   1019 
   1020     def _copy_value_from_environ_if_set(self, clean_env, name):
   1021         if name in os.environ:
   1022             clean_env[name] = os.environ[name]
   1023 
   1024     def setup_environ_for_server(self, server_name=None):
   1025         # We intentionally copy only a subset of os.environ when
   1026         # launching subprocesses to ensure consistent test results.
   1027         clean_env = {
   1028             'LOCAL_RESOURCE_ROOT': self.layout_tests_dir(),  # FIXME: Is this used?
   1029         }
   1030         variables_to_copy = [
   1031             'WEBKIT_TESTFONTS',  # FIXME: Is this still used?
   1032             'WEBKITOUTPUTDIR',   # FIXME: Is this still used?
   1033             'CHROME_DEVEL_SANDBOX',
   1034             'CHROME_IPC_LOGGING',
   1035             'ASAN_OPTIONS',
   1036             'VALGRIND_LIB',
   1037             'VALGRIND_LIB_INNER',
   1038         ]
   1039         if self.host.platform.is_linux() or self.host.platform.is_freebsd():
   1040             variables_to_copy += [
   1041                 'XAUTHORITY',
   1042                 'HOME',
   1043                 'LANG',
   1044                 'LD_LIBRARY_PATH',
   1045                 'DBUS_SESSION_BUS_ADDRESS',
   1046                 'XDG_DATA_DIRS',
   1047             ]
   1048             clean_env['DISPLAY'] = self._value_or_default_from_environ('DISPLAY', ':1')
   1049         if self.host.platform.is_mac():
   1050             clean_env['DYLD_LIBRARY_PATH'] = self._build_path()
   1051             clean_env['DYLD_FRAMEWORK_PATH'] = self._build_path()
   1052             variables_to_copy += [
   1053                 'HOME',
   1054             ]
   1055         if self.host.platform.is_win():
   1056             variables_to_copy += [
   1057                 'PATH',
   1058                 'GYP_DEFINES',  # Required to locate win sdk.
   1059             ]
   1060         if self.host.platform.is_cygwin():
   1061             variables_to_copy += [
   1062                 'HOMEDRIVE',
   1063                 'HOMEPATH',
   1064                 '_NT_SYMBOL_PATH',
   1065             ]
   1066 
   1067         for variable in variables_to_copy:
   1068             self._copy_value_from_environ_if_set(clean_env, variable)
   1069 
   1070         for string_variable in self.get_option('additional_env_var', []):
   1071             [name, value] = string_variable.split('=', 1)
   1072             clean_env[name] = value
   1073 
   1074         return clean_env
   1075 
   1076     def show_results_html_file(self, results_filename):
   1077         """This routine should display the HTML file pointed at by
   1078         results_filename in a users' browser."""
   1079         return self.host.user.open_url(path.abspath_to_uri(self.host.platform, results_filename))
   1080 
   1081     def create_driver(self, worker_number, no_timeout=False):
   1082         """Return a newly created Driver subclass for starting/stopping the test driver."""
   1083         return self._driver_class()(self, worker_number, pixel_tests=self.get_option('pixel_tests'), no_timeout=no_timeout)
   1084 
   1085     def start_helper(self):
   1086         """If a port needs to reconfigure graphics settings or do other
   1087         things to ensure a known test configuration, it should override this
   1088         method."""
   1089         helper_path = self._path_to_helper()
   1090         if helper_path:
   1091             _log.debug("Starting layout helper %s" % helper_path)
   1092             # Note: Not thread safe: http://bugs.python.org/issue2320
   1093             self._helper = self._executive.popen([helper_path],
   1094                 stdin=self._executive.PIPE, stdout=self._executive.PIPE, stderr=None)
   1095             is_ready = self._helper.stdout.readline()
   1096             if not is_ready.startswith('ready'):
   1097                 _log.error("layout_test_helper failed to be ready")
   1098 
   1099     def requires_http_server(self):
   1100         """Does the port require an HTTP server for running tests? This could
   1101         be the case when the tests aren't run on the host platform."""
   1102         return False
   1103 
   1104     def start_http_server(self, additional_dirs, number_of_drivers):
   1105         """Start a web server. Raise an error if it can't start or is already running.
   1106 
   1107         Ports can stub this out if they don't need a web server to be running."""
   1108         assert not self._http_server, 'Already running an http server.'
   1109 
   1110         if self.uses_apache():
   1111             server = apache_http.ApacheHTTP(self, self.results_directory(), additional_dirs=additional_dirs, number_of_servers=(number_of_drivers * 4))
   1112         else:
   1113             server = lighttpd.Lighttpd(self, self.results_directory())
   1114 
   1115         server.start()
   1116         self._http_server = server
   1117 
   1118     def start_websocket_server(self):
   1119         """Start a web server. Raise an error if it can't start or is already running.
   1120 
   1121         Ports can stub this out if they don't need a websocket server to be running."""
   1122         assert not self._websocket_server, 'Already running a websocket server.'
   1123 
   1124         server = pywebsocket.PyWebSocket(self, self.results_directory())
   1125         server.start()
   1126         self._websocket_server = server
   1127 
   1128     def http_server_supports_ipv6(self):
   1129         # Apache < 2.4 on win32 does not support IPv6, nor does cygwin apache.
   1130         if self.host.platform.is_cygwin() or self.get_option('use_apache') and self.host.platform.is_win():
   1131             return False
   1132         return True
   1133 
   1134     def stop_helper(self):
   1135         """Shut down the test helper if it is running. Do nothing if
   1136         it isn't, or it isn't available. If a port overrides start_helper()
   1137         it must override this routine as well."""
   1138         if self._helper:
   1139             _log.debug("Stopping layout test helper")
   1140             try:
   1141                 self._helper.stdin.write("x\n")
   1142                 self._helper.stdin.close()
   1143                 self._helper.wait()
   1144             except IOError, e:
   1145                 pass
   1146             finally:
   1147                 self._helper = None
   1148 
   1149     def stop_http_server(self):
   1150         """Shut down the http server if it is running. Do nothing if it isn't."""
   1151         if self._http_server:
   1152             self._http_server.stop()
   1153             self._http_server = None
   1154 
   1155     def stop_websocket_server(self):
   1156         """Shut down the websocket server if it is running. Do nothing if it isn't."""
   1157         if self._websocket_server:
   1158             self._websocket_server.stop()
   1159             self._websocket_server = None
   1160 
   1161     #
   1162     # TEST EXPECTATION-RELATED METHODS
   1163     #
   1164 
   1165     def test_configuration(self):
   1166         """Returns the current TestConfiguration for the port."""
   1167         if not self._test_configuration:
   1168             self._test_configuration = TestConfiguration(self._version, self._architecture, self._options.configuration.lower())
   1169         return self._test_configuration
   1170 
   1171     # FIXME: Belongs on a Platform object.
   1172     @memoized
   1173     def all_test_configurations(self):
   1174         """Returns a list of TestConfiguration instances, representing all available
   1175         test configurations for this port."""
   1176         return self._generate_all_test_configurations()
   1177 
   1178     # FIXME: Belongs on a Platform object.
   1179     def configuration_specifier_macros(self):
   1180         """Ports may provide a way to abbreviate configuration specifiers to conveniently
   1181         refer to them as one term or alias specific values to more generic ones. For example:
   1182 
   1183         (xp, vista, win7) -> win # Abbreviate all Windows versions into one namesake.
   1184         (lucid) -> linux  # Change specific name of the Linux distro to a more generic term.
   1185 
   1186         Returns a dictionary, each key representing a macro term ('win', for example),
   1187         and value being a list of valid configuration specifiers (such as ['xp', 'vista', 'win7'])."""
   1188         return self.CONFIGURATION_SPECIFIER_MACROS
   1189 
   1190     def all_baseline_variants(self):
   1191         """Returns a list of platform names sufficient to cover all the baselines.
   1192 
   1193         The list should be sorted so that a later platform  will reuse
   1194         an earlier platform's baselines if they are the same (e.g.,
   1195         'snowleopard' should precede 'leopard')."""
   1196         return self.ALL_BASELINE_VARIANTS
   1197 
   1198     def _generate_all_test_configurations(self):
   1199         """Returns a sequence of the TestConfigurations the port supports."""
   1200         # By default, we assume we want to test every graphics type in
   1201         # every configuration on every system.
   1202         test_configurations = []
   1203         for version, architecture in self.ALL_SYSTEMS:
   1204             for build_type in self.ALL_BUILD_TYPES:
   1205                 test_configurations.append(TestConfiguration(version, architecture, build_type))
   1206         return test_configurations
   1207 
   1208     try_builder_names = frozenset([
   1209         'linux_layout',
   1210         'mac_layout',
   1211         'win_layout',
   1212         'linux_layout_rel',
   1213         'mac_layout_rel',
   1214         'win_layout_rel',
   1215     ])
   1216 
   1217     def warn_if_bug_missing_in_test_expectations(self):
   1218         return True
   1219 
   1220     def _port_specific_expectations_files(self):
   1221         paths = []
   1222         paths.append(self.path_from_chromium_base('skia', 'skia_test_expectations.txt'))
   1223         paths.append(self.path_from_chromium_base('webkit', 'tools', 'layout_tests', 'test_expectations_w3c.txt'))
   1224         paths.append(self._filesystem.join(self.layout_tests_dir(), 'NeverFixTests'))
   1225         paths.append(self._filesystem.join(self.layout_tests_dir(), 'StaleTestExpectations'))
   1226         paths.append(self._filesystem.join(self.layout_tests_dir(), 'SlowTests'))
   1227         paths.append(self._filesystem.join(self.layout_tests_dir(), 'FlakyTests'))
   1228 
   1229         builder_name = self.get_option('builder_name', 'DUMMY_BUILDER_NAME')
   1230         if builder_name == 'DUMMY_BUILDER_NAME' or '(deps)' in builder_name or builder_name in self.try_builder_names:
   1231             paths.append(self.path_from_chromium_base('webkit', 'tools', 'layout_tests', 'test_expectations.txt'))
   1232         return paths
   1233 
   1234     def expectations_dict(self):
   1235         """Returns an OrderedDict of name -> expectations strings.
   1236         The names are expected to be (but not required to be) paths in the filesystem.
   1237         If the name is a path, the file can be considered updatable for things like rebaselining,
   1238         so don't use names that are paths if they're not paths.
   1239         Generally speaking the ordering should be files in the filesystem in cascade order
   1240         (TestExpectations followed by Skipped, if the port honors both formats),
   1241         then any built-in expectations (e.g., from compile-time exclusions), then --additional-expectations options."""
   1242         # FIXME: rename this to test_expectations() once all the callers are updated to know about the ordered dict.
   1243         expectations = OrderedDict()
   1244 
   1245         for path in self.expectations_files():
   1246             if self._filesystem.exists(path):
   1247                 expectations[path] = self._filesystem.read_text_file(path)
   1248 
   1249         for path in self.get_option('additional_expectations', []):
   1250             expanded_path = self._filesystem.expanduser(path)
   1251             if self._filesystem.exists(expanded_path):
   1252                 _log.debug("reading additional_expectations from path '%s'" % path)
   1253                 expectations[path] = self._filesystem.read_text_file(expanded_path)
   1254             else:
   1255                 _log.warning("additional_expectations path '%s' does not exist" % path)
   1256         return expectations
   1257 
   1258     def bot_expectations(self):
   1259         if not self.get_option('ignore_flaky_tests'):
   1260             return {}
   1261 
   1262         full_port_name = self.determine_full_port_name(self.host, self._options, self.port_name)
   1263         builder_category = self.get_option('ignore_builder_category', 'layout')
   1264         factory = BotTestExpectationsFactory()
   1265         # FIXME: This only grabs release builder's flakiness data. If we're running debug,
   1266         # when we should grab the debug builder's data.
   1267         expectations = factory.expectations_for_port(full_port_name, builder_category)
   1268 
   1269         if not expectations:
   1270             return {}
   1271 
   1272         ignore_mode = self.get_option('ignore_flaky_tests')
   1273         if ignore_mode == 'very-flaky' or ignore_mode == 'maybe-flaky':
   1274             return expectations.flakes_by_path(ignore_mode == 'very-flaky')
   1275         if ignore_mode == 'unexpected':
   1276             return expectations.unexpected_results_by_path()
   1277         _log.warning("Unexpected ignore mode: '%s'." % ignore_mode)
   1278         return {}
   1279 
   1280     def expectations_files(self):
   1281         return [self.path_to_generic_test_expectations_file()] + self._port_specific_expectations_files()
   1282 
   1283     def repository_paths(self):
   1284         """Returns a list of (repository_name, repository_path) tuples of its depending code base."""
   1285         return [('blink', self.layout_tests_dir()),
   1286                 ('chromium', self.path_from_chromium_base('build'))]
   1287 
   1288     _WDIFF_DEL = '##WDIFF_DEL##'
   1289     _WDIFF_ADD = '##WDIFF_ADD##'
   1290     _WDIFF_END = '##WDIFF_END##'
   1291 
   1292     def _format_wdiff_output_as_html(self, wdiff):
   1293         wdiff = cgi.escape(wdiff)
   1294         wdiff = wdiff.replace(self._WDIFF_DEL, "<span class=del>")
   1295         wdiff = wdiff.replace(self._WDIFF_ADD, "<span class=add>")
   1296         wdiff = wdiff.replace(self._WDIFF_END, "</span>")
   1297         html = "<head><style>.del { background: #faa; } "
   1298         html += ".add { background: #afa; }</style></head>"
   1299         html += "<pre>%s</pre>" % wdiff
   1300         return html
   1301 
   1302     def _wdiff_command(self, actual_filename, expected_filename):
   1303         executable = self._path_to_wdiff()
   1304         return [executable,
   1305                 "--start-delete=%s" % self._WDIFF_DEL,
   1306                 "--end-delete=%s" % self._WDIFF_END,
   1307                 "--start-insert=%s" % self._WDIFF_ADD,
   1308                 "--end-insert=%s" % self._WDIFF_END,
   1309                 actual_filename,
   1310                 expected_filename]
   1311 
   1312     @staticmethod
   1313     def _handle_wdiff_error(script_error):
   1314         # Exit 1 means the files differed, any other exit code is an error.
   1315         if script_error.exit_code != 1:
   1316             raise script_error
   1317 
   1318     def _run_wdiff(self, actual_filename, expected_filename):
   1319         """Runs wdiff and may throw exceptions.
   1320         This is mostly a hook for unit testing."""
   1321         # Diffs are treated as binary as they may include multiple files
   1322         # with conflicting encodings.  Thus we do not decode the output.
   1323         command = self._wdiff_command(actual_filename, expected_filename)
   1324         wdiff = self._executive.run_command(command, decode_output=False,
   1325             error_handler=self._handle_wdiff_error)
   1326         return self._format_wdiff_output_as_html(wdiff)
   1327 
   1328     _wdiff_error_html = "Failed to run wdiff, see error log."
   1329 
   1330     def wdiff_text(self, actual_filename, expected_filename):
   1331         """Returns a string of HTML indicating the word-level diff of the
   1332         contents of the two filenames. Returns an empty string if word-level
   1333         diffing isn't available."""
   1334         if not self.wdiff_available():
   1335             return ""
   1336         try:
   1337             # It's possible to raise a ScriptError we pass wdiff invalid paths.
   1338             return self._run_wdiff(actual_filename, expected_filename)
   1339         except OSError as e:
   1340             if e.errno in [errno.ENOENT, errno.EACCES, errno.ECHILD]:
   1341                 # Silently ignore cases where wdiff is missing.
   1342                 self._wdiff_available = False
   1343                 return ""
   1344             raise
   1345         except ScriptError as e:
   1346             _log.error("Failed to run wdiff: %s" % e)
   1347             self._wdiff_available = False
   1348             return self._wdiff_error_html
   1349 
   1350     # This is a class variable so we can test error output easily.
   1351     _pretty_patch_error_html = "Failed to run PrettyPatch, see error log."
   1352 
   1353     def pretty_patch_text(self, diff_path):
   1354         if self._pretty_patch_available is None:
   1355             self._pretty_patch_available = self.check_pretty_patch(logging=False)
   1356         if not self._pretty_patch_available:
   1357             return self._pretty_patch_error_html
   1358         command = ("ruby", "-I", self._filesystem.dirname(self._pretty_patch_path),
   1359                    self._pretty_patch_path, diff_path)
   1360         try:
   1361             # Diffs are treated as binary (we pass decode_output=False) as they
   1362             # may contain multiple files of conflicting encodings.
   1363             return self._executive.run_command(command, decode_output=False)
   1364         except OSError, e:
   1365             # If the system is missing ruby log the error and stop trying.
   1366             self._pretty_patch_available = False
   1367             _log.error("Failed to run PrettyPatch (%s): %s" % (command, e))
   1368             return self._pretty_patch_error_html
   1369         except ScriptError, e:
   1370             # If ruby failed to run for some reason, log the command
   1371             # output and stop trying.
   1372             self._pretty_patch_available = False
   1373             _log.error("Failed to run PrettyPatch (%s):\n%s" % (command, e.message_with_output()))
   1374             return self._pretty_patch_error_html
   1375 
   1376     def default_configuration(self):
   1377         return self._config.default_configuration()
   1378 
   1379     def clobber_old_port_specific_results(self):
   1380         pass
   1381 
   1382     def uses_apache(self):
   1383         return True
   1384 
   1385     # FIXME: This does not belong on the port object.
   1386     @memoized
   1387     def path_to_apache(self):
   1388         """Returns the full path to the apache binary.
   1389 
   1390         This is needed only by ports that use the apache_http_server module."""
   1391         raise NotImplementedError('Port.path_to_apache')
   1392 
   1393     def path_to_apache_config_file(self):
   1394         """Returns the full path to the apache configuration file.
   1395 
   1396         If the WEBKIT_HTTP_SERVER_CONF_PATH environment variable is set, its
   1397         contents will be used instead.
   1398 
   1399         This is needed only by ports that use the apache_http_server module."""
   1400         config_file_from_env = os.environ.get('WEBKIT_HTTP_SERVER_CONF_PATH')
   1401         if config_file_from_env:
   1402             if not self._filesystem.exists(config_file_from_env):
   1403                 raise IOError('%s was not found on the system' % config_file_from_env)
   1404             return config_file_from_env
   1405 
   1406         config_file_name = self._apache_config_file_name_for_platform(sys.platform)
   1407         return self._filesystem.join(self.layout_tests_dir(), 'http', 'conf', config_file_name)
   1408 
   1409     def path_to_lighttpd(self):
   1410         """Returns the path to the LigHTTPd binary.
   1411 
   1412         This is needed only by ports that use the http_server.py module."""
   1413         raise NotImplementedError('Port._path_to_lighttpd')
   1414 
   1415     def path_to_lighttpd_modules(self):
   1416         """Returns the path to the LigHTTPd modules directory.
   1417 
   1418         This is needed only by ports that use the http_server.py module."""
   1419         raise NotImplementedError('Port._path_to_lighttpd_modules')
   1420 
   1421     def path_to_lighttpd_php(self):
   1422         """Returns the path to the LigHTTPd PHP executable.
   1423 
   1424         This is needed only by ports that use the http_server.py module."""
   1425         raise NotImplementedError('Port._path_to_lighttpd_php')
   1426 
   1427 
   1428     #
   1429     # PROTECTED ROUTINES
   1430     #
   1431     # The routines below should only be called by routines in this class
   1432     # or any of its subclasses.
   1433     #
   1434 
   1435     # FIXME: This belongs on some platform abstraction instead of Port.
   1436     def _is_redhat_based(self):
   1437         return self._filesystem.exists('/etc/redhat-release')
   1438 
   1439     def _is_debian_based(self):
   1440         return self._filesystem.exists('/etc/debian_version')
   1441 
   1442     def _apache_version(self):
   1443         config = self._executive.run_command([self.path_to_apache(), '-v'])
   1444         return re.sub(r'(?:.|\n)*Server version: Apache/(\d+\.\d+)(?:.|\n)*', r'\1', config)
   1445 
   1446     # We pass sys_platform into this method to make it easy to unit test.
   1447     def _apache_config_file_name_for_platform(self, sys_platform):
   1448         if sys_platform == 'cygwin':
   1449             return 'cygwin-httpd.conf'  # CYGWIN is the only platform to still use Apache 1.3.
   1450         if sys_platform.startswith('linux'):
   1451             if self._is_redhat_based():
   1452                 return 'fedora-httpd-' + self._apache_version() + '.conf'
   1453             if self._is_debian_based():
   1454                 return 'debian-httpd-' + self._apache_version() + '.conf'
   1455         # All platforms use apache2 except for CYGWIN (and Mac OS X Tiger and prior, which we no longer support).
   1456         return "apache2-httpd.conf"
   1457 
   1458     def _path_to_driver(self, configuration=None):
   1459         """Returns the full path to the test driver."""
   1460         return self._build_path(self.driver_name())
   1461 
   1462     def _path_to_webcore_library(self):
   1463         """Returns the full path to a built copy of WebCore."""
   1464         return None
   1465 
   1466     def _path_to_helper(self):
   1467         """Returns the full path to the layout_test_helper binary, which
   1468         is used to help configure the system for the test run, or None
   1469         if no helper is needed.
   1470 
   1471         This is likely only used by start/stop_helper()."""
   1472         return None
   1473 
   1474     def _path_to_image_diff(self):
   1475         """Returns the full path to the image_diff binary, or None if it is not available.
   1476 
   1477         This is likely used only by diff_image()"""
   1478         return self._build_path('image_diff')
   1479 
   1480     @memoized
   1481     def _path_to_wdiff(self):
   1482         """Returns the full path to the wdiff binary, or None if it is not available.
   1483 
   1484         This is likely used only by wdiff_text()"""
   1485         for path in ("/usr/bin/wdiff", "/usr/bin/dwdiff"):
   1486             if self._filesystem.exists(path):
   1487                 return path
   1488         return None
   1489 
   1490     def _webkit_baseline_path(self, platform):
   1491         """Return the  full path to the top of the baseline tree for a
   1492         given platform."""
   1493         return self._filesystem.join(self.layout_tests_dir(), 'platform', platform)
   1494 
   1495     def _driver_class(self):
   1496         """Returns the port's driver implementation."""
   1497         return driver.Driver
   1498 
   1499     def _get_crash_log(self, name, pid, stdout, stderr, newer_than):
   1500         if stderr and 'AddressSanitizer' in stderr:
   1501             # Running the AddressSanitizer take a lot of memory, so we need to
   1502             # serialize access to it across all the concurrently running drivers.
   1503 
   1504             # FIXME: investigate using LLVM_SYMBOLIZER_PATH here to reduce the overhead.
   1505             asan_filter_path = self.path_from_chromium_base('tools', 'valgrind', 'asan', 'asan_symbolize.py')
   1506             if self._filesystem.exists(asan_filter_path):
   1507                 output = self._executive.run_command(['flock', sys.executable, asan_filter_path], input=stderr, decode_output=False)
   1508                 stderr = self._executive.run_command(['c++filt'], input=output, decode_output=False)
   1509 
   1510         name_str = name or '<unknown process name>'
   1511         pid_str = str(pid or '<unknown>')
   1512         stdout_lines = (stdout or '<empty>').decode('utf8', 'replace').splitlines()
   1513         stderr_lines = (stderr or '<empty>').decode('utf8', 'replace').splitlines()
   1514         return (stderr, 'crash log for %s (pid %s):\n%s\n%s\n' % (name_str, pid_str,
   1515             '\n'.join(('STDOUT: ' + l) for l in stdout_lines),
   1516             '\n'.join(('STDERR: ' + l) for l in stderr_lines)))
   1517 
   1518     def look_for_new_crash_logs(self, crashed_processes, start_time):
   1519         pass
   1520 
   1521     def look_for_new_samples(self, unresponsive_processes, start_time):
   1522         pass
   1523 
   1524     def sample_process(self, name, pid):
   1525         pass
   1526 
   1527     def physical_test_suites(self):
   1528         return [
   1529             # For example, to turn on force-compositing-mode in the svg/ directory:
   1530             # PhysicalTestSuite('svg',
   1531             #                   ['--force-compositing-mode']),
   1532             ]
   1533 
   1534     def virtual_test_suites(self):
   1535         return [
   1536             VirtualTestSuite('gpu',
   1537                              'fast/canvas',
   1538                              ['--enable-accelerated-2d-canvas',
   1539                               '--force-compositing-mode']),
   1540             VirtualTestSuite('gpu',
   1541                              'canvas/philip',
   1542                              ['--enable-accelerated-2d-canvas',
   1543                               '--force-compositing-mode']),
   1544             VirtualTestSuite('threaded',
   1545                              'compositing/visibility',
   1546                              ['--enable-threaded-compositing',
   1547                               '--force-compositing-mode']),
   1548             VirtualTestSuite('threaded',
   1549                              'compositing/webgl',
   1550                              ['--enable-threaded-compositing',
   1551                               '--force-compositing-mode']),
   1552             VirtualTestSuite('gpu',
   1553                              'fast/hidpi',
   1554                              ['--force-compositing-mode']),
   1555             VirtualTestSuite('softwarecompositing',
   1556                              'compositing',
   1557                              ['--disable-gpu',
   1558                               '--disable-gpu-compositing',
   1559                               '--force-compositing-mode'],
   1560                              use_legacy_naming=True),
   1561             VirtualTestSuite('deferred',
   1562                              'fast/images',
   1563                              ['--enable-deferred-image-decoding', '--enable-per-tile-painting', '--force-compositing-mode']),
   1564             VirtualTestSuite('deferred',
   1565                              'inspector/timeline',
   1566                              ['--enable-deferred-image-decoding', '--enable-per-tile-painting', '--force-compositing-mode']),
   1567             VirtualTestSuite('gpu/compositedscrolling/overflow',
   1568                              'compositing/overflow',
   1569                              ['--enable-accelerated-overflow-scroll',
   1570                               '--force-compositing-mode'],
   1571                              use_legacy_naming=True),
   1572             VirtualTestSuite('gpu/compositedscrolling/scrollbars',
   1573                              'scrollbars',
   1574                              ['--enable-accelerated-overflow-scroll',
   1575                               '--force-compositing-mode'],
   1576                              use_legacy_naming=True),
   1577             VirtualTestSuite('threaded',
   1578                              'animations',
   1579                              ['--enable-threaded-compositing',
   1580                               '--force-compositing-mode']),
   1581             VirtualTestSuite('threaded',
   1582                              'transitions',
   1583                              ['--enable-threaded-compositing',
   1584                               '--force-compositing-mode']),
   1585             VirtualTestSuite('stable',
   1586                              'webexposed',
   1587                              ['--stable-release-mode',
   1588                               '--force-compositing-mode']),
   1589             VirtualTestSuite('stable',
   1590                              'animations-unprefixed',
   1591                              ['--stable-release-mode',
   1592                               '--force-compositing-mode']),
   1593             VirtualTestSuite('stable',
   1594                              'media/stable',
   1595                              ['--stable-release-mode',
   1596                               '--force-compositing-mode']),
   1597             VirtualTestSuite('android',
   1598                              'fullscreen',
   1599                              ['--force-compositing-mode', '--enable-threaded-compositing',
   1600                               '--enable-fixed-position-compositing', '--enable-accelerated-overflow-scroll', '--enable-accelerated-scrollable-frames',
   1601                               '--enable-composited-scrolling-for-frames', '--enable-gesture-tap-highlight', '--enable-pinch',
   1602                               '--enable-overlay-fullscreen-video', '--enable-overlay-scrollbars', '--enable-overscroll-notifications',
   1603                               '--enable-fixed-layout', '--enable-viewport', '--disable-canvas-aa',
   1604                               '--disable-composited-antialiasing', '--enable-accelerated-fixed-root-background']),
   1605             VirtualTestSuite('implsidepainting',
   1606                              'inspector/timeline',
   1607                              ['--enable-threaded-compositing', '--enable-impl-side-painting', '--force-compositing-mode']),
   1608             VirtualTestSuite('serviceworker',
   1609                              'http/tests/serviceworker',
   1610                              ['--enable-service-worker',
   1611                               '--force-compositing-mode']),
   1612             VirtualTestSuite('targetedstylerecalc',
   1613                              'fast/css/invalidation',
   1614                              ['--enable-targeted-style-recalc',
   1615                               '--force-compositing-mode']),
   1616             VirtualTestSuite('stable',
   1617                              'fast/css3-text/css3-text-decoration/stable',
   1618                              ['--stable-release-mode',
   1619                               '--force-compositing-mode']),
   1620             VirtualTestSuite('stable',
   1621                              'http/tests/websocket',
   1622                              ['--stable-release-mode',
   1623                               '--force-compositing-mode']),
   1624             VirtualTestSuite('stable',
   1625                              'web-animations-api',
   1626                              ['--stable-release-mode',
   1627                               '--force-compositing-mode']),
   1628             VirtualTestSuite('linux-subpixel',
   1629                              'platform/linux/fast/text/subpixel',
   1630                              ['--enable-webkit-text-subpixel-positioning',
   1631                               '--force-compositing-mode']),
   1632             VirtualTestSuite('antialiasedtext',
   1633                              'fast/text',
   1634                              ['--enable-direct-write',
   1635                               '--enable-font-antialiasing',
   1636                               '--force-compositing-mode']),
   1637 
   1638         ]
   1639 
   1640     @memoized
   1641     def populated_virtual_test_suites(self):
   1642         suites = self.virtual_test_suites()
   1643 
   1644         # Sanity-check the suites to make sure they don't point to other suites.
   1645         suite_dirs = [suite.name for suite in suites]
   1646         for suite in suites:
   1647             assert suite.base not in suite_dirs
   1648 
   1649         for suite in suites:
   1650             base_tests = self._real_tests([suite.base])
   1651             suite.tests = {}
   1652             for test in base_tests:
   1653                 suite.tests[test.replace(suite.base, suite.name, 1)] = test
   1654         return suites
   1655 
   1656     def _virtual_tests(self, paths, suites):
   1657         virtual_tests = list()
   1658         for suite in suites:
   1659             if paths:
   1660                 for test in suite.tests:
   1661                     if any(test.startswith(p) for p in paths):
   1662                         virtual_tests.append(test)
   1663             else:
   1664                 virtual_tests.extend(suite.tests.keys())
   1665         return virtual_tests
   1666 
   1667     def is_virtual_test(self, test_name):
   1668         return bool(self.lookup_virtual_suite(test_name))
   1669 
   1670     def lookup_virtual_suite(self, test_name):
   1671         for suite in self.populated_virtual_test_suites():
   1672             if test_name.startswith(suite.name):
   1673                 return suite
   1674         return None
   1675 
   1676     def lookup_virtual_test_base(self, test_name):
   1677         suite = self.lookup_virtual_suite(test_name)
   1678         if not suite:
   1679             return None
   1680         return test_name.replace(suite.name, suite.base, 1)
   1681 
   1682     def lookup_virtual_test_args(self, test_name):
   1683         for suite in self.populated_virtual_test_suites():
   1684             if test_name.startswith(suite.name):
   1685                 return suite.args
   1686         return []
   1687 
   1688     def lookup_physical_test_args(self, test_name):
   1689         for suite in self.physical_test_suites():
   1690             if test_name.startswith(suite.name):
   1691                 return suite.args
   1692         return []
   1693 
   1694     def should_run_as_pixel_test(self, test_input):
   1695         if not self._options.pixel_tests:
   1696             return False
   1697         if self._options.pixel_test_directories:
   1698             return any(test_input.test_name.startswith(directory) for directory in self._options.pixel_test_directories)
   1699         return True
   1700 
   1701     def _modules_to_search_for_symbols(self):
   1702         path = self._path_to_webcore_library()
   1703         if path:
   1704             return [path]
   1705         return []
   1706 
   1707     def _symbols_string(self):
   1708         symbols = ''
   1709         for path_to_module in self._modules_to_search_for_symbols():
   1710             try:
   1711                 symbols += self._executive.run_command(['nm', path_to_module], error_handler=self._executive.ignore_error)
   1712             except OSError, e:
   1713                 _log.warn("Failed to run nm: %s.  Can't determine supported features correctly." % e)
   1714         return symbols
   1715 
   1716     # Ports which use compile-time feature detection should define this method and return
   1717     # a dictionary mapping from symbol substrings to possibly disabled test directories.
   1718     # When the symbol substrings are not matched, the directories will be skipped.
   1719     # If ports don't ever enable certain features, then those directories can just be
   1720     # in the Skipped list instead of compile-time-checked here.
   1721     def _missing_symbol_to_skipped_tests(self):
   1722         if self.PORT_HAS_AUDIO_CODECS_BUILT_IN:
   1723             return {}
   1724         else:
   1725             return {
   1726                 "ff_mp3_decoder": ["webaudio/codec-tests/mp3"],
   1727                 "ff_aac_decoder": ["webaudio/codec-tests/aac"],
   1728             }
   1729 
   1730     def _has_test_in_directories(self, directory_lists, test_list):
   1731         if not test_list:
   1732             return False
   1733 
   1734         directories = itertools.chain.from_iterable(directory_lists)
   1735         for directory, test in itertools.product(directories, test_list):
   1736             if test.startswith(directory):
   1737                 return True
   1738         return False
   1739 
   1740     def _skipped_tests_for_unsupported_features(self, test_list):
   1741         # Only check the symbols of there are tests in the test_list that might get skipped.
   1742         # This is a performance optimization to avoid the calling nm.
   1743         # Runtime feature detection not supported, fallback to static detection:
   1744         # Disable any tests for symbols missing from the executable or libraries.
   1745         if self._has_test_in_directories(self._missing_symbol_to_skipped_tests().values(), test_list):
   1746             symbols_string = self._symbols_string()
   1747             if symbols_string is not None:
   1748                 return reduce(operator.add, [directories for symbol_substring, directories in self._missing_symbol_to_skipped_tests().items() if symbol_substring not in symbols_string], [])
   1749         return []
   1750 
   1751     def _convert_path(self, path):
   1752         """Handles filename conversion for subprocess command line args."""
   1753         # See note above in diff_image() for why we need this.
   1754         if sys.platform == 'cygwin':
   1755             return cygpath(path)
   1756         return path
   1757 
   1758     def _build_path(self, *comps):
   1759         return self._build_path_with_configuration(None, *comps)
   1760 
   1761     def _build_path_with_configuration(self, configuration, *comps):
   1762         # Note that we don't do the option caching that the
   1763         # base class does, because finding the right directory is relatively
   1764         # fast.
   1765         configuration = configuration or self.get_option('configuration')
   1766         return self._static_build_path(self._filesystem, self.get_option('build_directory'),
   1767             self.path_from_chromium_base(), configuration, comps)
   1768 
   1769     def _check_driver_build_up_to_date(self, configuration):
   1770         if configuration in ('Debug', 'Release'):
   1771             try:
   1772                 debug_path = self._path_to_driver('Debug')
   1773                 release_path = self._path_to_driver('Release')
   1774 
   1775                 debug_mtime = self._filesystem.mtime(debug_path)
   1776                 release_mtime = self._filesystem.mtime(release_path)
   1777 
   1778                 if (debug_mtime > release_mtime and configuration == 'Release' or
   1779                     release_mtime > debug_mtime and configuration == 'Debug'):
   1780                     most_recent_binary = 'Release' if configuration == 'Debug' else 'Debug'
   1781                     _log.warning('You are running the %s binary. However the %s binary appears to be more recent. '
   1782                                  'Please pass --%s.', configuration, most_recent_binary, most_recent_binary.lower())
   1783                     _log.warning('')
   1784             # This will fail if we don't have both a debug and release binary.
   1785             # That's fine because, in this case, we must already be running the
   1786             # most up-to-date one.
   1787             except OSError:
   1788                 pass
   1789         return True
   1790 
   1791     def _chromium_baseline_path(self, platform):
   1792         if platform is None:
   1793             platform = self.name()
   1794         return self.path_from_webkit_base('LayoutTests', 'platform', platform)
   1795 
   1796 class VirtualTestSuite(object):
   1797     def __init__(self, name, base, args, use_legacy_naming=False, tests=None):
   1798         if use_legacy_naming:
   1799             self.name = 'virtual/' + name
   1800         else:
   1801             if name.find('/') != -1:
   1802                 _log.error("Virtual test suites names cannot contain /'s: %s" % name)
   1803                 return
   1804             self.name = 'virtual/' + name + '/' + base
   1805         self.base = base
   1806         self.args = args
   1807         self.tests = tests or set()
   1808 
   1809     def __repr__(self):
   1810         return "VirtualTestSuite('%s', '%s', %s)" % (self.name, self.base, self.args)
   1811 
   1812 
   1813 class PhysicalTestSuite(object):
   1814     def __init__(self, base, args):
   1815         self.name = base
   1816         self.base = base
   1817         self.args = args
   1818         self.tests = set()
   1819 
   1820     def __repr__(self):
   1821         return "PhysicalTestSuite('%s', '%s', %s)" % (self.name, self.base, self.args)
   1822