1 # Copyright (C) 2011 Google Inc. All rights reserved. 2 # 3 # Redistribution and use in source and binary forms, with or without 4 # modification, are permitted provided that the following conditions are 5 # met: 6 # 7 # * Redistributions of source code must retain the above copyright 8 # notice, this list of conditions and the following disclaimer. 9 # * Redistributions in binary form must reproduce the above 10 # copyright notice, this list of conditions and the following disclaimer 11 # in the documentation and/or other materials provided with the 12 # distribution. 13 # * Neither the name of Google Inc. nor the names of its 14 # contributors may be used to endorse or promote products derived from 15 # this software without specific prior written permission. 16 # 17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 30 import logging 31 import time 32 33 from webkitpy.layout_tests.port import base 34 from webkitpy.layout_tests.layout_package import test_failures 35 from webkitpy.layout_tests.layout_package import test_result_writer 36 from webkitpy.layout_tests.layout_package.test_results import TestResult 37 38 39 _log = logging.getLogger(__name__) 40 41 42 def run_single_test(port, options, test_input, driver, worker_name): 43 runner = SingleTestRunner(options, port, driver, test_input, worker_name) 44 return runner.run() 45 46 47 class SingleTestRunner: 48 49 def __init__(self, options, port, driver, test_input, worker_name): 50 self._options = options 51 self._port = port 52 self._driver = driver 53 self._filename = test_input.filename 54 self._timeout = test_input.timeout 55 self._worker_name = worker_name 56 self._testname = port.relative_test_filename(test_input.filename) 57 58 self._is_reftest = False 59 self._is_mismatch_reftest = False 60 self._reference_filename = None 61 62 fs = port._filesystem 63 reftest_expected_filename = port.reftest_expected_filename(self._filename) 64 if fs.exists(reftest_expected_filename): 65 self._is_reftest = True 66 self._reference_filename = reftest_expected_filename 67 68 reftest_expected_mismatch_filename = port.reftest_expected_mismatch_filename(self._filename) 69 if fs.exists(reftest_expected_mismatch_filename): 70 if self._is_reftest: 71 _log.error('It is not allowed that one test file has both' 72 ' expected.html file and expected-mismatch.html file' 73 ' at the same time. Please remove either %s or %s.', 74 reftest_expected_filename, reftest_expected_mismatch_filename) 75 else: 76 self._is_reftest = True 77 self._is_mismatch_reftest = True 78 self._reference_filename = reftest_expected_mismatch_filename 79 80 if self._is_reftest: 81 # Detect and report a test which has a wrong combination of expectation files. 82 # For example, if 'foo.html' has two expectation files, 'foo-expected.html' and 83 # 'foo-expected.txt', we should warn users. One test file must be used exclusively 84 # in either layout tests or reftests, but not in both. 85 for suffix in ('.txt', '.checksum', '.png', '.wav'): 86 expected_filename = self._port.expected_filename(self._filename, suffix) 87 if fs.exists(expected_filename): 88 _log.error('The reftest (%s) can not have an expectation file (%s).' 89 ' Please remove that file.', self._testname, expected_filename) 90 91 def _expected_driver_output(self): 92 return base.DriverOutput(self._port.expected_text(self._filename), 93 self._port.expected_image(self._filename), 94 self._port.expected_checksum(self._filename), 95 self._port.expected_audio(self._filename)) 96 97 def _should_fetch_expected_checksum(self): 98 return (self._options.pixel_tests and 99 not (self._options.new_baseline or self._options.reset_results)) 100 101 def _driver_input(self): 102 # The image hash is used to avoid doing an image dump if the 103 # checksums match, so it should be set to a blank value if we 104 # are generating a new baseline. (Otherwise, an image from a 105 # previous run will be copied into the baseline.""" 106 image_hash = None 107 if self._should_fetch_expected_checksum(): 108 image_hash = self._port.expected_checksum(self._filename) 109 return base.DriverInput(self._filename, self._timeout, image_hash) 110 111 def run(self): 112 if self._options.new_baseline or self._options.reset_results: 113 if self._is_reftest: 114 # Returns a dummy TestResult. We don't have to rebase for reftests. 115 return TestResult(self._filename) 116 else: 117 return self._run_rebaseline() 118 if self._is_reftest: 119 return self._run_reftest() 120 return self._run_compare_test() 121 122 def _run_compare_test(self): 123 driver_output = self._driver.run_test(self._driver_input()) 124 expected_driver_output = self._expected_driver_output() 125 test_result = self._compare_output(driver_output, expected_driver_output) 126 test_result_writer.write_test_result(self._port, self._filename, 127 driver_output, expected_driver_output, test_result.failures) 128 return test_result 129 130 def _run_rebaseline(self): 131 driver_output = self._driver.run_test(self._driver_input()) 132 failures = self._handle_error(driver_output) 133 test_result_writer.write_test_result(self._port, self._filename, 134 driver_output, None, failures) 135 # FIXME: It the test crashed or timed out, it might be bettter to avoid 136 # to write new baselines. 137 self._save_baselines(driver_output) 138 return TestResult(self._filename, failures, driver_output.test_time) 139 140 def _save_baselines(self, driver_output): 141 # Although all test_shell/DumpRenderTree output should be utf-8, 142 # we do not ever decode it inside run-webkit-tests. For some tests 143 # DumpRenderTree may not output utf-8 text (e.g. webarchives). 144 self._save_baseline_data(driver_output.text, ".txt", 145 generate_new_baseline=self._options.new_baseline) 146 if driver_output.audio: 147 self._save_baseline_data(driver_output.audio, '.wav', 148 generate_new_baseline=self._options.new_baseline) 149 if self._options.pixel_tests and driver_output.image_hash: 150 self._save_baseline_data(driver_output.image, ".png", 151 generate_new_baseline=self._options.new_baseline) 152 self._save_baseline_data(driver_output.image_hash, ".checksum", 153 generate_new_baseline=self._options.new_baseline) 154 155 def _save_baseline_data(self, data, modifier, generate_new_baseline=True): 156 """Saves a new baseline file into the port's baseline directory. 157 158 The file will be named simply "<test>-expected<modifier>", suitable for 159 use as the expected results in a later run. 160 161 Args: 162 data: result to be saved as the new baseline 163 modifier: type of the result file, e.g. ".txt" or ".png" 164 generate_new_baseline: whether to enerate a new, platform-specific 165 baseline, or update the existing one 166 """ 167 assert data is not None 168 port = self._port 169 fs = port._filesystem 170 if generate_new_baseline: 171 relative_dir = fs.dirname(self._testname) 172 baseline_path = port.baseline_path() 173 output_dir = fs.join(baseline_path, relative_dir) 174 output_file = fs.basename(fs.splitext(self._filename)[0] + 175 "-expected" + modifier) 176 fs.maybe_make_directory(output_dir) 177 output_path = fs.join(output_dir, output_file) 178 _log.debug('writing new baseline result "%s"' % (output_path)) 179 else: 180 output_path = port.expected_filename(self._filename, modifier) 181 _log.debug('resetting baseline result "%s"' % output_path) 182 183 port.update_baseline(output_path, data) 184 185 def _handle_error(self, driver_output, reference_filename=None): 186 """Returns test failures if some unusual errors happen in driver's run. 187 188 Args: 189 driver_output: The output from the driver. 190 reference_filename: The full path to the reference file which produced the driver_output. 191 This arg is optional and should be used only in reftests until we have a better way to know 192 which html file is used for producing the driver_output. 193 """ 194 failures = [] 195 fs = self._port._filesystem 196 if driver_output.timeout: 197 failures.append(test_failures.FailureTimeout(bool(reference_filename))) 198 199 if reference_filename: 200 testname = self._port.relative_test_filename(reference_filename) 201 else: 202 testname = self._testname 203 204 if driver_output.crash: 205 failures.append(test_failures.FailureCrash(bool(reference_filename))) 206 _log.debug("%s Stacktrace for %s:\n%s" % (self._worker_name, testname, 207 driver_output.error)) 208 elif driver_output.error: 209 _log.debug("%s %s output stderr lines:\n%s" % (self._worker_name, testname, 210 driver_output.error)) 211 return failures 212 213 def _compare_output(self, driver_output, expected_driver_output): 214 failures = [] 215 failures.extend(self._handle_error(driver_output)) 216 217 if driver_output.crash: 218 # Don't continue any more if we already have a crash. 219 # In case of timeouts, we continue since we still want to see the text and image output. 220 return TestResult(self._filename, failures, driver_output.test_time) 221 222 failures.extend(self._compare_text(driver_output.text, expected_driver_output.text)) 223 failures.extend(self._compare_audio(driver_output.audio, expected_driver_output.audio)) 224 if self._options.pixel_tests: 225 failures.extend(self._compare_image(driver_output, expected_driver_output)) 226 return TestResult(self._filename, failures, driver_output.test_time) 227 228 def _compare_text(self, actual_text, expected_text): 229 failures = [] 230 if (expected_text and actual_text and 231 # Assuming expected_text is already normalized. 232 self._port.compare_text(self._get_normalized_output_text(actual_text), expected_text)): 233 failures.append(test_failures.FailureTextMismatch()) 234 elif actual_text and not expected_text: 235 failures.append(test_failures.FailureMissingResult()) 236 return failures 237 238 def _compare_audio(self, actual_audio, expected_audio): 239 failures = [] 240 if (expected_audio and actual_audio and 241 self._port.compare_audio(actual_audio, expected_audio)): 242 failures.append(test_failures.FailureAudioMismatch()) 243 elif actual_audio and not expected_audio: 244 failures.append(test_failures.FailureMissingAudio()) 245 return failures 246 247 def _get_normalized_output_text(self, output): 248 """Returns the normalized text output, i.e. the output in which 249 the end-of-line characters are normalized to "\n".""" 250 # Running tests on Windows produces "\r\n". The "\n" part is helpfully 251 # changed to "\r\n" by our system (Python/Cygwin), resulting in 252 # "\r\r\n", when, in fact, we wanted to compare the text output with 253 # the normalized text expectation files. 254 return output.replace("\r\r\n", "\r\n").replace("\r\n", "\n") 255 256 def _compare_image(self, driver_output, expected_driver_outputs): 257 failures = [] 258 # If we didn't produce a hash file, this test must be text-only. 259 if driver_output.image_hash is None: 260 return failures 261 if not expected_driver_outputs.image: 262 failures.append(test_failures.FailureMissingImage()) 263 elif not expected_driver_outputs.image_hash: 264 failures.append(test_failures.FailureMissingImageHash()) 265 elif driver_output.image_hash != expected_driver_outputs.image_hash: 266 failures.append(test_failures.FailureImageHashMismatch()) 267 return failures 268 269 def _run_reftest(self): 270 driver_output1 = self._driver.run_test(self._driver_input()) 271 driver_output2 = self._driver.run_test( 272 base.DriverInput(self._reference_filename, self._timeout, driver_output1.image_hash)) 273 test_result = self._compare_output_with_reference(driver_output1, driver_output2) 274 275 test_result_writer.write_test_result(self._port, self._filename, 276 driver_output1, driver_output2, test_result.failures) 277 return test_result 278 279 def _compare_output_with_reference(self, driver_output1, driver_output2): 280 total_test_time = driver_output1.test_time + driver_output2.test_time 281 failures = [] 282 failures.extend(self._handle_error(driver_output1)) 283 if failures: 284 # Don't continue any more if we already have crash or timeout. 285 return TestResult(self._filename, failures, total_test_time) 286 failures.extend(self._handle_error(driver_output2, reference_filename=self._reference_filename)) 287 if failures: 288 return TestResult(self._filename, failures, total_test_time) 289 290 if self._is_mismatch_reftest: 291 if driver_output1.image_hash == driver_output2.image_hash: 292 failures.append(test_failures.FailureReftestMismatchDidNotOccur()) 293 elif driver_output1.image_hash != driver_output2.image_hash: 294 failures.append(test_failures.FailureReftestMismatch()) 295 return TestResult(self._filename, failures, total_test_time) 296
