1 # Copyright (C) 2011 Google Inc. All rights reserved. 2 # 3 # Redistribution and use in source and binary forms, with or without 4 # modification, are permitted provided that the following conditions are 5 # met: 6 # 7 # * Redistributions of source code must retain the above copyright 8 # notice, this list of conditions and the following disclaimer. 9 # * Redistributions in binary form must reproduce the above 10 # copyright notice, this list of conditions and the following disclaimer 11 # in the documentation and/or other materials provided with the 12 # distribution. 13 # * Neither the name of Google Inc. nor the names of its 14 # contributors may be used to endorse or promote products derived from 15 # this software without specific prior written permission. 16 # 17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 30 import logging 31 import re 32 import time 33 34 from webkitpy.layout_tests.controllers import test_result_writer 35 from webkitpy.layout_tests.port.driver import DriverInput, DriverOutput 36 from webkitpy.layout_tests.models import test_expectations 37 from webkitpy.layout_tests.models import test_failures 38 from webkitpy.layout_tests.models.test_results import TestResult 39 40 41 _log = logging.getLogger(__name__) 42 43 44 def run_single_test(port, options, results_directory, worker_name, driver, test_input, stop_when_done): 45 runner = SingleTestRunner(port, options, results_directory, worker_name, driver, test_input, stop_when_done) 46 return runner.run() 47 48 49 class SingleTestRunner(object): 50 (ALONGSIDE_TEST, PLATFORM_DIR, VERSION_DIR, UPDATE) = ('alongside', 'platform', 'version', 'update') 51 52 def __init__(self, port, options, results_directory, worker_name, driver, test_input, stop_when_done): 53 self._port = port 54 self._filesystem = port.host.filesystem 55 self._options = options 56 self._results_directory = results_directory 57 self._driver = driver 58 self._timeout = test_input.timeout 59 self._worker_name = worker_name 60 self._test_name = test_input.test_name 61 self._should_run_pixel_test = test_input.should_run_pixel_test 62 self._reference_files = test_input.reference_files 63 self._should_add_missing_baselines = test_input.should_add_missing_baselines 64 self._stop_when_done = stop_when_done 65 66 if self._reference_files: 67 # Detect and report a test which has a wrong combination of expectation files. 68 # For example, if 'foo.html' has two expectation files, 'foo-expected.html' and 69 # 'foo-expected.txt', we should warn users. One test file must be used exclusively 70 # in either layout tests or reftests, but not in both. 71 for suffix in ('.txt', '.png', '.wav'): 72 expected_filename = self._port.expected_filename(self._test_name, suffix) 73 if self._filesystem.exists(expected_filename): 74 _log.error('%s is a reftest, but has an unused expectation file. Please remove %s.', 75 self._test_name, expected_filename) 76 77 def _expected_driver_output(self): 78 return DriverOutput(self._port.expected_text(self._test_name), 79 self._port.expected_image(self._test_name), 80 self._port.expected_checksum(self._test_name), 81 self._port.expected_audio(self._test_name)) 82 83 def _should_fetch_expected_checksum(self): 84 return self._should_run_pixel_test and not (self._options.new_baseline or self._options.reset_results) 85 86 def _driver_input(self): 87 # The image hash is used to avoid doing an image dump if the 88 # checksums match, so it should be set to a blank value if we 89 # are generating a new baseline. (Otherwise, an image from a 90 # previous run will be copied into the baseline.""" 91 image_hash = None 92 if self._should_fetch_expected_checksum(): 93 image_hash = self._port.expected_checksum(self._test_name) 94 return DriverInput(self._test_name, self._timeout, image_hash, self._should_run_pixel_test) 95 96 def run(self): 97 if self._reference_files: 98 if self._options.reset_results: 99 reftest_type = set([reference_file[0] for reference_file in self._reference_files]) 100 result = TestResult(self._test_name, reftest_type=reftest_type) 101 result.type = test_expectations.SKIP 102 return result 103 return self._run_reftest() 104 if self._options.reset_results: 105 return self._run_rebaseline() 106 return self._run_compare_test() 107 108 def _run_compare_test(self): 109 driver_output = self._driver.run_test(self._driver_input(), self._stop_when_done) 110 expected_driver_output = self._expected_driver_output() 111 112 test_result = self._compare_output(expected_driver_output, driver_output) 113 if self._should_add_missing_baselines: 114 self._add_missing_baselines(test_result, driver_output) 115 test_result_writer.write_test_result(self._filesystem, self._port, self._results_directory, self._test_name, driver_output, expected_driver_output, test_result.failures) 116 return test_result 117 118 def _run_rebaseline(self): 119 driver_output = self._driver.run_test(self._driver_input(), self._stop_when_done) 120 failures = self._handle_error(driver_output) 121 test_result_writer.write_test_result(self._filesystem, self._port, self._results_directory, self._test_name, driver_output, None, failures) 122 # FIXME: It the test crashed or timed out, it might be better to avoid 123 # to write new baselines. 124 self._overwrite_baselines(driver_output) 125 return TestResult(self._test_name, failures, driver_output.test_time, driver_output.has_stderr(), pid=driver_output.pid) 126 127 _render_tree_dump_pattern = re.compile(r"^layer at \(\d+,\d+\) size \d+x\d+\n") 128 129 def _add_missing_baselines(self, test_result, driver_output): 130 missingImage = test_result.has_failure_matching_types(test_failures.FailureMissingImage, test_failures.FailureMissingImageHash) 131 if test_result.has_failure_matching_types(test_failures.FailureMissingResult): 132 self._save_baseline_data(driver_output.text, '.txt', self._location_for_new_baseline(driver_output.text, '.txt')) 133 if test_result.has_failure_matching_types(test_failures.FailureMissingAudio): 134 self._save_baseline_data(driver_output.audio, '.wav', self._location_for_new_baseline(driver_output.audio, '.wav')) 135 if missingImage: 136 self._save_baseline_data(driver_output.image, '.png', self._location_for_new_baseline(driver_output.image, '.png')) 137 138 def _location_for_new_baseline(self, data, extension): 139 if self._options.add_platform_exceptions: 140 return self.VERSION_DIR 141 if extension == '.png': 142 return self.PLATFORM_DIR 143 if extension == '.wav': 144 return self.ALONGSIDE_TEST 145 if extension == '.txt' and self._render_tree_dump_pattern.match(data): 146 return self.PLATFORM_DIR 147 return self.ALONGSIDE_TEST 148 149 def _overwrite_baselines(self, driver_output): 150 location = self.VERSION_DIR if self._options.add_platform_exceptions else self.UPDATE 151 self._save_baseline_data(driver_output.text, '.txt', location) 152 self._save_baseline_data(driver_output.audio, '.wav', location) 153 if self._should_run_pixel_test: 154 self._save_baseline_data(driver_output.image, '.png', location) 155 156 def _save_baseline_data(self, data, extension, location): 157 if data is None: 158 return 159 port = self._port 160 fs = self._filesystem 161 if location == self.ALONGSIDE_TEST: 162 output_dir = fs.dirname(port.abspath_for_test(self._test_name)) 163 elif location == self.VERSION_DIR: 164 output_dir = fs.join(port.baseline_version_dir(), fs.dirname(self._test_name)) 165 elif location == self.PLATFORM_DIR: 166 output_dir = fs.join(port.baseline_platform_dir(), fs.dirname(self._test_name)) 167 elif location == self.UPDATE: 168 output_dir = fs.dirname(port.expected_filename(self._test_name, extension)) 169 else: 170 raise AssertionError('unrecognized baseline location: %s' % location) 171 172 fs.maybe_make_directory(output_dir) 173 output_basename = fs.basename(fs.splitext(self._test_name)[0] + "-expected" + extension) 174 output_path = fs.join(output_dir, output_basename) 175 _log.info('Writing new expected result "%s"' % port.relative_test_filename(output_path)) 176 port.update_baseline(output_path, data) 177 178 def _handle_error(self, driver_output, reference_filename=None): 179 """Returns test failures if some unusual errors happen in driver's run. 180 181 Args: 182 driver_output: The output from the driver. 183 reference_filename: The full path to the reference file which produced the driver_output. 184 This arg is optional and should be used only in reftests until we have a better way to know 185 which html file is used for producing the driver_output. 186 """ 187 failures = [] 188 fs = self._filesystem 189 if driver_output.timeout: 190 failures.append(test_failures.FailureTimeout(bool(reference_filename))) 191 192 if reference_filename: 193 testname = self._port.relative_test_filename(reference_filename) 194 else: 195 testname = self._test_name 196 197 if driver_output.crash: 198 failures.append(test_failures.FailureCrash(bool(reference_filename), 199 driver_output.crashed_process_name, 200 driver_output.crashed_pid)) 201 if driver_output.error: 202 _log.debug("%s %s crashed, (stderr lines):" % (self._worker_name, testname)) 203 else: 204 _log.debug("%s %s crashed, (no stderr)" % (self._worker_name, testname)) 205 elif driver_output.error: 206 _log.debug("%s %s output stderr lines:" % (self._worker_name, testname)) 207 for line in driver_output.error.splitlines(): 208 _log.debug(" %s" % line) 209 return failures 210 211 def _compare_output(self, expected_driver_output, driver_output): 212 failures = [] 213 failures.extend(self._handle_error(driver_output)) 214 215 if driver_output.crash: 216 # Don't continue any more if we already have a crash. 217 # In case of timeouts, we continue since we still want to see the text and image output. 218 return TestResult(self._test_name, failures, driver_output.test_time, driver_output.has_stderr(), pid=driver_output.pid) 219 220 failures.extend(self._compare_text(expected_driver_output.text, driver_output.text)) 221 failures.extend(self._compare_audio(expected_driver_output.audio, driver_output.audio)) 222 if self._should_run_pixel_test: 223 failures.extend(self._compare_image(expected_driver_output, driver_output)) 224 return TestResult(self._test_name, failures, driver_output.test_time, driver_output.has_stderr(), pid=driver_output.pid) 225 226 def _compare_text(self, expected_text, actual_text): 227 failures = [] 228 if (expected_text and actual_text and 229 # Assuming expected_text is already normalized. 230 self._port.do_text_results_differ(expected_text, self._get_normalized_output_text(actual_text))): 231 failures.append(test_failures.FailureTextMismatch()) 232 elif actual_text and not expected_text: 233 failures.append(test_failures.FailureMissingResult()) 234 return failures 235 236 def _compare_audio(self, expected_audio, actual_audio): 237 failures = [] 238 if (expected_audio and actual_audio and 239 self._port.do_audio_results_differ(expected_audio, actual_audio)): 240 failures.append(test_failures.FailureAudioMismatch()) 241 elif actual_audio and not expected_audio: 242 failures.append(test_failures.FailureMissingAudio()) 243 return failures 244 245 def _get_normalized_output_text(self, output): 246 """Returns the normalized text output, i.e. the output in which 247 the end-of-line characters are normalized to "\n".""" 248 # Running tests on Windows produces "\r\n". The "\n" part is helpfully 249 # changed to "\r\n" by our system (Python/Cygwin), resulting in 250 # "\r\r\n", when, in fact, we wanted to compare the text output with 251 # the normalized text expectation files. 252 return output.replace("\r\r\n", "\r\n").replace("\r\n", "\n") 253 254 # FIXME: This function also creates the image diff. Maybe that work should 255 # be handled elsewhere? 256 def _compare_image(self, expected_driver_output, driver_output): 257 failures = [] 258 # If we didn't produce a hash file, this test must be text-only. 259 if driver_output.image_hash is None: 260 return failures 261 if not expected_driver_output.image: 262 failures.append(test_failures.FailureMissingImage()) 263 elif not expected_driver_output.image_hash: 264 failures.append(test_failures.FailureMissingImageHash()) 265 elif driver_output.image_hash != expected_driver_output.image_hash: 266 diff, err_str = self._port.diff_image(expected_driver_output.image, driver_output.image) 267 if err_str: 268 _log.warning(' %s : %s' % (self._test_name, err_str)) 269 failures.append(test_failures.FailureImageHashMismatch()) 270 driver_output.error = (driver_output.error or '') + err_str 271 else: 272 driver_output.image_diff = diff 273 if driver_output.image_diff: 274 failures.append(test_failures.FailureImageHashMismatch()) 275 else: 276 # See https://bugs.webkit.org/show_bug.cgi?id=69444 for why this isn't a full failure. 277 _log.warning(' %s -> pixel hash failed (but diff passed)' % self._test_name) 278 return failures 279 280 def _run_reftest(self): 281 test_output = self._driver.run_test(self._driver_input(), self._stop_when_done) 282 total_test_time = 0 283 reference_output = None 284 test_result = None 285 286 # A reftest can have multiple match references and multiple mismatch references; 287 # the test fails if any mismatch matches and all of the matches don't match. 288 # To minimize the number of references we have to check, we run all of the mismatches first, 289 # then the matches, and short-circuit out as soon as we can. 290 # Note that sorting by the expectation sorts "!=" before "==" so this is easy to do. 291 292 putAllMismatchBeforeMatch = sorted 293 reference_test_names = [] 294 for expectation, reference_filename in putAllMismatchBeforeMatch(self._reference_files): 295 reference_test_name = self._port.relative_test_filename(reference_filename) 296 reference_test_names.append(reference_test_name) 297 reference_output = self._driver.run_test(DriverInput(reference_test_name, self._timeout, None, should_run_pixel_test=True), self._stop_when_done) 298 test_result = self._compare_output_with_reference(reference_output, test_output, reference_filename, expectation == '!=') 299 300 if (expectation == '!=' and test_result.failures) or (expectation == '==' and not test_result.failures): 301 break 302 total_test_time += test_result.test_run_time 303 304 assert(reference_output) 305 test_result_writer.write_test_result(self._filesystem, self._port, self._results_directory, self._test_name, test_output, reference_output, test_result.failures) 306 reftest_type = set([reference_file[0] for reference_file in self._reference_files]) 307 return TestResult(self._test_name, test_result.failures, total_test_time + test_result.test_run_time, test_result.has_stderr, reftest_type=reftest_type, pid=test_result.pid, references=reference_test_names) 308 309 def _compare_output_with_reference(self, reference_driver_output, actual_driver_output, reference_filename, mismatch): 310 total_test_time = reference_driver_output.test_time + actual_driver_output.test_time 311 has_stderr = reference_driver_output.has_stderr() or actual_driver_output.has_stderr() 312 failures = [] 313 failures.extend(self._handle_error(actual_driver_output)) 314 if failures: 315 # Don't continue any more if we already have crash or timeout. 316 return TestResult(self._test_name, failures, total_test_time, has_stderr) 317 failures.extend(self._handle_error(reference_driver_output, reference_filename=reference_filename)) 318 if failures: 319 return TestResult(self._test_name, failures, total_test_time, has_stderr, pid=actual_driver_output.pid) 320 321 if not reference_driver_output.image_hash and not actual_driver_output.image_hash: 322 failures.append(test_failures.FailureReftestNoImagesGenerated(reference_filename)) 323 elif mismatch: 324 if reference_driver_output.image_hash == actual_driver_output.image_hash: 325 diff, err_str = self._port.diff_image(reference_driver_output.image, actual_driver_output.image) 326 if not diff: 327 failures.append(test_failures.FailureReftestMismatchDidNotOccur(reference_filename)) 328 else: 329 _log.warning(" %s -> ref test hashes matched but diff failed" % self._test_name) 330 331 elif reference_driver_output.image_hash != actual_driver_output.image_hash: 332 diff, err_str = self._port.diff_image(reference_driver_output.image, actual_driver_output.image) 333 if diff: 334 failures.append(test_failures.FailureReftestMismatch(reference_filename)) 335 else: 336 _log.warning(" %s -> ref test hashes didn't match but diff passed" % self._test_name) 337 338 return TestResult(self._test_name, failures, total_test_time, has_stderr, pid=actual_driver_output.pid) 339
