1 #!/usr/bin/env python 2 # Copyright (C) 2010 Google Inc. All rights reserved. 3 # 4 # Redistribution and use in source and binary forms, with or without 5 # modification, are permitted provided that the following conditions are 6 # met: 7 # 8 # * Redistributions of source code must retain the above copyright 9 # notice, this list of conditions and the following disclaimer. 10 # * Redistributions in binary form must reproduce the above 11 # copyright notice, this list of conditions and the following disclaimer 12 # in the documentation and/or other materials provided with the 13 # distribution. 14 # * Neither the name of Google Inc. nor the names of its 15 # contributors may be used to endorse or promote products derived from 16 # this software without specific prior written permission. 17 # 18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 30 """Chromium implementations of the Port interface.""" 31 32 import errno 33 import logging 34 import re 35 import signal 36 import subprocess 37 import sys 38 import time 39 import webbrowser 40 41 from webkitpy.common.system import executive 42 from webkitpy.common.system.path import cygpath 43 from webkitpy.layout_tests.layout_package import test_expectations 44 from webkitpy.layout_tests.port import base 45 from webkitpy.layout_tests.port import http_server 46 from webkitpy.layout_tests.port import websocket_server 47 48 _log = logging.getLogger("webkitpy.layout_tests.port.chromium") 49 50 51 # FIXME: This function doesn't belong in this package. 52 class ChromiumPort(base.Port): 53 """Abstract base class for Chromium implementations of the Port class.""" 54 ALL_BASELINE_VARIANTS = [ 55 'chromium-mac-snowleopard', 'chromium-mac-leopard', 56 'chromium-win-win7', 'chromium-win-vista', 'chromium-win-xp', 57 'chromium-linux-x86', 'chromium-linux-x86_64', 58 'chromium-gpu-mac-snowleopard', 'chromium-gpu-win-win7', 'chromium-gpu-linux-x86_64', 59 ] 60 61 def __init__(self, **kwargs): 62 base.Port.__init__(self, **kwargs) 63 self._chromium_base_dir = None 64 65 def _check_file_exists(self, path_to_file, file_description, 66 override_step=None, logging=True): 67 """Verify the file is present where expected or log an error. 68 69 Args: 70 file_name: The (human friendly) name or description of the file 71 you're looking for (e.g., "HTTP Server"). Used for error logging. 72 override_step: An optional string to be logged if the check fails. 73 logging: Whether or not log the error messages.""" 74 if not self._filesystem.exists(path_to_file): 75 if logging: 76 _log.error('Unable to find %s' % file_description) 77 _log.error(' at %s' % path_to_file) 78 if override_step: 79 _log.error(' %s' % override_step) 80 _log.error('') 81 return False 82 return True 83 84 85 def baseline_path(self): 86 return self._webkit_baseline_path(self._name) 87 88 def check_build(self, needs_http): 89 result = True 90 91 dump_render_tree_binary_path = self._path_to_driver() 92 result = self._check_file_exists(dump_render_tree_binary_path, 93 'test driver') and result 94 if result and self.get_option('build'): 95 result = self._check_driver_build_up_to_date( 96 self.get_option('configuration')) 97 else: 98 _log.error('') 99 100 helper_path = self._path_to_helper() 101 if helper_path: 102 result = self._check_file_exists(helper_path, 103 'layout test helper') and result 104 105 if self.get_option('pixel_tests'): 106 result = self.check_image_diff( 107 'To override, invoke with --no-pixel-tests') and result 108 109 # It's okay if pretty patch isn't available, but we will at 110 # least log a message. 111 self._pretty_patch_available = self.check_pretty_patch() 112 113 return result 114 115 def check_sys_deps(self, needs_http): 116 cmd = [self._path_to_driver(), '--check-layout-test-sys-deps'] 117 118 local_error = executive.ScriptError() 119 120 def error_handler(script_error): 121 local_error.exit_code = script_error.exit_code 122 123 output = self._executive.run_command(cmd, error_handler=error_handler) 124 if local_error.exit_code: 125 _log.error('System dependencies check failed.') 126 _log.error('To override, invoke with --nocheck-sys-deps') 127 _log.error('') 128 _log.error(output) 129 return False 130 return True 131 132 def check_image_diff(self, override_step=None, logging=True): 133 image_diff_path = self._path_to_image_diff() 134 return self._check_file_exists(image_diff_path, 'image diff exe', 135 override_step, logging) 136 137 def diff_image(self, expected_contents, actual_contents, 138 diff_filename=None): 139 # FIXME: need unit tests for this. 140 if not actual_contents and not expected_contents: 141 return False 142 if not actual_contents or not expected_contents: 143 return True 144 145 tempdir = self._filesystem.mkdtemp() 146 expected_filename = self._filesystem.join(str(tempdir), "expected.png") 147 self._filesystem.write_binary_file(expected_filename, expected_contents) 148 actual_filename = self._filesystem.join(str(tempdir), "actual.png") 149 self._filesystem.write_binary_file(actual_filename, actual_contents) 150 151 executable = self._path_to_image_diff() 152 if diff_filename: 153 cmd = [executable, '--diff', expected_filename, 154 actual_filename, diff_filename] 155 else: 156 cmd = [executable, expected_filename, actual_filename] 157 158 result = True 159 try: 160 exit_code = self._executive.run_command(cmd, return_exit_code=True) 161 if exit_code == 0: 162 # The images are the same. 163 result = False 164 elif exit_code != 1: 165 _log.error("image diff returned an exit code of " 166 + str(exit_code)) 167 # Returning False here causes the script to think that we 168 # successfully created the diff even though we didn't. If 169 # we return True, we think that the images match but the hashes 170 # don't match. 171 # FIXME: Figure out why image_diff returns other values. 172 result = False 173 except OSError, e: 174 if e.errno == errno.ENOENT or e.errno == errno.EACCES: 175 _compare_available = False 176 else: 177 raise e 178 finally: 179 self._filesystem.rmtree(str(tempdir)) 180 return result 181 182 def driver_name(self): 183 return "DumpRenderTree" 184 185 def path_from_chromium_base(self, *comps): 186 """Returns the full path to path made by joining the top of the 187 Chromium source tree and the list of path components in |*comps|.""" 188 if not self._chromium_base_dir: 189 abspath = self._filesystem.abspath(__file__) 190 offset = abspath.find('third_party') 191 if offset == -1: 192 self._chromium_base_dir = self._filesystem.join( 193 abspath[0:abspath.find('Tools')], 194 'Source', 'WebKit', 'chromium') 195 else: 196 self._chromium_base_dir = abspath[0:offset] 197 return self._filesystem.join(self._chromium_base_dir, *comps) 198 199 def path_to_test_expectations_file(self): 200 return self.path_from_webkit_base('LayoutTests', 'platform', 201 'chromium', 'test_expectations.txt') 202 203 def default_results_directory(self): 204 try: 205 return self.path_from_chromium_base('webkit', 206 self.get_option('configuration'), 207 'layout-test-results') 208 except AssertionError: 209 return self._build_path(self.get_option('configuration'), 210 'layout-test-results') 211 212 def setup_test_run(self): 213 # Delete the disk cache if any to ensure a clean test run. 214 dump_render_tree_binary_path = self._path_to_driver() 215 cachedir = self._filesystem.dirname(dump_render_tree_binary_path) 216 cachedir = self._filesystem.join(cachedir, "cache") 217 if self._filesystem.exists(cachedir): 218 self._filesystem.rmtree(cachedir) 219 220 def create_driver(self, worker_number): 221 """Starts a new Driver and returns a handle to it.""" 222 return ChromiumDriver(self, worker_number) 223 224 def start_helper(self): 225 helper_path = self._path_to_helper() 226 if helper_path: 227 _log.debug("Starting layout helper %s" % helper_path) 228 # Note: Not thread safe: http://bugs.python.org/issue2320 229 self._helper = subprocess.Popen([helper_path], 230 stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=None) 231 is_ready = self._helper.stdout.readline() 232 if not is_ready.startswith('ready'): 233 _log.error("layout_test_helper failed to be ready") 234 235 def stop_helper(self): 236 if self._helper: 237 _log.debug("Stopping layout test helper") 238 self._helper.stdin.write("x\n") 239 self._helper.stdin.close() 240 # wait() is not threadsafe and can throw OSError due to: 241 # http://bugs.python.org/issue1731717 242 self._helper.wait() 243 244 def all_baseline_variants(self): 245 return self.ALL_BASELINE_VARIANTS 246 247 def test_expectations(self): 248 """Returns the test expectations for this port. 249 250 Basically this string should contain the equivalent of a 251 test_expectations file. See test_expectations.py for more details.""" 252 expectations_path = self.path_to_test_expectations_file() 253 return self._filesystem.read_text_file(expectations_path) 254 255 def test_expectations_overrides(self): 256 try: 257 overrides_path = self.path_from_chromium_base('webkit', 'tools', 258 'layout_tests', 'test_expectations.txt') 259 except AssertionError: 260 return None 261 if not self._filesystem.exists(overrides_path): 262 return None 263 return self._filesystem.read_text_file(overrides_path) 264 265 def skipped_layout_tests(self, extra_test_files=None): 266 expectations_str = self.test_expectations() 267 overrides_str = self.test_expectations_overrides() 268 is_debug_mode = False 269 270 all_test_files = self.tests([]) 271 if extra_test_files: 272 all_test_files.update(extra_test_files) 273 274 expectations = test_expectations.TestExpectations( 275 self, all_test_files, expectations_str, self.test_configuration(), 276 is_lint_mode=False, overrides=overrides_str) 277 tests_dir = self.layout_tests_dir() 278 return [self.relative_test_filename(test) 279 for test in expectations.get_tests_with_result_type(test_expectations.SKIP)] 280 281 def test_repository_paths(self): 282 # Note: for JSON file's backward-compatibility we use 'chrome' rather 283 # than 'chromium' here. 284 repos = super(ChromiumPort, self).test_repository_paths() 285 repos.append(('chrome', self.path_from_chromium_base())) 286 return repos 287 288 # 289 # PROTECTED METHODS 290 # 291 # These routines should only be called by other methods in this file 292 # or any subclasses. 293 # 294 295 def _check_driver_build_up_to_date(self, configuration): 296 if configuration in ('Debug', 'Release'): 297 try: 298 debug_path = self._path_to_driver('Debug') 299 release_path = self._path_to_driver('Release') 300 301 debug_mtime = self._filesystem.mtime(debug_path) 302 release_mtime = self._filesystem.mtime(release_path) 303 304 if (debug_mtime > release_mtime and configuration == 'Release' or 305 release_mtime > debug_mtime and configuration == 'Debug'): 306 _log.warning('You are not running the most ' 307 'recent DumpRenderTree binary. You need to ' 308 'pass --debug or not to select between ' 309 'Debug and Release.') 310 _log.warning('') 311 # This will fail if we don't have both a debug and release binary. 312 # That's fine because, in this case, we must already be running the 313 # most up-to-date one. 314 except OSError: 315 pass 316 return True 317 318 def _chromium_baseline_path(self, platform): 319 if platform is None: 320 platform = self.name() 321 return self.path_from_webkit_base('LayoutTests', 'platform', platform) 322 323 def _convert_path(self, path): 324 """Handles filename conversion for subprocess command line args.""" 325 # See note above in diff_image() for why we need this. 326 if sys.platform == 'cygwin': 327 return cygpath(path) 328 return path 329 330 def _path_to_image_diff(self): 331 binary_name = 'ImageDiff' 332 return self._build_path(self.get_option('configuration'), binary_name) 333 334 335 class ChromiumDriver(base.Driver): 336 """Abstract interface for DRT.""" 337 338 def __init__(self, port, worker_number): 339 self._port = port 340 self._worker_number = worker_number 341 self._image_path = None 342 self.KILL_TIMEOUT = 3.0 343 if self._port.get_option('pixel_tests'): 344 self._image_path = self._port._filesystem.join(self._port.results_directory(), 345 'png_result%s.png' % self._worker_number) 346 347 def cmd_line(self): 348 cmd = self._command_wrapper(self._port.get_option('wrapper')) 349 cmd.append(self._port._path_to_driver()) 350 if self._port.get_option('pixel_tests'): 351 # See note above in diff_image() for why we need _convert_path(). 352 cmd.append("--pixel-tests=" + 353 self._port._convert_path(self._image_path)) 354 355 cmd.append('--test-shell') 356 357 if self._port.get_option('startup_dialog'): 358 cmd.append('--testshell-startup-dialog') 359 360 if self._port.get_option('gp_fault_error_box'): 361 cmd.append('--gp-fault-error-box') 362 363 if self._port.get_option('js_flags') is not None: 364 cmd.append('--js-flags="' + self._port.get_option('js_flags') + '"') 365 366 if self._port.get_option('stress_opt'): 367 cmd.append('--stress-opt') 368 369 if self._port.get_option('stress_deopt'): 370 cmd.append('--stress-deopt') 371 372 if self._port.get_option('accelerated_compositing'): 373 cmd.append('--enable-accelerated-compositing') 374 if self._port.get_option('accelerated_2d_canvas'): 375 cmd.append('--enable-accelerated-2d-canvas') 376 if self._port.get_option('enable_hardware_gpu'): 377 cmd.append('--enable-hardware-gpu') 378 379 cmd.extend(self._port.get_option('additional_drt_flag', [])) 380 return cmd 381 382 def start(self): 383 # FIXME: Should be an error to call this method twice. 384 cmd = self.cmd_line() 385 386 # We need to pass close_fds=True to work around Python bug #2320 387 # (otherwise we can hang when we kill DumpRenderTree when we are running 388 # multiple threads). See http://bugs.python.org/issue2320 . 389 # Note that close_fds isn't supported on Windows, but this bug only 390 # shows up on Mac and Linux. 391 close_flag = sys.platform not in ('win32', 'cygwin') 392 self._proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, 393 stdout=subprocess.PIPE, 394 stderr=subprocess.STDOUT, 395 close_fds=close_flag) 396 397 def poll(self): 398 # poll() is not threadsafe and can throw OSError due to: 399 # http://bugs.python.org/issue1731717 400 return self._proc.poll() 401 402 def _write_command_and_read_line(self, input=None): 403 """Returns a tuple: (line, did_crash)""" 404 try: 405 if input: 406 if isinstance(input, unicode): 407 # DRT expects utf-8 408 input = input.encode("utf-8") 409 self._proc.stdin.write(input) 410 # DumpRenderTree text output is always UTF-8. However some tests 411 # (e.g. webarchive) may spit out binary data instead of text so we 412 # don't bother to decode the output. 413 line = self._proc.stdout.readline() 414 # We could assert() here that line correctly decodes as UTF-8. 415 return (line, False) 416 except IOError, e: 417 _log.error("IOError communicating w/ DRT: " + str(e)) 418 return (None, True) 419 420 def _test_shell_command(self, uri, timeoutms, checksum): 421 cmd = uri 422 if timeoutms: 423 cmd += ' ' + str(timeoutms) 424 if checksum: 425 cmd += ' ' + checksum 426 cmd += "\n" 427 return cmd 428 429 def _output_image(self): 430 """Returns the image output which driver generated.""" 431 png_path = self._image_path 432 if png_path and self._port._filesystem.exists(png_path): 433 return self._port._filesystem.read_binary_file(png_path) 434 else: 435 return None 436 437 def _output_image_with_retry(self): 438 # Retry a few more times because open() sometimes fails on Windows, 439 # raising "IOError: [Errno 13] Permission denied:" 440 retry_num = 50 441 timeout_seconds = 5.0 442 for i in range(retry_num): 443 try: 444 return self._output_image() 445 except IOError, e: 446 if e.errno == errno.EACCES: 447 time.sleep(timeout_seconds / retry_num) 448 else: 449 raise e 450 return self._output_image() 451 452 def _clear_output_image(self): 453 png_path = self._image_path 454 if png_path and self._port._filesystem.exists(png_path): 455 self._port._filesystem.remove(png_path) 456 457 def run_test(self, driver_input): 458 output = [] 459 error = [] 460 crash = False 461 timeout = False 462 actual_uri = None 463 actual_checksum = None 464 self._clear_output_image() 465 start_time = time.time() 466 467 uri = self._port.filename_to_uri(driver_input.filename) 468 cmd = self._test_shell_command(uri, driver_input.timeout, 469 driver_input.image_hash) 470 (line, crash) = self._write_command_and_read_line(input=cmd) 471 472 while not crash and line.rstrip() != "#EOF": 473 # Make sure we haven't crashed. 474 if line == '' and self.poll() is not None: 475 # This is hex code 0xc000001d, which is used for abrupt 476 # termination. This happens if we hit ctrl+c from the prompt 477 # and we happen to be waiting on DRT. 478 # sdoyon: Not sure for which OS and in what circumstances the 479 # above code is valid. What works for me under Linux to detect 480 # ctrl+c is for the subprocess returncode to be negative 481 # SIGINT. And that agrees with the subprocess documentation. 482 if (-1073741510 == self._proc.returncode or 483 - signal.SIGINT == self._proc.returncode): 484 raise KeyboardInterrupt 485 crash = True 486 break 487 488 # Don't include #URL lines in our output 489 if line.startswith("#URL:"): 490 actual_uri = line.rstrip()[5:] 491 if uri != actual_uri: 492 # GURL capitalizes the drive letter of a file URL. 493 if (not re.search("^file:///[a-z]:", uri) or 494 uri.lower() != actual_uri.lower()): 495 _log.fatal("Test got out of sync:\n|%s|\n|%s|" % 496 (uri, actual_uri)) 497 raise AssertionError("test out of sync") 498 elif line.startswith("#MD5:"): 499 actual_checksum = line.rstrip()[5:] 500 elif line.startswith("#TEST_TIMED_OUT"): 501 timeout = True 502 # Test timed out, but we still need to read until #EOF. 503 elif actual_uri: 504 output.append(line) 505 else: 506 error.append(line) 507 508 (line, crash) = self._write_command_and_read_line(input=None) 509 510 # FIXME: Add support for audio when we're ready. 511 512 run_time = time.time() - start_time 513 output_image = self._output_image_with_retry() 514 text = ''.join(output) 515 if not text: 516 text = None 517 518 return base.DriverOutput(text, output_image, actual_checksum, audio=None, 519 crash=crash, test_time=run_time, timeout=timeout, error=''.join(error)) 520 521 def stop(self): 522 if self._proc: 523 self._proc.stdin.close() 524 self._proc.stdout.close() 525 if self._proc.stderr: 526 self._proc.stderr.close() 527 # Closing stdin/stdout/stderr hangs sometimes on OS X, 528 # (see __init__(), above), and anyway we don't want to hang 529 # the harness if DRT is buggy, so we wait a couple 530 # seconds to give DRT a chance to clean up, but then 531 # force-kill the process if necessary. 532 timeout = time.time() + self.KILL_TIMEOUT 533 while self._proc.poll() is None and time.time() < timeout: 534 time.sleep(0.1) 535 if self._proc.poll() is None: 536 _log.warning('stopping test driver timed out, ' 537 'killing it') 538 self._port._executive.kill_process(self._proc.pid) 539 # FIXME: This is sometime none. What is wrong? assert self._proc.poll() is not None 540 if self._proc.poll() is not None: 541 self._proc.wait() 542 self._proc = None 543
