1 # Copyright (C) 2011 Google Inc. All rights reserved. 2 # 3 # Redistribution and use in source and binary forms, with or without 4 # modification, are permitted provided that the following conditions are 5 # met: 6 # 7 # * Redistributions of source code must retain the above copyright 8 # notice, this list of conditions and the following disclaimer. 9 # * Redistributions in binary form must reproduce the above 10 # copyright notice, this list of conditions and the following disclaimer 11 # in the documentation and/or other materials provided with the 12 # distribution. 13 # * Neither the Google name nor the names of its 14 # contributors may be used to endorse or promote products derived from 15 # this software without specific prior written permission. 16 # 17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 import base64 30 import copy 31 import logging 32 import re 33 import shlex 34 import sys 35 import time 36 import os 37 38 from webkitpy.common.system import path 39 from webkitpy.common.system.profiler import ProfilerFactory 40 41 42 _log = logging.getLogger(__name__) 43 44 45 DRIVER_START_TIMEOUT_SECS = 30 46 47 48 class DriverInput(object): 49 def __init__(self, test_name, timeout, image_hash, should_run_pixel_test, args): 50 self.test_name = test_name 51 self.timeout = timeout # in ms 52 self.image_hash = image_hash 53 self.should_run_pixel_test = should_run_pixel_test 54 self.args = args 55 56 57 class DriverOutput(object): 58 """Groups information about a output from driver for easy passing 59 and post-processing of data.""" 60 61 def __init__(self, text, image, image_hash, audio, crash=False, 62 test_time=0, measurements=None, timeout=False, error='', crashed_process_name='??', 63 crashed_pid=None, crash_log=None, pid=None): 64 # FIXME: Args could be renamed to better clarify what they do. 65 self.text = text 66 self.image = image # May be empty-string if the test crashes. 67 self.image_hash = image_hash 68 self.image_diff = None # image_diff gets filled in after construction. 69 self.audio = audio # Binary format is port-dependent. 70 self.crash = crash 71 self.crashed_process_name = crashed_process_name 72 self.crashed_pid = crashed_pid 73 self.crash_log = crash_log 74 self.test_time = test_time 75 self.measurements = measurements 76 self.timeout = timeout 77 self.error = error # stderr output 78 self.pid = pid 79 80 def has_stderr(self): 81 return bool(self.error) 82 83 84 class DeviceFailure(Exception): 85 pass 86 87 88 class Driver(object): 89 """object for running test(s) using content_shell or other driver.""" 90 91 def __init__(self, port, worker_number, pixel_tests, no_timeout=False): 92 """Initialize a Driver to subsequently run tests. 93 94 Typically this routine will spawn content_shell in a config 95 ready for subsequent input. 96 97 port - reference back to the port object. 98 worker_number - identifier for a particular worker/driver instance 99 """ 100 self._port = port 101 self._worker_number = worker_number 102 self._no_timeout = no_timeout 103 104 self._driver_tempdir = None 105 # WebKitTestRunner can report back subprocess crashes by printing 106 # "#CRASHED - PROCESSNAME". Since those can happen at any time 107 # and ServerProcess won't be aware of them (since the actual tool 108 # didn't crash, just a subprocess) we record the crashed subprocess name here. 109 self._crashed_process_name = None 110 self._crashed_pid = None 111 112 # WebKitTestRunner can report back subprocesses that became unresponsive 113 # This could mean they crashed. 114 self._subprocess_was_unresponsive = False 115 116 # stderr reading is scoped on a per-test (not per-block) basis, so we store the accumulated 117 # stderr output, as well as if we've seen #EOF on this driver instance. 118 # FIXME: We should probably remove _read_first_block and _read_optional_image_block and 119 # instead scope these locally in run_test. 120 self.error_from_test = str() 121 self.err_seen_eof = False 122 self._server_process = None 123 self._current_cmd_line = None 124 125 self._measurements = {} 126 if self._port.get_option("profile"): 127 profiler_name = self._port.get_option("profiler") 128 self._profiler = ProfilerFactory.create_profiler(self._port.host, 129 self._port._path_to_driver(), self._port.results_directory(), profiler_name) 130 else: 131 self._profiler = None 132 133 def __del__(self): 134 self.stop() 135 136 def run_test(self, driver_input, stop_when_done): 137 """Run a single test and return the results. 138 139 Note that it is okay if a test times out or crashes and leaves 140 the driver in an indeterminate state. The upper layers of the program 141 are responsible for cleaning up and ensuring things are okay. 142 143 Returns a DriverOutput object. 144 """ 145 start_time = time.time() 146 self.start(driver_input.should_run_pixel_test, driver_input.args) 147 test_begin_time = time.time() 148 self.error_from_test = str() 149 self.err_seen_eof = False 150 151 command = self._command_from_driver_input(driver_input) 152 deadline = test_begin_time + int(driver_input.timeout) / 1000.0 153 154 self._server_process.write(command) 155 text, audio = self._read_first_block(deadline) # First block is either text or audio 156 image, actual_image_hash = self._read_optional_image_block(deadline) # The second (optional) block is image data. 157 158 crashed = self.has_crashed() 159 timed_out = self._server_process.timed_out 160 pid = self._server_process.pid() 161 162 if stop_when_done or crashed or timed_out: 163 # We call stop() even if we crashed or timed out in order to get any remaining stdout/stderr output. 164 # In the timeout case, we kill the hung process as well. 165 out, err = self._server_process.stop(self._port.driver_stop_timeout() if stop_when_done else 0.0) 166 if out: 167 text += out 168 if err: 169 self.error_from_test += err 170 self._server_process = None 171 172 crash_log = None 173 if crashed: 174 self.error_from_test, crash_log = self._get_crash_log(text, self.error_from_test, newer_than=start_time) 175 176 # If we don't find a crash log use a placeholder error message instead. 177 if not crash_log: 178 pid_str = str(self._crashed_pid) if self._crashed_pid else "unknown pid" 179 crash_log = 'No crash log found for %s:%s.\n' % (self._crashed_process_name, pid_str) 180 # If we were unresponsive append a message informing there may not have been a crash. 181 if self._subprocess_was_unresponsive: 182 crash_log += 'Process failed to become responsive before timing out.\n' 183 184 # Print stdout and stderr to the placeholder crash log; we want as much context as possible. 185 if self.error_from_test: 186 crash_log += '\nstdout:\n%s\nstderr:\n%s\n' % (text, self.error_from_test) 187 188 return DriverOutput(text, image, actual_image_hash, audio, 189 crash=crashed, test_time=time.time() - test_begin_time, measurements=self._measurements, 190 timeout=timed_out, error=self.error_from_test, 191 crashed_process_name=self._crashed_process_name, 192 crashed_pid=self._crashed_pid, crash_log=crash_log, pid=pid) 193 194 def _get_crash_log(self, stdout, stderr, newer_than): 195 return self._port._get_crash_log(self._crashed_process_name, self._crashed_pid, stdout, stderr, newer_than) 196 197 # FIXME: Seems this could just be inlined into callers. 198 @classmethod 199 def _command_wrapper(cls, wrapper_option): 200 # Hook for injecting valgrind or other runtime instrumentation, 201 # used by e.g. tools/valgrind/valgrind_tests.py. 202 return shlex.split(wrapper_option) if wrapper_option else [] 203 204 HTTP_DIR = "http/tests/" 205 HTTP_LOCAL_DIR = "http/tests/local/" 206 207 def is_http_test(self, test_name): 208 return test_name.startswith(self.HTTP_DIR) and not test_name.startswith(self.HTTP_LOCAL_DIR) 209 210 def test_to_uri(self, test_name): 211 """Convert a test name to a URI.""" 212 if not self.is_http_test(test_name): 213 return path.abspath_to_uri(self._port.host.platform, self._port.abspath_for_test(test_name)) 214 215 relative_path = test_name[len(self.HTTP_DIR):] 216 217 # TODO(dpranke): remove the SSL reference? 218 if relative_path.startswith("ssl/"): 219 return "https://127.0.0.1:8443/" + relative_path 220 return "http://127.0.0.1:8000/" + relative_path 221 222 def uri_to_test(self, uri): 223 """Return the base layout test name for a given URI. 224 225 This returns the test name for a given URI, e.g., if you passed in 226 "file:///src/LayoutTests/fast/html/keygen.html" it would return 227 "fast/html/keygen.html". 228 229 """ 230 if uri.startswith("file:///"): 231 prefix = path.abspath_to_uri(self._port.host.platform, self._port.layout_tests_dir()) 232 if not prefix.endswith('/'): 233 prefix += '/' 234 return uri[len(prefix):] 235 if uri.startswith("http://"): 236 return uri.replace('http://127.0.0.1:8000/', self.HTTP_DIR) 237 if uri.startswith("https://"): 238 return uri.replace('https://127.0.0.1:8443/', self.HTTP_DIR) 239 raise NotImplementedError('unknown url type: %s' % uri) 240 241 def has_crashed(self): 242 if self._server_process is None: 243 return False 244 if self._crashed_process_name: 245 return True 246 if self._server_process.has_crashed(): 247 self._crashed_process_name = self._server_process.name() 248 self._crashed_pid = self._server_process.pid() 249 return True 250 return False 251 252 def start(self, pixel_tests, per_test_args): 253 new_cmd_line = self.cmd_line(pixel_tests, per_test_args) 254 if not self._server_process or new_cmd_line != self._current_cmd_line: 255 self._start(pixel_tests, per_test_args) 256 self._run_post_start_tasks() 257 258 def _setup_environ_for_driver(self, environment): 259 if self._profiler: 260 environment = self._profiler.adjusted_environment(environment) 261 return environment 262 263 def _start(self, pixel_tests, per_test_args, wait_for_ready=True): 264 self.stop() 265 self._driver_tempdir = self._port._filesystem.mkdtemp(prefix='%s-' % self._port.driver_name()) 266 server_name = self._port.driver_name() 267 environment = self._port.setup_environ_for_server(server_name) 268 environment = self._setup_environ_for_driver(environment) 269 self._crashed_process_name = None 270 self._crashed_pid = None 271 cmd_line = self.cmd_line(pixel_tests, per_test_args) 272 self._server_process = self._port._server_process_constructor(self._port, server_name, cmd_line, environment, logging=self._port.get_option("driver_logging")) 273 self._server_process.start() 274 self._current_cmd_line = cmd_line 275 276 if wait_for_ready: 277 deadline = time.time() + DRIVER_START_TIMEOUT_SECS 278 if not self._wait_for_server_process_output(self._server_process, deadline, '#READY'): 279 _log.error("content_shell took too long to startup.") 280 281 def _wait_for_server_process_output(self, server_process, deadline, text): 282 output = '' 283 line = server_process.read_stdout_line(deadline) 284 while not server_process.timed_out and not server_process.has_crashed() and not text in line.rstrip(): 285 output += line 286 line = server_process.read_stdout_line(deadline) 287 288 if server_process.timed_out or server_process.has_crashed(): 289 _log.error('Failed to start the %s process: \n%s' % (server_process.name(), output)) 290 return False 291 292 return True 293 294 def _run_post_start_tasks(self): 295 # Remote drivers may override this to delay post-start tasks until the server has ack'd. 296 if self._profiler: 297 self._profiler.attach_to_pid(self._pid_on_target()) 298 299 def _pid_on_target(self): 300 # Remote drivers will override this method to return the pid on the device. 301 return self._server_process.pid() 302 303 def stop(self): 304 if self._server_process: 305 self._server_process.stop() 306 self._server_process = None 307 if self._profiler: 308 self._profiler.profile_after_exit() 309 310 if self._driver_tempdir: 311 self._port._filesystem.rmtree(str(self._driver_tempdir)) 312 self._driver_tempdir = None 313 314 self._current_cmd_line = None 315 316 def cmd_line(self, pixel_tests, per_test_args): 317 cmd = self._command_wrapper(self._port.get_option('wrapper')) 318 cmd.append(self._port._path_to_driver()) 319 if self._no_timeout: 320 cmd.append('--no-timeout') 321 cmd.extend(self._port.get_option('additional_drt_flag', [])) 322 cmd.extend(self._port.additional_drt_flag()) 323 cmd.extend(per_test_args) 324 cmd.append('-') 325 return cmd 326 327 def _check_for_driver_crash(self, error_line): 328 if error_line == "#CRASHED\n": 329 # This is used on Windows to report that the process has crashed 330 # See http://trac.webkit.org/changeset/65537. 331 self._crashed_process_name = self._server_process.name() 332 self._crashed_pid = self._server_process.pid() 333 elif (error_line.startswith("#CRASHED - ") 334 or error_line.startswith("#PROCESS UNRESPONSIVE - ")): 335 # WebKitTestRunner uses this to report that the WebProcess subprocess crashed. 336 match = re.match('#(?:CRASHED|PROCESS UNRESPONSIVE) - (\S+)', error_line) 337 self._crashed_process_name = match.group(1) if match else 'WebProcess' 338 match = re.search('pid (\d+)', error_line) 339 pid = int(match.group(1)) if match else None 340 self._crashed_pid = pid 341 # FIXME: delete this after we're sure this code is working :) 342 _log.debug('%s crash, pid = %s, error_line = %s' % (self._crashed_process_name, str(pid), error_line)) 343 if error_line.startswith("#PROCESS UNRESPONSIVE - "): 344 self._subprocess_was_unresponsive = True 345 self._port.sample_process(self._crashed_process_name, self._crashed_pid) 346 # We want to show this since it's not a regular crash and probably we don't have a crash log. 347 self.error_from_test += error_line 348 return True 349 return self.has_crashed() 350 351 def _command_from_driver_input(self, driver_input): 352 # FIXME: performance tests pass in full URLs instead of test names. 353 if driver_input.test_name.startswith('http://') or driver_input.test_name.startswith('https://') or driver_input.test_name == ('about:blank'): 354 command = driver_input.test_name 355 elif self.is_http_test(driver_input.test_name): 356 command = self.test_to_uri(driver_input.test_name) 357 else: 358 command = self._port.abspath_for_test(driver_input.test_name) 359 if sys.platform == 'cygwin': 360 command = path.cygpath(command) 361 362 assert not driver_input.image_hash or driver_input.should_run_pixel_test 363 364 # ' is the separator between arguments. 365 if self._port.supports_per_test_timeout(): 366 command += "'--timeout'%s" % driver_input.timeout 367 if driver_input.should_run_pixel_test: 368 command += "'--pixel-test" 369 if driver_input.image_hash: 370 command += "'" + driver_input.image_hash 371 return command + "\n" 372 373 def _read_first_block(self, deadline): 374 # returns (text_content, audio_content) 375 block = self._read_block(deadline) 376 if block.malloc: 377 self._measurements['Malloc'] = float(block.malloc) 378 if block.js_heap: 379 self._measurements['JSHeap'] = float(block.js_heap) 380 if block.content_type == 'audio/wav': 381 return (None, block.decoded_content) 382 return (block.decoded_content, None) 383 384 def _read_optional_image_block(self, deadline): 385 # returns (image, actual_image_hash) 386 block = self._read_block(deadline, wait_for_stderr_eof=True) 387 if block.content and block.content_type == 'image/png': 388 return (block.decoded_content, block.content_hash) 389 return (None, block.content_hash) 390 391 def _read_header(self, block, line, header_text, header_attr, header_filter=None): 392 if line.startswith(header_text) and getattr(block, header_attr) is None: 393 value = line.split()[1] 394 if header_filter: 395 value = header_filter(value) 396 setattr(block, header_attr, value) 397 return True 398 return False 399 400 def _process_stdout_line(self, block, line): 401 if (self._read_header(block, line, 'Content-Type: ', 'content_type') 402 or self._read_header(block, line, 'Content-Transfer-Encoding: ', 'encoding') 403 or self._read_header(block, line, 'Content-Length: ', '_content_length', int) 404 or self._read_header(block, line, 'ActualHash: ', 'content_hash') 405 or self._read_header(block, line, 'DumpMalloc: ', 'malloc') 406 or self._read_header(block, line, 'DumpJSHeap: ', 'js_heap')): 407 return 408 # Note, we're not reading ExpectedHash: here, but we could. 409 # If the line wasn't a header, we just append it to the content. 410 block.content += line 411 412 def _strip_eof(self, line): 413 if line and line.endswith("#EOF\n"): 414 return line[:-5], True 415 if line and line.endswith("#EOF\r\n"): 416 _log.error("Got a CRLF-terminated #EOF - this is a driver bug.") 417 return line[:-6], True 418 return line, False 419 420 def _read_block(self, deadline, wait_for_stderr_eof=False): 421 block = ContentBlock() 422 out_seen_eof = False 423 424 while not self.has_crashed(): 425 if out_seen_eof and (self.err_seen_eof or not wait_for_stderr_eof): 426 break 427 428 if self.err_seen_eof: 429 out_line = self._server_process.read_stdout_line(deadline) 430 err_line = None 431 elif out_seen_eof: 432 out_line = None 433 err_line = self._server_process.read_stderr_line(deadline) 434 else: 435 out_line, err_line = self._server_process.read_either_stdout_or_stderr_line(deadline) 436 437 if self._server_process.timed_out or self.has_crashed(): 438 break 439 440 if out_line: 441 assert not out_seen_eof 442 out_line, out_seen_eof = self._strip_eof(out_line) 443 if err_line: 444 assert not self.err_seen_eof 445 err_line, self.err_seen_eof = self._strip_eof(err_line) 446 447 if out_line: 448 if out_line[-1] != "\n": 449 _log.error("Last character read from DRT stdout line was not a newline! This indicates either a NRWT or DRT bug.") 450 content_length_before_header_check = block._content_length 451 self._process_stdout_line(block, out_line) 452 # FIXME: Unlike HTTP, DRT dumps the content right after printing a Content-Length header. 453 # Don't wait until we're done with headers, just read the binary blob right now. 454 if content_length_before_header_check != block._content_length: 455 if block._content_length > 0: 456 block.content = self._server_process.read_stdout(deadline, block._content_length) 457 else: 458 _log.error("Received content of type %s with Content-Length of 0! This indicates a bug in %s.", 459 block.content_type, self._server_process.name()) 460 461 if err_line: 462 if self._check_for_driver_crash(err_line): 463 break 464 self.error_from_test += err_line 465 466 block.decode_content() 467 return block 468 469 470 class ContentBlock(object): 471 def __init__(self): 472 self.content_type = None 473 self.encoding = None 474 self.content_hash = None 475 self._content_length = None 476 # Content is treated as binary data even though the text output is usually UTF-8. 477 self.content = str() # FIXME: Should be bytearray() once we require Python 2.6. 478 self.decoded_content = None 479 self.malloc = None 480 self.js_heap = None 481 482 def decode_content(self): 483 if self.encoding == 'base64' and self.content is not None: 484 self.decoded_content = base64.b64decode(self.content) 485 else: 486 self.decoded_content = self.content 487
