1 #!/usr/bin/env python 2 # Copyright (C) 2010 Google Inc. All rights reserved. 3 # 4 # Redistribution and use in source and binary forms, with or without 5 # modification, are permitted provided that the following conditions are 6 # met: 7 # 8 # * Redistributions of source code must retain the above copyright 9 # notice, this list of conditions and the following disclaimer. 10 # * Redistributions in binary form must reproduce the above 11 # copyright notice, this list of conditions and the following disclaimer 12 # in the documentation and/or other materials provided with the 13 # distribution. 14 # * Neither the name of Google Inc. nor the names of its 15 # contributors may be used to endorse or promote products derived from 16 # this software without specific prior written permission. 17 # 18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 30 """A Thread object for running the test shell and processing URLs from a 31 shared queue. 32 33 Each thread runs a separate instance of the test_shell binary and validates 34 the output. When there are no more URLs to process in the shared queue, the 35 thread exits. 36 """ 37 38 import copy 39 import logging 40 import os 41 import Queue 42 import signal 43 import sys 44 import thread 45 import threading 46 import time 47 48 import test_failures 49 50 51 def process_output(port, test_info, test_types, test_args, target, output_dir, 52 crash, timeout, test_run_time, actual_checksum, 53 output, error): 54 """Receives the output from a test_shell process, subjects it to a number 55 of tests, and returns a list of failure types the test produced. 56 57 Args: 58 port: port-specific hooks 59 proc: an active test_shell process 60 test_info: Object containing the test filename, uri and timeout 61 test_types: list of test types to subject the output to 62 test_args: arguments to be passed to each test 63 target: Debug or Release 64 output_dir: directory to put crash stack traces into 65 66 Returns: a list of failure objects and times for the test being processed 67 """ 68 failures = [] 69 70 # Some test args, such as the image hash, may be added or changed on a 71 # test-by-test basis. 72 local_test_args = copy.copy(test_args) 73 74 local_test_args.hash = actual_checksum 75 76 if crash: 77 failures.append(test_failures.FailureCrash()) 78 if timeout: 79 failures.append(test_failures.FailureTimeout()) 80 81 if crash: 82 logging.debug("Stacktrace for %s:\n%s" % (test_info.filename, error)) 83 # Strip off "file://" since RelativeTestFilename expects 84 # filesystem paths. 85 filename = os.path.join(output_dir, test_info.filename) 86 filename = os.path.splitext(filename)[0] + "-stack.txt" 87 port.maybe_make_directory(os.path.split(filename)[0]) 88 open(filename, "wb").write(error) 89 elif error: 90 logging.debug("Previous test output extra lines after dump:\n%s" % 91 error) 92 93 # Check the output and save the results. 94 start_time = time.time() 95 time_for_diffs = {} 96 for test_type in test_types: 97 start_diff_time = time.time() 98 new_failures = test_type.compare_output(port, test_info.filename, 99 output, local_test_args, 100 target) 101 # Don't add any more failures if we already have a crash, so we don't 102 # double-report those tests. We do double-report for timeouts since 103 # we still want to see the text and image output. 104 if not crash: 105 failures.extend(new_failures) 106 time_for_diffs[test_type.__class__.__name__] = ( 107 time.time() - start_diff_time) 108 109 total_time_for_all_diffs = time.time() - start_diff_time 110 return TestStats(test_info.filename, failures, test_run_time, 111 total_time_for_all_diffs, time_for_diffs) 112 113 114 class TestStats: 115 116 def __init__(self, filename, failures, test_run_time, 117 total_time_for_all_diffs, time_for_diffs): 118 self.filename = filename 119 self.failures = failures 120 self.test_run_time = test_run_time 121 self.total_time_for_all_diffs = total_time_for_all_diffs 122 self.time_for_diffs = time_for_diffs 123 124 125 class SingleTestThread(threading.Thread): 126 """Thread wrapper for running a single test file.""" 127 128 def __init__(self, port, image_path, shell_args, test_info, 129 test_types, test_args, target, output_dir): 130 """ 131 Args: 132 port: object implementing port-specific hooks 133 test_info: Object containing the test filename, uri and timeout 134 output_dir: Directory to put crash stacks into. 135 See TestShellThread for documentation of the remaining arguments. 136 """ 137 138 threading.Thread.__init__(self) 139 self._port = port 140 self._image_path = image_path 141 self._shell_args = shell_args 142 self._test_info = test_info 143 self._test_types = test_types 144 self._test_args = test_args 145 self._target = target 146 self._output_dir = output_dir 147 148 def run(self): 149 driver = self._port.start_test_driver(self._image_path, 150 self._shell_args) 151 start = time.time() 152 crash, timeout, actual_checksum, output, error = \ 153 driver.run_test(test_info.uri.strip(), test_info.timeout, 154 test_info.image_hash) 155 end = time.time() 156 self._test_stats = process_output(self._port, 157 self._test_info, self._test_types, self._test_args, 158 self._target, self._output_dir, crash, timeout, end - start, 159 actual_checksum, output, error) 160 driver.stop() 161 162 def get_test_stats(self): 163 return self._test_stats 164 165 166 class TestShellThread(threading.Thread): 167 168 def __init__(self, port, filename_list_queue, result_queue, 169 test_types, test_args, image_path, shell_args, options): 170 """Initialize all the local state for this test shell thread. 171 172 Args: 173 port: interface to port-specific hooks 174 filename_list_queue: A thread safe Queue class that contains lists 175 of tuples of (filename, uri) pairs. 176 result_queue: A thread safe Queue class that will contain tuples of 177 (test, failure lists) for the test results. 178 test_types: A list of TestType objects to run the test output 179 against. 180 test_args: A TestArguments object to pass to each TestType. 181 shell_args: Any extra arguments to be passed to test_shell.exe. 182 options: A property dictionary as produced by optparse. The 183 command-line options should match those expected by 184 run_webkit_tests; they are typically passed via the 185 run_webkit_tests.TestRunner class.""" 186 threading.Thread.__init__(self) 187 self._port = port 188 self._filename_list_queue = filename_list_queue 189 self._result_queue = result_queue 190 self._filename_list = [] 191 self._test_types = test_types 192 self._test_args = test_args 193 self._driver = None 194 self._image_path = image_path 195 self._shell_args = shell_args 196 self._options = options 197 self._canceled = False 198 self._exception_info = None 199 self._directory_timing_stats = {} 200 self._test_stats = [] 201 self._num_tests = 0 202 self._start_time = 0 203 self._stop_time = 0 204 205 # Current directory of tests we're running. 206 self._current_dir = None 207 # Number of tests in self._current_dir. 208 self._num_tests_in_current_dir = None 209 # Time at which we started running tests from self._current_dir. 210 self._current_dir_start_time = None 211 212 def get_directory_timing_stats(self): 213 """Returns a dictionary mapping test directory to a tuple of 214 (number of tests in that directory, time to run the tests)""" 215 return self._directory_timing_stats 216 217 def get_individual_test_stats(self): 218 """Returns a list of (test_filename, time_to_run_test, 219 total_time_for_all_diffs, time_for_diffs) tuples.""" 220 return self._test_stats 221 222 def cancel(self): 223 """Set a flag telling this thread to quit.""" 224 self._canceled = True 225 226 def get_exception_info(self): 227 """If run() terminated on an uncaught exception, return it here 228 ((type, value, traceback) tuple). 229 Returns None if run() terminated normally. Meant to be called after 230 joining this thread.""" 231 return self._exception_info 232 233 def get_total_time(self): 234 return max(self._stop_time - self._start_time, 0.0) 235 236 def get_num_tests(self): 237 return self._num_tests 238 239 def run(self): 240 """Delegate main work to a helper method and watch for uncaught 241 exceptions.""" 242 self._start_time = time.time() 243 self._num_tests = 0 244 try: 245 logging.debug('%s starting' % (self.getName())) 246 self._run(test_runner=None, result_summary=None) 247 logging.debug('%s done (%d tests)' % (self.getName(), 248 self.get_num_tests())) 249 except: 250 # Save the exception for our caller to see. 251 self._exception_info = sys.exc_info() 252 self._stop_time = time.time() 253 # Re-raise it and die. 254 logging.error('%s dying: %s' % (self.getName(), 255 self._exception_info)) 256 raise 257 self._stop_time = time.time() 258 259 def run_in_main_thread(self, test_runner, result_summary): 260 """This hook allows us to run the tests from the main thread if 261 --num-test-shells==1, instead of having to always run two or more 262 threads. This allows us to debug the test harness without having to 263 do multi-threaded debugging.""" 264 self._run(test_runner, result_summary) 265 266 def _run(self, test_runner, result_summary): 267 """Main work entry point of the thread. Basically we pull urls from the 268 filename queue and run the tests until we run out of urls. 269 270 If test_runner is not None, then we call test_runner.UpdateSummary() 271 with the results of each test.""" 272 batch_size = 0 273 batch_count = 0 274 if self._options.batch_size: 275 try: 276 batch_size = int(self._options.batch_size) 277 except: 278 logging.info("Ignoring invalid batch size '%s'" % 279 self._options.batch_size) 280 281 # Append tests we're running to the existing tests_run.txt file. 282 # This is created in run_webkit_tests.py:_PrepareListsAndPrintOutput. 283 tests_run_filename = os.path.join(self._options.results_directory, 284 "tests_run.txt") 285 tests_run_file = open(tests_run_filename, "a") 286 287 while True: 288 if self._canceled: 289 logging.info('Testing canceled') 290 tests_run_file.close() 291 return 292 293 if len(self._filename_list) is 0: 294 if self._current_dir is not None: 295 self._directory_timing_stats[self._current_dir] = \ 296 (self._num_tests_in_current_dir, 297 time.time() - self._current_dir_start_time) 298 299 try: 300 self._current_dir, self._filename_list = \ 301 self._filename_list_queue.get_nowait() 302 except Queue.Empty: 303 self._kill_test_shell() 304 tests_run_file.close() 305 return 306 307 self._num_tests_in_current_dir = len(self._filename_list) 308 self._current_dir_start_time = time.time() 309 310 test_info = self._filename_list.pop() 311 312 # We have a url, run tests. 313 batch_count += 1 314 self._num_tests += 1 315 if self._options.run_singly: 316 failures = self._run_test_singly(test_info) 317 else: 318 failures = self._run_test(test_info) 319 320 filename = test_info.filename 321 tests_run_file.write(filename + "\n") 322 if failures: 323 # Check and kill test shell if we need too. 324 if len([1 for f in failures if f.should_kill_test_shell()]): 325 self._kill_test_shell() 326 # Reset the batch count since the shell just bounced. 327 batch_count = 0 328 # Print the error message(s). 329 error_str = '\n'.join([' ' + f.message() for f in failures]) 330 logging.debug("%s %s failed:\n%s" % (self.getName(), 331 self._port.relative_test_filename(filename), 332 error_str)) 333 else: 334 logging.debug("%s %s passed" % (self.getName(), 335 self._port.relative_test_filename(filename))) 336 self._result_queue.put((filename, failures)) 337 338 if batch_size > 0 and batch_count > batch_size: 339 # Bounce the shell and reset count. 340 self._kill_test_shell() 341 batch_count = 0 342 343 if test_runner: 344 test_runner.update_summary(result_summary) 345 346 def _run_test_singly(self, test_info): 347 """Run a test in a separate thread, enforcing a hard time limit. 348 349 Since we can only detect the termination of a thread, not any internal 350 state or progress, we can only run per-test timeouts when running test 351 files singly. 352 353 Args: 354 test_info: Object containing the test filename, uri and timeout 355 356 Return: 357 A list of TestFailure objects describing the error. 358 """ 359 worker = SingleTestThread(self._port, self._image_path, 360 self._shell_args, 361 test_info, 362 self._test_types, 363 self._test_args, 364 self._options.target, 365 self._options.results_directory) 366 367 worker.start() 368 369 # When we're running one test per test_shell process, we can enforce 370 # a hard timeout. the test_shell watchdog uses 2.5x the timeout 371 # We want to be larger than that. 372 worker.join(int(test_info.timeout) * 3.0 / 1000.0) 373 if worker.isAlive(): 374 # If join() returned with the thread still running, the 375 # test_shell.exe is completely hung and there's nothing 376 # more we can do with it. We have to kill all the 377 # test_shells to free it up. If we're running more than 378 # one test_shell thread, we'll end up killing the other 379 # test_shells too, introducing spurious crashes. We accept that 380 # tradeoff in order to avoid losing the rest of this thread's 381 # results. 382 logging.error('Test thread hung: killing all test_shells') 383 worker._driver.stop() 384 385 try: 386 stats = worker.get_test_stats() 387 self._test_stats.append(stats) 388 failures = stats.failures 389 except AttributeError, e: 390 failures = [] 391 logging.error('Cannot get results of test: %s' % 392 test_info.filename) 393 394 return failures 395 396 def _run_test(self, test_info): 397 """Run a single test file using a shared test_shell process. 398 399 Args: 400 test_info: Object containing the test filename, uri and timeout 401 402 Return: 403 A list of TestFailure objects describing the error. 404 """ 405 self._ensure_test_shell_is_running() 406 # The pixel_hash is used to avoid doing an image dump if the 407 # checksums match, so it should be set to a blank value if we 408 # are generating a new baseline. (Otherwise, an image from a 409 # previous run will be copied into the baseline.) 410 image_hash = test_info.image_hash 411 if image_hash and self._test_args.new_baseline: 412 image_hash = "" 413 start = time.time() 414 crash, timeout, actual_checksum, output, error = \ 415 self._driver.run_test(test_info.uri, test_info.timeout, image_hash) 416 end = time.time() 417 418 stats = process_output(self._port, test_info, self._test_types, 419 self._test_args, self._options.target, 420 self._options.results_directory, crash, 421 timeout, end - start, actual_checksum, 422 output, error) 423 424 self._test_stats.append(stats) 425 return stats.failures 426 427 def _ensure_test_shell_is_running(self): 428 """Start the shared test shell, if it's not running. Not for use when 429 running tests singly, since those each start a separate test shell in 430 their own thread. 431 """ 432 if (not self._driver or self._driver.poll() is not None): 433 self._driver = self._port.start_driver( 434 self._image_path, self._shell_args) 435 436 def _kill_test_shell(self): 437 """Kill the test shell process if it's running.""" 438 if self._driver: 439 self._driver.stop() 440 self._driver = None 441
