1 #!/usr/bin/env python 2 # Copyright (C) 2010 Google Inc. All rights reserved. 3 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor (at] inf.u-szeged.hu), University of Szeged 4 # 5 # Redistribution and use in source and binary forms, with or without 6 # modification, are permitted provided that the following conditions are 7 # met: 8 # 9 # * Redistributions of source code must retain the above copyright 10 # notice, this list of conditions and the following disclaimer. 11 # * Redistributions in binary form must reproduce the above 12 # copyright notice, this list of conditions and the following disclaimer 13 # in the documentation and/or other materials provided with the 14 # distribution. 15 # * Neither the name of Google Inc. nor the names of its 16 # contributors may be used to endorse or promote products derived from 17 # this software without specific prior written permission. 18 # 19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 31 """Run layout tests.""" 32 33 import errno 34 import logging 35 import optparse 36 import os 37 import signal 38 import sys 39 40 from layout_package import json_results_generator 41 from layout_package import printing 42 from layout_package import test_runner 43 from layout_package import test_runner2 44 45 from webkitpy.common.system import user 46 from webkitpy.thirdparty import simplejson 47 48 import port 49 50 _log = logging.getLogger(__name__) 51 52 53 def run(port, options, args, regular_output=sys.stderr, 54 buildbot_output=sys.stdout): 55 """Run the tests. 56 57 Args: 58 port: Port object for port-specific behavior 59 options: a dictionary of command line options 60 args: a list of sub directories or files to test 61 regular_output: a stream-like object that we can send logging/debug 62 output to 63 buildbot_output: a stream-like object that we can write all output that 64 is intended to be parsed by the buildbot to 65 Returns: 66 the number of unexpected results that occurred, or -1 if there is an 67 error. 68 69 """ 70 warnings = _set_up_derived_options(port, options) 71 72 printer = printing.Printer(port, options, regular_output, buildbot_output, 73 int(options.child_processes), options.experimental_fully_parallel) 74 for w in warnings: 75 _log.warning(w) 76 77 if options.help_printing: 78 printer.help_printing() 79 printer.cleanup() 80 return 0 81 82 last_unexpected_results = _gather_unexpected_results(port) 83 if options.print_last_failures: 84 printer.write("\n".join(last_unexpected_results) + "\n") 85 printer.cleanup() 86 return 0 87 88 # We wrap any parts of the run that are slow or likely to raise exceptions 89 # in a try/finally to ensure that we clean up the logging configuration. 90 num_unexpected_results = -1 91 try: 92 runner = test_runner2.TestRunner2(port, options, printer) 93 runner._print_config() 94 95 printer.print_update("Collecting tests ...") 96 try: 97 runner.collect_tests(args, last_unexpected_results) 98 except IOError, e: 99 if e.errno == errno.ENOENT: 100 return -1 101 raise 102 103 if options.lint_test_files: 104 return runner.lint() 105 106 printer.print_update("Parsing expectations ...") 107 runner.parse_expectations() 108 109 printer.print_update("Checking build ...") 110 if not port.check_build(runner.needs_http()): 111 _log.error("Build check failed") 112 return -1 113 114 result_summary = runner.set_up_run() 115 if result_summary: 116 num_unexpected_results = runner.run(result_summary) 117 runner.clean_up_run() 118 _log.debug("Testing completed, Exit status: %d" % 119 num_unexpected_results) 120 finally: 121 printer.cleanup() 122 123 return num_unexpected_results 124 125 126 def _set_up_derived_options(port_obj, options): 127 """Sets the options values that depend on other options values.""" 128 # We return a list of warnings to print after the printer is initialized. 129 warnings = [] 130 131 if options.worker_model is None: 132 options.worker_model = port_obj.default_worker_model() 133 134 if options.worker_model == 'inline': 135 if options.child_processes and int(options.child_processes) > 1: 136 warnings.append("--worker-model=inline overrides --child-processes") 137 options.child_processes = "1" 138 if not options.child_processes: 139 options.child_processes = os.environ.get("WEBKIT_TEST_CHILD_PROCESSES", 140 str(port_obj.default_child_processes())) 141 142 if not options.configuration: 143 options.configuration = port_obj.default_configuration() 144 145 if options.pixel_tests is None: 146 options.pixel_tests = True 147 148 if not options.use_apache: 149 options.use_apache = sys.platform in ('darwin', 'linux2') 150 151 if not options.time_out_ms: 152 if options.configuration == "Debug": 153 options.time_out_ms = str(2 * test_runner.TestRunner.DEFAULT_TEST_TIMEOUT_MS) 154 else: 155 options.time_out_ms = str(test_runner.TestRunner.DEFAULT_TEST_TIMEOUT_MS) 156 157 options.slow_time_out_ms = str(5 * int(options.time_out_ms)) 158 159 if options.additional_platform_directory: 160 normalized_platform_directories = [] 161 for path in options.additional_platform_directory: 162 if not port_obj._filesystem.isabs(path): 163 warnings.append("--additional-platform-directory=%s is ignored since it is not absolute" % path) 164 continue 165 normalized_platform_directories.append(port_obj._filesystem.normpath(path)) 166 options.additional_platform_directory = normalized_platform_directories 167 168 return warnings 169 170 171 def _gather_unexpected_results(port): 172 """Returns the unexpected results from the previous run, if any.""" 173 filesystem = port._filesystem 174 results_directory = port.results_directory() 175 options = port._options 176 last_unexpected_results = [] 177 if options.print_last_failures or options.retest_last_failures: 178 unexpected_results_filename = filesystem.join(results_directory, "unexpected_results.json") 179 if filesystem.exists(unexpected_results_filename): 180 results = json_results_generator.load_json(filesystem, unexpected_results_filename) 181 last_unexpected_results = results['tests'].keys() 182 return last_unexpected_results 183 184 185 def _compat_shim_callback(option, opt_str, value, parser): 186 print "Ignoring unsupported option: %s" % opt_str 187 188 189 def _compat_shim_option(option_name, **kwargs): 190 return optparse.make_option(option_name, action="callback", 191 callback=_compat_shim_callback, 192 help="Ignored, for old-run-webkit-tests compat only.", **kwargs) 193 194 195 def parse_args(args=None): 196 """Provides a default set of command line args. 197 198 Returns a tuple of options, args from optparse""" 199 200 # FIXME: All of these options should be stored closer to the code which 201 # FIXME: actually uses them. configuration_options should move 202 # FIXME: to WebKitPort and be shared across all scripts. 203 configuration_options = [ 204 optparse.make_option("-t", "--target", dest="configuration", 205 help="(DEPRECATED)"), 206 # FIXME: --help should display which configuration is default. 207 optparse.make_option('--debug', action='store_const', const='Debug', 208 dest="configuration", 209 help='Set the configuration to Debug'), 210 optparse.make_option('--release', action='store_const', 211 const='Release', dest="configuration", 212 help='Set the configuration to Release'), 213 # old-run-webkit-tests also accepts -c, --configuration CONFIGURATION. 214 ] 215 216 print_options = printing.print_options() 217 218 # FIXME: These options should move onto the ChromiumPort. 219 chromium_options = [ 220 optparse.make_option("--chromium", action="store_true", default=False, 221 help="use the Chromium port"), 222 optparse.make_option("--startup-dialog", action="store_true", 223 default=False, help="create a dialog on DumpRenderTree startup"), 224 optparse.make_option("--gp-fault-error-box", action="store_true", 225 default=False, help="enable Windows GP fault error box"), 226 optparse.make_option("--js-flags", 227 type="string", help="JavaScript flags to pass to tests"), 228 optparse.make_option("--stress-opt", action="store_true", 229 default=False, 230 help="Enable additional stress test to JavaScript optimization"), 231 optparse.make_option("--stress-deopt", action="store_true", 232 default=False, 233 help="Enable additional stress test to JavaScript optimization"), 234 optparse.make_option("--nocheck-sys-deps", action="store_true", 235 default=False, 236 help="Don't check the system dependencies (themes)"), 237 optparse.make_option("--accelerated-compositing", 238 action="store_true", 239 help="Use hardware-accelerated compositing for rendering"), 240 optparse.make_option("--no-accelerated-compositing", 241 action="store_false", 242 dest="accelerated_compositing", 243 help="Don't use hardware-accelerated compositing for rendering"), 244 optparse.make_option("--accelerated-2d-canvas", 245 action="store_true", 246 help="Use hardware-accelerated 2D Canvas calls"), 247 optparse.make_option("--no-accelerated-2d-canvas", 248 action="store_false", 249 dest="accelerated_2d_canvas", 250 help="Don't use hardware-accelerated 2D Canvas calls"), 251 optparse.make_option("--enable-hardware-gpu", 252 action="store_true", 253 default=False, 254 help="Run graphics tests on real GPU hardware vs software"), 255 ] 256 257 # Missing Mac-specific old-run-webkit-tests options: 258 # FIXME: Need: -g, --guard for guard malloc support on Mac. 259 # FIXME: Need: -l --leaks Enable leaks checking. 260 # FIXME: Need: --sample-on-timeout Run sample on timeout 261 262 old_run_webkit_tests_compat = [ 263 # NRWT doesn't generate results by default anyway. 264 _compat_shim_option("--no-new-test-results"), 265 # NRWT doesn't sample on timeout yet anyway. 266 _compat_shim_option("--no-sample-on-timeout"), 267 # FIXME: NRWT needs to support remote links eventually. 268 _compat_shim_option("--use-remote-links-to-tests"), 269 ] 270 271 results_options = [ 272 # NEED for bots: --use-remote-links-to-tests Link to test files 273 # within the SVN repository in the results. 274 optparse.make_option("-p", "--pixel-tests", action="store_true", 275 dest="pixel_tests", help="Enable pixel-to-pixel PNG comparisons"), 276 optparse.make_option("--no-pixel-tests", action="store_false", 277 dest="pixel_tests", help="Disable pixel-to-pixel PNG comparisons"), 278 optparse.make_option("--tolerance", 279 help="Ignore image differences less than this percentage (some " 280 "ports may ignore this option)", type="float"), 281 optparse.make_option("--results-directory", help="Location of test results"), 282 optparse.make_option("--build-directory", 283 help="Path to the directory under which build files are kept (should not include configuration)"), 284 optparse.make_option("--new-baseline", action="store_true", 285 default=False, help="Save all generated results as new baselines " 286 "into the platform directory, overwriting whatever's " 287 "already there."), 288 optparse.make_option("--reset-results", action="store_true", 289 default=False, help="Reset any existing baselines to the " 290 "generated results"), 291 optparse.make_option("--additional-drt-flag", action="append", 292 default=[], help="Additional command line flag to pass to DumpRenderTree " 293 "Specify multiple times to add multiple flags."), 294 optparse.make_option("--additional-platform-directory", action="append", 295 default=[], help="Additional directory where to look for test " 296 "baselines (will take precendence over platform baselines). " 297 "Specify multiple times to add multiple search path entries."), 298 optparse.make_option("--no-show-results", action="store_false", 299 default=True, dest="show_results", 300 help="Don't launch a browser with results after the tests " 301 "are done"), 302 # FIXME: We should have a helper function to do this sort of 303 # deprectated mapping and automatically log, etc. 304 optparse.make_option("--noshow-results", action="store_false", 305 dest="show_results", 306 help="Deprecated, same as --no-show-results."), 307 optparse.make_option("--no-launch-safari", action="store_false", 308 dest="show_results", 309 help="old-run-webkit-tests compat, same as --noshow-results."), 310 # old-run-webkit-tests: 311 # --[no-]launch-safari Launch (or do not launch) Safari to display 312 # test results (default: launch) 313 optparse.make_option("--full-results-html", action="store_true", 314 default=False, 315 help="Show all failures in results.html, rather than only " 316 "regressions"), 317 optparse.make_option("--clobber-old-results", action="store_true", 318 default=False, help="Clobbers test results from previous runs."), 319 optparse.make_option("--platform", 320 help="Override the platform for expected results"), 321 optparse.make_option("--no-record-results", action="store_false", 322 default=True, dest="record_results", 323 help="Don't record the results."), 324 # old-run-webkit-tests also has HTTP toggle options: 325 # --[no-]http Run (or do not run) http tests 326 # (default: run) 327 ] 328 329 test_options = [ 330 optparse.make_option("--build", dest="build", 331 action="store_true", default=True, 332 help="Check to ensure the DumpRenderTree build is up-to-date " 333 "(default)."), 334 optparse.make_option("--no-build", dest="build", 335 action="store_false", help="Don't check to see if the " 336 "DumpRenderTree build is up-to-date."), 337 optparse.make_option("-n", "--dry-run", action="store_true", 338 default=False, 339 help="Do everything but actually run the tests or upload results."), 340 # old-run-webkit-tests has --valgrind instead of wrapper. 341 optparse.make_option("--wrapper", 342 help="wrapper command to insert before invocations of " 343 "DumpRenderTree; option is split on whitespace before " 344 "running. (Example: --wrapper='valgrind --smc-check=all')"), 345 # old-run-webkit-tests: 346 # -i|--ignore-tests Comma-separated list of directories 347 # or tests to ignore 348 optparse.make_option("--test-list", action="append", 349 help="read list of tests to run from file", metavar="FILE"), 350 # old-run-webkit-tests uses --skipped==[default|ignore|only] 351 # instead of --force: 352 optparse.make_option("--force", action="store_true", default=False, 353 help="Run all tests, even those marked SKIP in the test list"), 354 optparse.make_option("--use-apache", action="store_true", 355 default=False, help="Whether to use apache instead of lighttpd."), 356 optparse.make_option("--time-out-ms", 357 help="Set the timeout for each test"), 358 # old-run-webkit-tests calls --randomize-order --random: 359 optparse.make_option("--randomize-order", action="store_true", 360 default=False, help=("Run tests in random order (useful " 361 "for tracking down corruption)")), 362 optparse.make_option("--run-chunk", 363 help=("Run a specified chunk (n:l), the nth of len l, " 364 "of the layout tests")), 365 optparse.make_option("--run-part", help=("Run a specified part (n:m), " 366 "the nth of m parts, of the layout tests")), 367 # old-run-webkit-tests calls --batch-size: --nthly n 368 # Restart DumpRenderTree every n tests (default: 1000) 369 optparse.make_option("--batch-size", 370 help=("Run a the tests in batches (n), after every n tests, " 371 "DumpRenderTree is relaunched."), type="int", default=0), 372 # old-run-webkit-tests calls --run-singly: -1|--singly 373 # Isolate each test case run (implies --nthly 1 --verbose) 374 optparse.make_option("--run-singly", action="store_true", 375 default=False, help="run a separate DumpRenderTree for each test"), 376 optparse.make_option("--child-processes", 377 help="Number of DumpRenderTrees to run in parallel."), 378 # FIXME: Display default number of child processes that will run. 379 optparse.make_option("--worker-model", action="store", 380 default=None, help=("controls worker model. Valid values are " 381 "'inline', 'threads', and 'processes'.")), 382 optparse.make_option("--experimental-fully-parallel", 383 action="store_true", default=False, 384 help="run all tests in parallel"), 385 optparse.make_option("--exit-after-n-failures", type="int", default=500, 386 help="Exit after the first N failures instead of running all " 387 "tests"), 388 optparse.make_option("--exit-after-n-crashes-or-timeouts", type="int", 389 default=20, help="Exit after the first N crashes instead of " 390 "running all tests"), 391 # FIXME: consider: --iterations n 392 # Number of times to run the set of tests (e.g. ABCABCABC) 393 optparse.make_option("--print-last-failures", action="store_true", 394 default=False, help="Print the tests in the last run that " 395 "had unexpected failures (or passes) and then exit."), 396 optparse.make_option("--retest-last-failures", action="store_true", 397 default=False, help="re-test the tests in the last run that " 398 "had unexpected failures (or passes)."), 399 optparse.make_option("--retry-failures", action="store_true", 400 default=True, 401 help="Re-try any tests that produce unexpected results (default)"), 402 optparse.make_option("--no-retry-failures", action="store_false", 403 dest="retry_failures", 404 help="Don't re-try any tests that produce unexpected results."), 405 ] 406 407 misc_options = [ 408 optparse.make_option("--lint-test-files", action="store_true", 409 default=False, help=("Makes sure the test files parse for all " 410 "configurations. Does not run any tests.")), 411 ] 412 413 # FIXME: Move these into json_results_generator.py 414 results_json_options = [ 415 optparse.make_option("--master-name", help="The name of the buildbot master."), 416 optparse.make_option("--builder-name", default="DUMMY_BUILDER_NAME", 417 help=("The name of the builder shown on the waterfall running " 418 "this script e.g. WebKit.")), 419 optparse.make_option("--build-name", default="DUMMY_BUILD_NAME", 420 help=("The name of the builder used in its path, e.g. " 421 "webkit-rel.")), 422 optparse.make_option("--build-number", default="DUMMY_BUILD_NUMBER", 423 help=("The build number of the builder running this script.")), 424 optparse.make_option("--test-results-server", default="", 425 help=("If specified, upload results json files to this appengine " 426 "server.")), 427 ] 428 429 option_list = (configuration_options + print_options + 430 chromium_options + results_options + test_options + 431 misc_options + results_json_options + 432 old_run_webkit_tests_compat) 433 option_parser = optparse.OptionParser(option_list=option_list) 434 435 return option_parser.parse_args(args) 436 437 438 def main(): 439 options, args = parse_args() 440 port_obj = port.get(options.platform, options) 441 return run(port_obj, options, args) 442 443 444 if '__main__' == __name__: 445 try: 446 sys.exit(main()) 447 except KeyboardInterrupt: 448 # this mirrors what the shell normally does 449 sys.exit(signal.SIGINT + 128) 450
