1 #!/usr/bin/python 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. 3 # Use of this source code is governed by a BSD-style license that can be 4 # found in the LICENSE file. 5 6 import glob 7 import optparse 8 import os.path 9 import socket 10 import sys 11 import thread 12 import time 13 import urllib 14 15 # Allow the import of third party modules 16 script_dir = os.path.dirname(os.path.abspath(__file__)) 17 sys.path.insert(0, os.path.join(script_dir, '../../../../third_party/')) 18 sys.path.insert(0, os.path.join(script_dir, '../../../../tools/valgrind/')) 19 sys.path.insert(0, os.path.join(script_dir, '../../../../testing/')) 20 21 import browsertester.browserlauncher 22 import browsertester.rpclistener 23 import browsertester.server 24 25 import memcheck_analyze 26 import tsan_analyze 27 28 import test_env 29 30 def BuildArgParser(): 31 usage = 'usage: %prog [options]' 32 parser = optparse.OptionParser(usage) 33 34 parser.add_option('-p', '--port', dest='port', action='store', type='int', 35 default='0', help='The TCP port the server will bind to. ' 36 'The default is to pick an unused port number.') 37 parser.add_option('--browser_path', dest='browser_path', action='store', 38 type='string', default=None, 39 help='Use the browser located here.') 40 parser.add_option('--map_file', dest='map_files', action='append', 41 type='string', nargs=2, default=[], 42 metavar='DEST SRC', 43 help='Add file SRC to be served from the HTTP server, ' 44 'to be made visible under the path DEST.') 45 parser.add_option('--serving_dir', dest='serving_dirs', action='append', 46 type='string', default=[], 47 metavar='DIRNAME', 48 help='Add directory DIRNAME to be served from the HTTP ' 49 'server to be made visible under the root.') 50 parser.add_option('--output_dir', dest='output_dir', action='store', 51 type='string', default=None, 52 metavar='DIRNAME', 53 help='Set directory DIRNAME to be the output directory ' 54 'when POSTing data to the server. NOTE: if this flag is ' 55 'not set, POSTs will fail.') 56 parser.add_option('--test_arg', dest='test_args', action='append', 57 type='string', nargs=2, default=[], 58 metavar='KEY VALUE', 59 help='Parameterize the test with a key/value pair.') 60 parser.add_option('--redirect_url', dest='map_redirects', action='append', 61 type='string', nargs=2, default=[], 62 metavar='DEST SRC', 63 help='Add a redirect to the HTTP server, ' 64 'requests for SRC will result in a redirect (302) to DEST.') 65 parser.add_option('-f', '--file', dest='files', action='append', 66 type='string', default=[], 67 metavar='FILENAME', 68 help='Add a file to serve from the HTTP server, to be ' 69 'made visible in the root directory. ' 70 '"--file path/to/foo.html" is equivalent to ' 71 '"--map_file foo.html path/to/foo.html"') 72 parser.add_option('--mime_type', dest='mime_types', action='append', 73 type='string', nargs=2, default=[], metavar='DEST SRC', 74 help='Map file extension SRC to MIME type DEST when ' 75 'serving it from the HTTP server.') 76 parser.add_option('-u', '--url', dest='url', action='store', 77 type='string', default=None, 78 help='The webpage to load.') 79 parser.add_option('--ppapi_plugin', dest='ppapi_plugin', action='store', 80 type='string', default=None, 81 help='Use the browser plugin located here.') 82 parser.add_option('--ppapi_plugin_mimetype', dest='ppapi_plugin_mimetype', 83 action='store', type='string', default='application/x-nacl', 84 help='Associate this mimetype with the browser plugin. ' 85 'Unused if --ppapi_plugin is not specified.') 86 parser.add_option('--sel_ldr', dest='sel_ldr', action='store', 87 type='string', default=None, 88 help='Use the sel_ldr located here.') 89 parser.add_option('--sel_ldr_bootstrap', dest='sel_ldr_bootstrap', 90 action='store', type='string', default=None, 91 help='Use the bootstrap loader located here.') 92 parser.add_option('--irt_library', dest='irt_library', action='store', 93 type='string', default=None, 94 help='Use the integrated runtime (IRT) library ' 95 'located here.') 96 parser.add_option('--interactive', dest='interactive', action='store_true', 97 default=False, help='Do not quit after testing is done. ' 98 'Handy for iterative development. Disables timeout.') 99 parser.add_option('--debug', dest='debug', action='store_true', default=False, 100 help='Request debugging output from browser.') 101 parser.add_option('--timeout', dest='timeout', action='store', type='float', 102 default=5.0, 103 help='The maximum amount of time to wait, in seconds, for ' 104 'the browser to make a request. The timer resets with each ' 105 'request.') 106 parser.add_option('--hard_timeout', dest='hard_timeout', action='store', 107 type='float', default=None, 108 help='The maximum amount of time to wait, in seconds, for ' 109 'the entire test. This will kill runaway tests. ') 110 parser.add_option('--allow_404', dest='allow_404', action='store_true', 111 default=False, 112 help='Allow 404s to occur without failing the test.') 113 parser.add_option('-b', '--bandwidth', dest='bandwidth', action='store', 114 type='float', default='0.0', 115 help='The amount of bandwidth (megabits / second) to ' 116 'simulate between the client and the server. This used for ' 117 'replies with file payloads. All other responses are ' 118 'assumed to be short. Bandwidth values <= 0.0 are assumed ' 119 'to mean infinite bandwidth.') 120 parser.add_option('--extension', dest='browser_extensions', action='append', 121 type='string', default=[], 122 help='Load the browser extensions located at the list of ' 123 'paths. Note: this currently only works with the Chrome ' 124 'browser.') 125 parser.add_option('--tool', dest='tool', action='store', 126 type='string', default=None, 127 help='Run tests under a tool.') 128 parser.add_option('--browser_flag', dest='browser_flags', action='append', 129 type='string', default=[], 130 help='Additional flags for the chrome command.') 131 parser.add_option('--enable_ppapi_dev', dest='enable_ppapi_dev', 132 action='store', type='int', default=1, 133 help='Enable/disable PPAPI Dev interfaces while testing.') 134 parser.add_option('--nacl_exe_stdin', dest='nacl_exe_stdin', 135 type='string', default=None, 136 help='Redirect standard input of NaCl executable.') 137 parser.add_option('--nacl_exe_stdout', dest='nacl_exe_stdout', 138 type='string', default=None, 139 help='Redirect standard output of NaCl executable.') 140 parser.add_option('--nacl_exe_stderr', dest='nacl_exe_stderr', 141 type='string', default=None, 142 help='Redirect standard error of NaCl executable.') 143 parser.add_option('--expect_browser_process_crash', 144 dest='expect_browser_process_crash', 145 action='store_true', 146 help='Do not signal a failure if the browser process ' 147 'crashes') 148 parser.add_option('--enable_crash_reporter', dest='enable_crash_reporter', 149 action='store_true', default=False, 150 help='Force crash reporting on.') 151 parser.add_option('--enable_sockets', dest='enable_sockets', 152 action='store_true', default=False, 153 help='Pass --allow-nacl-socket-api=<host> to Chrome, where ' 154 '<host> is the name of the browser tester\'s web server.') 155 156 return parser 157 158 159 def ProcessToolLogs(options, logs_dir): 160 if options.tool == 'memcheck': 161 analyzer = memcheck_analyze.MemcheckAnalyzer('', use_gdb=True) 162 logs_wildcard = 'xml.*' 163 elif options.tool == 'tsan': 164 analyzer = tsan_analyze.TsanAnalyzer(use_gdb=True) 165 logs_wildcard = 'log.*' 166 files = glob.glob(os.path.join(logs_dir, logs_wildcard)) 167 retcode = analyzer.Report(files, options.url) 168 return retcode 169 170 171 # An exception that indicates possible flake. 172 class RetryTest(Exception): 173 pass 174 175 176 def DumpNetLog(netlog): 177 sys.stdout.write('\n') 178 if not os.path.isfile(netlog): 179 sys.stdout.write('Cannot find netlog, did Chrome actually launch?\n') 180 else: 181 sys.stdout.write('Netlog exists (%d bytes).\n' % os.path.getsize(netlog)) 182 sys.stdout.write('Dumping it to stdout.\n\n\n') 183 sys.stdout.write(open(netlog).read()) 184 sys.stdout.write('\n\n\n') 185 186 187 # Try to discover the real IP address of this machine. If we can't figure it 188 # out, fall back to localhost. 189 # A windows bug makes using the loopback interface flaky in rare cases. 190 # http://code.google.com/p/chromium/issues/detail?id=114369 191 def GetHostName(): 192 host = 'localhost' 193 try: 194 host = socket.gethostbyname(socket.gethostname()) 195 except Exception: 196 pass 197 if host == '0.0.0.0': 198 host = 'localhost' 199 return host 200 201 202 def RunTestsOnce(url, options): 203 # Set the default here so we're assured hard_timeout will be defined. 204 # Tests, such as run_inbrowser_trusted_crash_in_startup_test, may not use the 205 # RunFromCommand line entry point - and otherwise get stuck in an infinite 206 # loop when something goes wrong and the hard timeout is not set. 207 # http://code.google.com/p/chromium/issues/detail?id=105406 208 if options.hard_timeout is None: 209 options.hard_timeout = options.timeout * 4 210 211 options.files.append(os.path.join(script_dir, 'browserdata', 'nacltest.js')) 212 213 # Setup the environment with the setuid sandbox path. 214 test_env.enable_sandbox_if_required(os.environ) 215 216 # Create server 217 host = GetHostName() 218 try: 219 server = browsertester.server.Create(host, options.port) 220 except Exception: 221 sys.stdout.write('Could not bind %r, falling back to localhost.\n' % host) 222 server = browsertester.server.Create('localhost', options.port) 223 224 # If port 0 has been requested, an arbitrary port will be bound so we need to 225 # query it. Older version of Python do not set server_address correctly when 226 # The requested port is 0 so we need to break encapsulation and query the 227 # socket directly. 228 host, port = server.socket.getsockname() 229 230 file_mapping = dict(options.map_files) 231 for filename in options.files: 232 file_mapping[os.path.basename(filename)] = filename 233 for server_path, real_path in file_mapping.iteritems(): 234 if not os.path.exists(real_path): 235 raise AssertionError('\'%s\' does not exist.' % real_path) 236 mime_types = {} 237 for ext, mime_type in options.mime_types: 238 mime_types['.' + ext] = mime_type 239 240 def ShutdownCallback(): 241 server.TestingEnded() 242 close_browser = options.tool is not None and not options.interactive 243 return close_browser 244 245 listener = browsertester.rpclistener.RPCListener(ShutdownCallback) 246 server.Configure(file_mapping, 247 dict(options.map_redirects), 248 mime_types, 249 options.allow_404, 250 options.bandwidth, 251 listener, 252 options.serving_dirs, 253 options.output_dir) 254 255 browser = browsertester.browserlauncher.ChromeLauncher(options) 256 257 full_url = 'http://%s:%d/%s' % (host, port, url) 258 if len(options.test_args) > 0: 259 full_url += '?' + urllib.urlencode(options.test_args) 260 browser.Run(full_url, host, port) 261 server.TestingBegun(0.125) 262 263 # In Python 2.5, server.handle_request may block indefinitely. Serving pages 264 # is done in its own thread so the main thread can time out as needed. 265 def Serve(): 266 while server.test_in_progress or options.interactive: 267 server.handle_request() 268 thread.start_new_thread(Serve, ()) 269 270 tool_failed = False 271 time_started = time.time() 272 273 def HardTimeout(total_time): 274 return total_time >= 0.0 and time.time() - time_started >= total_time 275 276 try: 277 while server.test_in_progress or options.interactive: 278 if not browser.IsRunning(): 279 if options.expect_browser_process_crash: 280 break 281 listener.ServerError('Browser process ended during test ' 282 '(return code %r)' % browser.GetReturnCode()) 283 # If Chrome exits prematurely without making a single request to the 284 # web server, this is probally a Chrome crash-on-launch bug not related 285 # to the test at hand. Retry, unless we're in interactive mode. In 286 # interactive mode the user may manually close the browser, so don't 287 # retry (it would just be annoying.) 288 if not server.received_request and not options.interactive: 289 raise RetryTest('Chrome failed to launch.') 290 else: 291 break 292 elif not options.interactive and server.TimedOut(options.timeout): 293 js_time = server.TimeSinceJSHeartbeat() 294 err = 'Did not hear from the test for %.1f seconds.' % options.timeout 295 err += '\nHeard from Javascript %.1f seconds ago.' % js_time 296 if js_time > 2.0: 297 err += '\nThe renderer probably hung or crashed.' 298 else: 299 err += '\nThe test probably did not get a callback that it expected.' 300 listener.ServerError(err) 301 if not server.received_request: 302 raise RetryTest('Chrome hung before running the test.') 303 break 304 elif not options.interactive and HardTimeout(options.hard_timeout): 305 listener.ServerError('The test took over %.1f seconds. This is ' 306 'probably a runaway test.' % options.hard_timeout) 307 break 308 else: 309 # If Python 2.5 support is dropped, stick server.handle_request() here. 310 time.sleep(0.125) 311 312 if options.tool: 313 sys.stdout.write('##################### Waiting for the tool to exit\n') 314 browser.WaitForProcessDeath() 315 sys.stdout.write('##################### Processing tool logs\n') 316 tool_failed = ProcessToolLogs(options, browser.tool_log_dir) 317 318 finally: 319 try: 320 if listener.ever_failed and not options.interactive: 321 if not server.received_request: 322 sys.stdout.write('\nNo URLs were served by the test runner. It is ' 323 'unlikely this test failure has anything to do with ' 324 'this particular test.\n') 325 DumpNetLog(browser.NetLogName()) 326 except Exception: 327 listener.ever_failed = 1 328 # Try to let the browser clean itself up normally before killing it. 329 sys.stdout.write('##################### Terminating the browser\n') 330 browser.WaitForProcessDeath() 331 if browser.IsRunning(): 332 sys.stdout.write('##################### TERM failed, KILLING\n') 333 # Always call Cleanup; it kills the process, but also removes the 334 # user-data-dir. 335 browser.Cleanup() 336 # We avoid calling server.server_close() here because it causes 337 # the HTTP server thread to exit uncleanly with an EBADF error, 338 # which adds noise to the logs (though it does not cause the test 339 # to fail). server_close() does not attempt to tell the server 340 # loop to shut down before closing the socket FD it is 341 # select()ing. Since we are about to exit, we don't really need 342 # to close the socket FD. 343 344 if tool_failed: 345 return 2 346 elif listener.ever_failed: 347 return 1 348 else: 349 return 0 350 351 352 # This is an entrypoint for tests that treat the browser tester as a Python 353 # library rather than an opaque script. 354 # (e.g. run_inbrowser_trusted_crash_in_startup_test) 355 def Run(url, options): 356 result = 1 357 attempt = 1 358 while True: 359 try: 360 result = RunTestsOnce(url, options) 361 if result: 362 # Currently (2013/11/15) nacl_integration is fairly flaky and there is 363 # not enough time to look into it. Retry if the test fails for any 364 # reason. Note that in general this test runner tries to only retry 365 # when a known flake is encountered. (See the other raise 366 # RetryTest(..)s in this file.) This blanket retry means that those 367 # other cases could be removed without changing the behavior of the test 368 # runner, but it is hoped that this blanket retry will eventually be 369 # unnecessary and subsequently removed. The more precise retries have 370 # been left in place to preserve the knowledge. 371 raise RetryTest('HACK retrying failed test.') 372 break 373 except RetryTest: 374 # Only retry once. 375 if attempt < 2: 376 sys.stdout.write('\n@@@STEP_WARNINGS@@@\n') 377 sys.stdout.write('WARNING: suspected flake, retrying test!\n\n') 378 attempt += 1 379 continue 380 else: 381 sys.stdout.write('\nWARNING: failed too many times, not retrying.\n\n') 382 result = 1 383 break 384 return result 385 386 387 def RunFromCommandLine(): 388 parser = BuildArgParser() 389 options, args = parser.parse_args() 390 391 if len(args) != 0: 392 print args 393 parser.error('Invalid arguments') 394 395 # Validate the URL 396 url = options.url 397 if url is None: 398 parser.error('Must specify a URL') 399 400 return Run(url, options) 401 402 403 if __name__ == '__main__': 404 sys.exit(RunFromCommandLine()) 405
