1 # Copyright (c) 2009, Google Inc. All rights reserved. 2 # Copyright (c) 2009 Apple Inc. All rights reserved. 3 # 4 # Redistribution and use in source and binary forms, with or without 5 # modification, are permitted provided that the following conditions are 6 # met: 7 # 8 # * Redistributions of source code must retain the above copyright 9 # notice, this list of conditions and the following disclaimer. 10 # * Redistributions in binary form must reproduce the above 11 # copyright notice, this list of conditions and the following disclaimer 12 # in the documentation and/or other materials provided with the 13 # distribution. 14 # * Neither the name of Google Inc. nor the names of its 15 # contributors may be used to endorse or promote products derived from 16 # this software without specific prior written permission. 17 # 18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 30 import StringIO 31 import errno 32 import logging 33 import multiprocessing 34 import os 35 import signal 36 import subprocess 37 import sys 38 import time 39 40 from webkitpy.common.system.outputtee import Tee 41 from webkitpy.common.system.filesystem import FileSystem 42 43 44 _log = logging.getLogger(__name__) 45 46 47 class ScriptError(Exception): 48 49 def __init__(self, 50 message=None, 51 script_args=None, 52 exit_code=None, 53 output=None, 54 cwd=None, 55 output_limit=500): 56 shortened_output = output 57 if output and output_limit and len(output) > output_limit: 58 shortened_output = "Last %s characters of output:\n%s" % (output_limit, output[-output_limit:]) 59 60 if not message: 61 message = 'Failed to run "%s"' % repr(script_args) 62 if exit_code: 63 message += " exit_code: %d" % exit_code 64 if cwd: 65 message += " cwd: %s" % cwd 66 67 if shortened_output: 68 message += "\n\noutput: %s" % shortened_output 69 70 Exception.__init__(self, message) 71 self.script_args = script_args # 'args' is already used by Exception 72 self.exit_code = exit_code 73 self.output = output 74 self.cwd = cwd 75 76 def message_with_output(self): 77 return unicode(self) 78 79 def command_name(self): 80 command_path = self.script_args 81 if type(command_path) is list: 82 command_path = command_path[0] 83 return os.path.basename(command_path) 84 85 86 class Executive(object): 87 PIPE = subprocess.PIPE 88 STDOUT = subprocess.STDOUT 89 90 def _should_close_fds(self): 91 # We need to pass close_fds=True to work around Python bug #2320 92 # (otherwise we can hang when we kill DumpRenderTree when we are running 93 # multiple threads). See http://bugs.python.org/issue2320 . 94 # Note that close_fds isn't supported on Windows, but this bug only 95 # shows up on Mac and Linux. 96 return sys.platform not in ('win32', 'cygwin') 97 98 def _run_command_with_teed_output(self, args, teed_output, **kwargs): 99 child_process = self.popen(args, 100 stdout=self.PIPE, 101 stderr=self.STDOUT, 102 close_fds=self._should_close_fds(), 103 **kwargs) 104 105 # Use our own custom wait loop because Popen ignores a tee'd 106 # stderr/stdout. 107 # FIXME: This could be improved not to flatten output to stdout. 108 while True: 109 output_line = child_process.stdout.readline() 110 if output_line == "" and child_process.poll() != None: 111 # poll() is not threadsafe and can throw OSError due to: 112 # http://bugs.python.org/issue1731717 113 return child_process.poll() 114 # We assume that the child process wrote to us in utf-8, 115 # so no re-encoding is necessary before writing here. 116 teed_output.write(output_line) 117 118 # FIXME: Remove this deprecated method and move callers to run_command. 119 # FIXME: This method is a hack to allow running command which both 120 # capture their output and print out to stdin. Useful for things 121 # like "build-webkit" where we want to display to the user that we're building 122 # but still have the output to stuff into a log file. 123 def run_and_throw_if_fail(self, args, quiet=False, decode_output=True, **kwargs): 124 # Cache the child's output locally so it can be used for error reports. 125 child_out_file = StringIO.StringIO() 126 tee_stdout = sys.stdout 127 if quiet: 128 dev_null = open(os.devnull, "w") # FIXME: Does this need an encoding? 129 tee_stdout = dev_null 130 child_stdout = Tee(child_out_file, tee_stdout) 131 exit_code = self._run_command_with_teed_output(args, child_stdout, **kwargs) 132 if quiet: 133 dev_null.close() 134 135 child_output = child_out_file.getvalue() 136 child_out_file.close() 137 138 if decode_output: 139 child_output = child_output.decode(self._child_process_encoding()) 140 141 if exit_code: 142 raise ScriptError(script_args=args, 143 exit_code=exit_code, 144 output=child_output) 145 return child_output 146 147 def cpu_count(self): 148 return multiprocessing.cpu_count() 149 150 @staticmethod 151 def interpreter_for_script(script_path, fs=None): 152 fs = fs or FileSystem() 153 lines = fs.read_text_file(script_path).splitlines() 154 if not len(lines): 155 return None 156 first_line = lines[0] 157 if not first_line.startswith('#!'): 158 return None 159 if first_line.find('python') > -1: 160 return sys.executable 161 if first_line.find('perl') > -1: 162 return 'perl' 163 if first_line.find('ruby') > -1: 164 return 'ruby' 165 return None 166 167 @staticmethod 168 def shell_command_for_script(script_path, fs=None): 169 fs = fs or FileSystem() 170 # Win32 does not support shebang. We need to detect the interpreter ourself. 171 if sys.platform == 'win32': 172 interpreter = Executive.interpreter_for_script(script_path, fs) 173 if interpreter: 174 return [interpreter, script_path] 175 return [script_path] 176 177 def kill_process(self, pid): 178 """Attempts to kill the given pid. 179 Will fail silently if pid does not exist or insufficient permisssions.""" 180 if sys.platform == "win32": 181 # We only use taskkill.exe on windows (not cygwin) because subprocess.pid 182 # is a CYGWIN pid and taskkill.exe expects a windows pid. 183 # Thankfully os.kill on CYGWIN handles either pid type. 184 command = ["taskkill.exe", "/f", "/t", "/pid", pid] 185 # taskkill will exit 128 if the process is not found. We should log. 186 self.run_command(command, error_handler=self.ignore_error) 187 return 188 189 # According to http://docs.python.org/library/os.html 190 # os.kill isn't available on Windows. python 2.5.5 os.kill appears 191 # to work in cygwin, however it occasionally raises EAGAIN. 192 retries_left = 10 if sys.platform == "cygwin" else 1 193 while retries_left > 0: 194 try: 195 retries_left -= 1 196 os.kill(pid, signal.SIGKILL) 197 _ = os.waitpid(pid, os.WNOHANG) 198 except OSError, e: 199 if e.errno == errno.EAGAIN: 200 if retries_left <= 0: 201 _log.warn("Failed to kill pid %s. Too many EAGAIN errors." % pid) 202 continue 203 if e.errno == errno.ESRCH: # The process does not exist. 204 return 205 if e.errno == errno.EPIPE: # The process has exited already on cygwin 206 return 207 if e.errno == errno.ECHILD: 208 # Can't wait on a non-child process, but the kill worked. 209 return 210 if e.errno == errno.EACCES and sys.platform == 'cygwin': 211 # Cygwin python sometimes can't kill native processes. 212 return 213 raise 214 215 def _win32_check_running_pid(self, pid): 216 # importing ctypes at the top-level seems to cause weird crashes at 217 # exit under cygwin on apple's win port. Only win32 needs cygwin, so 218 # we import it here instead. See https://bugs.webkit.org/show_bug.cgi?id=91682 219 import ctypes 220 221 class PROCESSENTRY32(ctypes.Structure): 222 _fields_ = [("dwSize", ctypes.c_ulong), 223 ("cntUsage", ctypes.c_ulong), 224 ("th32ProcessID", ctypes.c_ulong), 225 ("th32DefaultHeapID", ctypes.POINTER(ctypes.c_ulong)), 226 ("th32ModuleID", ctypes.c_ulong), 227 ("cntThreads", ctypes.c_ulong), 228 ("th32ParentProcessID", ctypes.c_ulong), 229 ("pcPriClassBase", ctypes.c_ulong), 230 ("dwFlags", ctypes.c_ulong), 231 ("szExeFile", ctypes.c_char * 260)] 232 233 CreateToolhelp32Snapshot = ctypes.windll.kernel32.CreateToolhelp32Snapshot 234 Process32First = ctypes.windll.kernel32.Process32First 235 Process32Next = ctypes.windll.kernel32.Process32Next 236 CloseHandle = ctypes.windll.kernel32.CloseHandle 237 TH32CS_SNAPPROCESS = 0x00000002 # win32 magic number 238 hProcessSnap = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0) 239 pe32 = PROCESSENTRY32() 240 pe32.dwSize = ctypes.sizeof(PROCESSENTRY32) 241 result = False 242 if not Process32First(hProcessSnap, ctypes.byref(pe32)): 243 _log.debug("Failed getting first process.") 244 CloseHandle(hProcessSnap) 245 return result 246 while True: 247 if pe32.th32ProcessID == pid: 248 result = True 249 break 250 if not Process32Next(hProcessSnap, ctypes.byref(pe32)): 251 break 252 CloseHandle(hProcessSnap) 253 return result 254 255 def check_running_pid(self, pid): 256 """Return True if pid is alive, otherwise return False.""" 257 if sys.platform == 'win32': 258 return self._win32_check_running_pid(pid) 259 260 try: 261 os.kill(pid, 0) 262 return True 263 except OSError: 264 return False 265 266 def running_pids(self, process_name_filter=None): 267 if not process_name_filter: 268 process_name_filter = lambda process_name: True 269 270 running_pids = [] 271 272 if sys.platform in ("win32", "cygwin"): 273 # FIXME: running_pids isn't implemented on Windows yet... 274 return [] 275 276 ps_process = self.popen(['ps', '-eo', 'pid,comm'], stdout=self.PIPE, stderr=self.PIPE) 277 stdout, _ = ps_process.communicate() 278 for line in stdout.splitlines(): 279 try: 280 # In some cases the line can contain one or more 281 # leading white-spaces, so strip it before split. 282 pid, process_name = line.strip().split(' ', 1) 283 if process_name_filter(process_name): 284 running_pids.append(int(pid)) 285 except ValueError, e: 286 pass 287 288 return sorted(running_pids) 289 290 def wait_newest(self, process_name_filter=None): 291 if not process_name_filter: 292 process_name_filter = lambda process_name: True 293 294 running_pids = self.running_pids(process_name_filter) 295 if not running_pids: 296 return 297 pid = running_pids[-1] 298 299 while self.check_running_pid(pid): 300 time.sleep(0.25) 301 302 def wait_limited(self, pid, limit_in_seconds=None, check_frequency_in_seconds=None): 303 seconds_left = limit_in_seconds or 10 304 sleep_length = check_frequency_in_seconds or 1 305 while seconds_left > 0 and self.check_running_pid(pid): 306 seconds_left -= sleep_length 307 time.sleep(sleep_length) 308 309 def _windows_image_name(self, process_name): 310 name, extension = os.path.splitext(process_name) 311 if not extension: 312 # taskkill expects processes to end in .exe 313 # If necessary we could add a flag to disable appending .exe. 314 process_name = "%s.exe" % name 315 return process_name 316 317 def interrupt(self, pid): 318 interrupt_signal = signal.SIGINT 319 # FIXME: The python docs seem to imply that platform == 'win32' may need to use signal.CTRL_C_EVENT 320 # http://docs.python.org/2/library/signal.html 321 try: 322 os.kill(pid, interrupt_signal) 323 except OSError: 324 # Silently ignore when the pid doesn't exist. 325 # It's impossible for callers to avoid race conditions with process shutdown. 326 pass 327 328 # Error handlers do not need to be static methods once all callers are 329 # updated to use an Executive object. 330 331 @staticmethod 332 def default_error_handler(error): 333 raise error 334 335 @staticmethod 336 def ignore_error(error): 337 pass 338 339 def _compute_stdin(self, input): 340 """Returns (stdin, string_to_communicate)""" 341 # FIXME: We should be returning /dev/null for stdin 342 # or closing stdin after process creation to prevent 343 # child processes from getting input from the user. 344 if not input: 345 return (None, None) 346 if hasattr(input, "read"): # Check if the input is a file. 347 return (input, None) # Assume the file is in the right encoding. 348 349 # Popen in Python 2.5 and before does not automatically encode unicode objects. 350 # http://bugs.python.org/issue5290 351 # See https://bugs.webkit.org/show_bug.cgi?id=37528 352 # for an example of a regresion caused by passing a unicode string directly. 353 # FIXME: We may need to encode differently on different platforms. 354 if isinstance(input, unicode): 355 input = input.encode(self._child_process_encoding()) 356 return (self.PIPE, input) 357 358 def command_for_printing(self, args): 359 """Returns a print-ready string representing command args. 360 The string should be copy/paste ready for execution in a shell.""" 361 args = self._stringify_args(args) 362 escaped_args = [] 363 for arg in args: 364 if isinstance(arg, unicode): 365 # Escape any non-ascii characters for easy copy/paste 366 arg = arg.encode("unicode_escape") 367 # FIXME: Do we need to fix quotes here? 368 escaped_args.append(arg) 369 return " ".join(escaped_args) 370 371 # FIXME: run_and_throw_if_fail should be merged into this method. 372 def run_command(self, 373 args, 374 cwd=None, 375 env=None, 376 input=None, 377 error_handler=None, 378 return_exit_code=False, 379 return_stderr=True, 380 decode_output=True, debug_logging=True): 381 """Popen wrapper for convenience and to work around python bugs.""" 382 assert(isinstance(args, list) or isinstance(args, tuple)) 383 start_time = time.time() 384 385 stdin, string_to_communicate = self._compute_stdin(input) 386 stderr = self.STDOUT if return_stderr else None 387 388 process = self.popen(args, 389 stdin=stdin, 390 stdout=self.PIPE, 391 stderr=stderr, 392 cwd=cwd, 393 env=env, 394 close_fds=self._should_close_fds()) 395 output = process.communicate(string_to_communicate)[0] 396 397 # run_command automatically decodes to unicode() unless explicitly told not to. 398 if decode_output: 399 output = output.decode(self._child_process_encoding()) 400 401 # wait() is not threadsafe and can throw OSError due to: 402 # http://bugs.python.org/issue1731717 403 exit_code = process.wait() 404 405 if debug_logging: 406 _log.debug('"%s" took %.2fs' % (self.command_for_printing(args), time.time() - start_time)) 407 408 if return_exit_code: 409 return exit_code 410 411 if exit_code: 412 script_error = ScriptError(script_args=args, 413 exit_code=exit_code, 414 output=output, 415 cwd=cwd) 416 (error_handler or self.default_error_handler)(script_error) 417 return output 418 419 def _child_process_encoding(self): 420 # Win32 Python 2.x uses CreateProcessA rather than CreateProcessW 421 # to launch subprocesses, so we have to encode arguments using the 422 # current code page. 423 if sys.platform == 'win32' and sys.version < '3': 424 return 'mbcs' 425 # All other platforms use UTF-8. 426 # FIXME: Using UTF-8 on Cygwin will confuse Windows-native commands 427 # which will expect arguments to be encoded using the current code 428 # page. 429 return 'utf-8' 430 431 def _should_encode_child_process_arguments(self): 432 # Cygwin's Python's os.execv doesn't support unicode command 433 # arguments, and neither does Cygwin's execv itself. 434 if sys.platform == 'cygwin': 435 return True 436 437 # Win32 Python 2.x uses CreateProcessA rather than CreateProcessW 438 # to launch subprocesses, so we have to encode arguments using the 439 # current code page. 440 if sys.platform == 'win32' and sys.version < '3': 441 return True 442 443 return False 444 445 def _encode_argument_if_needed(self, argument): 446 if not self._should_encode_child_process_arguments(): 447 return argument 448 return argument.encode(self._child_process_encoding()) 449 450 def _stringify_args(self, args): 451 # Popen will throw an exception if args are non-strings (like int()) 452 string_args = map(unicode, args) 453 # The Windows implementation of Popen cannot handle unicode strings. :( 454 return map(self._encode_argument_if_needed, string_args) 455 456 # The only required arugment to popen is named "args", the rest are optional keyword arguments. 457 def popen(self, args, **kwargs): 458 # FIXME: We should always be stringifying the args, but callers who pass shell=True 459 # expect that the exact bytes passed will get passed to the shell (even if they're wrongly encoded). 460 # shell=True is wrong for many other reasons, and we should remove this 461 # hack as soon as we can fix all callers to not use shell=True. 462 if kwargs.get('shell') == True: 463 string_args = args 464 else: 465 string_args = self._stringify_args(args) 466 return subprocess.Popen(string_args, **kwargs) 467 468 def call(self, args, **kwargs): 469 return subprocess.call(self._stringify_args(args), **kwargs) 470 471 def run_in_parallel(self, command_lines_and_cwds, processes=None): 472 """Runs a list of (cmd_line list, cwd string) tuples in parallel and returns a list of (retcode, stdout, stderr) tuples.""" 473 assert len(command_lines_and_cwds) 474 return self.map(_run_command_thunk, command_lines_and_cwds, processes) 475 476 def map(self, thunk, arglist, processes=None): 477 if sys.platform in ('cygwin', 'win32') or len(arglist) == 1: 478 return map(thunk, arglist) 479 pool = multiprocessing.Pool(processes=(processes or multiprocessing.cpu_count())) 480 try: 481 return pool.map(thunk, arglist) 482 finally: 483 pool.close() 484 pool.join() 485 486 487 def _run_command_thunk(cmd_line_and_cwd): 488 # Note that this needs to be a bare module (and hence Picklable) method to work with multiprocessing.Pool. 489 (cmd_line, cwd) = cmd_line_and_cwd 490 proc = subprocess.Popen(cmd_line, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 491 stdout, stderr = proc.communicate() 492 return (proc.returncode, stdout, stderr) 493