1 # Copyright (c) 2009, Google Inc. All rights reserved. 2 # Copyright (c) 2009 Apple Inc. All rights reserved. 3 # 4 # Redistribution and use in source and binary forms, with or without 5 # modification, are permitted provided that the following conditions are 6 # met: 7 # 8 # * Redistributions of source code must retain the above copyright 9 # notice, this list of conditions and the following disclaimer. 10 # * Redistributions in binary form must reproduce the above 11 # copyright notice, this list of conditions and the following disclaimer 12 # in the documentation and/or other materials provided with the 13 # distribution. 14 # * Neither the name of Google Inc. nor the names of its 15 # contributors may be used to endorse or promote products derived from 16 # this software without specific prior written permission. 17 # 18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 30 try: 31 # This API exists only in Python 2.6 and higher. :( 32 import multiprocessing 33 except ImportError: 34 multiprocessing = None 35 36 import ctypes 37 import errno 38 import logging 39 import os 40 import platform 41 import StringIO 42 import signal 43 import subprocess 44 import sys 45 import time 46 47 from webkitpy.common.system.deprecated_logging import tee 48 from webkitpy.common.system.filesystem import FileSystem 49 from webkitpy.python24 import versioning 50 51 52 _log = logging.getLogger("webkitpy.common.system") 53 54 55 class ScriptError(Exception): 56 57 # This is a custom List.__str__ implementation to allow size limiting. 58 def _string_from_args(self, args, limit=100): 59 args_string = unicode(args) 60 # We could make this much fancier, but for now this is OK. 61 if len(args_string) > limit: 62 return args_string[:limit - 3] + "..." 63 return args_string 64 65 def __init__(self, 66 message=None, 67 script_args=None, 68 exit_code=None, 69 output=None, 70 cwd=None): 71 if not message: 72 message = 'Failed to run "%s"' % self._string_from_args(script_args) 73 if exit_code: 74 message += " exit_code: %d" % exit_code 75 if cwd: 76 message += " cwd: %s" % cwd 77 78 Exception.__init__(self, message) 79 self.script_args = script_args # 'args' is already used by Exception 80 self.exit_code = exit_code 81 self.output = output 82 self.cwd = cwd 83 84 def message_with_output(self, output_limit=500): 85 if self.output: 86 if output_limit and len(self.output) > output_limit: 87 return u"%s\n\nLast %s characters of output:\n%s" % \ 88 (self, output_limit, self.output[-output_limit:]) 89 return u"%s\n\n%s" % (self, self.output) 90 return unicode(self) 91 92 def command_name(self): 93 command_path = self.script_args 94 if type(command_path) is list: 95 command_path = command_path[0] 96 return os.path.basename(command_path) 97 98 99 def run_command(*args, **kwargs): 100 # FIXME: This should not be a global static. 101 # New code should use Executive.run_command directly instead 102 return Executive().run_command(*args, **kwargs) 103 104 105 class Executive(object): 106 107 def _should_close_fds(self): 108 # We need to pass close_fds=True to work around Python bug #2320 109 # (otherwise we can hang when we kill DumpRenderTree when we are running 110 # multiple threads). See http://bugs.python.org/issue2320 . 111 # Note that close_fds isn't supported on Windows, but this bug only 112 # shows up on Mac and Linux. 113 return sys.platform not in ('win32', 'cygwin') 114 115 def _run_command_with_teed_output(self, args, teed_output): 116 args = map(unicode, args) # Popen will throw an exception if args are non-strings (like int()) 117 args = map(self._encode_argument_if_needed, args) 118 119 child_process = subprocess.Popen(args, 120 stdout=subprocess.PIPE, 121 stderr=subprocess.STDOUT, 122 close_fds=self._should_close_fds()) 123 124 # Use our own custom wait loop because Popen ignores a tee'd 125 # stderr/stdout. 126 # FIXME: This could be improved not to flatten output to stdout. 127 while True: 128 output_line = child_process.stdout.readline() 129 if output_line == "" and child_process.poll() != None: 130 # poll() is not threadsafe and can throw OSError due to: 131 # http://bugs.python.org/issue1731717 132 return child_process.poll() 133 # We assume that the child process wrote to us in utf-8, 134 # so no re-encoding is necessary before writing here. 135 teed_output.write(output_line) 136 137 # FIXME: Remove this deprecated method and move callers to run_command. 138 # FIXME: This method is a hack to allow running command which both 139 # capture their output and print out to stdin. Useful for things 140 # like "build-webkit" where we want to display to the user that we're building 141 # but still have the output to stuff into a log file. 142 def run_and_throw_if_fail(self, args, quiet=False, decode_output=True): 143 # Cache the child's output locally so it can be used for error reports. 144 child_out_file = StringIO.StringIO() 145 tee_stdout = sys.stdout 146 if quiet: 147 dev_null = open(os.devnull, "w") # FIXME: Does this need an encoding? 148 tee_stdout = dev_null 149 child_stdout = tee(child_out_file, tee_stdout) 150 exit_code = self._run_command_with_teed_output(args, child_stdout) 151 if quiet: 152 dev_null.close() 153 154 child_output = child_out_file.getvalue() 155 child_out_file.close() 156 157 if decode_output: 158 child_output = child_output.decode(self._child_process_encoding()) 159 160 if exit_code: 161 raise ScriptError(script_args=args, 162 exit_code=exit_code, 163 output=child_output) 164 return child_output 165 166 def cpu_count(self): 167 if multiprocessing: 168 return multiprocessing.cpu_count() 169 # Darn. We don't have the multiprocessing package. 170 system_name = platform.system() 171 if system_name == "Darwin": 172 return int(self.run_command(["sysctl", "-n", "hw.ncpu"])) 173 elif system_name == "Windows": 174 return int(os.environ.get('NUMBER_OF_PROCESSORS', 1)) 175 elif system_name == "Linux": 176 num_cores = os.sysconf("SC_NPROCESSORS_ONLN") 177 if isinstance(num_cores, int) and num_cores > 0: 178 return num_cores 179 # This quantity is a lie but probably a reasonable guess for modern 180 # machines. 181 return 2 182 183 @staticmethod 184 def interpreter_for_script(script_path, fs=FileSystem()): 185 lines = fs.read_text_file(script_path).splitlines() 186 if not len(lines): 187 return None 188 first_line = lines[0] 189 if not first_line.startswith('#!'): 190 return None 191 if first_line.find('python') > -1: 192 return sys.executable 193 if first_line.find('perl') > -1: 194 return 'perl' 195 if first_line.find('ruby') > -1: 196 return 'ruby' 197 return None 198 199 def kill_process(self, pid): 200 """Attempts to kill the given pid. 201 Will fail silently if pid does not exist or insufficient permisssions.""" 202 if sys.platform == "win32": 203 # We only use taskkill.exe on windows (not cygwin) because subprocess.pid 204 # is a CYGWIN pid and taskkill.exe expects a windows pid. 205 # Thankfully os.kill on CYGWIN handles either pid type. 206 command = ["taskkill.exe", "/f", "/pid", pid] 207 # taskkill will exit 128 if the process is not found. We should log. 208 self.run_command(command, error_handler=self.ignore_error) 209 return 210 211 # According to http://docs.python.org/library/os.html 212 # os.kill isn't available on Windows. python 2.5.5 os.kill appears 213 # to work in cygwin, however it occasionally raises EAGAIN. 214 retries_left = 10 if sys.platform == "cygwin" else 1 215 while retries_left > 0: 216 try: 217 retries_left -= 1 218 os.kill(pid, signal.SIGKILL) 219 except OSError, e: 220 if e.errno == errno.EAGAIN: 221 if retries_left <= 0: 222 _log.warn("Failed to kill pid %s. Too many EAGAIN errors." % pid) 223 continue 224 if e.errno == errno.ESRCH: # The process does not exist. 225 _log.warn("Called kill_process with a non-existant pid %s" % pid) 226 return 227 raise 228 229 def _win32_check_running_pid(self, pid): 230 231 class PROCESSENTRY32(ctypes.Structure): 232 _fields_ = [("dwSize", ctypes.c_ulong), 233 ("cntUsage", ctypes.c_ulong), 234 ("th32ProcessID", ctypes.c_ulong), 235 ("th32DefaultHeapID", ctypes.c_ulong), 236 ("th32ModuleID", ctypes.c_ulong), 237 ("cntThreads", ctypes.c_ulong), 238 ("th32ParentProcessID", ctypes.c_ulong), 239 ("pcPriClassBase", ctypes.c_ulong), 240 ("dwFlags", ctypes.c_ulong), 241 ("szExeFile", ctypes.c_char * 260)] 242 243 CreateToolhelp32Snapshot = ctypes.windll.kernel32.CreateToolhelp32Snapshot 244 Process32First = ctypes.windll.kernel32.Process32First 245 Process32Next = ctypes.windll.kernel32.Process32Next 246 CloseHandle = ctypes.windll.kernel32.CloseHandle 247 TH32CS_SNAPPROCESS = 0x00000002 # win32 magic number 248 hProcessSnap = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0) 249 pe32 = PROCESSENTRY32() 250 pe32.dwSize = ctypes.sizeof(PROCESSENTRY32) 251 result = False 252 if not Process32First(hProcessSnap, ctypes.byref(pe32)): 253 _log.debug("Failed getting first process.") 254 CloseHandle(hProcessSnap) 255 return result 256 while True: 257 if pe32.th32ProcessID == pid: 258 result = True 259 break 260 if not Process32Next(hProcessSnap, ctypes.byref(pe32)): 261 break 262 CloseHandle(hProcessSnap) 263 return result 264 265 def check_running_pid(self, pid): 266 """Return True if pid is alive, otherwise return False.""" 267 if sys.platform in ('darwin', 'linux2', 'cygwin'): 268 try: 269 os.kill(pid, 0) 270 return True 271 except OSError: 272 return False 273 elif sys.platform == 'win32': 274 return self._win32_check_running_pid(pid) 275 276 assert(False) 277 278 def _windows_image_name(self, process_name): 279 name, extension = os.path.splitext(process_name) 280 if not extension: 281 # taskkill expects processes to end in .exe 282 # If necessary we could add a flag to disable appending .exe. 283 process_name = "%s.exe" % name 284 return process_name 285 286 def kill_all(self, process_name): 287 """Attempts to kill processes matching process_name. 288 Will fail silently if no process are found.""" 289 if sys.platform in ("win32", "cygwin"): 290 image_name = self._windows_image_name(process_name) 291 command = ["taskkill.exe", "/f", "/im", image_name] 292 # taskkill will exit 128 if the process is not found. We should log. 293 self.run_command(command, error_handler=self.ignore_error) 294 return 295 296 # FIXME: This is inconsistent that kill_all uses TERM and kill_process 297 # uses KILL. Windows is always using /f (which seems like -KILL). 298 # We should pick one mode, or add support for switching between them. 299 # Note: Mac OS X 10.6 requires -SIGNALNAME before -u USER 300 command = ["killall", "-TERM", "-u", os.getenv("USER"), process_name] 301 # killall returns 1 if no process can be found and 2 on command error. 302 # FIXME: We should pass a custom error_handler to allow only exit_code 1. 303 # We should log in exit_code == 1 304 self.run_command(command, error_handler=self.ignore_error) 305 306 # Error handlers do not need to be static methods once all callers are 307 # updated to use an Executive object. 308 309 @staticmethod 310 def default_error_handler(error): 311 raise error 312 313 @staticmethod 314 def ignore_error(error): 315 pass 316 317 def _compute_stdin(self, input): 318 """Returns (stdin, string_to_communicate)""" 319 # FIXME: We should be returning /dev/null for stdin 320 # or closing stdin after process creation to prevent 321 # child processes from getting input from the user. 322 if not input: 323 return (None, None) 324 if hasattr(input, "read"): # Check if the input is a file. 325 return (input, None) # Assume the file is in the right encoding. 326 327 # Popen in Python 2.5 and before does not automatically encode unicode objects. 328 # http://bugs.python.org/issue5290 329 # See https://bugs.webkit.org/show_bug.cgi?id=37528 330 # for an example of a regresion caused by passing a unicode string directly. 331 # FIXME: We may need to encode differently on different platforms. 332 if isinstance(input, unicode): 333 input = input.encode(self._child_process_encoding()) 334 return (subprocess.PIPE, input) 335 336 def _command_for_printing(self, args): 337 """Returns a print-ready string representing command args. 338 The string should be copy/paste ready for execution in a shell.""" 339 escaped_args = [] 340 for arg in args: 341 if isinstance(arg, unicode): 342 # Escape any non-ascii characters for easy copy/paste 343 arg = arg.encode("unicode_escape") 344 # FIXME: Do we need to fix quotes here? 345 escaped_args.append(arg) 346 return " ".join(escaped_args) 347 348 # FIXME: run_and_throw_if_fail should be merged into this method. 349 def run_command(self, 350 args, 351 cwd=None, 352 input=None, 353 error_handler=None, 354 return_exit_code=False, 355 return_stderr=True, 356 decode_output=True): 357 """Popen wrapper for convenience and to work around python bugs.""" 358 assert(isinstance(args, list) or isinstance(args, tuple)) 359 start_time = time.time() 360 args = map(unicode, args) # Popen will throw an exception if args are non-strings (like int()) 361 args = map(self._encode_argument_if_needed, args) 362 363 stdin, string_to_communicate = self._compute_stdin(input) 364 stderr = subprocess.STDOUT if return_stderr else None 365 366 process = subprocess.Popen(args, 367 stdin=stdin, 368 stdout=subprocess.PIPE, 369 stderr=stderr, 370 cwd=cwd, 371 close_fds=self._should_close_fds()) 372 output = process.communicate(string_to_communicate)[0] 373 374 # run_command automatically decodes to unicode() unless explicitly told not to. 375 if decode_output: 376 output = output.decode(self._child_process_encoding()) 377 378 # wait() is not threadsafe and can throw OSError due to: 379 # http://bugs.python.org/issue1731717 380 exit_code = process.wait() 381 382 _log.debug('"%s" took %.2fs' % (self._command_for_printing(args), time.time() - start_time)) 383 384 if return_exit_code: 385 return exit_code 386 387 if exit_code: 388 script_error = ScriptError(script_args=args, 389 exit_code=exit_code, 390 output=output, 391 cwd=cwd) 392 (error_handler or self.default_error_handler)(script_error) 393 return output 394 395 def _child_process_encoding(self): 396 # Win32 Python 2.x uses CreateProcessA rather than CreateProcessW 397 # to launch subprocesses, so we have to encode arguments using the 398 # current code page. 399 if sys.platform == 'win32' and versioning.compare_version(sys, '3.0')[0] < 0: 400 return 'mbcs' 401 # All other platforms use UTF-8. 402 # FIXME: Using UTF-8 on Cygwin will confuse Windows-native commands 403 # which will expect arguments to be encoded using the current code 404 # page. 405 return 'utf-8' 406 407 def _should_encode_child_process_arguments(self): 408 # Cygwin's Python's os.execv doesn't support unicode command 409 # arguments, and neither does Cygwin's execv itself. 410 if sys.platform == 'cygwin': 411 return True 412 413 # Win32 Python 2.x uses CreateProcessA rather than CreateProcessW 414 # to launch subprocesses, so we have to encode arguments using the 415 # current code page. 416 if sys.platform == 'win32' and versioning.compare_version(sys, '3.0')[0] < 0: 417 return True 418 419 return False 420 421 def _encode_argument_if_needed(self, argument): 422 if not self._should_encode_child_process_arguments(): 423 return argument 424 return argument.encode(self._child_process_encoding()) 425
