1 # Copyright (C) 2010 Google Inc. All rights reserved. 2 # 3 # Redistribution and use in source and binary forms, with or without 4 # modification, are permitted provided that the following conditions are 5 # met: 6 # 7 # * Redistributions of source code must retain the above copyright 8 # notice, this list of conditions and the following disclaimer. 9 # * Redistributions in binary form must reproduce the above 10 # copyright notice, this list of conditions and the following disclaimer 11 # in the documentation and/or other materials provided with the 12 # distribution. 13 # * Neither the Google name nor the names of its 14 # contributors may be used to endorse or promote products derived from 15 # this software without specific prior written permission. 16 # 17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 """Package that implements the ServerProcess wrapper class""" 30 31 import errno 32 import logging 33 import signal 34 import sys 35 import time 36 37 # Note that although win32 python does provide an implementation of 38 # the win32 select API, it only works on sockets, and not on the named pipes 39 # used by subprocess, so we have to use the native APIs directly. 40 if sys.platform == 'win32': 41 import msvcrt 42 import win32pipe 43 import win32file 44 else: 45 import fcntl 46 import os 47 import select 48 49 from webkitpy.common.system.executive import ScriptError 50 51 52 _log = logging.getLogger(__name__) 53 54 55 class ServerProcess(object): 56 """This class provides a wrapper around a subprocess that 57 implements a simple request/response usage model. The primary benefit 58 is that reading responses takes a deadline, so that we don't ever block 59 indefinitely. The class also handles transparently restarting processes 60 as necessary to keep issuing commands.""" 61 62 def __init__(self, port_obj, name, cmd, env=None, universal_newlines=False, treat_no_data_as_crash=False): 63 self._port = port_obj 64 self._name = name # Should be the command name (e.g. content_shell, image_diff) 65 self._cmd = cmd 66 self._env = env 67 # Set if the process outputs non-standard newlines like '\r\n' or '\r'. 68 # Don't set if there will be binary data or the data must be ASCII encoded. 69 self._universal_newlines = universal_newlines 70 self._treat_no_data_as_crash = treat_no_data_as_crash 71 self._host = self._port.host 72 self._pid = None 73 self._reset() 74 75 # See comment in imports for why we need the win32 APIs and can't just use select. 76 # FIXME: there should be a way to get win32 vs. cygwin from platforminfo. 77 self._use_win32_apis = sys.platform == 'win32' 78 79 def name(self): 80 return self._name 81 82 def pid(self): 83 return self._pid 84 85 def _reset(self): 86 if getattr(self, '_proc', None): 87 if self._proc.stdin: 88 self._proc.stdin.close() 89 self._proc.stdin = None 90 if self._proc.stdout: 91 self._proc.stdout.close() 92 self._proc.stdout = None 93 if self._proc.stderr: 94 self._proc.stderr.close() 95 self._proc.stderr = None 96 97 self._proc = None 98 self._output = str() # bytesarray() once we require Python 2.6 99 self._error = str() # bytesarray() once we require Python 2.6 100 self._crashed = False 101 self.timed_out = False 102 103 def process_name(self): 104 return self._name 105 106 def _start(self): 107 if self._proc: 108 raise ValueError("%s already running" % self._name) 109 self._reset() 110 # close_fds is a workaround for http://bugs.python.org/issue2320 111 close_fds = not self._host.platform.is_win() 112 self._proc = self._host.executive.popen(self._cmd, stdin=self._host.executive.PIPE, 113 stdout=self._host.executive.PIPE, 114 stderr=self._host.executive.PIPE, 115 close_fds=close_fds, 116 env=self._env, 117 universal_newlines=self._universal_newlines) 118 self._pid = self._proc.pid 119 fd = self._proc.stdout.fileno() 120 if not self._use_win32_apis: 121 fl = fcntl.fcntl(fd, fcntl.F_GETFL) 122 fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK) 123 fd = self._proc.stderr.fileno() 124 fl = fcntl.fcntl(fd, fcntl.F_GETFL) 125 fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK) 126 127 def _handle_possible_interrupt(self): 128 """This routine checks to see if the process crashed or exited 129 because of a keyboard interrupt and raises KeyboardInterrupt 130 accordingly.""" 131 # FIXME: Linux and Mac set the returncode to -signal.SIGINT if a 132 # subprocess is killed with a ctrl^C. Previous comments in this 133 # routine said that supposedly Windows returns 0xc000001d, but that's not what 134 # -1073741510 evaluates to. Figure out what the right value is 135 # for win32 and cygwin here ... 136 if self._proc.returncode in (-1073741510, -signal.SIGINT): 137 raise KeyboardInterrupt 138 139 def poll(self): 140 """Check to see if the underlying process is running; returns None 141 if it still is (wrapper around subprocess.poll).""" 142 if self._proc: 143 return self._proc.poll() 144 return None 145 146 def write(self, bytes): 147 """Write a request to the subprocess. The subprocess is (re-)start()'ed 148 if is not already running.""" 149 if not self._proc: 150 self._start() 151 try: 152 self._proc.stdin.write(bytes) 153 except IOError, e: 154 self.stop(0.0) 155 # stop() calls _reset(), so we have to set crashed to True after calling stop(). 156 self._crashed = True 157 158 def _pop_stdout_line_if_ready(self): 159 index_after_newline = self._output.find('\n') + 1 160 if index_after_newline > 0: 161 return self._pop_output_bytes(index_after_newline) 162 return None 163 164 def _pop_stderr_line_if_ready(self): 165 index_after_newline = self._error.find('\n') + 1 166 if index_after_newline > 0: 167 return self._pop_error_bytes(index_after_newline) 168 return None 169 170 def pop_all_buffered_stderr(self): 171 return self._pop_error_bytes(len(self._error)) 172 173 def read_stdout_line(self, deadline): 174 return self._read(deadline, self._pop_stdout_line_if_ready) 175 176 def read_stderr_line(self, deadline): 177 return self._read(deadline, self._pop_stderr_line_if_ready) 178 179 def read_either_stdout_or_stderr_line(self, deadline): 180 def retrieve_bytes_from_buffers(): 181 stdout_line = self._pop_stdout_line_if_ready() 182 if stdout_line: 183 return stdout_line, None 184 stderr_line = self._pop_stderr_line_if_ready() 185 if stderr_line: 186 return None, stderr_line 187 return None # Instructs the caller to keep waiting. 188 189 return_value = self._read(deadline, retrieve_bytes_from_buffers) 190 # FIXME: This is a bit of a hack around the fact that _read normally only returns one value, but this caller wants it to return two. 191 if return_value is None: 192 return None, None 193 return return_value 194 195 def read_stdout(self, deadline, size): 196 if size <= 0: 197 raise ValueError('ServerProcess.read() called with a non-positive size: %d ' % size) 198 199 def retrieve_bytes_from_stdout_buffer(): 200 if len(self._output) >= size: 201 return self._pop_output_bytes(size) 202 return None 203 204 return self._read(deadline, retrieve_bytes_from_stdout_buffer) 205 206 def _log(self, message): 207 # This is a bit of a hack, but we first log a blank line to avoid 208 # messing up the master process's output. 209 _log.info('') 210 _log.info(message) 211 212 def _handle_timeout(self): 213 self.timed_out = True 214 self._port.sample_process(self._name, self._proc.pid) 215 216 def _split_string_after_index(self, string, index): 217 return string[:index], string[index:] 218 219 def _pop_output_bytes(self, bytes_count): 220 output, self._output = self._split_string_after_index(self._output, bytes_count) 221 return output 222 223 def _pop_error_bytes(self, bytes_count): 224 output, self._error = self._split_string_after_index(self._error, bytes_count) 225 return output 226 227 def _wait_for_data_and_update_buffers_using_select(self, deadline, stopping=False): 228 if self._proc.stdout.closed or self._proc.stderr.closed: 229 # If the process crashed and is using FIFOs, like Chromium Android, the 230 # stdout and stderr pipes will be closed. 231 return 232 233 out_fd = self._proc.stdout.fileno() 234 err_fd = self._proc.stderr.fileno() 235 select_fds = (out_fd, err_fd) 236 try: 237 read_fds, _, _ = select.select(select_fds, [], select_fds, max(deadline - time.time(), 0)) 238 except select.error, e: 239 # We can ignore EINVAL since it's likely the process just crashed and we'll 240 # figure that out the next time through the loop in _read(). 241 if e.args[0] == errno.EINVAL: 242 return 243 raise 244 245 try: 246 # Note that we may get no data during read() even though 247 # select says we got something; see the select() man page 248 # on linux. I don't know if this happens on Mac OS and 249 # other Unixen as well, but we don't bother special-casing 250 # Linux because it's relatively harmless either way. 251 if out_fd in read_fds: 252 data = self._proc.stdout.read() 253 if not data and not stopping and (self._treat_no_data_as_crash or self._proc.poll()): 254 self._crashed = True 255 self._output += data 256 257 if err_fd in read_fds: 258 data = self._proc.stderr.read() 259 if not data and not stopping and (self._treat_no_data_as_crash or self._proc.poll()): 260 self._crashed = True 261 self._error += data 262 except IOError, e: 263 # We can ignore the IOErrors because we will detect if the subporcess crashed 264 # the next time through the loop in _read() 265 pass 266 267 def _wait_for_data_and_update_buffers_using_win32_apis(self, deadline): 268 # See http://code.activestate.com/recipes/440554-module-to-allow-asynchronous-subprocess-use-on-win/ 269 # and http://docs.activestate.com/activepython/2.6/pywin32/modules.html 270 # for documentation on all of these win32-specific modules. 271 now = time.time() 272 out_fh = msvcrt.get_osfhandle(self._proc.stdout.fileno()) 273 err_fh = msvcrt.get_osfhandle(self._proc.stderr.fileno()) 274 while (self._proc.poll() is None) and (now < deadline): 275 output = self._non_blocking_read_win32(out_fh) 276 error = self._non_blocking_read_win32(err_fh) 277 if output or error: 278 if output: 279 self._output += output 280 if error: 281 self._error += error 282 return 283 time.sleep(0.01) 284 now = time.time() 285 return 286 287 def _non_blocking_read_win32(self, handle): 288 try: 289 _, avail, _ = win32pipe.PeekNamedPipe(handle, 0) 290 if avail > 0: 291 _, buf = win32file.ReadFile(handle, avail, None) 292 return buf 293 except Exception, e: 294 if e[0] not in (109, errno.ESHUTDOWN): # 109 == win32 ERROR_BROKEN_PIPE 295 raise 296 return None 297 298 def has_crashed(self): 299 if not self._crashed and self.poll(): 300 self._crashed = True 301 self._handle_possible_interrupt() 302 return self._crashed 303 304 # This read function is a bit oddly-designed, as it polls both stdout and stderr, yet 305 # only reads/returns from one of them (buffering both in local self._output/self._error). 306 # It might be cleaner to pass in the file descriptor to poll instead. 307 def _read(self, deadline, fetch_bytes_from_buffers_callback): 308 while True: 309 if self.has_crashed(): 310 return None 311 312 if time.time() > deadline: 313 self._handle_timeout() 314 return None 315 316 bytes = fetch_bytes_from_buffers_callback() 317 if bytes is not None: 318 return bytes 319 320 if self._use_win32_apis: 321 self._wait_for_data_and_update_buffers_using_win32_apis(deadline) 322 else: 323 self._wait_for_data_and_update_buffers_using_select(deadline) 324 325 def start(self): 326 if not self._proc: 327 self._start() 328 329 def stop(self, timeout_secs=3.0): 330 if not self._proc: 331 return (None, None) 332 333 now = time.time() 334 if self._proc.stdin: 335 self._proc.stdin.close() 336 self._proc.stdin = None 337 killed = False 338 if timeout_secs: 339 deadline = now + timeout_secs 340 while self._proc.poll() is None and time.time() < deadline: 341 time.sleep(0.01) 342 if self._proc.poll() is None: 343 _log.warning('stopping %s(pid %d) timed out, killing it' % (self._name, self._proc.pid)) 344 345 if self._proc.poll() is None: 346 self._kill() 347 killed = True 348 _log.debug('killed pid %d' % self._proc.pid) 349 350 # read any remaining data on the pipes and return it. 351 if not killed: 352 if self._use_win32_apis: 353 self._wait_for_data_and_update_buffers_using_win32_apis(now) 354 else: 355 self._wait_for_data_and_update_buffers_using_select(now, stopping=True) 356 out, err = self._output, self._error 357 self._reset() 358 return (out, err) 359 360 def kill(self): 361 self.stop(0.0) 362 363 def _kill(self): 364 self._host.executive.kill_process(self._proc.pid) 365 if self._proc.poll() is not None: 366 self._proc.wait() 367 368 def replace_outputs(self, stdout, stderr): 369 assert self._proc 370 if stdout: 371 self._proc.stdout.close() 372 self._proc.stdout = stdout 373 if stderr: 374 self._proc.stderr.close() 375 self._proc.stderr = stderr 376