1 # subprocess - Subprocesses with accessible I/O streams 2 # 3 # For more information about this module, see PEP 324. 4 # 5 # Copyright (c) 2003-2005 by Peter Astrand <astrand (at] lysator.liu.se> 6 # 7 # Licensed to PSF under a Contributor Agreement. 8 # See http://www.python.org/2.4/license for licensing details. 9 10 r"""Subprocesses with accessible I/O streams 11 12 This module allows you to spawn processes, connect to their 13 input/output/error pipes, and obtain their return codes. 14 15 For a complete description of this module see the Python documentation. 16 17 Main API 18 ======== 19 call(...): Runs a command, waits for it to complete, then returns 20 the return code. 21 check_call(...): Same as call() but raises CalledProcessError() 22 if return code is not 0 23 check_output(...): Same as check_call() but returns the contents of 24 stdout instead of a return code 25 Popen(...): A class for flexibly executing a command in a new process 26 27 Constants 28 --------- 29 PIPE: Special value that indicates a pipe should be created 30 STDOUT: Special value that indicates that stderr should go to stdout 31 """ 32 33 import sys 34 mswindows = (sys.platform == "win32") 35 36 import os 37 import types 38 import traceback 39 import gc 40 import signal 41 import errno 42 43 # Exception classes used by this module. 44 class CalledProcessError(Exception): 45 """This exception is raised when a process run by check_call() or 46 check_output() returns a non-zero exit status. 47 48 Attributes: 49 cmd, returncode, output 50 """ 51 def __init__(self, returncode, cmd, output=None): 52 self.returncode = returncode 53 self.cmd = cmd 54 self.output = output 55 def __str__(self): 56 return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode) 57 58 59 if mswindows: 60 import threading 61 import msvcrt 62 import _subprocess 63 class STARTUPINFO: 64 dwFlags = 0 65 hStdInput = None 66 hStdOutput = None 67 hStdError = None 68 wShowWindow = 0 69 class pywintypes: 70 error = IOError 71 else: 72 import select 73 _has_poll = hasattr(select, 'poll') 74 import fcntl 75 import pickle 76 77 # When select or poll has indicated that the file is writable, 78 # we can write up to _PIPE_BUF bytes without risk of blocking. 79 # POSIX defines PIPE_BUF as >= 512. 80 _PIPE_BUF = getattr(select, 'PIPE_BUF', 512) 81 82 83 __all__ = ["Popen", "PIPE", "STDOUT", "call", "check_call", 84 "check_output", "CalledProcessError"] 85 86 if mswindows: 87 from _subprocess import (CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP, 88 STD_INPUT_HANDLE, STD_OUTPUT_HANDLE, 89 STD_ERROR_HANDLE, SW_HIDE, 90 STARTF_USESTDHANDLES, STARTF_USESHOWWINDOW) 91 92 __all__.extend(["CREATE_NEW_CONSOLE", "CREATE_NEW_PROCESS_GROUP", 93 "STD_INPUT_HANDLE", "STD_OUTPUT_HANDLE", 94 "STD_ERROR_HANDLE", "SW_HIDE", 95 "STARTF_USESTDHANDLES", "STARTF_USESHOWWINDOW"]) 96 try: 97 MAXFD = os.sysconf("SC_OPEN_MAX") 98 except: 99 MAXFD = 256 100 101 _active = [] 102 103 def _cleanup(): 104 for inst in _active[:]: 105 res = inst._internal_poll(_deadstate=sys.maxint) 106 if res is not None: 107 try: 108 _active.remove(inst) 109 except ValueError: 110 # This can happen if two threads create a new Popen instance. 111 # It's harmless that it was already removed, so ignore. 112 pass 113 114 PIPE = -1 115 STDOUT = -2 116 117 118 def _eintr_retry_call(func, *args): 119 while True: 120 try: 121 return func(*args) 122 except (OSError, IOError) as e: 123 if e.errno == errno.EINTR: 124 continue 125 raise 126 127 128 # XXX This function is only used by multiprocessing and the test suite, 129 # but it's here so that it can be imported when Python is compiled without 130 # threads. 131 132 def _args_from_interpreter_flags(): 133 """Return a list of command-line arguments reproducing the current 134 settings in sys.flags and sys.warnoptions.""" 135 flag_opt_map = { 136 'debug': 'd', 137 # 'inspect': 'i', 138 # 'interactive': 'i', 139 'optimize': 'O', 140 'dont_write_bytecode': 'B', 141 'no_user_site': 's', 142 'no_site': 'S', 143 'ignore_environment': 'E', 144 'verbose': 'v', 145 'bytes_warning': 'b', 146 'py3k_warning': '3', 147 } 148 args = [] 149 for flag, opt in flag_opt_map.items(): 150 v = getattr(sys.flags, flag) 151 if v > 0: 152 args.append('-' + opt * v) 153 if getattr(sys.flags, 'hash_randomization') != 0: 154 args.append('-R') 155 for opt in sys.warnoptions: 156 args.append('-W' + opt) 157 return args 158 159 160 def call(*popenargs, **kwargs): 161 """Run command with arguments. Wait for command to complete, then 162 return the returncode attribute. 163 164 The arguments are the same as for the Popen constructor. Example: 165 166 retcode = call(["ls", "-l"]) 167 """ 168 return Popen(*popenargs, **kwargs).wait() 169 170 171 def check_call(*popenargs, **kwargs): 172 """Run command with arguments. Wait for command to complete. If 173 the exit code was zero then return, otherwise raise 174 CalledProcessError. The CalledProcessError object will have the 175 return code in the returncode attribute. 176 177 The arguments are the same as for the Popen constructor. Example: 178 179 check_call(["ls", "-l"]) 180 """ 181 retcode = call(*popenargs, **kwargs) 182 if retcode: 183 cmd = kwargs.get("args") 184 if cmd is None: 185 cmd = popenargs[0] 186 raise CalledProcessError(retcode, cmd) 187 return 0 188 189 190 def check_output(*popenargs, **kwargs): 191 r"""Run command with arguments and return its output as a byte string. 192 193 If the exit code was non-zero it raises a CalledProcessError. The 194 CalledProcessError object will have the return code in the returncode 195 attribute and output in the output attribute. 196 197 The arguments are the same as for the Popen constructor. Example: 198 199 >>> check_output(["ls", "-l", "/dev/null"]) 200 'crw-rw-rw- 1 root root 1, 3 Oct 18 2007 /dev/null\n' 201 202 The stdout argument is not allowed as it is used internally. 203 To capture standard error in the result, use stderr=STDOUT. 204 205 >>> check_output(["/bin/sh", "-c", 206 ... "ls -l non_existent_file ; exit 0"], 207 ... stderr=STDOUT) 208 'ls: non_existent_file: No such file or directory\n' 209 """ 210 if 'stdout' in kwargs: 211 raise ValueError('stdout argument not allowed, it will be overridden.') 212 process = Popen(stdout=PIPE, *popenargs, **kwargs) 213 output, unused_err = process.communicate() 214 retcode = process.poll() 215 if retcode: 216 cmd = kwargs.get("args") 217 if cmd is None: 218 cmd = popenargs[0] 219 raise CalledProcessError(retcode, cmd, output=output) 220 return output 221 222 223 def list2cmdline(seq): 224 """ 225 Translate a sequence of arguments into a command line 226 string, using the same rules as the MS C runtime: 227 228 1) Arguments are delimited by white space, which is either a 229 space or a tab. 230 231 2) A string surrounded by double quotation marks is 232 interpreted as a single argument, regardless of white space 233 contained within. A quoted string can be embedded in an 234 argument. 235 236 3) A double quotation mark preceded by a backslash is 237 interpreted as a literal double quotation mark. 238 239 4) Backslashes are interpreted literally, unless they 240 immediately precede a double quotation mark. 241 242 5) If backslashes immediately precede a double quotation mark, 243 every pair of backslashes is interpreted as a literal 244 backslash. If the number of backslashes is odd, the last 245 backslash escapes the next double quotation mark as 246 described in rule 3. 247 """ 248 249 # See 250 # http://msdn.microsoft.com/en-us/library/17w5ykft.aspx 251 # or search http://msdn.microsoft.com for 252 # "Parsing C++ Command-Line Arguments" 253 result = [] 254 needquote = False 255 for arg in seq: 256 bs_buf = [] 257 258 # Add a space to separate this argument from the others 259 if result: 260 result.append(' ') 261 262 needquote = (" " in arg) or ("\t" in arg) or not arg 263 if needquote: 264 result.append('"') 265 266 for c in arg: 267 if c == '\\': 268 # Don't know if we need to double yet. 269 bs_buf.append(c) 270 elif c == '"': 271 # Double backslashes. 272 result.append('\\' * len(bs_buf)*2) 273 bs_buf = [] 274 result.append('\\"') 275 else: 276 # Normal char 277 if bs_buf: 278 result.extend(bs_buf) 279 bs_buf = [] 280 result.append(c) 281 282 # Add remaining backslashes, if any. 283 if bs_buf: 284 result.extend(bs_buf) 285 286 if needquote: 287 result.extend(bs_buf) 288 result.append('"') 289 290 return ''.join(result) 291 292 293 class Popen(object): 294 """ Execute a child program in a new process. 295 296 For a complete description of the arguments see the Python documentation. 297 298 Arguments: 299 args: A string, or a sequence of program arguments. 300 301 bufsize: supplied as the buffering argument to the open() function when 302 creating the stdin/stdout/stderr pipe file objects 303 304 executable: A replacement program to execute. 305 306 stdin, stdout and stderr: These specify the executed programs' standard 307 input, standard output and standard error file handles, respectively. 308 309 preexec_fn: (POSIX only) An object to be called in the child process 310 just before the child is executed. 311 312 close_fds: Controls closing or inheriting of file descriptors. 313 314 shell: If true, the command will be executed through the shell. 315 316 cwd: Sets the current directory before the child is executed. 317 318 env: Defines the environment variables for the new process. 319 320 universal_newlines: If true, use universal line endings for file 321 objects stdin, stdout and stderr. 322 323 startupinfo and creationflags (Windows only) 324 325 Attributes: 326 stdin, stdout, stderr, pid, returncode 327 """ 328 _child_created = False # Set here since __del__ checks it 329 330 def __init__(self, args, bufsize=0, executable=None, 331 stdin=None, stdout=None, stderr=None, 332 preexec_fn=None, close_fds=False, shell=False, 333 cwd=None, env=None, universal_newlines=False, 334 startupinfo=None, creationflags=0): 335 """Create new Popen instance.""" 336 _cleanup() 337 338 if not isinstance(bufsize, (int, long)): 339 raise TypeError("bufsize must be an integer") 340 341 if mswindows: 342 if preexec_fn is not None: 343 raise ValueError("preexec_fn is not supported on Windows " 344 "platforms") 345 if close_fds and (stdin is not None or stdout is not None or 346 stderr is not None): 347 raise ValueError("close_fds is not supported on Windows " 348 "platforms if you redirect stdin/stdout/stderr") 349 else: 350 # POSIX 351 if startupinfo is not None: 352 raise ValueError("startupinfo is only supported on Windows " 353 "platforms") 354 if creationflags != 0: 355 raise ValueError("creationflags is only supported on Windows " 356 "platforms") 357 358 self.stdin = None 359 self.stdout = None 360 self.stderr = None 361 self.pid = None 362 self.returncode = None 363 self.universal_newlines = universal_newlines 364 365 # Input and output objects. The general principle is like 366 # this: 367 # 368 # Parent Child 369 # ------ ----- 370 # p2cwrite ---stdin---> p2cread 371 # c2pread <--stdout--- c2pwrite 372 # errread <--stderr--- errwrite 373 # 374 # On POSIX, the child objects are file descriptors. On 375 # Windows, these are Windows file handles. The parent objects 376 # are file descriptors on both platforms. The parent objects 377 # are None when not using PIPEs. The child objects are None 378 # when not redirecting. 379 380 (p2cread, p2cwrite, 381 c2pread, c2pwrite, 382 errread, errwrite), to_close = self._get_handles(stdin, stdout, stderr) 383 384 try: 385 self._execute_child(args, executable, preexec_fn, close_fds, 386 cwd, env, universal_newlines, 387 startupinfo, creationflags, shell, to_close, 388 p2cread, p2cwrite, 389 c2pread, c2pwrite, 390 errread, errwrite) 391 except Exception: 392 # Preserve original exception in case os.close raises. 393 exc_type, exc_value, exc_trace = sys.exc_info() 394 395 for fd in to_close: 396 try: 397 if mswindows: 398 fd.Close() 399 else: 400 os.close(fd) 401 except EnvironmentError: 402 pass 403 404 raise exc_type, exc_value, exc_trace 405 406 if mswindows: 407 if p2cwrite is not None: 408 p2cwrite = msvcrt.open_osfhandle(p2cwrite.Detach(), 0) 409 if c2pread is not None: 410 c2pread = msvcrt.open_osfhandle(c2pread.Detach(), 0) 411 if errread is not None: 412 errread = msvcrt.open_osfhandle(errread.Detach(), 0) 413 414 if p2cwrite is not None: 415 self.stdin = os.fdopen(p2cwrite, 'wb', bufsize) 416 if c2pread is not None: 417 if universal_newlines: 418 self.stdout = os.fdopen(c2pread, 'rU', bufsize) 419 else: 420 self.stdout = os.fdopen(c2pread, 'rb', bufsize) 421 if errread is not None: 422 if universal_newlines: 423 self.stderr = os.fdopen(errread, 'rU', bufsize) 424 else: 425 self.stderr = os.fdopen(errread, 'rb', bufsize) 426 427 428 def _translate_newlines(self, data): 429 data = data.replace("\r\n", "\n") 430 data = data.replace("\r", "\n") 431 return data 432 433 434 def __del__(self, _maxint=sys.maxint): 435 # If __init__ hasn't had a chance to execute (e.g. if it 436 # was passed an undeclared keyword argument), we don't 437 # have a _child_created attribute at all. 438 if not self._child_created: 439 # We didn't get to successfully create a child process. 440 return 441 # In case the child hasn't been waited on, check if it's done. 442 self._internal_poll(_deadstate=_maxint) 443 if self.returncode is None and _active is not None: 444 # Child is still running, keep us alive until we can wait on it. 445 _active.append(self) 446 447 448 def communicate(self, input=None): 449 """Interact with process: Send data to stdin. Read data from 450 stdout and stderr, until end-of-file is reached. Wait for 451 process to terminate. The optional input argument should be a 452 string to be sent to the child process, or None, if no data 453 should be sent to the child. 454 455 communicate() returns a tuple (stdout, stderr).""" 456 457 # Optimization: If we are only using one pipe, or no pipe at 458 # all, using select() or threads is unnecessary. 459 if [self.stdin, self.stdout, self.stderr].count(None) >= 2: 460 stdout = None 461 stderr = None 462 if self.stdin: 463 if input: 464 try: 465 self.stdin.write(input) 466 except IOError as e: 467 if e.errno != errno.EPIPE and e.errno != errno.EINVAL: 468 raise 469 self.stdin.close() 470 elif self.stdout: 471 stdout = _eintr_retry_call(self.stdout.read) 472 self.stdout.close() 473 elif self.stderr: 474 stderr = _eintr_retry_call(self.stderr.read) 475 self.stderr.close() 476 self.wait() 477 return (stdout, stderr) 478 479 return self._communicate(input) 480 481 482 def poll(self): 483 """Check if child process has terminated. Set and return returncode 484 attribute.""" 485 return self._internal_poll() 486 487 488 if mswindows: 489 # 490 # Windows methods 491 # 492 def _get_handles(self, stdin, stdout, stderr): 493 """Construct and return tuple with IO objects: 494 p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite 495 """ 496 to_close = set() 497 if stdin is None and stdout is None and stderr is None: 498 return (None, None, None, None, None, None), to_close 499 500 p2cread, p2cwrite = None, None 501 c2pread, c2pwrite = None, None 502 errread, errwrite = None, None 503 504 if stdin is None: 505 p2cread = _subprocess.GetStdHandle(_subprocess.STD_INPUT_HANDLE) 506 if p2cread is None: 507 p2cread, _ = _subprocess.CreatePipe(None, 0) 508 elif stdin == PIPE: 509 p2cread, p2cwrite = _subprocess.CreatePipe(None, 0) 510 elif isinstance(stdin, int): 511 p2cread = msvcrt.get_osfhandle(stdin) 512 else: 513 # Assuming file-like object 514 p2cread = msvcrt.get_osfhandle(stdin.fileno()) 515 p2cread = self._make_inheritable(p2cread) 516 # We just duplicated the handle, it has to be closed at the end 517 to_close.add(p2cread) 518 if stdin == PIPE: 519 to_close.add(p2cwrite) 520 521 if stdout is None: 522 c2pwrite = _subprocess.GetStdHandle(_subprocess.STD_OUTPUT_HANDLE) 523 if c2pwrite is None: 524 _, c2pwrite = _subprocess.CreatePipe(None, 0) 525 elif stdout == PIPE: 526 c2pread, c2pwrite = _subprocess.CreatePipe(None, 0) 527 elif isinstance(stdout, int): 528 c2pwrite = msvcrt.get_osfhandle(stdout) 529 else: 530 # Assuming file-like object 531 c2pwrite = msvcrt.get_osfhandle(stdout.fileno()) 532 c2pwrite = self._make_inheritable(c2pwrite) 533 # We just duplicated the handle, it has to be closed at the end 534 to_close.add(c2pwrite) 535 if stdout == PIPE: 536 to_close.add(c2pread) 537 538 if stderr is None: 539 errwrite = _subprocess.GetStdHandle(_subprocess.STD_ERROR_HANDLE) 540 if errwrite is None: 541 _, errwrite = _subprocess.CreatePipe(None, 0) 542 elif stderr == PIPE: 543 errread, errwrite = _subprocess.CreatePipe(None, 0) 544 elif stderr == STDOUT: 545 errwrite = c2pwrite 546 elif isinstance(stderr, int): 547 errwrite = msvcrt.get_osfhandle(stderr) 548 else: 549 # Assuming file-like object 550 errwrite = msvcrt.get_osfhandle(stderr.fileno()) 551 errwrite = self._make_inheritable(errwrite) 552 # We just duplicated the handle, it has to be closed at the end 553 to_close.add(errwrite) 554 if stderr == PIPE: 555 to_close.add(errread) 556 557 return (p2cread, p2cwrite, 558 c2pread, c2pwrite, 559 errread, errwrite), to_close 560 561 562 def _make_inheritable(self, handle): 563 """Return a duplicate of handle, which is inheritable""" 564 return _subprocess.DuplicateHandle(_subprocess.GetCurrentProcess(), 565 handle, _subprocess.GetCurrentProcess(), 0, 1, 566 _subprocess.DUPLICATE_SAME_ACCESS) 567 568 569 def _find_w9xpopen(self): 570 """Find and return absolut path to w9xpopen.exe""" 571 w9xpopen = os.path.join( 572 os.path.dirname(_subprocess.GetModuleFileName(0)), 573 "w9xpopen.exe") 574 if not os.path.exists(w9xpopen): 575 # Eeek - file-not-found - possibly an embedding 576 # situation - see if we can locate it in sys.exec_prefix 577 w9xpopen = os.path.join(os.path.dirname(sys.exec_prefix), 578 "w9xpopen.exe") 579 if not os.path.exists(w9xpopen): 580 raise RuntimeError("Cannot locate w9xpopen.exe, which is " 581 "needed for Popen to work with your " 582 "shell or platform.") 583 return w9xpopen 584 585 586 def _execute_child(self, args, executable, preexec_fn, close_fds, 587 cwd, env, universal_newlines, 588 startupinfo, creationflags, shell, to_close, 589 p2cread, p2cwrite, 590 c2pread, c2pwrite, 591 errread, errwrite): 592 """Execute program (MS Windows version)""" 593 594 if not isinstance(args, types.StringTypes): 595 args = list2cmdline(args) 596 597 # Process startup details 598 if startupinfo is None: 599 startupinfo = STARTUPINFO() 600 if None not in (p2cread, c2pwrite, errwrite): 601 startupinfo.dwFlags |= _subprocess.STARTF_USESTDHANDLES 602 startupinfo.hStdInput = p2cread 603 startupinfo.hStdOutput = c2pwrite 604 startupinfo.hStdError = errwrite 605 606 if shell: 607 startupinfo.dwFlags |= _subprocess.STARTF_USESHOWWINDOW 608 startupinfo.wShowWindow = _subprocess.SW_HIDE 609 comspec = os.environ.get("COMSPEC", "cmd.exe") 610 args = '{} /c "{}"'.format (comspec, args) 611 if (_subprocess.GetVersion() >= 0x80000000 or 612 os.path.basename(comspec).lower() == "command.com"): 613 # Win9x, or using command.com on NT. We need to 614 # use the w9xpopen intermediate program. For more 615 # information, see KB Q150956 616 # (http://web.archive.org/web/20011105084002/http://support.microsoft.com/support/kb/articles/Q150/9/56.asp) 617 w9xpopen = self._find_w9xpopen() 618 args = '"%s" %s' % (w9xpopen, args) 619 # Not passing CREATE_NEW_CONSOLE has been known to 620 # cause random failures on win9x. Specifically a 621 # dialog: "Your program accessed mem currently in 622 # use at xxx" and a hopeful warning about the 623 # stability of your system. Cost is Ctrl+C wont 624 # kill children. 625 creationflags |= _subprocess.CREATE_NEW_CONSOLE 626 627 def _close_in_parent(fd): 628 fd.Close() 629 to_close.remove(fd) 630 631 # Start the process 632 try: 633 hp, ht, pid, tid = _subprocess.CreateProcess(executable, args, 634 # no special security 635 None, None, 636 int(not close_fds), 637 creationflags, 638 env, 639 cwd, 640 startupinfo) 641 except pywintypes.error, e: 642 # Translate pywintypes.error to WindowsError, which is 643 # a subclass of OSError. FIXME: We should really 644 # translate errno using _sys_errlist (or similar), but 645 # how can this be done from Python? 646 raise WindowsError(*e.args) 647 finally: 648 # Child is launched. Close the parent's copy of those pipe 649 # handles that only the child should have open. You need 650 # to make sure that no handles to the write end of the 651 # output pipe are maintained in this process or else the 652 # pipe will not close when the child process exits and the 653 # ReadFile will hang. 654 if p2cread is not None: 655 _close_in_parent(p2cread) 656 if c2pwrite is not None: 657 _close_in_parent(c2pwrite) 658 if errwrite is not None: 659 _close_in_parent(errwrite) 660 661 # Retain the process handle, but close the thread handle 662 self._child_created = True 663 self._handle = hp 664 self.pid = pid 665 ht.Close() 666 667 def _internal_poll(self, _deadstate=None, 668 _WaitForSingleObject=_subprocess.WaitForSingleObject, 669 _WAIT_OBJECT_0=_subprocess.WAIT_OBJECT_0, 670 _GetExitCodeProcess=_subprocess.GetExitCodeProcess): 671 """Check if child process has terminated. Returns returncode 672 attribute. 673 674 This method is called by __del__, so it can only refer to objects 675 in its local scope. 676 677 """ 678 if self.returncode is None: 679 if _WaitForSingleObject(self._handle, 0) == _WAIT_OBJECT_0: 680 self.returncode = _GetExitCodeProcess(self._handle) 681 return self.returncode 682 683 684 def wait(self): 685 """Wait for child process to terminate. Returns returncode 686 attribute.""" 687 if self.returncode is None: 688 _subprocess.WaitForSingleObject(self._handle, 689 _subprocess.INFINITE) 690 self.returncode = _subprocess.GetExitCodeProcess(self._handle) 691 return self.returncode 692 693 694 def _readerthread(self, fh, buffer): 695 buffer.append(fh.read()) 696 697 698 def _communicate(self, input): 699 stdout = None # Return 700 stderr = None # Return 701 702 if self.stdout: 703 stdout = [] 704 stdout_thread = threading.Thread(target=self._readerthread, 705 args=(self.stdout, stdout)) 706 stdout_thread.setDaemon(True) 707 stdout_thread.start() 708 if self.stderr: 709 stderr = [] 710 stderr_thread = threading.Thread(target=self._readerthread, 711 args=(self.stderr, stderr)) 712 stderr_thread.setDaemon(True) 713 stderr_thread.start() 714 715 if self.stdin: 716 if input is not None: 717 try: 718 self.stdin.write(input) 719 except IOError as e: 720 if e.errno == errno.EPIPE: 721 # communicate() should ignore broken pipe error 722 pass 723 elif (e.errno == errno.EINVAL 724 and self.poll() is not None): 725 # Issue #19612: stdin.write() fails with EINVAL 726 # if the process already exited before the write 727 pass 728 else: 729 raise 730 self.stdin.close() 731 732 if self.stdout: 733 stdout_thread.join() 734 if self.stderr: 735 stderr_thread.join() 736 737 # All data exchanged. Translate lists into strings. 738 if stdout is not None: 739 stdout = stdout[0] 740 if stderr is not None: 741 stderr = stderr[0] 742 743 # Translate newlines, if requested. We cannot let the file 744 # object do the translation: It is based on stdio, which is 745 # impossible to combine with select (unless forcing no 746 # buffering). 747 if self.universal_newlines and hasattr(file, 'newlines'): 748 if stdout: 749 stdout = self._translate_newlines(stdout) 750 if stderr: 751 stderr = self._translate_newlines(stderr) 752 753 self.wait() 754 return (stdout, stderr) 755 756 def send_signal(self, sig): 757 """Send a signal to the process 758 """ 759 if sig == signal.SIGTERM: 760 self.terminate() 761 elif sig == signal.CTRL_C_EVENT: 762 os.kill(self.pid, signal.CTRL_C_EVENT) 763 elif sig == signal.CTRL_BREAK_EVENT: 764 os.kill(self.pid, signal.CTRL_BREAK_EVENT) 765 else: 766 raise ValueError("Unsupported signal: {}".format(sig)) 767 768 def terminate(self): 769 """Terminates the process 770 """ 771 try: 772 _subprocess.TerminateProcess(self._handle, 1) 773 except OSError as e: 774 # ERROR_ACCESS_DENIED (winerror 5) is received when the 775 # process already died. 776 if e.winerror != 5: 777 raise 778 rc = _subprocess.GetExitCodeProcess(self._handle) 779 if rc == _subprocess.STILL_ACTIVE: 780 raise 781 self.returncode = rc 782 783 kill = terminate 784 785 else: 786 # 787 # POSIX methods 788 # 789 def _get_handles(self, stdin, stdout, stderr): 790 """Construct and return tuple with IO objects: 791 p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite 792 """ 793 to_close = set() 794 p2cread, p2cwrite = None, None 795 c2pread, c2pwrite = None, None 796 errread, errwrite = None, None 797 798 if stdin is None: 799 pass 800 elif stdin == PIPE: 801 p2cread, p2cwrite = self.pipe_cloexec() 802 to_close.update((p2cread, p2cwrite)) 803 elif isinstance(stdin, int): 804 p2cread = stdin 805 else: 806 # Assuming file-like object 807 p2cread = stdin.fileno() 808 809 if stdout is None: 810 pass 811 elif stdout == PIPE: 812 c2pread, c2pwrite = self.pipe_cloexec() 813 to_close.update((c2pread, c2pwrite)) 814 elif isinstance(stdout, int): 815 c2pwrite = stdout 816 else: 817 # Assuming file-like object 818 c2pwrite = stdout.fileno() 819 820 if stderr is None: 821 pass 822 elif stderr == PIPE: 823 errread, errwrite = self.pipe_cloexec() 824 to_close.update((errread, errwrite)) 825 elif stderr == STDOUT: 826 if c2pwrite is not None: 827 errwrite = c2pwrite 828 else: # child's stdout is not set, use parent's stdout 829 errwrite = sys.__stdout__.fileno() 830 elif isinstance(stderr, int): 831 errwrite = stderr 832 else: 833 # Assuming file-like object 834 errwrite = stderr.fileno() 835 836 return (p2cread, p2cwrite, 837 c2pread, c2pwrite, 838 errread, errwrite), to_close 839 840 841 def _set_cloexec_flag(self, fd, cloexec=True): 842 try: 843 cloexec_flag = fcntl.FD_CLOEXEC 844 except AttributeError: 845 cloexec_flag = 1 846 847 old = fcntl.fcntl(fd, fcntl.F_GETFD) 848 if cloexec: 849 fcntl.fcntl(fd, fcntl.F_SETFD, old | cloexec_flag) 850 else: 851 fcntl.fcntl(fd, fcntl.F_SETFD, old & ~cloexec_flag) 852 853 854 def pipe_cloexec(self): 855 """Create a pipe with FDs set CLOEXEC.""" 856 # Pipes' FDs are set CLOEXEC by default because we don't want them 857 # to be inherited by other subprocesses: the CLOEXEC flag is removed 858 # from the child's FDs by _dup2(), between fork() and exec(). 859 # This is not atomic: we would need the pipe2() syscall for that. 860 r, w = os.pipe() 861 self._set_cloexec_flag(r) 862 self._set_cloexec_flag(w) 863 return r, w 864 865 866 def _close_fds(self, but): 867 if hasattr(os, 'closerange'): 868 os.closerange(3, but) 869 os.closerange(but + 1, MAXFD) 870 else: 871 for i in xrange(3, MAXFD): 872 if i == but: 873 continue 874 try: 875 os.close(i) 876 except: 877 pass 878 879 880 def _execute_child(self, args, executable, preexec_fn, close_fds, 881 cwd, env, universal_newlines, 882 startupinfo, creationflags, shell, to_close, 883 p2cread, p2cwrite, 884 c2pread, c2pwrite, 885 errread, errwrite): 886 """Execute program (POSIX version)""" 887 888 if isinstance(args, types.StringTypes): 889 args = [args] 890 else: 891 args = list(args) 892 893 if shell: 894 args = ["/bin/sh", "-c"] + args 895 if executable: 896 args[0] = executable 897 898 if executable is None: 899 executable = args[0] 900 901 def _close_in_parent(fd): 902 os.close(fd) 903 to_close.remove(fd) 904 905 # For transferring possible exec failure from child to parent 906 # The first char specifies the exception type: 0 means 907 # OSError, 1 means some other error. 908 errpipe_read, errpipe_write = self.pipe_cloexec() 909 try: 910 try: 911 gc_was_enabled = gc.isenabled() 912 # Disable gc to avoid bug where gc -> file_dealloc -> 913 # write to stderr -> hang. http://bugs.python.org/issue1336 914 gc.disable() 915 try: 916 self.pid = os.fork() 917 except: 918 if gc_was_enabled: 919 gc.enable() 920 raise 921 self._child_created = True 922 if self.pid == 0: 923 # Child 924 try: 925 # Close parent's pipe ends 926 if p2cwrite is not None: 927 os.close(p2cwrite) 928 if c2pread is not None: 929 os.close(c2pread) 930 if errread is not None: 931 os.close(errread) 932 os.close(errpipe_read) 933 934 # When duping fds, if there arises a situation 935 # where one of the fds is either 0, 1 or 2, it 936 # is possible that it is overwritten (#12607). 937 if c2pwrite == 0: 938 c2pwrite = os.dup(c2pwrite) 939 if errwrite == 0 or errwrite == 1: 940 errwrite = os.dup(errwrite) 941 942 # Dup fds for child 943 def _dup2(a, b): 944 # dup2() removes the CLOEXEC flag but 945 # we must do it ourselves if dup2() 946 # would be a no-op (issue #10806). 947 if a == b: 948 self._set_cloexec_flag(a, False) 949 elif a is not None: 950 os.dup2(a, b) 951 _dup2(p2cread, 0) 952 _dup2(c2pwrite, 1) 953 _dup2(errwrite, 2) 954 955 # Close pipe fds. Make sure we don't close the 956 # same fd more than once, or standard fds. 957 closed = { None } 958 for fd in [p2cread, c2pwrite, errwrite]: 959 if fd not in closed and fd > 2: 960 os.close(fd) 961 closed.add(fd) 962 963 if cwd is not None: 964 os.chdir(cwd) 965 966 if preexec_fn: 967 preexec_fn() 968 969 # Close all other fds, if asked for - after 970 # preexec_fn(), which may open FDs. 971 if close_fds: 972 self._close_fds(but=errpipe_write) 973 974 if env is None: 975 os.execvp(executable, args) 976 else: 977 os.execvpe(executable, args, env) 978 979 except: 980 exc_type, exc_value, tb = sys.exc_info() 981 # Save the traceback and attach it to the exception object 982 exc_lines = traceback.format_exception(exc_type, 983 exc_value, 984 tb) 985 exc_value.child_traceback = ''.join(exc_lines) 986 os.write(errpipe_write, pickle.dumps(exc_value)) 987 988 # This exitcode won't be reported to applications, so it 989 # really doesn't matter what we return. 990 os._exit(255) 991 992 # Parent 993 if gc_was_enabled: 994 gc.enable() 995 finally: 996 # be sure the FD is closed no matter what 997 os.close(errpipe_write) 998 999 # Wait for exec to fail or succeed; possibly raising exception 1000 data = _eintr_retry_call(os.read, errpipe_read, 1048576) 1001 pickle_bits = [] 1002 while data: 1003 pickle_bits.append(data) 1004 data = _eintr_retry_call(os.read, errpipe_read, 1048576) 1005 data = "".join(pickle_bits) 1006 finally: 1007 if p2cread is not None and p2cwrite is not None: 1008 _close_in_parent(p2cread) 1009 if c2pwrite is not None and c2pread is not None: 1010 _close_in_parent(c2pwrite) 1011 if errwrite is not None and errread is not None: 1012 _close_in_parent(errwrite) 1013 1014 # be sure the FD is closed no matter what 1015 os.close(errpipe_read) 1016 1017 if data != "": 1018 try: 1019 _eintr_retry_call(os.waitpid, self.pid, 0) 1020 except OSError as e: 1021 if e.errno != errno.ECHILD: 1022 raise 1023 child_exception = pickle.loads(data) 1024 raise child_exception 1025 1026 1027 def _handle_exitstatus(self, sts, _WIFSIGNALED=os.WIFSIGNALED, 1028 _WTERMSIG=os.WTERMSIG, _WIFEXITED=os.WIFEXITED, 1029 _WEXITSTATUS=os.WEXITSTATUS): 1030 # This method is called (indirectly) by __del__, so it cannot 1031 # refer to anything outside of its local scope. 1032 if _WIFSIGNALED(sts): 1033 self.returncode = -_WTERMSIG(sts) 1034 elif _WIFEXITED(sts): 1035 self.returncode = _WEXITSTATUS(sts) 1036 else: 1037 # Should never happen 1038 raise RuntimeError("Unknown child exit status!") 1039 1040 1041 def _internal_poll(self, _deadstate=None, _waitpid=os.waitpid, 1042 _WNOHANG=os.WNOHANG, _os_error=os.error, _ECHILD=errno.ECHILD): 1043 """Check if child process has terminated. Returns returncode 1044 attribute. 1045 1046 This method is called by __del__, so it cannot reference anything 1047 outside of the local scope (nor can any methods it calls). 1048 1049 """ 1050 if self.returncode is None: 1051 try: 1052 pid, sts = _waitpid(self.pid, _WNOHANG) 1053 if pid == self.pid: 1054 self._handle_exitstatus(sts) 1055 except _os_error as e: 1056 if _deadstate is not None: 1057 self.returncode = _deadstate 1058 if e.errno == _ECHILD: 1059 # This happens if SIGCLD is set to be ignored or 1060 # waiting for child processes has otherwise been 1061 # disabled for our process. This child is dead, we 1062 # can't get the status. 1063 # http://bugs.python.org/issue15756 1064 self.returncode = 0 1065 return self.returncode 1066 1067 1068 def wait(self): 1069 """Wait for child process to terminate. Returns returncode 1070 attribute.""" 1071 while self.returncode is None: 1072 try: 1073 pid, sts = _eintr_retry_call(os.waitpid, self.pid, 0) 1074 except OSError as e: 1075 if e.errno != errno.ECHILD: 1076 raise 1077 # This happens if SIGCLD is set to be ignored or waiting 1078 # for child processes has otherwise been disabled for our 1079 # process. This child is dead, we can't get the status. 1080 pid = self.pid 1081 sts = 0 1082 # Check the pid and loop as waitpid has been known to return 1083 # 0 even without WNOHANG in odd situations. issue14396. 1084 if pid == self.pid: 1085 self._handle_exitstatus(sts) 1086 return self.returncode 1087 1088 1089 def _communicate(self, input): 1090 if self.stdin: 1091 # Flush stdio buffer. This might block, if the user has 1092 # been writing to .stdin in an uncontrolled fashion. 1093 self.stdin.flush() 1094 if not input: 1095 self.stdin.close() 1096 1097 if _has_poll: 1098 stdout, stderr = self._communicate_with_poll(input) 1099 else: 1100 stdout, stderr = self._communicate_with_select(input) 1101 1102 # All data exchanged. Translate lists into strings. 1103 if stdout is not None: 1104 stdout = ''.join(stdout) 1105 if stderr is not None: 1106 stderr = ''.join(stderr) 1107 1108 # Translate newlines, if requested. We cannot let the file 1109 # object do the translation: It is based on stdio, which is 1110 # impossible to combine with select (unless forcing no 1111 # buffering). 1112 if self.universal_newlines and hasattr(file, 'newlines'): 1113 if stdout: 1114 stdout = self._translate_newlines(stdout) 1115 if stderr: 1116 stderr = self._translate_newlines(stderr) 1117 1118 self.wait() 1119 return (stdout, stderr) 1120 1121 1122 def _communicate_with_poll(self, input): 1123 stdout = None # Return 1124 stderr = None # Return 1125 fd2file = {} 1126 fd2output = {} 1127 1128 poller = select.poll() 1129 def register_and_append(file_obj, eventmask): 1130 poller.register(file_obj.fileno(), eventmask) 1131 fd2file[file_obj.fileno()] = file_obj 1132 1133 def close_unregister_and_remove(fd): 1134 poller.unregister(fd) 1135 fd2file[fd].close() 1136 fd2file.pop(fd) 1137 1138 if self.stdin and input: 1139 register_and_append(self.stdin, select.POLLOUT) 1140 1141 select_POLLIN_POLLPRI = select.POLLIN | select.POLLPRI 1142 if self.stdout: 1143 register_and_append(self.stdout, select_POLLIN_POLLPRI) 1144 fd2output[self.stdout.fileno()] = stdout = [] 1145 if self.stderr: 1146 register_and_append(self.stderr, select_POLLIN_POLLPRI) 1147 fd2output[self.stderr.fileno()] = stderr = [] 1148 1149 input_offset = 0 1150 while fd2file: 1151 try: 1152 ready = poller.poll() 1153 except select.error, e: 1154 if e.args[0] == errno.EINTR: 1155 continue 1156 raise 1157 1158 for fd, mode in ready: 1159 if mode & select.POLLOUT: 1160 chunk = input[input_offset : input_offset + _PIPE_BUF] 1161 try: 1162 input_offset += os.write(fd, chunk) 1163 except OSError as e: 1164 if e.errno == errno.EPIPE: 1165 close_unregister_and_remove(fd) 1166 else: 1167 raise 1168 else: 1169 if input_offset >= len(input): 1170 close_unregister_and_remove(fd) 1171 elif mode & select_POLLIN_POLLPRI: 1172 data = os.read(fd, 4096) 1173 if not data: 1174 close_unregister_and_remove(fd) 1175 fd2output[fd].append(data) 1176 else: 1177 # Ignore hang up or errors. 1178 close_unregister_and_remove(fd) 1179 1180 return (stdout, stderr) 1181 1182 1183 def _communicate_with_select(self, input): 1184 read_set = [] 1185 write_set = [] 1186 stdout = None # Return 1187 stderr = None # Return 1188 1189 if self.stdin and input: 1190 write_set.append(self.stdin) 1191 if self.stdout: 1192 read_set.append(self.stdout) 1193 stdout = [] 1194 if self.stderr: 1195 read_set.append(self.stderr) 1196 stderr = [] 1197 1198 input_offset = 0 1199 while read_set or write_set: 1200 try: 1201 rlist, wlist, xlist = select.select(read_set, write_set, []) 1202 except select.error, e: 1203 if e.args[0] == errno.EINTR: 1204 continue 1205 raise 1206 1207 if self.stdin in wlist: 1208 chunk = input[input_offset : input_offset + _PIPE_BUF] 1209 try: 1210 bytes_written = os.write(self.stdin.fileno(), chunk) 1211 except OSError as e: 1212 if e.errno == errno.EPIPE: 1213 self.stdin.close() 1214 write_set.remove(self.stdin) 1215 else: 1216 raise 1217 else: 1218 input_offset += bytes_written 1219 if input_offset >= len(input): 1220 self.stdin.close() 1221 write_set.remove(self.stdin) 1222 1223 if self.stdout in rlist: 1224 data = os.read(self.stdout.fileno(), 1024) 1225 if data == "": 1226 self.stdout.close() 1227 read_set.remove(self.stdout) 1228 stdout.append(data) 1229 1230 if self.stderr in rlist: 1231 data = os.read(self.stderr.fileno(), 1024) 1232 if data == "": 1233 self.stderr.close() 1234 read_set.remove(self.stderr) 1235 stderr.append(data) 1236 1237 return (stdout, stderr) 1238 1239 1240 def send_signal(self, sig): 1241 """Send a signal to the process 1242 """ 1243 os.kill(self.pid, sig) 1244 1245 def terminate(self): 1246 """Terminate the process with SIGTERM 1247 """ 1248 self.send_signal(signal.SIGTERM) 1249 1250 def kill(self): 1251 """Kill the process with SIGKILL 1252 """ 1253 self.send_signal(signal.SIGKILL) 1254 1255 1256 def _demo_posix(): 1257 # 1258 # Example 1: Simple redirection: Get process list 1259 # 1260 plist = Popen(["ps"], stdout=PIPE).communicate()[0] 1261 print "Process list:" 1262 print plist 1263 1264 # 1265 # Example 2: Change uid before executing child 1266 # 1267 if os.getuid() == 0: 1268 p = Popen(["id"], preexec_fn=lambda: os.setuid(100)) 1269 p.wait() 1270 1271 # 1272 # Example 3: Connecting several subprocesses 1273 # 1274 print "Looking for 'hda'..." 1275 p1 = Popen(["dmesg"], stdout=PIPE) 1276 p2 = Popen(["grep", "hda"], stdin=p1.stdout, stdout=PIPE) 1277 print repr(p2.communicate()[0]) 1278 1279 # 1280 # Example 4: Catch execution error 1281 # 1282 print 1283 print "Trying a weird file..." 1284 try: 1285 print Popen(["/this/path/does/not/exist"]).communicate() 1286 except OSError, e: 1287 if e.errno == errno.ENOENT: 1288 print "The file didn't exist. I thought so..." 1289 print "Child traceback:" 1290 print e.child_traceback 1291 else: 1292 print "Error", e.errno 1293 else: 1294 print >>sys.stderr, "Gosh. No error." 1295 1296 1297 def _demo_windows(): 1298 # 1299 # Example 1: Connecting several subprocesses 1300 # 1301 print "Looking for 'PROMPT' in set output..." 1302 p1 = Popen("set", stdout=PIPE, shell=True) 1303 p2 = Popen('find "PROMPT"', stdin=p1.stdout, stdout=PIPE) 1304 print repr(p2.communicate()[0]) 1305 1306 # 1307 # Example 2: Simple execution of program 1308 # 1309 print "Executing calc..." 1310 p = Popen("calc") 1311 p.wait() 1312 1313 1314 if __name__ == "__main__": 1315 if mswindows: 1316 _demo_windows() 1317 else: 1318 _demo_posix() 1319
