1 from __future__ import absolute_import 2 import os, signal, subprocess, sys 3 import re 4 import platform 5 import tempfile 6 7 import lit.ShUtil as ShUtil 8 import lit.Test as Test 9 import lit.util 10 from lit.util import to_bytes, to_string 11 12 class InternalShellError(Exception): 13 def __init__(self, command, message): 14 self.command = command 15 self.message = message 16 17 kIsWindows = platform.system() == 'Windows' 18 19 # Don't use close_fds on Windows. 20 kUseCloseFDs = not kIsWindows 21 22 # Use temporary files to replace /dev/null on Windows. 23 kAvoidDevNull = kIsWindows 24 25 class ShellEnvironment(object): 26 27 """Mutable shell environment containing things like CWD and env vars. 28 29 Environment variables are not implemented, but cwd tracking is. 30 """ 31 32 def __init__(self, cwd, env): 33 self.cwd = cwd 34 self.env = env 35 36 def executeShCmd(cmd, shenv, results): 37 if isinstance(cmd, ShUtil.Seq): 38 if cmd.op == ';': 39 res = executeShCmd(cmd.lhs, shenv, results) 40 return executeShCmd(cmd.rhs, shenv, results) 41 42 if cmd.op == '&': 43 raise InternalShellError(cmd,"unsupported shell operator: '&'") 44 45 if cmd.op == '||': 46 res = executeShCmd(cmd.lhs, shenv, results) 47 if res != 0: 48 res = executeShCmd(cmd.rhs, shenv, results) 49 return res 50 51 if cmd.op == '&&': 52 res = executeShCmd(cmd.lhs, shenv, results) 53 if res is None: 54 return res 55 56 if res == 0: 57 res = executeShCmd(cmd.rhs, shenv, results) 58 return res 59 60 raise ValueError('Unknown shell command: %r' % cmd.op) 61 assert isinstance(cmd, ShUtil.Pipeline) 62 63 # Handle shell builtins first. 64 if cmd.commands[0].args[0] == 'cd': 65 # Update the cwd in the environment. 66 if len(cmd.commands[0].args) != 2: 67 raise ValueError('cd supports only one argument') 68 newdir = cmd.commands[0].args[1] 69 if os.path.isabs(newdir): 70 shenv.cwd = newdir 71 else: 72 shenv.cwd = os.path.join(shenv.cwd, newdir) 73 return 0 74 75 procs = [] 76 input = subprocess.PIPE 77 stderrTempFiles = [] 78 opened_files = [] 79 named_temp_files = [] 80 # To avoid deadlock, we use a single stderr stream for piped 81 # output. This is null until we have seen some output using 82 # stderr. 83 for i,j in enumerate(cmd.commands): 84 # Apply the redirections, we use (N,) as a sentinel to indicate stdin, 85 # stdout, stderr for N equal to 0, 1, or 2 respectively. Redirects to or 86 # from a file are represented with a list [file, mode, file-object] 87 # where file-object is initially None. 88 redirects = [(0,), (1,), (2,)] 89 for r in j.redirects: 90 if r[0] == ('>',2): 91 redirects[2] = [r[1], 'w', None] 92 elif r[0] == ('>>',2): 93 redirects[2] = [r[1], 'a', None] 94 elif r[0] == ('>&',2) and r[1] in '012': 95 redirects[2] = redirects[int(r[1])] 96 elif r[0] == ('>&',) or r[0] == ('&>',): 97 redirects[1] = redirects[2] = [r[1], 'w', None] 98 elif r[0] == ('>',): 99 redirects[1] = [r[1], 'w', None] 100 elif r[0] == ('>>',): 101 redirects[1] = [r[1], 'a', None] 102 elif r[0] == ('<',): 103 redirects[0] = [r[1], 'r', None] 104 else: 105 raise InternalShellError(j,"Unsupported redirect: %r" % (r,)) 106 107 # Map from the final redirections to something subprocess can handle. 108 final_redirects = [] 109 for index,r in enumerate(redirects): 110 if r == (0,): 111 result = input 112 elif r == (1,): 113 if index == 0: 114 raise InternalShellError(j,"Unsupported redirect for stdin") 115 elif index == 1: 116 result = subprocess.PIPE 117 else: 118 result = subprocess.STDOUT 119 elif r == (2,): 120 if index != 2: 121 raise InternalShellError(j,"Unsupported redirect on stdout") 122 result = subprocess.PIPE 123 else: 124 if r[2] is None: 125 if kAvoidDevNull and r[0] == '/dev/null': 126 r[2] = tempfile.TemporaryFile(mode=r[1]) 127 else: 128 # Make sure relative paths are relative to the cwd. 129 redir_filename = os.path.join(shenv.cwd, r[0]) 130 r[2] = open(redir_filename, r[1]) 131 # Workaround a Win32 and/or subprocess bug when appending. 132 # 133 # FIXME: Actually, this is probably an instance of PR6753. 134 if r[1] == 'a': 135 r[2].seek(0, 2) 136 opened_files.append(r[2]) 137 result = r[2] 138 final_redirects.append(result) 139 140 stdin, stdout, stderr = final_redirects 141 142 # If stderr wants to come from stdout, but stdout isn't a pipe, then put 143 # stderr on a pipe and treat it as stdout. 144 if (stderr == subprocess.STDOUT and stdout != subprocess.PIPE): 145 stderr = subprocess.PIPE 146 stderrIsStdout = True 147 else: 148 stderrIsStdout = False 149 150 # Don't allow stderr on a PIPE except for the last 151 # process, this could deadlock. 152 # 153 # FIXME: This is slow, but so is deadlock. 154 if stderr == subprocess.PIPE and j != cmd.commands[-1]: 155 stderr = tempfile.TemporaryFile(mode='w+b') 156 stderrTempFiles.append((i, stderr)) 157 158 # Resolve the executable path ourselves. 159 args = list(j.args) 160 executable = lit.util.which(args[0], shenv.env['PATH']) 161 if not executable: 162 raise InternalShellError(j, '%r: command not found' % j.args[0]) 163 164 # Replace uses of /dev/null with temporary files. 165 if kAvoidDevNull: 166 for i,arg in enumerate(args): 167 if arg == "/dev/null": 168 f = tempfile.NamedTemporaryFile(delete=False) 169 f.close() 170 named_temp_files.append(f.name) 171 args[i] = f.name 172 173 try: 174 procs.append(subprocess.Popen(args, cwd=shenv.cwd, 175 executable = executable, 176 stdin = stdin, 177 stdout = stdout, 178 stderr = stderr, 179 env = shenv.env, 180 close_fds = kUseCloseFDs)) 181 except OSError as e: 182 raise InternalShellError(j, 'Could not create process due to {}'.format(e)) 183 184 # Immediately close stdin for any process taking stdin from us. 185 if stdin == subprocess.PIPE: 186 procs[-1].stdin.close() 187 procs[-1].stdin = None 188 189 # Update the current stdin source. 190 if stdout == subprocess.PIPE: 191 input = procs[-1].stdout 192 elif stderrIsStdout: 193 input = procs[-1].stderr 194 else: 195 input = subprocess.PIPE 196 197 # Explicitly close any redirected files. We need to do this now because we 198 # need to release any handles we may have on the temporary files (important 199 # on Win32, for example). Since we have already spawned the subprocess, our 200 # handles have already been transferred so we do not need them anymore. 201 for f in opened_files: 202 f.close() 203 204 # FIXME: There is probably still deadlock potential here. Yawn. 205 procData = [None] * len(procs) 206 procData[-1] = procs[-1].communicate() 207 208 for i in range(len(procs) - 1): 209 if procs[i].stdout is not None: 210 out = procs[i].stdout.read() 211 else: 212 out = '' 213 if procs[i].stderr is not None: 214 err = procs[i].stderr.read() 215 else: 216 err = '' 217 procData[i] = (out,err) 218 219 # Read stderr out of the temp files. 220 for i,f in stderrTempFiles: 221 f.seek(0, 0) 222 procData[i] = (procData[i][0], f.read()) 223 224 def to_string(bytes): 225 if isinstance(bytes, str): 226 return bytes 227 return bytes.encode('utf-8') 228 229 exitCode = None 230 for i,(out,err) in enumerate(procData): 231 res = procs[i].wait() 232 # Detect Ctrl-C in subprocess. 233 if res == -signal.SIGINT: 234 raise KeyboardInterrupt 235 236 # Ensure the resulting output is always of string type. 237 try: 238 out = to_string(out.decode('utf-8')) 239 except: 240 out = str(out) 241 try: 242 err = to_string(err.decode('utf-8')) 243 except: 244 err = str(err) 245 246 results.append((cmd.commands[i], out, err, res)) 247 if cmd.pipe_err: 248 # Python treats the exit code as a signed char. 249 if exitCode is None: 250 exitCode = res 251 elif res < 0: 252 exitCode = min(exitCode, res) 253 else: 254 exitCode = max(exitCode, res) 255 else: 256 exitCode = res 257 258 # Remove any named temporary files we created. 259 for f in named_temp_files: 260 try: 261 os.remove(f) 262 except OSError: 263 pass 264 265 if cmd.negate: 266 exitCode = not exitCode 267 268 return exitCode 269 270 def executeScriptInternal(test, litConfig, tmpBase, commands, cwd): 271 cmds = [] 272 for ln in commands: 273 try: 274 cmds.append(ShUtil.ShParser(ln, litConfig.isWindows, 275 test.config.pipefail).parse()) 276 except: 277 return lit.Test.Result(Test.FAIL, "shell parser error on: %r" % ln) 278 279 cmd = cmds[0] 280 for c in cmds[1:]: 281 cmd = ShUtil.Seq(cmd, '&&', c) 282 283 results = [] 284 try: 285 shenv = ShellEnvironment(cwd, test.config.environment) 286 exitCode = executeShCmd(cmd, shenv, results) 287 except InternalShellError: 288 e = sys.exc_info()[1] 289 exitCode = 127 290 results.append((e.command, '', e.message, exitCode)) 291 292 out = err = '' 293 for i,(cmd, cmd_out,cmd_err,res) in enumerate(results): 294 out += 'Command %d: %s\n' % (i, ' '.join('"%s"' % s for s in cmd.args)) 295 out += 'Command %d Result: %r\n' % (i, res) 296 out += 'Command %d Output:\n%s\n\n' % (i, cmd_out) 297 out += 'Command %d Stderr:\n%s\n\n' % (i, cmd_err) 298 299 return out, err, exitCode 300 301 def executeScript(test, litConfig, tmpBase, commands, cwd): 302 bashPath = litConfig.getBashPath(); 303 isWin32CMDEXE = (litConfig.isWindows and not bashPath) 304 script = tmpBase + '.script' 305 if isWin32CMDEXE: 306 script += '.bat' 307 308 # Write script file 309 mode = 'w' 310 if litConfig.isWindows and not isWin32CMDEXE: 311 mode += 'b' # Avoid CRLFs when writing bash scripts. 312 f = open(script, mode) 313 if isWin32CMDEXE: 314 f.write('\nif %ERRORLEVEL% NEQ 0 EXIT\n'.join(commands)) 315 else: 316 if test.config.pipefail: 317 f.write('set -o pipefail;') 318 f.write('{ ' + '; } &&\n{ '.join(commands) + '; }') 319 f.write('\n') 320 f.close() 321 322 if isWin32CMDEXE: 323 command = ['cmd','/c', script] 324 else: 325 if bashPath: 326 command = [bashPath, script] 327 else: 328 command = ['/bin/sh', script] 329 if litConfig.useValgrind: 330 # FIXME: Running valgrind on sh is overkill. We probably could just 331 # run on clang with no real loss. 332 command = litConfig.valgrindArgs + command 333 334 return lit.util.executeCommand(command, cwd=cwd, 335 env=test.config.environment) 336 337 def parseIntegratedTestScriptCommands(source_path): 338 """ 339 parseIntegratedTestScriptCommands(source_path) -> commands 340 341 Parse the commands in an integrated test script file into a list of 342 (line_number, command_type, line). 343 """ 344 345 # This code is carefully written to be dual compatible with Python 2.5+ and 346 # Python 3 without requiring input files to always have valid codings. The 347 # trick we use is to open the file in binary mode and use the regular 348 # expression library to find the commands, with it scanning strings in 349 # Python2 and bytes in Python3. 350 # 351 # Once we find a match, we do require each script line to be decodable to 352 # UTF-8, so we convert the outputs to UTF-8 before returning. This way the 353 # remaining code can work with "strings" agnostic of the executing Python 354 # version. 355 356 keywords = ['RUN:', 'XFAIL:', 'REQUIRES:', 'UNSUPPORTED:', 'END.'] 357 keywords_re = re.compile( 358 to_bytes("(%s)(.*)\n" % ("|".join(k for k in keywords),))) 359 360 f = open(source_path, 'rb') 361 try: 362 # Read the entire file contents. 363 data = f.read() 364 365 # Ensure the data ends with a newline. 366 if not data.endswith(to_bytes('\n')): 367 data = data + to_bytes('\n') 368 369 # Iterate over the matches. 370 line_number = 1 371 last_match_position = 0 372 for match in keywords_re.finditer(data): 373 # Compute the updated line number by counting the intervening 374 # newlines. 375 match_position = match.start() 376 line_number += data.count(to_bytes('\n'), last_match_position, 377 match_position) 378 last_match_position = match_position 379 380 # Convert the keyword and line to UTF-8 strings and yield the 381 # command. Note that we take care to return regular strings in 382 # Python 2, to avoid other code having to differentiate between the 383 # str and unicode types. 384 keyword,ln = match.groups() 385 yield (line_number, to_string(keyword[:-1].decode('utf-8')), 386 to_string(ln.decode('utf-8'))) 387 finally: 388 f.close() 389 390 391 def parseIntegratedTestScript(test, normalize_slashes=False, 392 extra_substitutions=[], require_script=True): 393 """parseIntegratedTestScript - Scan an LLVM/Clang style integrated test 394 script and extract the lines to 'RUN' as well as 'XFAIL' and 'REQUIRES' 395 and 'UNSUPPORTED' information. The RUN lines also will have variable 396 substitution performed. If 'require_script' is False an empty script may be 397 returned. This can be used for test formats where the actual script is 398 optional or ignored. 399 """ 400 401 # Get the temporary location, this is always relative to the test suite 402 # root, not test source root. 403 # 404 # FIXME: This should not be here? 405 sourcepath = test.getSourcePath() 406 sourcedir = os.path.dirname(sourcepath) 407 execpath = test.getExecPath() 408 execdir,execbase = os.path.split(execpath) 409 tmpDir = os.path.join(execdir, 'Output') 410 tmpBase = os.path.join(tmpDir, execbase) 411 412 # Normalize slashes, if requested. 413 if normalize_slashes: 414 sourcepath = sourcepath.replace('\\', '/') 415 sourcedir = sourcedir.replace('\\', '/') 416 tmpDir = tmpDir.replace('\\', '/') 417 tmpBase = tmpBase.replace('\\', '/') 418 419 # We use #_MARKER_# to hide %% while we do the other substitutions. 420 substitutions = list(extra_substitutions) 421 substitutions.extend([('%%', '#_MARKER_#')]) 422 substitutions.extend(test.config.substitutions) 423 substitutions.extend([('%s', sourcepath), 424 ('%S', sourcedir), 425 ('%p', sourcedir), 426 ('%{pathsep}', os.pathsep), 427 ('%t', tmpBase + '.tmp'), 428 ('%T', tmpDir), 429 ('#_MARKER_#', '%')]) 430 431 # "%/[STpst]" should be normalized. 432 substitutions.extend([ 433 ('%/s', sourcepath.replace('\\', '/')), 434 ('%/S', sourcedir.replace('\\', '/')), 435 ('%/p', sourcedir.replace('\\', '/')), 436 ('%/t', tmpBase.replace('\\', '/') + '.tmp'), 437 ('%/T', tmpDir.replace('\\', '/')), 438 ]) 439 440 # Collect the test lines from the script. 441 script = [] 442 requires = [] 443 unsupported = [] 444 for line_number, command_type, ln in \ 445 parseIntegratedTestScriptCommands(sourcepath): 446 if command_type == 'RUN': 447 # Trim trailing whitespace. 448 ln = ln.rstrip() 449 450 # Substitute line number expressions 451 ln = re.sub('%\(line\)', str(line_number), ln) 452 def replace_line_number(match): 453 if match.group(1) == '+': 454 return str(line_number + int(match.group(2))) 455 if match.group(1) == '-': 456 return str(line_number - int(match.group(2))) 457 ln = re.sub('%\(line *([\+-]) *(\d+)\)', replace_line_number, ln) 458 459 # Collapse lines with trailing '\\'. 460 if script and script[-1][-1] == '\\': 461 script[-1] = script[-1][:-1] + ln 462 else: 463 script.append(ln) 464 elif command_type == 'XFAIL': 465 test.xfails.extend([s.strip() for s in ln.split(',')]) 466 elif command_type == 'REQUIRES': 467 requires.extend([s.strip() for s in ln.split(',')]) 468 elif command_type == 'UNSUPPORTED': 469 unsupported.extend([s.strip() for s in ln.split(',')]) 470 elif command_type == 'END': 471 # END commands are only honored if the rest of the line is empty. 472 if not ln.strip(): 473 break 474 else: 475 raise ValueError("unknown script command type: %r" % ( 476 command_type,)) 477 478 # Apply substitutions to the script. Allow full regular 479 # expression syntax. Replace each matching occurrence of regular 480 # expression pattern a with substitution b in line ln. 481 def processLine(ln): 482 # Apply substitutions 483 for a,b in substitutions: 484 if kIsWindows: 485 b = b.replace("\\","\\\\") 486 ln = re.sub(a, b, ln) 487 488 # Strip the trailing newline and any extra whitespace. 489 return ln.strip() 490 script = [processLine(ln) 491 for ln in script] 492 493 # Verify the script contains a run line. 494 if require_script and not script: 495 return lit.Test.Result(Test.UNRESOLVED, "Test has no run line!") 496 497 # Check for unterminated run lines. 498 if script and script[-1][-1] == '\\': 499 return lit.Test.Result(Test.UNRESOLVED, 500 "Test has unterminated run lines (with '\\')") 501 502 # Check that we have the required features: 503 missing_required_features = [f for f in requires 504 if f not in test.config.available_features] 505 if missing_required_features: 506 msg = ', '.join(missing_required_features) 507 return lit.Test.Result(Test.UNSUPPORTED, 508 "Test requires the following features: %s" % msg) 509 unsupported_features = [f for f in unsupported 510 if f in test.config.available_features] 511 if unsupported_features: 512 msg = ', '.join(unsupported_features) 513 return lit.Test.Result(Test.UNSUPPORTED, 514 "Test is unsupported with the following features: %s" % msg) 515 516 return script,tmpBase,execdir 517 518 def _runShTest(test, litConfig, useExternalSh, 519 script, tmpBase, execdir): 520 # Create the output directory if it does not already exist. 521 lit.util.mkdir_p(os.path.dirname(tmpBase)) 522 523 if useExternalSh: 524 res = executeScript(test, litConfig, tmpBase, script, execdir) 525 else: 526 res = executeScriptInternal(test, litConfig, tmpBase, script, execdir) 527 if isinstance(res, lit.Test.Result): 528 return res 529 530 out,err,exitCode = res 531 if exitCode == 0: 532 status = Test.PASS 533 else: 534 status = Test.FAIL 535 536 # Form the output log. 537 output = """Script:\n--\n%s\n--\nExit Code: %d\n\n""" % ( 538 '\n'.join(script), exitCode) 539 540 # Append the outputs, if present. 541 if out: 542 output += """Command Output (stdout):\n--\n%s\n--\n""" % (out,) 543 if err: 544 output += """Command Output (stderr):\n--\n%s\n--\n""" % (err,) 545 546 return lit.Test.Result(status, output) 547 548 549 def executeShTest(test, litConfig, useExternalSh, 550 extra_substitutions=[]): 551 if test.config.unsupported: 552 return (Test.UNSUPPORTED, 'Test is unsupported') 553 554 res = parseIntegratedTestScript(test, useExternalSh, extra_substitutions) 555 if isinstance(res, lit.Test.Result): 556 return res 557 if litConfig.noExecute: 558 return lit.Test.Result(Test.PASS) 559 560 script, tmpBase, execdir = res 561 return _runShTest(test, litConfig, useExternalSh, script, tmpBase, execdir) 562 563
