1 from __future__ import absolute_import 2 import os, signal, subprocess, sys 3 import re 4 import platform 5 import tempfile 6 7 import lit.ShUtil as ShUtil 8 import lit.Test as Test 9 import lit.util 10 11 class InternalShellError(Exception): 12 def __init__(self, command, message): 13 self.command = command 14 self.message = message 15 16 kIsWindows = platform.system() == 'Windows' 17 18 # Don't use close_fds on Windows. 19 kUseCloseFDs = not kIsWindows 20 21 # Use temporary files to replace /dev/null on Windows. 22 kAvoidDevNull = kIsWindows 23 24 def executeShCmd(cmd, cfg, cwd, results): 25 if isinstance(cmd, ShUtil.Seq): 26 if cmd.op == ';': 27 res = executeShCmd(cmd.lhs, cfg, cwd, results) 28 return executeShCmd(cmd.rhs, cfg, cwd, results) 29 30 if cmd.op == '&': 31 raise InternalShellError(cmd,"unsupported shell operator: '&'") 32 33 if cmd.op == '||': 34 res = executeShCmd(cmd.lhs, cfg, cwd, results) 35 if res != 0: 36 res = executeShCmd(cmd.rhs, cfg, cwd, results) 37 return res 38 39 if cmd.op == '&&': 40 res = executeShCmd(cmd.lhs, cfg, cwd, results) 41 if res is None: 42 return res 43 44 if res == 0: 45 res = executeShCmd(cmd.rhs, cfg, cwd, results) 46 return res 47 48 raise ValueError('Unknown shell command: %r' % cmd.op) 49 50 assert isinstance(cmd, ShUtil.Pipeline) 51 procs = [] 52 input = subprocess.PIPE 53 stderrTempFiles = [] 54 opened_files = [] 55 named_temp_files = [] 56 # To avoid deadlock, we use a single stderr stream for piped 57 # output. This is null until we have seen some output using 58 # stderr. 59 for i,j in enumerate(cmd.commands): 60 # Apply the redirections, we use (N,) as a sentinel to indicate stdin, 61 # stdout, stderr for N equal to 0, 1, or 2 respectively. Redirects to or 62 # from a file are represented with a list [file, mode, file-object] 63 # where file-object is initially None. 64 redirects = [(0,), (1,), (2,)] 65 for r in j.redirects: 66 if r[0] == ('>',2): 67 redirects[2] = [r[1], 'w', None] 68 elif r[0] == ('>>',2): 69 redirects[2] = [r[1], 'a', None] 70 elif r[0] == ('>&',2) and r[1] in '012': 71 redirects[2] = redirects[int(r[1])] 72 elif r[0] == ('>&',) or r[0] == ('&>',): 73 redirects[1] = redirects[2] = [r[1], 'w', None] 74 elif r[0] == ('>',): 75 redirects[1] = [r[1], 'w', None] 76 elif r[0] == ('>>',): 77 redirects[1] = [r[1], 'a', None] 78 elif r[0] == ('<',): 79 redirects[0] = [r[1], 'r', None] 80 else: 81 raise InternalShellError(j,"Unsupported redirect: %r" % (r,)) 82 83 # Map from the final redirections to something subprocess can handle. 84 final_redirects = [] 85 for index,r in enumerate(redirects): 86 if r == (0,): 87 result = input 88 elif r == (1,): 89 if index == 0: 90 raise InternalShellError(j,"Unsupported redirect for stdin") 91 elif index == 1: 92 result = subprocess.PIPE 93 else: 94 result = subprocess.STDOUT 95 elif r == (2,): 96 if index != 2: 97 raise InternalShellError(j,"Unsupported redirect on stdout") 98 result = subprocess.PIPE 99 else: 100 if r[2] is None: 101 if kAvoidDevNull and r[0] == '/dev/null': 102 r[2] = tempfile.TemporaryFile(mode=r[1]) 103 else: 104 r[2] = open(r[0], r[1]) 105 # Workaround a Win32 and/or subprocess bug when appending. 106 # 107 # FIXME: Actually, this is probably an instance of PR6753. 108 if r[1] == 'a': 109 r[2].seek(0, 2) 110 opened_files.append(r[2]) 111 result = r[2] 112 final_redirects.append(result) 113 114 stdin, stdout, stderr = final_redirects 115 116 # If stderr wants to come from stdout, but stdout isn't a pipe, then put 117 # stderr on a pipe and treat it as stdout. 118 if (stderr == subprocess.STDOUT and stdout != subprocess.PIPE): 119 stderr = subprocess.PIPE 120 stderrIsStdout = True 121 else: 122 stderrIsStdout = False 123 124 # Don't allow stderr on a PIPE except for the last 125 # process, this could deadlock. 126 # 127 # FIXME: This is slow, but so is deadlock. 128 if stderr == subprocess.PIPE and j != cmd.commands[-1]: 129 stderr = tempfile.TemporaryFile(mode='w+b') 130 stderrTempFiles.append((i, stderr)) 131 132 # Resolve the executable path ourselves. 133 args = list(j.args) 134 executable = lit.util.which(args[0], cfg.environment['PATH']) 135 if not executable: 136 raise InternalShellError(j, '%r: command not found' % j.args[0]) 137 138 # Replace uses of /dev/null with temporary files. 139 if kAvoidDevNull: 140 for i,arg in enumerate(args): 141 if arg == "/dev/null": 142 f = tempfile.NamedTemporaryFile(delete=False) 143 f.close() 144 named_temp_files.append(f.name) 145 args[i] = f.name 146 147 procs.append(subprocess.Popen(args, cwd=cwd, 148 executable = executable, 149 stdin = stdin, 150 stdout = stdout, 151 stderr = stderr, 152 env = cfg.environment, 153 close_fds = kUseCloseFDs)) 154 155 # Immediately close stdin for any process taking stdin from us. 156 if stdin == subprocess.PIPE: 157 procs[-1].stdin.close() 158 procs[-1].stdin = None 159 160 # Update the current stdin source. 161 if stdout == subprocess.PIPE: 162 input = procs[-1].stdout 163 elif stderrIsStdout: 164 input = procs[-1].stderr 165 else: 166 input = subprocess.PIPE 167 168 # Explicitly close any redirected files. We need to do this now because we 169 # need to release any handles we may have on the temporary files (important 170 # on Win32, for example). Since we have already spawned the subprocess, our 171 # handles have already been transferred so we do not need them anymore. 172 for f in opened_files: 173 f.close() 174 175 # FIXME: There is probably still deadlock potential here. Yawn. 176 procData = [None] * len(procs) 177 procData[-1] = procs[-1].communicate() 178 179 for i in range(len(procs) - 1): 180 if procs[i].stdout is not None: 181 out = procs[i].stdout.read() 182 else: 183 out = '' 184 if procs[i].stderr is not None: 185 err = procs[i].stderr.read() 186 else: 187 err = '' 188 procData[i] = (out,err) 189 190 # Read stderr out of the temp files. 191 for i,f in stderrTempFiles: 192 f.seek(0, 0) 193 procData[i] = (procData[i][0], f.read()) 194 195 exitCode = None 196 for i,(out,err) in enumerate(procData): 197 res = procs[i].wait() 198 # Detect Ctrl-C in subprocess. 199 if res == -signal.SIGINT: 200 raise KeyboardInterrupt 201 202 # Ensure the resulting output is always of string type. 203 try: 204 out = str(out.decode('ascii')) 205 except: 206 out = str(out) 207 try: 208 err = str(err.decode('ascii')) 209 except: 210 err = str(err) 211 212 results.append((cmd.commands[i], out, err, res)) 213 if cmd.pipe_err: 214 # Python treats the exit code as a signed char. 215 if exitCode is None: 216 exitCode = res 217 elif res < 0: 218 exitCode = min(exitCode, res) 219 else: 220 exitCode = max(exitCode, res) 221 else: 222 exitCode = res 223 224 # Remove any named temporary files we created. 225 for f in named_temp_files: 226 try: 227 os.remove(f) 228 except OSError: 229 pass 230 231 if cmd.negate: 232 exitCode = not exitCode 233 234 return exitCode 235 236 def executeScriptInternal(test, litConfig, tmpBase, commands, cwd): 237 cmds = [] 238 for ln in commands: 239 try: 240 cmds.append(ShUtil.ShParser(ln, litConfig.isWindows, 241 test.config.pipefail).parse()) 242 except: 243 return lit.Test.Result(Test.FAIL, "shell parser error on: %r" % ln) 244 245 cmd = cmds[0] 246 for c in cmds[1:]: 247 cmd = ShUtil.Seq(cmd, '&&', c) 248 249 results = [] 250 try: 251 exitCode = executeShCmd(cmd, test.config, cwd, results) 252 except InternalShellError: 253 e = sys.exc_info()[1] 254 exitCode = 127 255 results.append((e.command, '', e.message, exitCode)) 256 257 out = err = '' 258 for i,(cmd, cmd_out,cmd_err,res) in enumerate(results): 259 out += 'Command %d: %s\n' % (i, ' '.join('"%s"' % s for s in cmd.args)) 260 out += 'Command %d Result: %r\n' % (i, res) 261 out += 'Command %d Output:\n%s\n\n' % (i, cmd_out) 262 out += 'Command %d Stderr:\n%s\n\n' % (i, cmd_err) 263 264 return out, err, exitCode 265 266 def executeScript(test, litConfig, tmpBase, commands, cwd): 267 bashPath = litConfig.getBashPath(); 268 isWin32CMDEXE = (litConfig.isWindows and not bashPath) 269 script = tmpBase + '.script' 270 if isWin32CMDEXE: 271 script += '.bat' 272 273 # Write script file 274 mode = 'w' 275 if litConfig.isWindows and not isWin32CMDEXE: 276 mode += 'b' # Avoid CRLFs when writing bash scripts. 277 f = open(script, mode) 278 if isWin32CMDEXE: 279 f.write('\nif %ERRORLEVEL% NEQ 0 EXIT\n'.join(commands)) 280 else: 281 if test.config.pipefail: 282 f.write('set -o pipefail;') 283 f.write('{ ' + '; } &&\n{ '.join(commands) + '; }') 284 f.write('\n') 285 f.close() 286 287 if isWin32CMDEXE: 288 command = ['cmd','/c', script] 289 else: 290 if bashPath: 291 command = [bashPath, script] 292 else: 293 command = ['/bin/sh', script] 294 if litConfig.useValgrind: 295 # FIXME: Running valgrind on sh is overkill. We probably could just 296 # run on clang with no real loss. 297 command = litConfig.valgrindArgs + command 298 299 return lit.util.executeCommand(command, cwd=cwd, 300 env=test.config.environment) 301 302 def parseIntegratedTestScriptCommands(source_path): 303 """ 304 parseIntegratedTestScriptCommands(source_path) -> commands 305 306 Parse the commands in an integrated test script file into a list of 307 (line_number, command_type, line). 308 """ 309 310 # This code is carefully written to be dual compatible with Python 2.5+ and 311 # Python 3 without requiring input files to always have valid codings. The 312 # trick we use is to open the file in binary mode and use the regular 313 # expression library to find the commands, with it scanning strings in 314 # Python2 and bytes in Python3. 315 # 316 # Once we find a match, we do require each script line to be decodable to 317 # ascii, so we convert the outputs to ascii before returning. This way the 318 # remaining code can work with "strings" agnostic of the executing Python 319 # version. 320 321 def to_bytes(str): 322 # Encode to Latin1 to get binary data. 323 return str.encode('ISO-8859-1') 324 keywords = ('RUN:', 'XFAIL:', 'REQUIRES:', 'END.') 325 keywords_re = re.compile( 326 to_bytes("(%s)(.*)\n" % ("|".join(k for k in keywords),))) 327 328 f = open(source_path, 'rb') 329 try: 330 # Read the entire file contents. 331 data = f.read() 332 333 # Iterate over the matches. 334 line_number = 1 335 last_match_position = 0 336 for match in keywords_re.finditer(data): 337 # Compute the updated line number by counting the intervening 338 # newlines. 339 match_position = match.start() 340 line_number += data.count(to_bytes('\n'), last_match_position, 341 match_position) 342 last_match_position = match_position 343 344 # Convert the keyword and line to ascii strings and yield the 345 # command. Note that we take care to return regular strings in 346 # Python 2, to avoid other code having to differentiate between the 347 # str and unicode types. 348 keyword,ln = match.groups() 349 yield (line_number, str(keyword[:-1].decode('ascii')), 350 str(ln.decode('ascii'))) 351 finally: 352 f.close() 353 354 def parseIntegratedTestScript(test, normalize_slashes=False, 355 extra_substitutions=[]): 356 """parseIntegratedTestScript - Scan an LLVM/Clang style integrated test 357 script and extract the lines to 'RUN' as well as 'XFAIL' and 'REQUIRES' 358 information. The RUN lines also will have variable substitution performed. 359 """ 360 361 # Get the temporary location, this is always relative to the test suite 362 # root, not test source root. 363 # 364 # FIXME: This should not be here? 365 sourcepath = test.getSourcePath() 366 sourcedir = os.path.dirname(sourcepath) 367 execpath = test.getExecPath() 368 execdir,execbase = os.path.split(execpath) 369 tmpDir = os.path.join(execdir, 'Output') 370 tmpBase = os.path.join(tmpDir, execbase) 371 372 # Normalize slashes, if requested. 373 if normalize_slashes: 374 sourcepath = sourcepath.replace('\\', '/') 375 sourcedir = sourcedir.replace('\\', '/') 376 tmpDir = tmpDir.replace('\\', '/') 377 tmpBase = tmpBase.replace('\\', '/') 378 379 # We use #_MARKER_# to hide %% while we do the other substitutions. 380 substitutions = list(extra_substitutions) 381 substitutions.extend([('%%', '#_MARKER_#')]) 382 substitutions.extend(test.config.substitutions) 383 substitutions.extend([('%s', sourcepath), 384 ('%S', sourcedir), 385 ('%p', sourcedir), 386 ('%{pathsep}', os.pathsep), 387 ('%t', tmpBase + '.tmp'), 388 ('%T', tmpDir), 389 ('#_MARKER_#', '%')]) 390 391 # "%/[STpst]" should be normalized. 392 substitutions.extend([ 393 ('%/s', sourcepath.replace('\\', '/')), 394 ('%/S', sourcedir.replace('\\', '/')), 395 ('%/p', sourcedir.replace('\\', '/')), 396 ('%/t', tmpBase.replace('\\', '/') + '.tmp'), 397 ('%/T', tmpDir.replace('\\', '/')), 398 ]) 399 400 # Collect the test lines from the script. 401 script = [] 402 requires = [] 403 for line_number, command_type, ln in \ 404 parseIntegratedTestScriptCommands(sourcepath): 405 if command_type == 'RUN': 406 # Trim trailing whitespace. 407 ln = ln.rstrip() 408 409 # Substitute line number expressions 410 ln = re.sub('%\(line\)', str(line_number), ln) 411 def replace_line_number(match): 412 if match.group(1) == '+': 413 return str(line_number + int(match.group(2))) 414 if match.group(1) == '-': 415 return str(line_number - int(match.group(2))) 416 ln = re.sub('%\(line *([\+-]) *(\d+)\)', replace_line_number, ln) 417 418 # Collapse lines with trailing '\\'. 419 if script and script[-1][-1] == '\\': 420 script[-1] = script[-1][:-1] + ln 421 else: 422 script.append(ln) 423 elif command_type == 'XFAIL': 424 test.xfails.extend([s.strip() for s in ln.split(',')]) 425 elif command_type == 'REQUIRES': 426 requires.extend([s.strip() for s in ln.split(',')]) 427 elif command_type == 'END': 428 # END commands are only honored if the rest of the line is empty. 429 if not ln.strip(): 430 break 431 else: 432 raise ValueError("unknown script command type: %r" % ( 433 command_type,)) 434 435 # Apply substitutions to the script. Allow full regular 436 # expression syntax. Replace each matching occurrence of regular 437 # expression pattern a with substitution b in line ln. 438 def processLine(ln): 439 # Apply substitutions 440 for a,b in substitutions: 441 if kIsWindows: 442 b = b.replace("\\","\\\\") 443 ln = re.sub(a, b, ln) 444 445 # Strip the trailing newline and any extra whitespace. 446 return ln.strip() 447 script = [processLine(ln) 448 for ln in script] 449 450 # Verify the script contains a run line. 451 if not script: 452 return lit.Test.Result(Test.UNRESOLVED, "Test has no run line!") 453 454 # Check for unterminated run lines. 455 if script[-1][-1] == '\\': 456 return lit.Test.Result(Test.UNRESOLVED, 457 "Test has unterminated run lines (with '\\')") 458 459 # Check that we have the required features: 460 missing_required_features = [f for f in requires 461 if f not in test.config.available_features] 462 if missing_required_features: 463 msg = ', '.join(missing_required_features) 464 return lit.Test.Result(Test.UNSUPPORTED, 465 "Test requires the following features: %s" % msg) 466 467 return script,tmpBase,execdir 468 469 def executeShTest(test, litConfig, useExternalSh, 470 extra_substitutions=[]): 471 if test.config.unsupported: 472 return (Test.UNSUPPORTED, 'Test is unsupported') 473 474 res = parseIntegratedTestScript(test, useExternalSh, extra_substitutions) 475 if isinstance(res, lit.Test.Result): 476 return res 477 if litConfig.noExecute: 478 return lit.Test.Result(Test.PASS) 479 480 script, tmpBase, execdir = res 481 482 # Create the output directory if it does not already exist. 483 lit.util.mkdir_p(os.path.dirname(tmpBase)) 484 485 if useExternalSh: 486 res = executeScript(test, litConfig, tmpBase, script, execdir) 487 else: 488 res = executeScriptInternal(test, litConfig, tmpBase, script, execdir) 489 if isinstance(res, lit.Test.Result): 490 return res 491 492 out,err,exitCode = res 493 if exitCode == 0: 494 status = Test.PASS 495 else: 496 status = Test.FAIL 497 498 # Form the output log. 499 output = """Script:\n--\n%s\n--\nExit Code: %d\n\n""" % ( 500 '\n'.join(script), exitCode) 501 502 # Append the outputs, if present. 503 if out: 504 output += """Command Output (stdout):\n--\n%s\n--\n""" % (out,) 505 if err: 506 output += """Command Output (stderr):\n--\n%s\n--\n""" % (err,) 507 508 return lit.Test.Result(status, output) 509
