1 # -*- coding: utf-8 -*- 2 # The LLVM Compiler Infrastructure 3 # 4 # This file is distributed under the University of Illinois Open Source 5 # License. See LICENSE.TXT for details. 6 """ This module is responsible to run the analyzer commands. """ 7 8 import re 9 import os 10 import os.path 11 import tempfile 12 import functools 13 import subprocess 14 import logging 15 from libscanbuild.compilation import classify_source, compiler_language 16 from libscanbuild.clang import get_version, get_arguments 17 from libscanbuild.shell import decode 18 19 __all__ = ['run'] 20 21 # To have good results from static analyzer certain compiler options shall be 22 # omitted. The compiler flag filtering only affects the static analyzer run. 23 # 24 # Keys are the option name, value number of options to skip 25 IGNORED_FLAGS = { 26 '-c': 0, # compile option will be overwritten 27 '-fsyntax-only': 0, # static analyzer option will be overwritten 28 '-o': 1, # will set up own output file 29 # flags below are inherited from the perl implementation. 30 '-g': 0, 31 '-save-temps': 0, 32 '-install_name': 1, 33 '-exported_symbols_list': 1, 34 '-current_version': 1, 35 '-compatibility_version': 1, 36 '-init': 1, 37 '-e': 1, 38 '-seg1addr': 1, 39 '-bundle_loader': 1, 40 '-multiply_defined': 1, 41 '-sectorder': 3, 42 '--param': 1, 43 '--serialize-diagnostics': 1 44 } 45 46 47 def require(required): 48 """ Decorator for checking the required values in state. 49 50 It checks the required attributes in the passed state and stop when 51 any of those is missing. """ 52 53 def decorator(function): 54 @functools.wraps(function) 55 def wrapper(*args, **kwargs): 56 for key in required: 57 if key not in args[0]: 58 raise KeyError('{0} not passed to {1}'.format( 59 key, function.__name__)) 60 61 return function(*args, **kwargs) 62 63 return wrapper 64 65 return decorator 66 67 68 @require(['command', # entry from compilation database 69 'directory', # entry from compilation database 70 'file', # entry from compilation database 71 'clang', # clang executable name (and path) 72 'direct_args', # arguments from command line 73 'force_debug', # kill non debug macros 74 'output_dir', # where generated report files shall go 75 'output_format', # it's 'plist' or 'html' or both 76 'output_failures']) # generate crash reports or not 77 def run(opts): 78 """ Entry point to run (or not) static analyzer against a single entry 79 of the compilation database. 80 81 This complex task is decomposed into smaller methods which are calling 82 each other in chain. If the analyzis is not possibe the given method 83 just return and break the chain. 84 85 The passed parameter is a python dictionary. Each method first check 86 that the needed parameters received. (This is done by the 'require' 87 decorator. It's like an 'assert' to check the contract between the 88 caller and the called method.) """ 89 90 try: 91 command = opts.pop('command') 92 command = command if isinstance(command, list) else decode(command) 93 logging.debug("Run analyzer against '%s'", command) 94 opts.update(classify_parameters(command)) 95 96 return arch_check(opts) 97 except Exception: 98 logging.error("Problem occured during analyzis.", exc_info=1) 99 return None 100 101 102 @require(['clang', 'directory', 'flags', 'file', 'output_dir', 'language', 103 'error_type', 'error_output', 'exit_code']) 104 def report_failure(opts): 105 """ Create report when analyzer failed. 106 107 The major report is the preprocessor output. The output filename generated 108 randomly. The compiler output also captured into '.stderr.txt' file. 109 And some more execution context also saved into '.info.txt' file. """ 110 111 def extension(opts): 112 """ Generate preprocessor file extension. """ 113 114 mapping = {'objective-c++': '.mii', 'objective-c': '.mi', 'c++': '.ii'} 115 return mapping.get(opts['language'], '.i') 116 117 def destination(opts): 118 """ Creates failures directory if not exits yet. """ 119 120 name = os.path.join(opts['output_dir'], 'failures') 121 if not os.path.isdir(name): 122 os.makedirs(name) 123 return name 124 125 error = opts['error_type'] 126 (handle, name) = tempfile.mkstemp(suffix=extension(opts), 127 prefix='clang_' + error + '_', 128 dir=destination(opts)) 129 os.close(handle) 130 cwd = opts['directory'] 131 cmd = get_arguments([opts['clang'], '-fsyntax-only', '-E'] + 132 opts['flags'] + [opts['file'], '-o', name], cwd) 133 logging.debug('exec command in %s: %s', cwd, ' '.join(cmd)) 134 subprocess.call(cmd, cwd=cwd) 135 # write general information about the crash 136 with open(name + '.info.txt', 'w') as handle: 137 handle.write(opts['file'] + os.linesep) 138 handle.write(error.title().replace('_', ' ') + os.linesep) 139 handle.write(' '.join(cmd) + os.linesep) 140 handle.write(' '.join(os.uname()) + os.linesep) 141 handle.write(get_version(opts['clang'])) 142 handle.close() 143 # write the captured output too 144 with open(name + '.stderr.txt', 'w') as handle: 145 handle.writelines(opts['error_output']) 146 handle.close() 147 # return with the previous step exit code and output 148 return { 149 'error_output': opts['error_output'], 150 'exit_code': opts['exit_code'] 151 } 152 153 154 @require(['clang', 'directory', 'flags', 'direct_args', 'file', 'output_dir', 155 'output_format']) 156 def run_analyzer(opts, continuation=report_failure): 157 """ It assembles the analysis command line and executes it. Capture the 158 output of the analysis and returns with it. If failure reports are 159 requested, it calls the continuation to generate it. """ 160 161 def output(): 162 """ Creates output file name for reports. """ 163 if opts['output_format'] in {'plist', 'plist-html'}: 164 (handle, name) = tempfile.mkstemp(prefix='report-', 165 suffix='.plist', 166 dir=opts['output_dir']) 167 os.close(handle) 168 return name 169 return opts['output_dir'] 170 171 cwd = opts['directory'] 172 cmd = get_arguments([opts['clang'], '--analyze'] + opts['direct_args'] + 173 opts['flags'] + [opts['file'], '-o', output()], 174 cwd) 175 logging.debug('exec command in %s: %s', cwd, ' '.join(cmd)) 176 child = subprocess.Popen(cmd, 177 cwd=cwd, 178 universal_newlines=True, 179 stdout=subprocess.PIPE, 180 stderr=subprocess.STDOUT) 181 output = child.stdout.readlines() 182 child.stdout.close() 183 # do report details if it were asked 184 child.wait() 185 if opts.get('output_failures', False) and child.returncode: 186 error_type = 'crash' if child.returncode & 127 else 'other_error' 187 opts.update({ 188 'error_type': error_type, 189 'error_output': output, 190 'exit_code': child.returncode 191 }) 192 return continuation(opts) 193 # return the output for logging and exit code for testing 194 return {'error_output': output, 'exit_code': child.returncode} 195 196 197 @require(['flags', 'force_debug']) 198 def filter_debug_flags(opts, continuation=run_analyzer): 199 """ Filter out nondebug macros when requested. """ 200 201 if opts.pop('force_debug'): 202 # lazy implementation just append an undefine macro at the end 203 opts.update({'flags': opts['flags'] + ['-UNDEBUG']}) 204 205 return continuation(opts) 206 207 208 @require(['file', 'directory']) 209 def set_file_path_relative(opts, continuation=filter_debug_flags): 210 """ Set source file path to relative to the working directory. 211 212 The only purpose of this function is to pass the SATestBuild.py tests. """ 213 214 opts.update({'file': os.path.relpath(opts['file'], opts['directory'])}) 215 216 return continuation(opts) 217 218 219 @require(['language', 'compiler', 'file', 'flags']) 220 def language_check(opts, continuation=set_file_path_relative): 221 """ Find out the language from command line parameters or file name 222 extension. The decision also influenced by the compiler invocation. """ 223 224 accepted = frozenset({ 225 'c', 'c++', 'objective-c', 'objective-c++', 'c-cpp-output', 226 'c++-cpp-output', 'objective-c-cpp-output' 227 }) 228 229 # language can be given as a parameter... 230 language = opts.pop('language') 231 compiler = opts.pop('compiler') 232 # ... or find out from source file extension 233 if language is None and compiler is not None: 234 language = classify_source(opts['file'], compiler == 'c') 235 236 if language is None: 237 logging.debug('skip analysis, language not known') 238 return None 239 elif language not in accepted: 240 logging.debug('skip analysis, language not supported') 241 return None 242 else: 243 logging.debug('analysis, language: %s', language) 244 opts.update({'language': language, 245 'flags': ['-x', language] + opts['flags']}) 246 return continuation(opts) 247 248 249 @require(['arch_list', 'flags']) 250 def arch_check(opts, continuation=language_check): 251 """ Do run analyzer through one of the given architectures. """ 252 253 disabled = frozenset({'ppc', 'ppc64'}) 254 255 received_list = opts.pop('arch_list') 256 if received_list: 257 # filter out disabled architectures and -arch switches 258 filtered_list = [a for a in received_list if a not in disabled] 259 if filtered_list: 260 # There should be only one arch given (or the same multiple 261 # times). If there are multiple arch are given and are not 262 # the same, those should not change the pre-processing step. 263 # But that's the only pass we have before run the analyzer. 264 current = filtered_list.pop() 265 logging.debug('analysis, on arch: %s', current) 266 267 opts.update({'flags': ['-arch', current] + opts['flags']}) 268 return continuation(opts) 269 else: 270 logging.debug('skip analysis, found not supported arch') 271 return None 272 else: 273 logging.debug('analysis, on default arch') 274 return continuation(opts) 275 276 277 def classify_parameters(command): 278 """ Prepare compiler flags (filters some and add others) and take out 279 language (-x) and architecture (-arch) flags for future processing. """ 280 281 result = { 282 'flags': [], # the filtered compiler flags 283 'arch_list': [], # list of architecture flags 284 'language': None, # compilation language, None, if not specified 285 'compiler': compiler_language(command) # 'c' or 'c++' 286 } 287 288 # iterate on the compile options 289 args = iter(command[1:]) 290 for arg in args: 291 # take arch flags into a separate basket 292 if arg == '-arch': 293 result['arch_list'].append(next(args)) 294 # take language 295 elif arg == '-x': 296 result['language'] = next(args) 297 # parameters which looks source file are not flags 298 elif re.match(r'^[^-].+', arg) and classify_source(arg): 299 pass 300 # ignore some flags 301 elif arg in IGNORED_FLAGS: 302 count = IGNORED_FLAGS[arg] 303 for _ in range(count): 304 next(args) 305 # we don't care about extra warnings, but we should suppress ones 306 # that we don't want to see. 307 elif re.match(r'^-W.+', arg) and not re.match(r'^-Wno-.+', arg): 308 pass 309 # and consider everything else as compilation flag. 310 else: 311 result['flags'].append(arg) 312 313 return result 314
