1 #!/usr/bin/env python2.7 2 3 from __future__ import print_function 4 5 import yaml 6 # Try to use the C parser. 7 try: 8 from yaml import CLoader as Loader 9 except ImportError: 10 print("For faster parsing, you may want to install libYAML for PyYAML") 11 from yaml import Loader 12 13 import cgi 14 from collections import defaultdict 15 import fnmatch 16 import functools 17 from multiprocessing import Lock 18 import os, os.path 19 import subprocess 20 try: 21 # The previously builtin function `intern()` was moved 22 # to the `sys` module in Python 3. 23 from sys import intern 24 except: 25 pass 26 27 import optpmap 28 29 try: 30 dict.iteritems 31 except AttributeError: 32 # Python 3 33 def itervalues(d): 34 return iter(d.values()) 35 def iteritems(d): 36 return iter(d.items()) 37 else: 38 # Python 2 39 def itervalues(d): 40 return d.itervalues() 41 def iteritems(d): 42 return d.iteritems() 43 44 45 def html_file_name(filename): 46 return filename.replace('/', '_').replace('#', '_') + ".html" 47 48 49 def make_link(File, Line): 50 return "\"{}#L{}\"".format(html_file_name(File), Line) 51 52 53 class Remark(yaml.YAMLObject): 54 # Work-around for http://pyyaml.org/ticket/154. 55 yaml_loader = Loader 56 57 default_demangler = 'c++filt -n' 58 demangler_proc = None 59 60 @classmethod 61 def set_demangler(cls, demangler): 62 cls.demangler_proc = subprocess.Popen(demangler.split(), stdin=subprocess.PIPE, stdout=subprocess.PIPE) 63 cls.demangler_lock = Lock() 64 65 @classmethod 66 def demangle(cls, name): 67 with cls.demangler_lock: 68 cls.demangler_proc.stdin.write((name + '\n').encode('utf-8')) 69 cls.demangler_proc.stdin.flush() 70 return cls.demangler_proc.stdout.readline().rstrip().decode('utf-8') 71 72 # Intern all strings since we have lot of duplication across filenames, 73 # remark text. 74 # 75 # Change Args from a list of dicts to a tuple of tuples. This saves 76 # memory in two ways. One, a small tuple is significantly smaller than a 77 # small dict. Two, using tuple instead of list allows Args to be directly 78 # used as part of the key (in Python only immutable types are hashable). 79 def _reduce_memory(self): 80 self.Pass = intern(self.Pass) 81 self.Name = intern(self.Name) 82 try: 83 # Can't intern unicode strings. 84 self.Function = intern(self.Function) 85 except: 86 pass 87 88 def _reduce_memory_dict(old_dict): 89 new_dict = dict() 90 for (k, v) in iteritems(old_dict): 91 if type(k) is str: 92 k = intern(k) 93 94 if type(v) is str: 95 v = intern(v) 96 elif type(v) is dict: 97 # This handles [{'Caller': ..., 'DebugLoc': { 'File': ... }}] 98 v = _reduce_memory_dict(v) 99 new_dict[k] = v 100 return tuple(new_dict.items()) 101 102 self.Args = tuple([_reduce_memory_dict(arg_dict) for arg_dict in self.Args]) 103 104 # The inverse operation of the dictonary-related memory optimization in 105 # _reduce_memory_dict. E.g. 106 # (('DebugLoc', (('File', ...) ... ))) -> [{'DebugLoc': {'File': ...} ....}] 107 def recover_yaml_structure(self): 108 def tuple_to_dict(t): 109 d = dict() 110 for (k, v) in t: 111 if type(v) is tuple: 112 v = tuple_to_dict(v) 113 d[k] = v 114 return d 115 116 self.Args = [tuple_to_dict(arg_tuple) for arg_tuple in self.Args] 117 118 def canonicalize(self): 119 if not hasattr(self, 'Hotness'): 120 self.Hotness = 0 121 if not hasattr(self, 'Args'): 122 self.Args = [] 123 self._reduce_memory() 124 125 @property 126 def File(self): 127 return self.DebugLoc['File'] 128 129 @property 130 def Line(self): 131 return int(self.DebugLoc['Line']) 132 133 @property 134 def Column(self): 135 return self.DebugLoc['Column'] 136 137 @property 138 def DebugLocString(self): 139 return "{}:{}:{}".format(self.File, self.Line, self.Column) 140 141 @property 142 def DemangledFunctionName(self): 143 return self.demangle(self.Function) 144 145 @property 146 def Link(self): 147 return make_link(self.File, self.Line) 148 149 def getArgString(self, mapping): 150 mapping = dict(list(mapping)) 151 dl = mapping.get('DebugLoc') 152 if dl: 153 del mapping['DebugLoc'] 154 155 assert(len(mapping) == 1) 156 (key, value) = list(mapping.items())[0] 157 158 if key == 'Caller' or key == 'Callee' or key == 'DirectCallee': 159 value = cgi.escape(self.demangle(value)) 160 161 if dl and key != 'Caller': 162 dl_dict = dict(list(dl)) 163 return u"<a href={}>{}</a>".format( 164 make_link(dl_dict['File'], dl_dict['Line']), value) 165 else: 166 return value 167 168 # Return a cached dictionary for the arguments. The key for each entry is 169 # the argument key (e.g. 'Callee' for inlining remarks. The value is a 170 # list containing the value (e.g. for 'Callee' the function) and 171 # optionally a DebugLoc. 172 def getArgDict(self): 173 if hasattr(self, 'ArgDict'): 174 return self.ArgDict 175 self.ArgDict = {} 176 for arg in self.Args: 177 if len(arg) == 2: 178 if arg[0][0] == 'DebugLoc': 179 dbgidx = 0 180 else: 181 assert(arg[1][0] == 'DebugLoc') 182 dbgidx = 1 183 184 key = arg[1 - dbgidx][0] 185 entry = (arg[1 - dbgidx][1], arg[dbgidx][1]) 186 else: 187 arg = arg[0] 188 key = arg[0] 189 entry = (arg[1], ) 190 191 self.ArgDict[key] = entry 192 return self.ArgDict 193 194 def getDiffPrefix(self): 195 if hasattr(self, 'Added'): 196 if self.Added: 197 return '+' 198 else: 199 return '-' 200 return '' 201 202 @property 203 def PassWithDiffPrefix(self): 204 return self.getDiffPrefix() + self.Pass 205 206 @property 207 def message(self): 208 # Args is a list of mappings (dictionaries) 209 values = [self.getArgString(mapping) for mapping in self.Args] 210 return "".join(values) 211 212 @property 213 def RelativeHotness(self): 214 if self.max_hotness: 215 return "{0:.2f}%".format(self.Hotness * 100. / self.max_hotness) 216 else: 217 return '' 218 219 @property 220 def key(self): 221 return (self.__class__, self.PassWithDiffPrefix, self.Name, self.File, 222 self.Line, self.Column, self.Function, self.Args) 223 224 def __hash__(self): 225 return hash(self.key) 226 227 def __eq__(self, other): 228 return self.key == other.key 229 230 def __repr__(self): 231 return str(self.key) 232 233 234 class Analysis(Remark): 235 yaml_tag = '!Analysis' 236 237 @property 238 def color(self): 239 return "white" 240 241 242 class AnalysisFPCommute(Analysis): 243 yaml_tag = '!AnalysisFPCommute' 244 245 246 class AnalysisAliasing(Analysis): 247 yaml_tag = '!AnalysisAliasing' 248 249 250 class Passed(Remark): 251 yaml_tag = '!Passed' 252 253 @property 254 def color(self): 255 return "green" 256 257 258 class Missed(Remark): 259 yaml_tag = '!Missed' 260 261 @property 262 def color(self): 263 return "red" 264 265 266 def get_remarks(input_file): 267 max_hotness = 0 268 all_remarks = dict() 269 file_remarks = defaultdict(functools.partial(defaultdict, list)) 270 271 with open(input_file) as f: 272 docs = yaml.load_all(f, Loader=Loader) 273 for remark in docs: 274 remark.canonicalize() 275 # Avoid remarks withoug debug location or if they are duplicated 276 if not hasattr(remark, 'DebugLoc') or remark.key in all_remarks: 277 continue 278 all_remarks[remark.key] = remark 279 280 file_remarks[remark.File][remark.Line].append(remark) 281 282 # If we're reading a back a diff yaml file, max_hotness is already 283 # captured which may actually be less than the max hotness found 284 # in the file. 285 if hasattr(remark, 'max_hotness'): 286 max_hotness = remark.max_hotness 287 max_hotness = max(max_hotness, remark.Hotness) 288 289 return max_hotness, all_remarks, file_remarks 290 291 292 def gather_results(filenames, num_jobs, should_print_progress): 293 if should_print_progress: 294 print('Reading YAML files...') 295 if not Remark.demangler_proc: 296 Remark.set_demangler(Remark.default_demangler) 297 remarks = optpmap.pmap( 298 get_remarks, filenames, num_jobs, should_print_progress) 299 max_hotness = max(entry[0] for entry in remarks) 300 301 def merge_file_remarks(file_remarks_job, all_remarks, merged): 302 for filename, d in iteritems(file_remarks_job): 303 for line, remarks in iteritems(d): 304 for remark in remarks: 305 # Bring max_hotness into the remarks so that 306 # RelativeHotness does not depend on an external global. 307 remark.max_hotness = max_hotness 308 if remark.key not in all_remarks: 309 merged[filename][line].append(remark) 310 311 all_remarks = dict() 312 file_remarks = defaultdict(functools.partial(defaultdict, list)) 313 for _, all_remarks_job, file_remarks_job in remarks: 314 merge_file_remarks(file_remarks_job, all_remarks, file_remarks) 315 all_remarks.update(all_remarks_job) 316 317 return all_remarks, file_remarks, max_hotness != 0 318 319 320 def find_opt_files(*dirs_or_files): 321 all = [] 322 for dir_or_file in dirs_or_files: 323 if os.path.isfile(dir_or_file): 324 all.append(dir_or_file) 325 else: 326 for dir, subdirs, files in os.walk(dir_or_file): 327 # Exclude mounted directories and symlinks (os.walk default). 328 subdirs[:] = [d for d in subdirs 329 if not os.path.ismount(os.path.join(dir, d))] 330 for file in files: 331 if fnmatch.fnmatch(file, "*.opt.yaml*"): 332 all.append(os.path.join(dir, file)) 333 return all 334
