1 #!/usr/bin/env python2.7 2 3 from __future__ import print_function 4 5 import yaml 6 # Try to use the C parser. 7 try: 8 from yaml import CLoader as Loader 9 except ImportError: 10 print("For faster parsing, you may want to install libYAML for PyYAML") 11 from yaml import Loader 12 13 import cgi 14 from collections import defaultdict 15 import fnmatch 16 import functools 17 from multiprocessing import Lock 18 import os, os.path 19 import subprocess 20 try: 21 # The previously builtin function `intern()` was moved 22 # to the `sys` module in Python 3. 23 from sys import intern 24 except: 25 pass 26 27 import optpmap 28 29 30 p = subprocess.Popen(['c++filt', '-n'], stdin=subprocess.PIPE, stdout=subprocess.PIPE) 31 p_lock = Lock() 32 33 34 try: 35 dict.iteritems 36 except AttributeError: 37 # Python 3 38 def itervalues(d): 39 return iter(d.values()) 40 def iteritems(d): 41 return iter(d.items()) 42 else: 43 # Python 2 44 def itervalues(d): 45 return d.itervalues() 46 def iteritems(d): 47 return d.iteritems() 48 49 50 def demangle(name): 51 with p_lock: 52 p.stdin.write((name + '\n').encode('utf-8')) 53 p.stdin.flush() 54 return p.stdout.readline().rstrip().decode('utf-8') 55 56 57 def html_file_name(filename): 58 return filename.replace('/', '_').replace('#', '_') + ".html" 59 60 61 def make_link(File, Line): 62 return "\"{}#L{}\"".format(html_file_name(File), Line) 63 64 65 class Remark(yaml.YAMLObject): 66 # Work-around for http://pyyaml.org/ticket/154. 67 yaml_loader = Loader 68 69 # Intern all strings since we have lot of duplication across filenames, 70 # remark text. 71 # 72 # Change Args from a list of dicts to a tuple of tuples. This saves 73 # memory in two ways. One, a small tuple is significantly smaller than a 74 # small dict. Two, using tuple instead of list allows Args to be directly 75 # used as part of the key (in Python only immutable types are hashable). 76 def _reduce_memory(self): 77 self.Pass = intern(self.Pass) 78 self.Name = intern(self.Name) 79 self.Function = intern(self.Function) 80 81 def _reduce_memory_dict(old_dict): 82 new_dict = dict() 83 for (k, v) in iteritems(old_dict): 84 if type(k) is str: 85 k = intern(k) 86 87 if type(v) is str: 88 v = intern(v) 89 elif type(v) is dict: 90 # This handles [{'Caller': ..., 'DebugLoc': { 'File': ... }}] 91 v = _reduce_memory_dict(v) 92 new_dict[k] = v 93 return tuple(new_dict.items()) 94 95 self.Args = tuple([_reduce_memory_dict(arg_dict) for arg_dict in self.Args]) 96 97 # The inverse operation of the dictonary-related memory optimization in 98 # _reduce_memory_dict. E.g. 99 # (('DebugLoc', (('File', ...) ... ))) -> [{'DebugLoc': {'File': ...} ....}] 100 def recover_yaml_structure(self): 101 def tuple_to_dict(t): 102 d = dict() 103 for (k, v) in t: 104 if type(v) is tuple: 105 v = tuple_to_dict(v) 106 d[k] = v 107 return d 108 109 self.Args = [tuple_to_dict(arg_tuple) for arg_tuple in self.Args] 110 111 def canonicalize(self): 112 if not hasattr(self, 'Hotness'): 113 self.Hotness = 0 114 if not hasattr(self, 'Args'): 115 self.Args = [] 116 self._reduce_memory() 117 118 @property 119 def File(self): 120 return self.DebugLoc['File'] 121 122 @property 123 def Line(self): 124 return int(self.DebugLoc['Line']) 125 126 @property 127 def Column(self): 128 return self.DebugLoc['Column'] 129 130 @property 131 def DebugLocString(self): 132 return "{}:{}:{}".format(self.File, self.Line, self.Column) 133 134 @property 135 def DemangledFunctionName(self): 136 return demangle(self.Function) 137 138 @property 139 def Link(self): 140 return make_link(self.File, self.Line) 141 142 def getArgString(self, mapping): 143 mapping = dict(list(mapping)) 144 dl = mapping.get('DebugLoc') 145 if dl: 146 del mapping['DebugLoc'] 147 148 assert(len(mapping) == 1) 149 (key, value) = list(mapping.items())[0] 150 151 if key == 'Caller' or key == 'Callee': 152 value = cgi.escape(demangle(value)) 153 154 if dl and key != 'Caller': 155 dl_dict = dict(list(dl)) 156 return "<a href={}>{}</a>".format( 157 make_link(dl_dict['File'], dl_dict['Line']), value) 158 else: 159 return value 160 161 def getDiffPrefix(self): 162 if hasattr(self, 'Added'): 163 if self.Added: 164 return '+' 165 else: 166 return '-' 167 return '' 168 169 @property 170 def PassWithDiffPrefix(self): 171 return self.getDiffPrefix() + self.Pass 172 173 @property 174 def message(self): 175 # Args is a list of mappings (dictionaries) 176 values = [self.getArgString(mapping) for mapping in self.Args] 177 return "".join(values) 178 179 @property 180 def RelativeHotness(self): 181 if self.max_hotness: 182 return "{0:.2f}%".format(self.Hotness * 100. / self.max_hotness) 183 else: 184 return '' 185 186 @property 187 def key(self): 188 return (self.__class__, self.PassWithDiffPrefix, self.Name, self.File, 189 self.Line, self.Column, self.Function, self.Args) 190 191 def __hash__(self): 192 return hash(self.key) 193 194 def __eq__(self, other): 195 return self.key == other.key 196 197 def __repr__(self): 198 return str(self.key) 199 200 201 class Analysis(Remark): 202 yaml_tag = '!Analysis' 203 204 @property 205 def color(self): 206 return "white" 207 208 209 class AnalysisFPCommute(Analysis): 210 yaml_tag = '!AnalysisFPCommute' 211 212 213 class AnalysisAliasing(Analysis): 214 yaml_tag = '!AnalysisAliasing' 215 216 217 class Passed(Remark): 218 yaml_tag = '!Passed' 219 220 @property 221 def color(self): 222 return "green" 223 224 225 class Missed(Remark): 226 yaml_tag = '!Missed' 227 228 @property 229 def color(self): 230 return "red" 231 232 233 def get_remarks(input_file): 234 max_hotness = 0 235 all_remarks = dict() 236 file_remarks = defaultdict(functools.partial(defaultdict, list)) 237 238 with open(input_file) as f: 239 docs = yaml.load_all(f, Loader=Loader) 240 for remark in docs: 241 remark.canonicalize() 242 # Avoid remarks withoug debug location or if they are duplicated 243 if not hasattr(remark, 'DebugLoc') or remark.key in all_remarks: 244 continue 245 all_remarks[remark.key] = remark 246 247 file_remarks[remark.File][remark.Line].append(remark) 248 249 # If we're reading a back a diff yaml file, max_hotness is already 250 # captured which may actually be less than the max hotness found 251 # in the file. 252 if hasattr(remark, 'max_hotness'): 253 max_hotness = remark.max_hotness 254 max_hotness = max(max_hotness, remark.Hotness) 255 256 return max_hotness, all_remarks, file_remarks 257 258 259 def gather_results(filenames, num_jobs, should_print_progress): 260 if should_print_progress: 261 print('Reading YAML files...') 262 remarks = optpmap.pmap( 263 get_remarks, filenames, num_jobs, should_print_progress) 264 max_hotness = max(entry[0] for entry in remarks) 265 266 def merge_file_remarks(file_remarks_job, all_remarks, merged): 267 for filename, d in iteritems(file_remarks_job): 268 for line, remarks in iteritems(d): 269 for remark in remarks: 270 # Bring max_hotness into the remarks so that 271 # RelativeHotness does not depend on an external global. 272 remark.max_hotness = max_hotness 273 if remark.key not in all_remarks: 274 merged[filename][line].append(remark) 275 276 all_remarks = dict() 277 file_remarks = defaultdict(functools.partial(defaultdict, list)) 278 for _, all_remarks_job, file_remarks_job in remarks: 279 merge_file_remarks(file_remarks_job, all_remarks, file_remarks) 280 all_remarks.update(all_remarks_job) 281 282 return all_remarks, file_remarks, max_hotness != 0 283 284 285 def find_opt_files(*dirs_or_files): 286 all = [] 287 for dir_or_file in dirs_or_files: 288 if os.path.isfile(dir_or_file): 289 all.append(dir_or_file) 290 else: 291 for dir, subdirs, files in os.walk(dir_or_file): 292 # Exclude mounted directories and symlinks (os.walk default). 293 subdirs[:] = [d for d in subdirs 294 if not os.path.ismount(os.path.join(dir, d))] 295 for file in files: 296 if fnmatch.fnmatch(file, "*.opt.yaml"): 297 all.append(os.path.join(dir, file)) 298 return all 299
