1 #!/usr/bin/env python2.7 2 3 from __future__ import print_function 4 5 import argparse 6 import cgi 7 import codecs 8 import errno 9 import functools 10 from multiprocessing import cpu_count 11 import os.path 12 import re 13 import shutil 14 import sys 15 16 from pygments import highlight 17 from pygments.lexers.c_cpp import CppLexer 18 from pygments.formatters import HtmlFormatter 19 20 import optpmap 21 import optrecord 22 23 24 desc = '''Generate HTML output to visualize optimization records from the YAML files 25 generated with -fsave-optimization-record and -fdiagnostics-show-hotness. 26 27 The tools requires PyYAML and Pygments Python packages.''' 28 29 30 # This allows passing the global context to the child processes. 31 class Context: 32 def __init__(self, caller_loc = dict()): 33 # Map function names to their source location for function where inlining happened 34 self.caller_loc = caller_loc 35 36 context = Context() 37 38 def suppress(remark): 39 if remark.Name == 'sil.Specialized': 40 return remark.getArgDict()['Function'][0].startswith('\"Swift.') 41 elif remark.Name == 'sil.Inlined': 42 return remark.getArgDict()['Callee'][0].startswith(('\"Swift.', '\"specialized Swift.')) 43 return False 44 45 class SourceFileRenderer: 46 def __init__(self, source_dir, output_dir, filename, no_highlight): 47 self.filename = filename 48 existing_filename = None 49 if os.path.exists(filename): 50 existing_filename = filename 51 else: 52 fn = os.path.join(source_dir, filename) 53 if os.path.exists(fn): 54 existing_filename = fn 55 56 self.no_highlight = no_highlight 57 self.stream = codecs.open(os.path.join(output_dir, optrecord.html_file_name(filename)), 'w', encoding='utf-8') 58 if existing_filename: 59 self.source_stream = open(existing_filename) 60 else: 61 self.source_stream = None 62 print(''' 63 <html> 64 <h1>Unable to locate file {}</h1> 65 </html> 66 '''.format(filename), file=self.stream) 67 68 self.html_formatter = HtmlFormatter(encoding='utf-8') 69 self.cpp_lexer = CppLexer(stripnl=False) 70 71 def render_source_lines(self, stream, line_remarks): 72 file_text = stream.read() 73 74 if self.no_highlight: 75 html_highlighted = file_text.decode('utf-8') 76 else: 77 html_highlighted = highlight( 78 file_text, 79 self.cpp_lexer, 80 self.html_formatter) 81 82 # Note that the API is different between Python 2 and 3. On 83 # Python 3, pygments.highlight() returns a bytes object, so we 84 # have to decode. On Python 2, the output is str but since we 85 # support unicode characters and the output streams is unicode we 86 # decode too. 87 html_highlighted = html_highlighted.decode('utf-8') 88 89 # Take off the header and footer, these must be 90 # reapplied line-wise, within the page structure 91 html_highlighted = html_highlighted.replace('<div class="highlight"><pre>', '') 92 html_highlighted = html_highlighted.replace('</pre></div>', '') 93 94 for (linenum, html_line) in enumerate(html_highlighted.split('\n'), start=1): 95 print(u''' 96 <tr> 97 <td><a name=\"L{linenum}\">{linenum}</a></td> 98 <td></td> 99 <td></td> 100 <td><div class="highlight"><pre>{html_line}</pre></div></td> 101 </tr>'''.format(**locals()), file=self.stream) 102 103 for remark in line_remarks.get(linenum, []): 104 if not suppress(remark): 105 self.render_inline_remarks(remark, html_line) 106 107 def render_inline_remarks(self, r, line): 108 inlining_context = r.DemangledFunctionName 109 dl = context.caller_loc.get(r.Function) 110 if dl: 111 dl_dict = dict(list(dl)) 112 link = optrecord.make_link(dl_dict['File'], dl_dict['Line'] - 2) 113 inlining_context = "<a href={link}>{r.DemangledFunctionName}</a>".format(**locals()) 114 115 # Column is the number of characters *including* tabs, keep those and 116 # replace everything else with spaces. 117 indent = line[:max(r.Column, 1) - 1] 118 indent = re.sub('\S', ' ', indent) 119 120 print(u''' 121 <tr> 122 <td></td> 123 <td>{r.RelativeHotness}</td> 124 <td class=\"column-entry-{r.color}\">{r.PassWithDiffPrefix}</td> 125 <td><pre style="display:inline">{indent}</pre><span class=\"column-entry-yellow\"> {r.message} </span></td> 126 <td class=\"column-entry-yellow\">{inlining_context}</td> 127 </tr>'''.format(**locals()), file=self.stream) 128 129 def render(self, line_remarks): 130 if not self.source_stream: 131 return 132 133 print(''' 134 <html> 135 <title>{}</title> 136 <meta charset="utf-8" /> 137 <head> 138 <link rel='stylesheet' type='text/css' href='style.css'> 139 </head> 140 <body> 141 <div class="centered"> 142 <table class="source"> 143 <thead> 144 <tr> 145 <th style="width: 2%">Line</td> 146 <th style="width: 3%">Hotness</td> 147 <th style="width: 10%">Optimization</td> 148 <th style="width: 70%">Source</td> 149 <th style="width: 15%">Inline Context</td> 150 </tr> 151 </thead> 152 <tbody>'''.format(os.path.basename(self.filename)), file=self.stream) 153 self.render_source_lines(self.source_stream, line_remarks) 154 155 print(''' 156 </tbody> 157 </table> 158 </body> 159 </html>''', file=self.stream) 160 161 162 class IndexRenderer: 163 def __init__(self, output_dir, should_display_hotness, max_hottest_remarks_on_index): 164 self.stream = codecs.open(os.path.join(output_dir, 'index.html'), 'w', encoding='utf-8') 165 self.should_display_hotness = should_display_hotness 166 self.max_hottest_remarks_on_index = max_hottest_remarks_on_index 167 168 def render_entry(self, r, odd): 169 escaped_name = cgi.escape(r.DemangledFunctionName) 170 print(u''' 171 <tr> 172 <td class=\"column-entry-{odd}\"><a href={r.Link}>{r.DebugLocString}</a></td> 173 <td class=\"column-entry-{odd}\">{r.RelativeHotness}</td> 174 <td class=\"column-entry-{odd}\">{escaped_name}</td> 175 <td class=\"column-entry-{r.color}\">{r.PassWithDiffPrefix}</td> 176 </tr>'''.format(**locals()), file=self.stream) 177 178 def render(self, all_remarks): 179 print(''' 180 <html> 181 <meta charset="utf-8" /> 182 <head> 183 <link rel='stylesheet' type='text/css' href='style.css'> 184 </head> 185 <body> 186 <div class="centered"> 187 <table> 188 <tr> 189 <td>Source Location</td> 190 <td>Hotness</td> 191 <td>Function</td> 192 <td>Pass</td> 193 </tr>''', file=self.stream) 194 195 max_entries = None 196 if self.should_display_hotness: 197 max_entries = self.max_hottest_remarks_on_index 198 199 for i, remark in enumerate(all_remarks[:max_entries]): 200 if not suppress(remark): 201 self.render_entry(remark, i % 2) 202 print(''' 203 </table> 204 </body> 205 </html>''', file=self.stream) 206 207 208 def _render_file(source_dir, output_dir, ctx, no_highlight, entry): 209 global context 210 context = ctx 211 filename, remarks = entry 212 SourceFileRenderer(source_dir, output_dir, filename, no_highlight).render(remarks) 213 214 215 def map_remarks(all_remarks): 216 # Set up a map between function names and their source location for 217 # function where inlining happened 218 for remark in optrecord.itervalues(all_remarks): 219 if isinstance(remark, optrecord.Passed) and remark.Pass == "inline" and remark.Name == "Inlined": 220 for arg in remark.Args: 221 arg_dict = dict(list(arg)) 222 caller = arg_dict.get('Caller') 223 if caller: 224 try: 225 context.caller_loc[caller] = arg_dict['DebugLoc'] 226 except KeyError: 227 pass 228 229 230 def generate_report(all_remarks, 231 file_remarks, 232 source_dir, 233 output_dir, 234 no_highlight, 235 should_display_hotness, 236 max_hottest_remarks_on_index, 237 num_jobs, 238 should_print_progress): 239 try: 240 os.makedirs(output_dir) 241 except OSError as e: 242 if e.errno == errno.EEXIST and os.path.isdir(output_dir): 243 pass 244 else: 245 raise 246 247 if should_print_progress: 248 print('Rendering index page...') 249 if should_display_hotness: 250 sorted_remarks = sorted(optrecord.itervalues(all_remarks), key=lambda r: (r.Hotness, r.File, r.Line, r.Column, r.PassWithDiffPrefix, r.yaml_tag, r.Function), reverse=True) 251 else: 252 sorted_remarks = sorted(optrecord.itervalues(all_remarks), key=lambda r: (r.File, r.Line, r.Column, r.PassWithDiffPrefix, r.yaml_tag, r.Function)) 253 IndexRenderer(output_dir, should_display_hotness, max_hottest_remarks_on_index).render(sorted_remarks) 254 255 shutil.copy(os.path.join(os.path.dirname(os.path.realpath(__file__)), 256 "style.css"), output_dir) 257 258 _render_file_bound = functools.partial(_render_file, source_dir, output_dir, context, no_highlight) 259 if should_print_progress: 260 print('Rendering HTML files...') 261 optpmap.pmap(_render_file_bound, 262 file_remarks.items(), 263 num_jobs, 264 should_print_progress) 265 266 267 def main(): 268 parser = argparse.ArgumentParser(description=desc) 269 parser.add_argument( 270 'yaml_dirs_or_files', 271 nargs='+', 272 help='List of optimization record files or directories searched ' 273 'for optimization record files.') 274 parser.add_argument( 275 '--output-dir', 276 '-o', 277 default='html', 278 help='Path to a directory where generated HTML files will be output. ' 279 'If the directory does not already exist, it will be created. ' 280 '"%(default)s" by default.') 281 parser.add_argument( 282 '--jobs', 283 '-j', 284 default=None, 285 type=int, 286 help='Max job count (defaults to %(default)s, the current CPU count)') 287 parser.add_argument( 288 '--source-dir', 289 '-s', 290 default='', 291 help='set source directory') 292 parser.add_argument( 293 '--no-progress-indicator', 294 '-n', 295 action='store_true', 296 default=False, 297 help='Do not display any indicator of how many YAML files were read ' 298 'or rendered into HTML.') 299 parser.add_argument( 300 '--max-hottest-remarks-on-index', 301 default=1000, 302 type=int, 303 help='Maximum number of the hottest remarks to appear on the index page') 304 parser.add_argument( 305 '--no-highlight', 306 action='store_true', 307 default=False, 308 help='Do not use a syntax highlighter when rendering the source code') 309 parser.add_argument( 310 '--demangler', 311 help='Set the demangler to be used (defaults to %s)' % optrecord.Remark.default_demangler) 312 313 # Do not make this a global variable. Values needed to be propagated through 314 # to individual classes and functions to be portable with multiprocessing across 315 # Windows and non-Windows. 316 args = parser.parse_args() 317 318 print_progress = not args.no_progress_indicator 319 if args.demangler: 320 optrecord.Remark.set_demangler(args.demangler) 321 322 files = optrecord.find_opt_files(*args.yaml_dirs_or_files) 323 if not files: 324 parser.error("No *.opt.yaml files found") 325 sys.exit(1) 326 327 all_remarks, file_remarks, should_display_hotness = \ 328 optrecord.gather_results(files, args.jobs, print_progress) 329 330 map_remarks(all_remarks) 331 332 generate_report(all_remarks, 333 file_remarks, 334 args.source_dir, 335 args.output_dir, 336 args.no_highlight, 337 should_display_hotness, 338 args.max_hottest_remarks_on_index, 339 args.jobs, 340 print_progress) 341 342 if __name__ == '__main__': 343 main() 344