Home | History | Annotate | Download | only in fdo_scripts 
      1 # Copyright 2011 Google Inc. All Rights Reserved.
      2 """Summarize hottest basic blocks found while doing a ChromeOS FDO build.
      3 
      4 Here is an example execution:
      5 
      6   summarize_hot_blocks.py
      7    --data_dir=~/chromeos/chroot/var/cache/chromeos-chrome/ --cutoff=10000
      8    --output_dir=/home/x/y
      9 
     10 With the cutoff, it will ignore any basic blocks that have a count less
     11 than what is specified (in this example 10000)
     12 The script looks inside the directory (this is typically a directory where
     13 the object files are generated) for files with *.profile and *.optimized
     14 suffixes. To get these, the following flags were added to the compiler
     15 invokation within vanilla_vs_fdo.py in the profile-use phase.
     16 
     17               "-fdump-tree-optimized-blocks-lineno "
     18               "-fdump-ipa-profile-blocks-lineno "
     19 
     20 Here is an example of the *.profile and *.optimized files contents:
     21 
     22 # BLOCK 7 freq:3901 count:60342, starting at line 92
     23 # PRED: 6 [39.0%]  count:60342 (true,exec)
     24   [url_canon_internal.cc : 92:28] MEM[(const char * *)source_6(D) + 16B] =
     25   D.28080_17;
     26   [url_canon_internal.cc : 93:41] MEM[(struct Component *)parsed_4(D) + 16B] =
     27   MEM[(const struct Component &)repl_1(D) + 80];
     28 # SUCC: 8 [100.0%]  count:60342 (fallthru,exec)
     29 # BLOCK 8 freq:10000 count:154667, starting at line 321
     30 # PRED: 7 [100.0%]  count:60342 (fallthru,exec) 6 [61.0%]  count:94325
     31 (false,exec)
     32   [url_canon_internal.cc : 321:51] # DEBUG D#10 =>
     33   [googleurl/src/url_canon_internal.cc : 321] &parsed_4(D)->host
     34 
     35 this script finds the blocks with highest count and shows the first line
     36 of each block so that it is easy to identify the origin of the basic block.
     37 
     38 """
     39 
     40 __author__ = 'llozano (at] google.com (Luis Lozano)'
     41 
     42 import optparse
     43 import os
     44 import re
     45 import shutil
     46 import sys
     47 import tempfile
     48 
     49 from cros_utils import command_executer
     50 
     51 
     52 # Given a line, check if it has a block count and return it.
     53 # Return -1 if there is no match
     54 def GetBlockCount(line):
     55   match_obj = re.match('.*# BLOCK \d+ .*count:(\d+)', line)
     56   if match_obj:
     57     return int(match_obj.group(1))
     58   else:
     59     return -1
     60 
     61 
     62 class Collector(object):
     63 
     64   def __init__(self, data_dir, cutoff, output_dir, tempdir):
     65     self._data_dir = data_dir
     66     self._cutoff = cutoff
     67     self._output_dir = output_dir
     68     self._tempdir = tempdir
     69     self._ce = command_executer.GetCommandExecuter()
     70 
     71   def CollectFileList(self, file_exp, list_file):
     72     command = ("find %s -type f -name '%s' > %s" %
     73                (self._data_dir, file_exp,
     74                 os.path.join(self._tempdir, list_file)))
     75     ret = self._ce.RunCommand(command)
     76     if ret:
     77       raise RuntimeError('Failed: %s' % command)
     78 
     79   def SummarizeLines(self, data_file):
     80     sum_lines = []
     81     search_lno = False
     82     for line in data_file:
     83       count = GetBlockCount(line)
     84       if count != -1:
     85         if count >= self._cutoff:
     86           search_lno = True
     87           sum_line = line.strip()
     88           sum_count = count
     89       # look for a line that starts with line number information
     90       elif search_lno and re.match('^\s*\[.*: \d*:\d*]', line):
     91         search_lno = False
     92         sum_lines.append('%d:%s: %s %s' %
     93                          (sum_count, data_file.name, sum_line, line))
     94     return sum_lines
     95 
     96   # Look for blocks in the data file that have a count larger than the cutoff
     97   # and generate a sorted summary file of the hottest blocks.
     98   def SummarizeFile(self, data_file, sum_file):
     99     with open(data_file, 'r') as f:
    100       sum_lines = self.SummarizeLines(f)
    101 
    102     # sort reverse the list in place by the block count number
    103     sum_lines.sort(key=GetBlockCount, reverse=True)
    104 
    105     with open(sum_file, 'w') as sf:
    106       sf.write(''.join(sum_lines))
    107 
    108     print 'Generated file Summary: ', sum_file
    109 
    110   # Find hottest blocks in the list of files, generate a sorted summary for
    111   # each file and then do a sorted merge of all the summaries.
    112   def SummarizeList(self, list_file, summary_file):
    113     with open(os.path.join(self._tempdir, list_file)) as f:
    114       sort_list = []
    115       for file_name in f:
    116         file_name = file_name.strip()
    117         sum_file = '%s.sum' % file_name
    118         sort_list.append('%s%s' % (sum_file, chr(0)))
    119         self.SummarizeFile(file_name, sum_file)
    120 
    121     tmp_list_file = os.path.join(self._tempdir, 'file_list.dat')
    122     with open(tmp_list_file, 'w') as file_list_file:
    123       for x in sort_list:
    124         file_list_file.write(x)
    125 
    126     merge_command = ('sort -nr -t: -k1 --merge --files0-from=%s > %s ' %
    127                      (tmp_list_file, summary_file))
    128 
    129     ret = self._ce.RunCommand(merge_command)
    130     if ret:
    131       raise RuntimeError('Failed: %s' % merge_command)
    132     print 'Generated general summary: ', summary_file
    133 
    134   def SummarizePreOptimized(self, summary_file):
    135     self.CollectFileList('*.profile', 'chrome.profile.list')
    136     self.SummarizeList('chrome.profile.list',
    137                        os.path.join(self._output_dir, summary_file))
    138 
    139   def SummarizeOptimized(self, summary_file):
    140     self.CollectFileList('*.optimized', 'chrome.optimized.list')
    141     self.SummarizeList('chrome.optimized.list',
    142                        os.path.join(self._output_dir, summary_file))
    143 
    144 
    145 def Main(argv):
    146   command_executer.InitCommandExecuter()
    147   usage = ('usage: %prog --data_dir=<dir> --cutoff=<value> '
    148            '--output_dir=<dir> [--keep_tmp]')
    149   parser = optparse.OptionParser(usage=usage)
    150   parser.add_option('--data_dir',
    151                     dest='data_dir',
    152                     help=('directory where the FDO (*.profile and '
    153                           '*.optimized) files are located'))
    154   parser.add_option('--cutoff',
    155                     dest='cutoff',
    156                     help='Minimum count to consider for each basic block')
    157   parser.add_option('--output_dir',
    158                     dest='output_dir',
    159                     help=('directory where summary data will be generated'
    160                           '(pre_optimized.txt, optimized.txt)'))
    161   parser.add_option('--keep_tmp',
    162                     action='store_true',
    163                     dest='keep_tmp',
    164                     default=False,
    165                     help=('Keep directory with temporary files'
    166                           '(for debugging purposes)'))
    167   options = parser.parse_args(argv)[0]
    168   if not all((options.data_dir, options.cutoff, options.output_dir)):
    169     parser.print_help()
    170     sys.exit(1)
    171 
    172   tempdir = tempfile.mkdtemp()
    173 
    174   co = Collector(options.data_dir, int(options.cutoff), options.output_dir,
    175                  tempdir)
    176   co.SummarizePreOptimized('pre_optimized.txt')
    177   co.SummarizeOptimized('optimized.txt')
    178 
    179   if not options.keep_tmp:
    180     shutil.rmtree(tempdir, ignore_errors=True)
    181 
    182   return 0
    183 
    184 
    185 if __name__ == '__main__':
    186   retval = Main(sys.argv)
    187   sys.exit(retval)
    188