Home | History | Annotate | Download | only in tools 
      1 #!/usr/bin/env python
      2 #
      3 # Copyright 2010 the V8 project authors. All rights reserved.
      4 # Redistribution and use in source and binary forms, with or without
      5 # modification, are permitted provided that the following conditions are
      6 # met:
      7 #
      8 #     * Redistributions of source code must retain the above copyright
      9 #       notice, this list of conditions and the following disclaimer.
     10 #     * Redistributions in binary form must reproduce the above
     11 #       copyright notice, this list of conditions and the following
     12 #       disclaimer in the documentation and/or other materials provided
     13 #       with the distribution.
     14 #     * Neither the name of Google Inc. nor the names of its
     15 #       contributors may be used to endorse or promote products derived
     16 #       from this software without specific prior written permission.
     17 #
     18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
     19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
     20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
     21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
     22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
     23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
     24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
     25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
     26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
     28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     29 #
     30 
     31 #
     32 # This is an utility for plotting charts based on GC traces produced by V8 when
     33 # run with flags --trace-gc --trace-gc-nvp. Relies on gnuplot for actual
     34 # plotting.
     35 #
     36 # Usage: gc-nvp-trace-processor.py <GC-trace-filename>
     37 #
     38 
     39 
     40 from __future__ import with_statement
     41 import sys, types, subprocess, math
     42 import gc_nvp_common
     43 
     44 def flatten(l):
     45   flat = []
     46   for i in l: flat.extend(i)
     47   return flat
     48 
     49 def gnuplot(script):
     50   gnuplot = subprocess.Popen(["gnuplot"], stdin=subprocess.PIPE)
     51   gnuplot.stdin.write(script)
     52   gnuplot.stdin.close()
     53   gnuplot.wait()
     54 
     55 x1y1 = 'x1y1'
     56 x1y2 = 'x1y2'
     57 x2y1 = 'x2y1'
     58 x2y2 = 'x2y2'
     59 
     60 class Item(object):
     61   def __init__(self, title, field, axis = x1y1, **keywords):
     62     self.title = title
     63     self.axis = axis
     64     self.props = keywords
     65     if type(field) is types.ListType:
     66       self.field = field
     67     else:
     68       self.field = [field]
     69 
     70   def fieldrefs(self):
     71     return self.field
     72 
     73   def to_gnuplot(self, context):
     74     args = ['"%s"' % context.datafile,
     75             'using %s' % context.format_fieldref(self.field),
     76             'title "%s"' % self.title,
     77             'axis %s' % self.axis]
     78     if 'style' in self.props:
     79       args.append('with %s' % self.props['style'])
     80     if 'lc' in self.props:
     81       args.append('lc rgb "%s"' % self.props['lc'])
     82     if 'fs' in self.props:
     83       args.append('fs %s' % self.props['fs'])
     84     return ' '.join(args)
     85 
     86 class Plot(object):
     87   def __init__(self, *items):
     88     self.items = items
     89 
     90   def fieldrefs(self):
     91     return flatten([item.fieldrefs() for item in self.items])
     92 
     93   def to_gnuplot(self, ctx):
     94     return 'plot ' + ', '.join([item.to_gnuplot(ctx) for item in self.items])
     95 
     96 class Set(object):
     97   def __init__(self, value):
     98     self.value = value
     99 
    100   def to_gnuplot(self, ctx):
    101     return 'set ' + self.value
    102 
    103   def fieldrefs(self):
    104     return []
    105 
    106 class Context(object):
    107   def __init__(self, datafile, field_to_index):
    108     self.datafile = datafile
    109     self.field_to_index = field_to_index
    110 
    111   def format_fieldref(self, fieldref):
    112     return ':'.join([str(self.field_to_index[field]) for field in fieldref])
    113 
    114 def collect_fields(plot):
    115   field_to_index = {}
    116   fields = []
    117 
    118   def add_field(field):
    119     if field not in field_to_index:
    120       fields.append(field)
    121       field_to_index[field] = len(fields)
    122 
    123   for field in flatten([item.fieldrefs() for item in plot]):
    124     add_field(field)
    125 
    126   return (fields, field_to_index)
    127 
    128 def is_y2_used(plot):
    129   for subplot in plot:
    130     if isinstance(subplot, Plot):
    131       for item in subplot.items:
    132         if item.axis == x1y2 or item.axis == x2y2:
    133           return True
    134   return False
    135 
    136 def get_field(trace_line, field):
    137   t = type(field)
    138   if t is types.StringType:
    139     return trace_line[field]
    140   elif t is types.FunctionType:
    141     return field(trace_line)
    142 
    143 def generate_datafile(datafile_name, trace, fields):
    144   with open(datafile_name, 'w') as datafile:
    145     for line in trace:
    146       data_line = [str(get_field(line, field)) for field in fields]
    147       datafile.write('\t'.join(data_line))
    148       datafile.write('\n')
    149 
    150 def generate_script_and_datafile(plot, trace, datafile, output):
    151   (fields, field_to_index) = collect_fields(plot)
    152   generate_datafile(datafile, trace, fields)
    153   script = [
    154       'set terminal png',
    155       'set output "%s"' % output,
    156       'set autoscale',
    157       'set ytics nomirror',
    158       'set xtics nomirror',
    159       'set key below'
    160   ]
    161 
    162   if is_y2_used(plot):
    163     script.append('set autoscale y2')
    164     script.append('set y2tics')
    165 
    166   context = Context(datafile, field_to_index)
    167 
    168   for item in plot:
    169     script.append(item.to_gnuplot(context))
    170 
    171   return '\n'.join(script)
    172 
    173 def plot_all(plots, trace, prefix):
    174   charts = []
    175 
    176   for plot in plots:
    177     outfilename = "%s_%d.png" % (prefix, len(charts))
    178     charts.append(outfilename)
    179     script = generate_script_and_datafile(plot, trace, '~datafile', outfilename)
    180     print 'Plotting %s...' % outfilename
    181     gnuplot(script)
    182 
    183   return charts
    184 
    185 def reclaimed_bytes(row):
    186   return row['total_size_before'] - row['total_size_after']
    187 
    188 def other_scope(r):
    189   if r['gc'] == 's':
    190     # there is no 'other' scope for scavenging collections.
    191     return 0
    192   return r['pause'] - r['mark'] - r['sweep'] - r['external']
    193 
    194 def scavenge_scope(r):
    195   if r['gc'] == 's':
    196     return r['pause'] - r['external']
    197   return 0
    198 
    199 
    200 def real_mutator(r):
    201   return r['mutator'] - r['steps_took']
    202 
    203 plots = [
    204   [
    205     Set('style fill solid 0.5 noborder'),
    206     Set('style histogram rowstacked'),
    207     Set('style data histograms'),
    208     Plot(Item('Scavenge', scavenge_scope, lc = 'green'),
    209          Item('Marking', 'mark', lc = 'purple'),
    210          Item('Sweep', 'sweep', lc = 'blue'),
    211          Item('External', 'external', lc = '#489D43'),
    212          Item('Other', other_scope, lc = 'grey'),
    213          Item('IGC Steps', 'steps_took', lc = '#FF6347'))
    214   ],
    215   [
    216     Set('style fill solid 0.5 noborder'),
    217     Set('style histogram rowstacked'),
    218     Set('style data histograms'),
    219     Plot(Item('Scavenge', scavenge_scope, lc = 'green'),
    220          Item('Marking', 'mark', lc = 'purple'),
    221          Item('Sweep', 'sweep', lc = 'blue'),
    222          Item('External', 'external', lc = '#489D43'),
    223          Item('Other', other_scope, lc = '#ADD8E6'),
    224          Item('External', 'external', lc = '#D3D3D3'))
    225   ],
    226 
    227   [
    228     Plot(Item('Mutator', real_mutator, lc = 'black', style = 'lines'))
    229   ],
    230   [
    231     Set('style histogram rowstacked'),
    232     Set('style data histograms'),
    233     Plot(Item('Heap Size (before GC)', 'total_size_before', x1y2,
    234               fs = 'solid 0.4 noborder',
    235               lc = 'green'),
    236          Item('Total holes (after GC)', 'holes_size_before', x1y2,
    237               fs = 'solid 0.4 noborder',
    238               lc = 'red'),
    239          Item('GC Time', ['i', 'pause'], style = 'lines', lc = 'red'))
    240   ],
    241   [
    242     Set('style histogram rowstacked'),
    243     Set('style data histograms'),
    244     Plot(Item('Heap Size (after GC)', 'total_size_after', x1y2,
    245               fs = 'solid 0.4 noborder',
    246               lc = 'green'),
    247          Item('Total holes (after GC)', 'holes_size_after', x1y2,
    248               fs = 'solid 0.4 noborder',
    249               lc = 'red'),
    250          Item('GC Time', ['i', 'pause'],
    251               style = 'lines',
    252               lc = 'red'))
    253   ],
    254   [
    255     Set('style fill solid 0.5 noborder'),
    256     Set('style data histograms'),
    257     Plot(Item('Allocated', 'allocated'),
    258          Item('Reclaimed', reclaimed_bytes),
    259          Item('Promoted', 'promoted', style = 'lines', lc = 'black'))
    260   ],
    261 ]
    262 
    263 def freduce(f, field, trace, init):
    264   return reduce(lambda t,r: f(t, r[field]), trace, init)
    265 
    266 def calc_total(trace, field):
    267   return freduce(lambda t,v: t + long(v), field, trace, long(0))
    268 
    269 def calc_max(trace, field):
    270   return freduce(lambda t,r: max(t, r), field, trace, 0)
    271 
    272 def count_nonzero(trace, field):
    273   return freduce(lambda t,r: t if r == 0 else t + 1, field, trace, 0)
    274 
    275 
    276 def process_trace(filename):
    277   trace = gc_nvp_common.parse_gc_trace(filename)
    278 
    279   marksweeps = filter(lambda r: r['gc'] == 'ms', trace)
    280   scavenges = filter(lambda r: r['gc'] == 's', trace)
    281   globalgcs = filter(lambda r: r['gc'] != 's', trace)
    282 
    283 
    284   charts = plot_all(plots, trace, filename)
    285 
    286   def stats(out, prefix, trace, field):
    287     n = len(trace)
    288     total = calc_total(trace, field)
    289     max = calc_max(trace, field)
    290     if n > 0:
    291       avg = total / n
    292     else:
    293       avg = 0
    294     if n > 1:
    295       dev = math.sqrt(freduce(lambda t,r: t + (r - avg) ** 2, field, trace, 0) /
    296                       (n - 1))
    297     else:
    298       dev = 0
    299 
    300     out.write('<tr><td>%s</td><td>%d</td><td>%d</td>'
    301               '<td>%d</td><td>%d [dev %f]</td></tr>' %
    302               (prefix, n, total, max, avg, dev))
    303 
    304   def HumanReadable(size):
    305     suffixes = ['bytes', 'kB', 'MB', 'GB']
    306     power = 1
    307     for i in range(len(suffixes)):
    308       if size < power*1024:
    309         return "%.1f" % (float(size) / power) + " " + suffixes[i]
    310       power *= 1024
    311 
    312   def throughput(name, trace):
    313     total_live_after = calc_total(trace, 'total_size_after')
    314     total_live_before = calc_total(trace, 'total_size_before')
    315     total_gc = calc_total(trace, 'pause')
    316     if total_gc == 0:
    317       return
    318     out.write('GC %s Throughput (after): %s / %s ms = %s/ms<br/>' %
    319               (name,
    320                HumanReadable(total_live_after),
    321                total_gc,
    322                HumanReadable(total_live_after / total_gc)))
    323     out.write('GC %s Throughput (before): %s / %s ms = %s/ms<br/>' %
    324               (name,
    325                HumanReadable(total_live_before),
    326                total_gc,
    327                HumanReadable(total_live_before / total_gc)))
    328 
    329 
    330   with open(filename + '.html', 'w') as out:
    331     out.write('<html><body>')
    332     out.write('<table>')
    333     out.write('<tr><td>Phase</td><td>Count</td><td>Time (ms)</td>')
    334     out.write('<td>Max</td><td>Avg</td></tr>')
    335     stats(out, 'Total in GC', trace, 'pause')
    336     stats(out, 'Scavenge', scavenges, 'pause')
    337     stats(out, 'MarkSweep', marksweeps, 'pause')
    338     stats(out, 'Mark', filter(lambda r: r['mark'] != 0, trace), 'mark')
    339     stats(out, 'Sweep', filter(lambda r: r['sweep'] != 0, trace), 'sweep')
    340     stats(out,
    341           'External',
    342           filter(lambda r: r['external'] != 0, trace),
    343           'external')
    344     out.write('</table>')
    345     throughput('TOTAL', trace)
    346     throughput('MS', marksweeps)
    347     throughput('OLDSPACE', globalgcs)
    348     out.write('<br/>')
    349     for chart in charts:
    350       out.write('<img src="%s">' % chart)
    351       out.write('</body></html>')
    352 
    353   print "%s generated." % (filename + '.html')
    354 
    355 if len(sys.argv) != 2:
    356   print "Usage: %s <GC-trace-filename>" % sys.argv[0]
    357   sys.exit(1)
    358 
    359 process_trace(sys.argv[1])
    360