1 #! /usr/bin/env python 2 # 3 # btt_plot.py: Generate matplotlib plots for BTT generate data files 4 # 5 # (C) Copyright 2009 Hewlett-Packard Development Company, L.P. 6 # 7 # This program is free software; you can redistribute it and/or modify 8 # it under the terms of the GNU General Public License as published by 9 # the Free Software Foundation; either version 2 of the License, or 10 # (at your option) any later version. 11 # 12 # This program is distributed in the hope that it will be useful, 13 # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 # GNU General Public License for more details. 16 # 17 # You should have received a copy of the GNU General Public License 18 # along with this program; if not, write to the Free Software 19 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 20 # 21 22 """ 23 btt_plot.py: Generate matplotlib plots for BTT generated data files 24 25 Files handled: 26 AQD - Average Queue Depth Running average of queue depths 27 28 BNOS - Block numbers accessed Markers for each block 29 30 Q2D - Queue to Issue latencies Running averages 31 D2C - Issue to Complete latencies Running averages 32 Q2C - Queue to Complete latencies Running averages 33 34 Usage: 35 btt_plot_aqd.py equivalent to: btt_plot.py -t aqd <type>=aqd 36 btt_plot_bnos.py equivalent to: btt_plot.py -t bnos <type>=bnos 37 btt_plot_q2d.py equivalent to: btt_plot.py -t q2d <type>=q2d 38 btt_plot_d2c.py equivalent to: btt_plot.py -t d2c <type>=d2c 39 btt_plot_q2c.py equivalent to: btt_plot.py -t q2c <type>=q2c 40 41 Arguments: 42 [ -A | --generate-all ] Default: False 43 [ -L | --no-legend ] Default: Legend table produced 44 [ -o <file> | --output=<file> ] Default: <type>.png 45 [ -T <string> | --title=<string> ] Default: Based upon <type> 46 [ -v | --verbose ] Default: False 47 <data-files...> 48 49 The -A (--generate-all) argument is different: when this is specified, 50 an attempt is made to generate default plots for all 5 types (aqd, bnos, 51 q2d, d2c and q2c). It will find files with the appropriate suffix for 52 each type ('aqd.dat' for example). If such files are found, a plot for 53 that type will be made. The output file name will be the default for 54 each type. The -L (--no-legend) option will be obeyed for all plots, 55 but the -o (--output) and -T (--title) options will be ignored. 56 """ 57 58 __author__ = 'Alan D. Brunelle <alan.brunelle (at] hp.com>' 59 60 #------------------------------------------------------------------------------ 61 62 import matplotlib 63 matplotlib.use('Agg') 64 import getopt, glob, os, sys 65 import matplotlib.pyplot as plt 66 67 plot_size = [10.9, 8.4] # inches... 68 69 add_legend = True 70 generate_all = False 71 output_file = None 72 title_str = None 73 type = None 74 verbose = False 75 76 types = [ 'aqd', 'q2d', 'd2c', 'q2c', 'bnos' ] 77 progs = [ 'btt_plot_%s.py' % t for t in types ] 78 79 get_base = lambda file: file[file.find('_')+1:file.rfind('_')] 80 81 #------------------------------------------------------------------------------ 82 def fatal(msg): 83 """Generate fatal error message and exit""" 84 85 print >>sys.stderr, 'FATAL: %s' % msg 86 sys.exit(1) 87 88 #---------------------------------------------------------------------- 89 def get_data(files): 90 """Retrieve data from files provided. 91 92 Returns a database containing: 93 'min_x', 'max_x' - Minimum and maximum X values found 94 'min_y', 'max_y' - Minimum and maximum Y values found 95 'x', 'y' - X & Y value arrays 96 'ax', 'ay' - Running average over X & Y -- 97 if > 10 values provided... 98 """ 99 #-------------------------------------------------------------- 100 def check(mn, mx, v): 101 """Returns new min, max, and float value for those passed in""" 102 103 v = float(v) 104 if mn == None or v < mn: mn = v 105 if mx == None or v > mx: mx = v 106 return mn, mx, v 107 108 #-------------------------------------------------------------- 109 def avg(xs, ys): 110 """Computes running average for Xs and Ys""" 111 112 #------------------------------------------------------ 113 def _avg(vals): 114 """Computes average for array of values passed""" 115 116 total = 0.0 117 for val in vals: 118 total += val 119 return total / len(vals) 120 121 #------------------------------------------------------ 122 if len(xs) < 1000: 123 return xs, ys 124 125 axs = [xs[0]] 126 ays = [ys[0]] 127 _xs = [xs[0]] 128 _ys = [ys[0]] 129 130 x_range = (xs[-1] - xs[0]) / 100 131 for idx in range(1, len(ys)): 132 if (xs[idx] - _xs[0]) > x_range: 133 axs.append(_avg(_xs)) 134 ays.append(_avg(_ys)) 135 del _xs, _ys 136 137 _xs = [xs[idx]] 138 _ys = [ys[idx]] 139 else: 140 _xs.append(xs[idx]) 141 _ys.append(ys[idx]) 142 143 if len(_xs) > 1: 144 axs.append(_avg(_xs)) 145 ays.append(_avg(_ys)) 146 147 return axs, ays 148 149 #-------------------------------------------------------------- 150 global verbose 151 152 db = {} 153 min_x = max_x = min_y = max_y = None 154 for file in files: 155 if not os.path.exists(file): 156 fatal('%s not found' % file) 157 elif verbose: 158 print 'Processing %s' % file 159 160 xs = [] 161 ys = [] 162 for line in open(file, 'r'): 163 f = line.rstrip().split(None) 164 if line.find('#') == 0 or len(f) < 2: 165 continue 166 (min_x, max_x, x) = check(min_x, max_x, f[0]) 167 (min_y, max_y, y) = check(min_y, max_y, f[1]) 168 xs.append(x) 169 ys.append(y) 170 171 db[file] = {'x':xs, 'y':ys} 172 if len(xs) > 10: 173 db[file]['ax'], db[file]['ay'] = avg(xs, ys) 174 else: 175 db[file]['ax'] = db[file]['ay'] = None 176 177 db['min_x'] = min_x 178 db['max_x'] = max_x 179 db['min_y'] = min_y 180 db['max_y'] = max_y 181 return db 182 183 #---------------------------------------------------------------------- 184 def parse_args(args): 185 """Parse command line arguments. 186 187 Returns list of (data) files that need to be processed -- /unless/ 188 the -A (--generate-all) option is passed, in which case superfluous 189 data files are ignored... 190 """ 191 192 global add_legend, output_file, title_str, type, verbose 193 global generate_all 194 195 prog = args[0][args[0].rfind('/')+1:] 196 if prog == 'btt_plot.py': 197 pass 198 elif not prog in progs: 199 fatal('%s not a valid command name' % prog) 200 else: 201 type = prog[prog.rfind('_')+1:prog.rfind('.py')] 202 203 s_opts = 'ALo:t:T:v' 204 l_opts = [ 'generate-all', 'type', 'no-legend', 'output', 'title', 205 'verbose' ] 206 207 try: 208 (opts, args) = getopt.getopt(args[1:], s_opts, l_opts) 209 except getopt.error, msg: 210 print >>sys.stderr, msg 211 fatal(__doc__) 212 213 for (o, a) in opts: 214 if o in ('-A', '--generate-all'): 215 generate_all = True 216 elif o in ('-L', '--no-legend'): 217 add_legend = False 218 elif o in ('-o', '--output'): 219 output_file = a 220 elif o in ('-t', '--type'): 221 if not a in types: 222 fatal('Type %s not supported' % a) 223 type = a 224 elif o in ('-T', '--title'): 225 title_str = a 226 elif o in ('-v', '--verbose'): 227 verbose = True 228 229 if type == None and not generate_all: 230 fatal('Need type of data files to process - (-t <type>)') 231 232 return args 233 234 #------------------------------------------------------------------------------ 235 def gen_title(fig, type, title_str): 236 """Sets the title for the figure based upon the type /or/ user title""" 237 238 if title_str != None: 239 pass 240 elif type == 'aqd': 241 title_str = 'Average Queue Depth' 242 elif type == 'bnos': 243 title_str = 'Block Numbers Accessed' 244 elif type == 'q2d': 245 title_str = 'Queue (Q) To Issue (D) Average Latencies' 246 elif type == 'd2c': 247 title_str = 'Issue (D) To Complete (C) Average Latencies' 248 elif type == 'q2c': 249 title_str = 'Queue (Q) To Complete (C) Average Latencies' 250 251 title = fig.text(.5, .95, title_str, horizontalalignment='center') 252 title.set_fontsize('large') 253 254 #------------------------------------------------------------------------------ 255 def gen_labels(db, ax, type): 256 """Generate X & Y 'axis'""" 257 258 #---------------------------------------------------------------------- 259 def gen_ylabel(ax, type): 260 """Set the Y axis label based upon the type""" 261 262 if type == 'aqd': 263 str = 'Number of Requests Queued' 264 elif type == 'bnos': 265 str = 'Block Number' 266 else: 267 str = 'Seconds' 268 ax.set_ylabel(str) 269 270 #---------------------------------------------------------------------- 271 xdelta = 0.1 * (db['max_x'] - db['min_x']) 272 ydelta = 0.1 * (db['max_y'] - db['min_y']) 273 274 ax.set_xlim(db['min_x'] - xdelta, db['max_x'] + xdelta) 275 ax.set_ylim(db['min_y'] - ydelta, db['max_y'] + ydelta) 276 ax.set_xlabel('Runtime (seconds)') 277 ax.grid(True) 278 gen_ylabel(ax, type) 279 280 #------------------------------------------------------------------------------ 281 def generate_output(type, db): 282 """Generate the output plot based upon the type and database""" 283 284 #---------------------------------------------------------------------- 285 def color(idx, style): 286 """Returns a color/symbol type based upon the index passed.""" 287 288 colors = [ 'b', 'g', 'r', 'c', 'm', 'y', 'k' ] 289 l_styles = [ '-', ':', '--', '-.' ] 290 m_styles = [ 'o', '+', '.', ',', 's', 'v', 'x', '<', '>' ] 291 292 color = colors[idx % len(colors)] 293 if style == 'line': 294 style = l_styles[(idx / len(l_styles)) % len(l_styles)] 295 elif style == 'marker': 296 style = m_styles[(idx / len(m_styles)) % len(m_styles)] 297 298 return '%s%s' % (color, style) 299 300 #---------------------------------------------------------------------- 301 def gen_legends(a, legends): 302 leg = ax.legend(legends, 'best', shadow=True) 303 frame = leg.get_frame() 304 frame.set_facecolor('0.80') 305 for t in leg.get_texts(): 306 t.set_fontsize('xx-small') 307 308 #---------------------------------------------------------------------- 309 global add_legend, output_file, title_str, verbose 310 311 if output_file != None: 312 ofile = output_file 313 else: 314 ofile = '%s.png' % type 315 316 if verbose: 317 print 'Generating plot into %s' % ofile 318 319 fig = plt.figure(figsize=plot_size) 320 ax = fig.add_subplot(111) 321 322 gen_title(fig, type, title_str) 323 gen_labels(db, ax, type) 324 325 idx = 0 326 if add_legend: 327 legends = [] 328 else: 329 legends = None 330 331 keys = [] 332 for file in db.iterkeys(): 333 if not file in ['min_x', 'max_x', 'min_y', 'max_y']: 334 keys.append(file) 335 336 keys.sort() 337 for file in keys: 338 dat = db[file] 339 if type == 'bnos': 340 ax.plot(dat['x'], dat['y'], color(idx, 'marker'), 341 markersize=1) 342 elif dat['ax'] == None: 343 continue # Don't add legend 344 else: 345 ax.plot(dat['ax'], dat['ay'], color(idx, 'line'), 346 linewidth=1.0) 347 if add_legend: 348 legends.append(get_base(file)) 349 idx += 1 350 351 if add_legend and len(legends) > 0: 352 gen_legends(ax, legends) 353 plt.savefig(ofile) 354 355 #------------------------------------------------------------------------------ 356 def get_files(type): 357 """Returns the list of files for the -A option based upon type""" 358 359 if type == 'bnos': 360 files = [] 361 for fn in glob.glob('*c.dat'): 362 for t in [ 'q2q', 'd2d', 'q2c', 'd2c' ]: 363 if fn.find(t) >= 0: 364 break 365 else: 366 files.append(fn) 367 else: 368 files = glob.glob('*%s.dat' % type) 369 return files 370 371 #------------------------------------------------------------------------------ 372 if __name__ == '__main__': 373 files = parse_args(sys.argv) 374 375 if generate_all: 376 output_file = title_str = type = None 377 for t in types: 378 files = get_files(t) 379 if len(files) == 0: 380 continue 381 elif t != 'bnos': 382 generate_output(t, get_data(files)) 383 continue 384 385 for file in files: 386 base = get_base(file) 387 title_str = 'Block Numbers Accessed: %s' % base 388 output_file = 'bnos_%s.png' % base 389 generate_output(t, get_data([file])) 390 elif len(files) < 1: 391 fatal('Need data files to process') 392 else: 393 generate_output(type, get_data(files)) 394 sys.exit(0) 395
