1 #!/usr/bin/env python 2 # 3 # Copyright (C) 2013 eNovance SAS <licensing (at] enovance.com> 4 # Author: Erwan Velu <erwan (at] enovance.com> 5 # 6 # The license below covers all files distributed with fio unless otherwise 7 # noted in the file itself. 8 # 9 # This program is free software; you can redistribute it and/or modify 10 # it under the terms of the GNU General Public License version 2 as 11 # published by the Free Software Foundation. 12 # 13 # This program is distributed in the hope that it will be useful, 14 # but WITHOUT ANY WARRANTY; without even the implied warranty of 15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 # GNU General Public License for more details. 17 # 18 # You should have received a copy of the GNU General Public License 19 # along with this program; if not, write to the Free Software 20 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 21 22 import os 23 import fnmatch 24 import sys 25 import getopt 26 import re 27 import math 28 import shutil 29 30 def find_file(path, pattern): 31 fio_data_file=[] 32 # For all the local files 33 for file in os.listdir(path): 34 # If the file math the regexp 35 if fnmatch.fnmatch(file, pattern): 36 # Let's consider this file 37 fio_data_file.append(file) 38 39 return fio_data_file 40 41 def generate_gnuplot_script(fio_data_file,title,gnuplot_output_filename,gnuplot_output_dir,mode,disk_perf,gpm_dir): 42 if verbose: print "Generating rendering scripts" 43 filename=gnuplot_output_dir+'mygraph' 44 temporary_files.append(filename) 45 f=open(filename,'w') 46 47 # Plotting 3D or comparing graphs doesn't have a meaning unless if there is at least 2 traces 48 if len(fio_data_file) > 1: 49 f.write("call \'%s/graph3D.gpm\' \'%s' \'%s\' \'\' \'%s\' \'%s\'\n" % (gpm_dir,title,gnuplot_output_filename,gnuplot_output_filename,mode)) 50 51 # Setting up the compare files that will be plot later 52 compare=open(gnuplot_output_dir + 'compare.gnuplot','w') 53 compare.write(''' 54 set title '%s' 55 set terminal png size 1280,1024 56 set ytics axis out auto 57 set key top left reverse 58 set xlabel "Time (Seconds)" 59 set ylabel '%s' 60 set yrange [0:] 61 set style line 1 lt 1 lw 3 pt 3 linecolor rgb "green" 62 '''% (title,mode)) 63 compare.close() 64 #Copying the common file for all kind of graph (raw/smooth/trend) 65 compare_raw_filename="compare-%s-2Draw" % (gnuplot_output_filename) 66 compare_smooth_filename="compare-%s-2Dsmooth" % (gnuplot_output_filename) 67 compare_trend_filename="compare-%s-2Dtrend" % (gnuplot_output_filename) 68 69 shutil.copy(gnuplot_output_dir+'compare.gnuplot',gnuplot_output_dir+compare_raw_filename+".gnuplot") 70 shutil.copy(gnuplot_output_dir+'compare.gnuplot',gnuplot_output_dir+compare_smooth_filename+".gnuplot") 71 shutil.copy(gnuplot_output_dir+'compare.gnuplot',gnuplot_output_dir+compare_trend_filename+".gnuplot") 72 temporary_files.append(gnuplot_output_dir+compare_raw_filename+".gnuplot") 73 temporary_files.append(gnuplot_output_dir+compare_smooth_filename+".gnuplot") 74 temporary_files.append(gnuplot_output_dir+compare_trend_filename+".gnuplot") 75 76 #Setting up a different output filename for each kind of graph 77 compare_raw=open(gnuplot_output_dir+compare_raw_filename + ".gnuplot",'a') 78 compare_raw.write("set output '%s.png'\n" % compare_raw_filename) 79 compare_smooth=open(gnuplot_output_dir+compare_smooth_filename+".gnuplot",'a') 80 compare_smooth.write("set output '%s.png'\n" % compare_smooth_filename) 81 compare_trend=open(gnuplot_output_dir+compare_trend_filename+".gnuplot",'a') 82 compare_trend.write("set output '%s.png'\n" % compare_trend_filename) 83 84 # Let's plot the average value for all the traces 85 global_disk_perf = sum(disk_perf, []) 86 global_avg = average(global_disk_perf) 87 compare_raw.write("plot %s w l ls 1 ti 'Global average value (%.2f)'" % (global_avg,global_avg)); 88 compare_smooth.write("plot %s w l ls 1 ti 'Global average value (%.2f)'" % (global_avg,global_avg)); 89 compare_trend.write("plot %s w l ls 1 ti 'Global average value (%.2f)'" % (global_avg,global_avg)); 90 91 pos=0 92 # Let's create a temporary file for each selected fio file 93 for file in fio_data_file: 94 tmp_filename = "gnuplot_temp_file.%d" % pos 95 96 # Plotting comparing graphs doesn't have a meaning unless if there is at least 2 traces 97 if len(fio_data_file) > 1: 98 # Adding the plot instruction for each kind of comparing graphs 99 compare_raw.write(",\\\n'%s' using 2:3 with linespoints title '%s'" % (tmp_filename,fio_data_file[pos])) 100 compare_smooth.write(",\\\n'%s' using 2:3 smooth csplines title '%s'" % (tmp_filename,fio_data_file[pos])) 101 compare_trend.write(",\\\n'%s' using 2:3 smooth bezier title '%s'" % (tmp_filename,fio_data_file[pos])) 102 103 png_file=file.replace('.log','') 104 raw_filename = "%s-2Draw" % (png_file) 105 smooth_filename = "%s-2Dsmooth" % (png_file) 106 trend_filename = "%s-2Dtrend" % (png_file) 107 avg = average(disk_perf[pos]) 108 f.write("call \'%s/graph2D.gpm\' \'%s' \'%s\' \'%s\' \'%s\' \'%s\' \'%s\' \'%s\' \'%f\'\n" % (gpm_dir,title,tmp_filename,fio_data_file[pos],raw_filename,mode,smooth_filename,trend_filename,avg)) 109 pos = pos +1 110 111 # Plotting comparing graphs doesn't have a meaning unless if there is at least 2 traces 112 if len(fio_data_file) > 1: 113 os.remove(gnuplot_output_dir+"compare.gnuplot") 114 compare_raw.close() 115 compare_smooth.close() 116 compare_trend.close() 117 f.close() 118 119 def generate_gnuplot_math_script(title,gnuplot_output_filename,mode,average,gnuplot_output_dir,gpm_dir): 120 filename=gnuplot_output_dir+'mymath'; 121 temporary_files.append(filename) 122 f=open(filename,'a') 123 f.write("call \'%s/math.gpm\' \'%s' \'%s\' \'\' \'%s\' \'%s\' %s\n" % (gpm_dir,title,gnuplot_output_filename,gnuplot_output_filename,mode,average)) 124 f.close() 125 126 def compute_aggregated_file(fio_data_file, gnuplot_output_filename, gnuplot_output_dir): 127 if verbose: print "Processing data file 2/2" 128 temp_files=[] 129 pos=0 130 131 # Let's create a temporary file for each selected fio file 132 for file in fio_data_file: 133 tmp_filename = "%sgnuplot_temp_file.%d" % (gnuplot_output_dir, pos) 134 temp_files.append(open(tmp_filename,'r')) 135 pos = pos +1 136 137 f = open(gnuplot_output_dir+gnuplot_output_filename, "w") 138 temporary_files.append(gnuplot_output_dir+gnuplot_output_filename) 139 index=0 140 # Let's add some information 141 for tempfile in temp_files: 142 f.write("# Disk%d was coming from %s\n" % (index,fio_data_file[index])) 143 f.write(tempfile.read()) 144 f.write("\n") 145 tempfile.close() 146 index = index + 1 147 f.close() 148 149 def average(s): return sum(s) * 1.0 / len(s) 150 151 def compute_temp_file(fio_data_file,disk_perf,gnuplot_output_dir, min_time, max_time): 152 end_time=max_time 153 if end_time == -1: 154 end_time="infinite" 155 if verbose: print "Processing data file 1/2 with %s<time<%s" % (min_time,end_time) 156 files=[] 157 temp_outfile=[] 158 blk_size=0 159 for file in fio_data_file: 160 files.append(open(file)) 161 pos = len(files) - 1 162 tmp_filename = "%sgnuplot_temp_file.%d" % (gnuplot_output_dir,pos) 163 temporary_files.append(tmp_filename) 164 gnuplot_file=open(tmp_filename,'w') 165 temp_outfile.append(gnuplot_file) 166 gnuplot_file.write("#Temporary file based on file %s\n" % file) 167 disk_perf.append([]) 168 169 shall_break = False 170 while True: 171 current_line=[] 172 nb_empty_files=0 173 nb_files=len(files) 174 for myfile in files: 175 s=myfile.readline().replace(',',' ').split() 176 if not s: 177 nb_empty_files+=1 178 s="-1, 0, 0, 0".replace(',',' ').split() 179 180 if (nb_empty_files == nb_files): 181 shall_break=True 182 break; 183 184 current_line.append(s); 185 186 if shall_break == True: 187 break 188 189 last_time = -1 190 index=-1 191 perfs=[] 192 for line in enumerate(current_line): 193 # Index will be used to remember what file was featuring what value 194 index=index+1 195 196 time, perf, x, block_size = line[1] 197 if (blk_size == 0): 198 try: 199 blk_size=int(block_size) 200 except: 201 print "Error while reading the following line :" 202 print line 203 sys.exit(1); 204 205 # We ignore the first 500msec as it doesn't seems to be part of the real benchmark 206 # Time < 500 usually reports BW=0 breaking the min computing 207 if (min_time == 0): 208 min_time==0.5 209 210 # Then we estimate if the data we got is part of the time range we want to plot 211 if ((float(time)>(float(min_time)*1000)) and ((int(time) < (int(max_time)*1000)) or max_time==-1)): 212 disk_perf[index].append(int(perf)) 213 perfs.append("%d %s %s"% (index, time, perf)) 214 215 # If we reach this point, it means that all the traces are coherent 216 for p in enumerate(perfs): 217 index, perf_time,perf = p[1].split() 218 temp_outfile[int(index)].write("%s %.2f %s\n" % (index, float(float(perf_time)/1000), perf)) 219 220 221 for file in files: 222 file.close() 223 for file in temp_outfile: 224 file.close() 225 return blk_size 226 227 def compute_math(fio_data_file, title,gnuplot_output_filename,gnuplot_output_dir,mode,disk_perf,gpm_dir): 228 if verbose: print "Computing Maths" 229 global_min=[] 230 global_max=[] 231 average_file=open(gnuplot_output_dir+gnuplot_output_filename+'.average', 'w') 232 min_file=open(gnuplot_output_dir+gnuplot_output_filename+'.min', 'w') 233 max_file=open(gnuplot_output_dir+gnuplot_output_filename+'.max', 'w') 234 stddev_file=open(gnuplot_output_dir+gnuplot_output_filename+'.stddev', 'w') 235 global_file=open(gnuplot_output_dir+gnuplot_output_filename+'.global','w') 236 temporary_files.append(gnuplot_output_dir+gnuplot_output_filename+'.average') 237 temporary_files.append(gnuplot_output_dir+gnuplot_output_filename+'.min') 238 temporary_files.append(gnuplot_output_dir+gnuplot_output_filename+'.max') 239 temporary_files.append(gnuplot_output_dir+gnuplot_output_filename+'.stddev') 240 temporary_files.append(gnuplot_output_dir+gnuplot_output_filename+'.global') 241 242 min_file.write('DiskName %s\n' % mode) 243 max_file.write('DiskName %s\n'% mode) 244 average_file.write('DiskName %s\n'% mode) 245 stddev_file.write('DiskName %s\n'% mode ) 246 for disk in xrange(len(fio_data_file)): 247 # print disk_perf[disk] 248 min_file.write("# Disk%d was coming from %s\n" % (disk,fio_data_file[disk])) 249 max_file.write("# Disk%d was coming from %s\n" % (disk,fio_data_file[disk])) 250 average_file.write("# Disk%d was coming from %s\n" % (disk,fio_data_file[disk])) 251 stddev_file.write("# Disk%d was coming from %s\n" % (disk,fio_data_file[disk])) 252 avg = average(disk_perf[disk]) 253 variance = map(lambda x: (x - avg)**2, disk_perf[disk]) 254 standard_deviation = math.sqrt(average(variance)) 255 # print "Disk%d [ min=%.2f max=%.2f avg=%.2f stddev=%.2f \n" % (disk,min(disk_perf[disk]),max(disk_perf[disk]),avg, standard_deviation) 256 average_file.write('%d %d\n' % (disk, avg)) 257 stddev_file.write('%d %d\n' % (disk, standard_deviation)) 258 local_min=min(disk_perf[disk]) 259 local_max=max(disk_perf[disk]) 260 min_file.write('%d %d\n' % (disk, local_min)) 261 max_file.write('%d %d\n' % (disk, local_max)) 262 global_min.append(int(local_min)) 263 global_max.append(int(local_max)) 264 265 global_disk_perf = sum(disk_perf, []) 266 avg = average(global_disk_perf) 267 variance = map(lambda x: (x - avg)**2, global_disk_perf) 268 standard_deviation = math.sqrt(average(variance)) 269 270 global_file.write('min=%.2f\n' % min(global_disk_perf)) 271 global_file.write('max=%.2f\n' % max(global_disk_perf)) 272 global_file.write('avg=%.2f\n' % avg) 273 global_file.write('stddev=%.2f\n' % standard_deviation) 274 global_file.write('values_count=%d\n' % len(global_disk_perf)) 275 global_file.write('disks_count=%d\n' % len(fio_data_file)) 276 #print "Global [ min=%.2f max=%.2f avg=%.2f stddev=%.2f \n" % (min(global_disk_perf),max(global_disk_perf),avg, standard_deviation) 277 278 average_file.close() 279 min_file.close() 280 max_file.close() 281 stddev_file.close() 282 global_file.close() 283 try: 284 os.remove(gnuplot_output_dir+'mymath') 285 except: 286 True 287 288 generate_gnuplot_math_script("Average values of "+title,gnuplot_output_filename+'.average',mode,int(avg),gnuplot_output_dir,gpm_dir) 289 generate_gnuplot_math_script("Min values of "+title,gnuplot_output_filename+'.min',mode,average(global_min),gnuplot_output_dir,gpm_dir) 290 generate_gnuplot_math_script("Max values of "+title,gnuplot_output_filename+'.max',mode,average(global_max),gnuplot_output_dir,gpm_dir) 291 generate_gnuplot_math_script("Standard Deviation of "+title,gnuplot_output_filename+'.stddev',mode,int(standard_deviation),gnuplot_output_dir,gpm_dir) 292 293 def parse_global_files(fio_data_file, global_search): 294 max_result=0 295 max_file='' 296 for file in fio_data_file: 297 f=open(file) 298 disk_count=0 299 search_value=-1 300 301 # Let's read the complete file 302 while True: 303 try: 304 # We do split the name from the value 305 name,value=f.readline().split("=") 306 except: 307 f.close() 308 break 309 # If we ended the file 310 if not name: 311 # Let's process what we have 312 f.close() 313 break 314 else: 315 # disks_count is not global_search item 316 # As we need it for some computation, let's save it 317 if name=="disks_count": 318 disks_count=int(value) 319 320 # Let's catch the searched item 321 if global_search in name: 322 search_value=float(value) 323 324 # Let's process the avg value by estimated the global bandwidth per file 325 # We keep the biggest in memory for reporting 326 if global_search == "avg": 327 if (disks_count > 0) and (search_value != -1): 328 result=disks_count*search_value 329 if (result > max_result): 330 max_result=result 331 max_file=file 332 # Let's print the avg output 333 if global_search == "avg": 334 print "Biggest aggregated value of %s was %2.f in file %s\n" % (global_search, max_result, max_file) 335 else: 336 print "Global search %s is not yet implemented\n" % global_search 337 338 def render_gnuplot(fio_data_file, gnuplot_output_dir): 339 print "Running gnuplot Rendering" 340 try: 341 # Let's render all the compared files if some 342 if len(fio_data_file) > 1: 343 if verbose: print " |-> Rendering comparing traces" 344 os.system("cd %s; for i in *.gnuplot; do gnuplot $i; done" % gnuplot_output_dir) 345 if verbose: print " |-> Rendering math traces" 346 os.system("cd %s; gnuplot mymath" % gnuplot_output_dir) 347 if verbose: print " |-> Rendering 2D & 3D traces" 348 os.system("cd %s; gnuplot mygraph" % gnuplot_output_dir) 349 350 name_of_directory="the current" 351 if gnuplot_output_dir != "./": 352 name_of_directory=gnuplot_output_dir 353 print "\nRendering traces are available in %s directory" % name_of_directory 354 global keep_temp_files 355 keep_temp_files=False 356 except: 357 print "Could not run gnuplot on mymath or mygraph !\n" 358 sys.exit(1); 359 360 def print_help(): 361 print 'fio2gnuplot -ghbiodvk -t <title> -o <outputfile> -p <pattern> -G <type> -m <time> -M <time>' 362 print 363 print '-h --help : Print this help' 364 print '-p <pattern> or --pattern <pattern> : A pattern in regexp to select fio input files' 365 print '-b or --bandwidth : A predefined pattern for selecting *_bw.log files' 366 print '-i or --iops : A predefined pattern for selecting *_iops.log files' 367 print '-g or --gnuplot : Render gnuplot traces before exiting' 368 print '-o or --outputfile <file> : The basename for gnuplot traces' 369 print ' - Basename is set with the pattern if defined' 370 print '-d or --outputdir <dir> : The directory where gnuplot shall render files' 371 print '-t or --title <title> : The title of the gnuplot traces' 372 print ' - Title is set with the block size detected in fio traces' 373 print '-G or --Global <type> : Search for <type> in .global files match by a pattern' 374 print ' - Available types are : min, max, avg, stddev' 375 print ' - The .global extension is added automatically to the pattern' 376 print '-m or --min_time <time> : Only consider data starting from <time> seconds (default is 0)' 377 print '-M or --max_time <time> : Only consider data ending before <time> seconds (default is -1 aka nolimit)' 378 print '-v or --verbose : Increasing verbosity' 379 print '-k or --keep : Keep all temporary files from gnuplot\'s output dir' 380 381 def main(argv): 382 mode='unknown' 383 pattern='' 384 pattern_set_by_user=False 385 title='No title' 386 gnuplot_output_filename='result' 387 gnuplot_output_dir='./' 388 gpm_dir="/usr/share/fio/" 389 disk_perf=[] 390 run_gnuplot=False 391 parse_global=False 392 global_search='' 393 min_time=0 394 max_time=-1 395 global verbose 396 verbose=False 397 global temporary_files 398 temporary_files=[] 399 global keep_temp_files 400 keep_temp_files=True 401 force_keep_temp_files=False 402 403 if not os.path.isfile(gpm_dir+'math.gpm'): 404 gpm_dir="/usr/local/share/fio/" 405 if not os.path.isfile(gpm_dir+'math.gpm'): 406 print "Looks like fio didn't get installed properly as no gpm files found in '/usr/share/fio' or '/usr/local/share/fio'\n" 407 sys.exit(3) 408 409 try: 410 opts, args = getopt.getopt(argv[1:],"ghkbivo:d:t:p:G:m:M:",['bandwidth', 'iops', 'pattern', 'outputfile', 'outputdir', 'title', 'min_time', 'max_time', 'gnuplot', 'Global', 'help', 'verbose','keep']) 411 except getopt.GetoptError: 412 print "Error: One of the options passed to the cmdline was not supported" 413 print "Please fix your command line or read the help (-h option)" 414 sys.exit(2) 415 416 for opt, arg in opts: 417 if opt in ("-b", "--bandwidth"): 418 pattern='*_bw.log' 419 elif opt in ("-i", "--iops"): 420 pattern='*_iops.log' 421 elif opt in ("-v", "--verbose"): 422 verbose=True 423 elif opt in ("-k", "--keep"): 424 #User really wants to keep the temporary files 425 force_keep_temp_files=True 426 elif opt in ("-p", "--pattern"): 427 pattern_set_by_user=True 428 pattern=arg 429 pattern=pattern.replace('\\','') 430 elif opt in ("-o", "--outputfile"): 431 gnuplot_output_filename=arg 432 elif opt in ("-d", "--outputdir"): 433 gnuplot_output_dir=arg 434 if not gnuplot_output_dir.endswith('/'): 435 gnuplot_output_dir=gnuplot_output_dir+'/' 436 if not os.path.exists(gnuplot_output_dir): 437 os.makedirs(gnuplot_output_dir) 438 elif opt in ("-t", "--title"): 439 title=arg 440 elif opt in ("-m", "--min_time"): 441 min_time=arg 442 elif opt in ("-M", "--max_time"): 443 max_time=arg 444 elif opt in ("-g", "--gnuplot"): 445 run_gnuplot=True 446 elif opt in ("-G", "--Global"): 447 parse_global=True 448 global_search=arg 449 elif opt in ("-h", "--help"): 450 print_help() 451 sys.exit(1) 452 453 # Adding .global extension to the file 454 if parse_global==True: 455 if not gnuplot_output_filename.endswith('.global'): 456 pattern = pattern+'.global' 457 458 fio_data_file=find_file('.',pattern) 459 if len(fio_data_file) == 0: 460 print "No log file found with pattern %s!" % pattern 461 sys.exit(1) 462 else: 463 print "%d files Selected with pattern '%s'" % (len(fio_data_file), pattern) 464 465 fio_data_file=sorted(fio_data_file, key=str.lower) 466 for file in fio_data_file: 467 print ' |-> %s' % file 468 if "_bw.log" in file : 469 mode="Bandwidth (KB/sec)" 470 if "_iops.log" in file : 471 mode="IO per Seconds (IO/sec)" 472 if (title == 'No title') and (mode != 'unknown'): 473 if "Bandwidth" in mode: 474 title='Bandwidth benchmark with %d fio results' % len(fio_data_file) 475 if "IO" in mode: 476 title='IO benchmark with %d fio results' % len(fio_data_file) 477 478 print 479 #We need to adjust the output filename regarding the pattern required by the user 480 if (pattern_set_by_user == True): 481 gnuplot_output_filename=pattern 482 # As we do have some regexp in the pattern, let's make this simpliest 483 # We do remove the simpliest parts of the expression to get a clear file name 484 gnuplot_output_filename=gnuplot_output_filename.replace('-*-','-') 485 gnuplot_output_filename=gnuplot_output_filename.replace('*','-') 486 gnuplot_output_filename=gnuplot_output_filename.replace('--','-') 487 gnuplot_output_filename=gnuplot_output_filename.replace('.log','') 488 # Insure that we don't have any starting or trailing dash to the filename 489 gnuplot_output_filename = gnuplot_output_filename[:-1] if gnuplot_output_filename.endswith('-') else gnuplot_output_filename 490 gnuplot_output_filename = gnuplot_output_filename[1:] if gnuplot_output_filename.startswith('-') else gnuplot_output_filename 491 492 if parse_global==True: 493 parse_global_files(fio_data_file, global_search) 494 else: 495 blk_size=compute_temp_file(fio_data_file,disk_perf,gnuplot_output_dir,min_time,max_time) 496 title="%s @ Blocksize = %dK" % (title,blk_size/1024) 497 compute_aggregated_file(fio_data_file, gnuplot_output_filename, gnuplot_output_dir) 498 compute_math(fio_data_file,title,gnuplot_output_filename,gnuplot_output_dir,mode,disk_perf,gpm_dir) 499 generate_gnuplot_script(fio_data_file,title,gnuplot_output_filename,gnuplot_output_dir,mode,disk_perf,gpm_dir) 500 501 if (run_gnuplot==True): 502 render_gnuplot(fio_data_file, gnuplot_output_dir) 503 504 # Shall we clean the temporary files ? 505 if keep_temp_files==False and force_keep_temp_files==False: 506 # Cleaning temporary files 507 if verbose: print "Cleaning temporary files" 508 for f in enumerate(temporary_files): 509 if verbose: print " -> %s"%f[1] 510 try: 511 os.remove(f[1]) 512 except: 513 True 514 515 #Main 516 if __name__ == "__main__": 517 sys.exit(main(sys.argv)) 518
