1 #!/usr/bin/env python 2 # Copyright 2015 the V8 project authors. All rights reserved. 3 # Use of this source code is governed by a BSD-style license that can be 4 # found in the LICENSE file. 5 ''' 6 python %prog 7 8 Convert a perf trybot JSON file into a pleasing HTML page. It can read 9 from standard input or via the --filename option. Examples: 10 11 cat results.json | %prog --title "ia32 results" 12 %prog -f results.json -t "ia32 results" -o results.html 13 ''' 14 15 import json 16 import math 17 from optparse import OptionParser 18 import os 19 import shutil 20 import sys 21 import tempfile 22 23 PERCENT_CONSIDERED_SIGNIFICANT = 0.5 24 PROBABILITY_CONSIDERED_SIGNIFICANT = 0.02 25 PROBABILITY_CONSIDERED_MEANINGLESS = 0.05 26 27 28 def ComputeZ(baseline_avg, baseline_sigma, mean, n): 29 if baseline_sigma == 0: 30 return 1000.0; 31 return abs((mean - baseline_avg) / (baseline_sigma / math.sqrt(n))) 32 33 34 # Values from http://www.fourmilab.ch/rpkp/experiments/analysis/zCalc.html 35 def ComputeProbability(z): 36 if z > 2.575829: # p 0.005: two sided < 0.01 37 return 0 38 if z > 2.326348: # p 0.010 39 return 0.01 40 if z > 2.170091: # p 0.015 41 return 0.02 42 if z > 2.053749: # p 0.020 43 return 0.03 44 if z > 1.959964: # p 0.025: two sided < 0.05 45 return 0.04 46 if z > 1.880793: # p 0.030 47 return 0.05 48 if z > 1.811910: # p 0.035 49 return 0.06 50 if z > 1.750686: # p 0.040 51 return 0.07 52 if z > 1.695397: # p 0.045 53 return 0.08 54 if z > 1.644853: # p 0.050: two sided < 0.10 55 return 0.09 56 if z > 1.281551: # p 0.100: two sided < 0.20 57 return 0.10 58 return 0.20 # two sided p >= 0.20 59 60 61 class Result: 62 def __init__(self, test_name, count, hasScoreUnits, result, sigma, 63 master_result, master_sigma): 64 self.result_ = float(result) 65 self.sigma_ = float(sigma) 66 self.master_result_ = float(master_result) 67 self.master_sigma_ = float(master_sigma) 68 self.significant_ = False 69 self.notable_ = 0 70 self.percentage_string_ = "" 71 # compute notability and significance. 72 try: 73 if hasScoreUnits: 74 compare_num = 100*self.result_/self.master_result_ - 100 75 else: 76 compare_num = 100*self.master_result_/self.result_ - 100 77 if abs(compare_num) > 0.1: 78 self.percentage_string_ = "%3.1f" % (compare_num) 79 z = ComputeZ(self.master_result_, self.master_sigma_, self.result_, count) 80 p = ComputeProbability(z) 81 if p < PROBABILITY_CONSIDERED_SIGNIFICANT: 82 self.significant_ = True 83 if compare_num >= PERCENT_CONSIDERED_SIGNIFICANT: 84 self.notable_ = 1 85 elif compare_num <= -PERCENT_CONSIDERED_SIGNIFICANT: 86 self.notable_ = -1 87 except ZeroDivisionError: 88 self.percentage_string_ = "NaN" 89 self.significant_ = True 90 91 def result(self): 92 return self.result_ 93 94 def sigma(self): 95 return self.sigma_ 96 97 def master_result(self): 98 return self.master_result_ 99 100 def master_sigma(self): 101 return self.master_sigma_ 102 103 def percentage_string(self): 104 return self.percentage_string_; 105 106 def isSignificant(self): 107 return self.significant_ 108 109 def isNotablyPositive(self): 110 return self.notable_ > 0 111 112 def isNotablyNegative(self): 113 return self.notable_ < 0 114 115 116 class Benchmark: 117 def __init__(self, name, data): 118 self.name_ = name 119 self.tests_ = {} 120 for test in data: 121 # strip off "<name>/" prefix, allowing for subsequent "/"s 122 test_name = test.split("/", 1)[1] 123 self.appendResult(test_name, data[test]) 124 125 # tests is a dictionary of Results 126 def tests(self): 127 return self.tests_ 128 129 def SortedTestKeys(self): 130 keys = self.tests_.keys() 131 keys.sort() 132 t = "Total" 133 if t in keys: 134 keys.remove(t) 135 keys.append(t) 136 return keys 137 138 def name(self): 139 return self.name_ 140 141 def appendResult(self, test_name, test_data): 142 with_string = test_data["result with patch "] 143 data = with_string.split() 144 master_string = test_data["result without patch"] 145 master_data = master_string.split() 146 runs = int(test_data["runs"]) 147 units = test_data["units"] 148 hasScoreUnits = units == "score" 149 self.tests_[test_name] = Result(test_name, 150 runs, 151 hasScoreUnits, 152 data[0], data[2], 153 master_data[0], master_data[2]) 154 155 156 class BenchmarkRenderer: 157 def __init__(self, output_file): 158 self.print_output_ = [] 159 self.output_file_ = output_file 160 161 def Print(self, str_data): 162 self.print_output_.append(str_data) 163 164 def FlushOutput(self): 165 string_data = "\n".join(self.print_output_) 166 print_output = [] 167 if self.output_file_: 168 # create a file 169 with open(self.output_file_, "w") as text_file: 170 text_file.write(string_data) 171 else: 172 print(string_data) 173 174 def RenderOneBenchmark(self, benchmark): 175 self.Print("<h2>") 176 self.Print("<a name=\"" + benchmark.name() + "\">") 177 self.Print(benchmark.name() + "</a> <a href=\"#top\">(top)</a>") 178 self.Print("</h2>"); 179 self.Print("<table class=\"benchmark\">") 180 self.Print("<thead>") 181 self.Print(" <th>Test</th>") 182 self.Print(" <th>Result</th>") 183 self.Print(" <th>Master</th>") 184 self.Print(" <th>%</th>") 185 self.Print("</thead>") 186 self.Print("<tbody>") 187 tests = benchmark.tests() 188 for test in benchmark.SortedTestKeys(): 189 t = tests[test] 190 self.Print(" <tr>") 191 self.Print(" <td>" + test + "</td>") 192 self.Print(" <td>" + str(t.result()) + "</td>") 193 self.Print(" <td>" + str(t.master_result()) + "</td>") 194 t = tests[test] 195 res = t.percentage_string() 196 if t.isSignificant(): 197 res = self.bold(res) 198 if t.isNotablyPositive(): 199 res = self.green(res) 200 elif t.isNotablyNegative(): 201 res = self.red(res) 202 self.Print(" <td>" + res + "</td>") 203 self.Print(" </tr>") 204 self.Print("</tbody>") 205 self.Print("</table>") 206 207 def ProcessJSONData(self, data, title): 208 self.Print("<h1>" + title + "</h1>") 209 self.Print("<ul>") 210 for benchmark in data: 211 if benchmark != "errors": 212 self.Print("<li><a href=\"#" + benchmark + "\">" + benchmark + "</a></li>") 213 self.Print("</ul>") 214 for benchmark in data: 215 if benchmark != "errors": 216 benchmark_object = Benchmark(benchmark, data[benchmark]) 217 self.RenderOneBenchmark(benchmark_object) 218 219 def bold(self, data): 220 return "<b>" + data + "</b>" 221 222 def red(self, data): 223 return "<font color=\"red\">" + data + "</font>" 224 225 226 def green(self, data): 227 return "<font color=\"green\">" + data + "</font>" 228 229 def PrintHeader(self): 230 data = """<html> 231 <head> 232 <title>Output</title> 233 <style type="text/css"> 234 /* 235 Style inspired by Andy Ferra's gist at https://gist.github.com/andyferra/2554919 236 */ 237 body { 238 font-family: Helvetica, arial, sans-serif; 239 font-size: 14px; 240 line-height: 1.6; 241 padding-top: 10px; 242 padding-bottom: 10px; 243 background-color: white; 244 padding: 30px; 245 } 246 h1, h2, h3, h4, h5, h6 { 247 margin: 20px 0 10px; 248 padding: 0; 249 font-weight: bold; 250 -webkit-font-smoothing: antialiased; 251 cursor: text; 252 position: relative; 253 } 254 h1 { 255 font-size: 28px; 256 color: black; 257 } 258 259 h2 { 260 font-size: 24px; 261 border-bottom: 1px solid #cccccc; 262 color: black; 263 } 264 265 h3 { 266 font-size: 18px; 267 } 268 269 h4 { 270 font-size: 16px; 271 } 272 273 h5 { 274 font-size: 14px; 275 } 276 277 h6 { 278 color: #777777; 279 font-size: 14px; 280 } 281 282 p, blockquote, ul, ol, dl, li, table, pre { 283 margin: 15px 0; 284 } 285 286 li p.first { 287 display: inline-block; 288 } 289 290 ul, ol { 291 padding-left: 30px; 292 } 293 294 ul :first-child, ol :first-child { 295 margin-top: 0; 296 } 297 298 ul :last-child, ol :last-child { 299 margin-bottom: 0; 300 } 301 302 table { 303 padding: 0; 304 } 305 306 table tr { 307 border-top: 1px solid #cccccc; 308 background-color: white; 309 margin: 0; 310 padding: 0; 311 } 312 313 table tr:nth-child(2n) { 314 background-color: #f8f8f8; 315 } 316 317 table tr th { 318 font-weight: bold; 319 border: 1px solid #cccccc; 320 text-align: left; 321 margin: 0; 322 padding: 6px 13px; 323 } 324 table tr td { 325 border: 1px solid #cccccc; 326 text-align: left; 327 margin: 0; 328 padding: 6px 13px; 329 } 330 table tr th :first-child, table tr td :first-child { 331 margin-top: 0; 332 } 333 table tr th :last-child, table tr td :last-child { 334 margin-bottom: 0; 335 } 336 </style> 337 </head> 338 <body> 339 """ 340 self.Print(data) 341 342 def PrintFooter(self): 343 data = """</body> 344 </html> 345 """ 346 self.Print(data) 347 348 349 def Render(opts, args): 350 if opts.filename: 351 with open(opts.filename) as json_data: 352 data = json.load(json_data) 353 else: 354 # load data from stdin 355 data = json.load(sys.stdin) 356 357 if opts.title: 358 title = opts.title 359 elif opts.filename: 360 title = opts.filename 361 else: 362 title = "Benchmark results" 363 renderer = BenchmarkRenderer(opts.output) 364 renderer.PrintHeader() 365 renderer.ProcessJSONData(data, title) 366 renderer.PrintFooter() 367 renderer.FlushOutput() 368 369 370 if __name__ == '__main__': 371 parser = OptionParser(usage=__doc__) 372 parser.add_option("-f", "--filename", dest="filename", 373 help="Specifies the filename for the JSON results " 374 "rather than reading from stdin.") 375 parser.add_option("-t", "--title", dest="title", 376 help="Optional title of the web page.") 377 parser.add_option("-o", "--output", dest="output", 378 help="Write html output to this file rather than stdout.") 379 380 (opts, args) = parser.parse_args() 381 Render(opts, args) 382
