1 #!/usr/bin/env python 2 # Copyright 2015 the V8 project authors. All rights reserved. 3 # Use of this source code is governed by a BSD-style license that can be 4 # found in the LICENSE file. 5 ''' 6 python %prog 7 8 Convert a perf trybot JSON file into a pleasing HTML page. It can read 9 from standard input or via the --filename option. Examples: 10 11 cat results.json | %prog --title "ia32 results" 12 %prog -f results.json -t "ia32 results" -o results.html 13 ''' 14 15 import commands 16 import json 17 import math 18 from optparse import OptionParser 19 import os 20 import shutil 21 import sys 22 import tempfile 23 24 PERCENT_CONSIDERED_SIGNIFICANT = 0.5 25 PROBABILITY_CONSIDERED_SIGNIFICANT = 0.02 26 PROBABILITY_CONSIDERED_MEANINGLESS = 0.05 27 28 29 def ComputeZ(baseline_avg, baseline_sigma, mean, n): 30 if baseline_sigma == 0: 31 return 1000.0; 32 return abs((mean - baseline_avg) / (baseline_sigma / math.sqrt(n))) 33 34 35 # Values from http://www.fourmilab.ch/rpkp/experiments/analysis/zCalc.html 36 def ComputeProbability(z): 37 if z > 2.575829: # p 0.005: two sided < 0.01 38 return 0 39 if z > 2.326348: # p 0.010 40 return 0.01 41 if z > 2.170091: # p 0.015 42 return 0.02 43 if z > 2.053749: # p 0.020 44 return 0.03 45 if z > 1.959964: # p 0.025: two sided < 0.05 46 return 0.04 47 if z > 1.880793: # p 0.030 48 return 0.05 49 if z > 1.811910: # p 0.035 50 return 0.06 51 if z > 1.750686: # p 0.040 52 return 0.07 53 if z > 1.695397: # p 0.045 54 return 0.08 55 if z > 1.644853: # p 0.050: two sided < 0.10 56 return 0.09 57 if z > 1.281551: # p 0.100: two sided < 0.20 58 return 0.10 59 return 0.20 # two sided p >= 0.20 60 61 62 class Result: 63 def __init__(self, test_name, count, hasScoreUnits, result, sigma, 64 master_result, master_sigma): 65 self.result_ = float(result) 66 self.sigma_ = float(sigma) 67 self.master_result_ = float(master_result) 68 self.master_sigma_ = float(master_sigma) 69 self.significant_ = False 70 self.notable_ = 0 71 self.percentage_string_ = "" 72 # compute notability and significance. 73 if hasScoreUnits: 74 compare_num = 100*self.result_/self.master_result_ - 100 75 else: 76 compare_num = 100*self.master_result_/self.result_ - 100 77 if abs(compare_num) > 0.1: 78 self.percentage_string_ = "%3.1f" % (compare_num) 79 z = ComputeZ(self.master_result_, self.master_sigma_, self.result_, count) 80 p = ComputeProbability(z) 81 if p < PROBABILITY_CONSIDERED_SIGNIFICANT: 82 self.significant_ = True 83 if compare_num >= PERCENT_CONSIDERED_SIGNIFICANT: 84 self.notable_ = 1 85 elif compare_num <= -PERCENT_CONSIDERED_SIGNIFICANT: 86 self.notable_ = -1 87 88 def result(self): 89 return self.result_ 90 91 def sigma(self): 92 return self.sigma_ 93 94 def master_result(self): 95 return self.master_result_ 96 97 def master_sigma(self): 98 return self.master_sigma_ 99 100 def percentage_string(self): 101 return self.percentage_string_; 102 103 def isSignificant(self): 104 return self.significant_ 105 106 def isNotablyPositive(self): 107 return self.notable_ > 0 108 109 def isNotablyNegative(self): 110 return self.notable_ < 0 111 112 113 class Benchmark: 114 def __init__(self, name, data): 115 self.name_ = name 116 self.tests_ = {} 117 for test in data: 118 # strip off "<name>/" prefix, allowing for subsequent "/"s 119 test_name = test.split("/", 1)[1] 120 self.appendResult(test_name, data[test]) 121 122 # tests is a dictionary of Results 123 def tests(self): 124 return self.tests_ 125 126 def SortedTestKeys(self): 127 keys = self.tests_.keys() 128 keys.sort() 129 t = "Total" 130 if t in keys: 131 keys.remove(t) 132 keys.append(t) 133 return keys 134 135 def name(self): 136 return self.name_ 137 138 def appendResult(self, test_name, test_data): 139 with_string = test_data["result with patch "] 140 data = with_string.split() 141 master_string = test_data["result without patch"] 142 master_data = master_string.split() 143 runs = int(test_data["runs"]) 144 units = test_data["units"] 145 hasScoreUnits = units == "score" 146 self.tests_[test_name] = Result(test_name, 147 runs, 148 hasScoreUnits, 149 data[0], data[2], 150 master_data[0], master_data[2]) 151 152 153 class BenchmarkRenderer: 154 def __init__(self, output_file): 155 self.print_output_ = [] 156 self.output_file_ = output_file 157 158 def Print(self, str_data): 159 self.print_output_.append(str_data) 160 161 def FlushOutput(self): 162 string_data = "\n".join(self.print_output_) 163 print_output = [] 164 if self.output_file_: 165 # create a file 166 with open(self.output_file_, "w") as text_file: 167 text_file.write(string_data) 168 else: 169 print(string_data) 170 171 def RenderOneBenchmark(self, benchmark): 172 self.Print("<h2>") 173 self.Print("<a name=\"" + benchmark.name() + "\">") 174 self.Print(benchmark.name() + "</a> <a href=\"#top\">(top)</a>") 175 self.Print("</h2>"); 176 self.Print("<table class=\"benchmark\">") 177 self.Print("<thead>") 178 self.Print(" <th>Test</th>") 179 self.Print(" <th>Result</th>") 180 self.Print(" <th>Master</th>") 181 self.Print(" <th>%</th>") 182 self.Print("</thead>") 183 self.Print("<tbody>") 184 tests = benchmark.tests() 185 for test in benchmark.SortedTestKeys(): 186 t = tests[test] 187 self.Print(" <tr>") 188 self.Print(" <td>" + test + "</td>") 189 self.Print(" <td>" + str(t.result()) + "</td>") 190 self.Print(" <td>" + str(t.master_result()) + "</td>") 191 t = tests[test] 192 res = t.percentage_string() 193 if t.isSignificant(): 194 res = self.bold(res) 195 if t.isNotablyPositive(): 196 res = self.green(res) 197 elif t.isNotablyNegative(): 198 res = self.red(res) 199 self.Print(" <td>" + res + "</td>") 200 self.Print(" </tr>") 201 self.Print("</tbody>") 202 self.Print("</table>") 203 204 def ProcessJSONData(self, data, title): 205 self.Print("<h1>" + title + "</h1>") 206 self.Print("<ul>") 207 for benchmark in data: 208 if benchmark != "errors": 209 self.Print("<li><a href=\"#" + benchmark + "\">" + benchmark + "</a></li>") 210 self.Print("</ul>") 211 for benchmark in data: 212 if benchmark != "errors": 213 benchmark_object = Benchmark(benchmark, data[benchmark]) 214 self.RenderOneBenchmark(benchmark_object) 215 216 def bold(self, data): 217 return "<b>" + data + "</b>" 218 219 def red(self, data): 220 return "<font color=\"red\">" + data + "</font>" 221 222 223 def green(self, data): 224 return "<font color=\"green\">" + data + "</font>" 225 226 def PrintHeader(self): 227 data = """<html> 228 <head> 229 <title>Output</title> 230 <style type="text/css"> 231 /* 232 Style inspired by Andy Ferra's gist at https://gist.github.com/andyferra/2554919 233 */ 234 body { 235 font-family: Helvetica, arial, sans-serif; 236 font-size: 14px; 237 line-height: 1.6; 238 padding-top: 10px; 239 padding-bottom: 10px; 240 background-color: white; 241 padding: 30px; 242 } 243 h1, h2, h3, h4, h5, h6 { 244 margin: 20px 0 10px; 245 padding: 0; 246 font-weight: bold; 247 -webkit-font-smoothing: antialiased; 248 cursor: text; 249 position: relative; 250 } 251 h1 { 252 font-size: 28px; 253 color: black; 254 } 255 256 h2 { 257 font-size: 24px; 258 border-bottom: 1px solid #cccccc; 259 color: black; 260 } 261 262 h3 { 263 font-size: 18px; 264 } 265 266 h4 { 267 font-size: 16px; 268 } 269 270 h5 { 271 font-size: 14px; 272 } 273 274 h6 { 275 color: #777777; 276 font-size: 14px; 277 } 278 279 p, blockquote, ul, ol, dl, li, table, pre { 280 margin: 15px 0; 281 } 282 283 li p.first { 284 display: inline-block; 285 } 286 287 ul, ol { 288 padding-left: 30px; 289 } 290 291 ul :first-child, ol :first-child { 292 margin-top: 0; 293 } 294 295 ul :last-child, ol :last-child { 296 margin-bottom: 0; 297 } 298 299 table { 300 padding: 0; 301 } 302 303 table tr { 304 border-top: 1px solid #cccccc; 305 background-color: white; 306 margin: 0; 307 padding: 0; 308 } 309 310 table tr:nth-child(2n) { 311 background-color: #f8f8f8; 312 } 313 314 table tr th { 315 font-weight: bold; 316 border: 1px solid #cccccc; 317 text-align: left; 318 margin: 0; 319 padding: 6px 13px; 320 } 321 table tr td { 322 border: 1px solid #cccccc; 323 text-align: left; 324 margin: 0; 325 padding: 6px 13px; 326 } 327 table tr th :first-child, table tr td :first-child { 328 margin-top: 0; 329 } 330 table tr th :last-child, table tr td :last-child { 331 margin-bottom: 0; 332 } 333 </style> 334 </head> 335 <body> 336 """ 337 self.Print(data) 338 339 def PrintFooter(self): 340 data = """</body> 341 </html> 342 """ 343 self.Print(data) 344 345 346 def Render(opts, args): 347 if opts.filename: 348 with open(opts.filename) as json_data: 349 data = json.load(json_data) 350 else: 351 # load data from stdin 352 data = json.load(sys.stdin) 353 354 if opts.title: 355 title = opts.title 356 elif opts.filename: 357 title = opts.filename 358 else: 359 title = "Benchmark results" 360 renderer = BenchmarkRenderer(opts.output) 361 renderer.PrintHeader() 362 renderer.ProcessJSONData(data, title) 363 renderer.PrintFooter() 364 renderer.FlushOutput() 365 366 367 if __name__ == '__main__': 368 parser = OptionParser(usage=__doc__) 369 parser.add_option("-f", "--filename", dest="filename", 370 help="Specifies the filename for the JSON results " 371 "rather than reading from stdin.") 372 parser.add_option("-t", "--title", dest="title", 373 help="Optional title of the web page.") 374 parser.add_option("-o", "--output", dest="output", 375 help="Write html output to this file rather than stdout.") 376 377 (opts, args) = parser.parse_args() 378 Render(opts, args) 379
