Home | History | Annotate | Download | only in html_output 
      1 <!DOCTYPE html>
      2 <html>
      3 <head>
      4 <title>Telemetry Performance Test Results</title>
      5 <style type="text/css">
      6 
      7 section {
      8     background: white;
      9     padding: 10px;
     10     position: relative;
     11 }
     12 
     13 .time-plots {
     14     padding-left: 25px;
     15 }
     16 
     17 .time-plots > div {
     18     display: inline-block;
     19     width: 90px;
     20     height: 40px;
     21     margin-right: 10px;
     22 }
     23 
     24 section h1 {
     25     text-align: center;
     26     font-size: 1em;
     27 }
     28 
     29 section .tooltip {
     30     position: absolute;
     31     text-align: center;
     32     background: #ffcc66;
     33     border-radius: 5px;
     34     padding: 0px 5px;
     35 }
     36 
     37 body {
     38     padding: 0px;
     39     margin: 0px;
     40     font-family: sans-serif;
     41 }
     42 
     43 table {
     44     background: white;
     45     width: 100%;
     46 }
     47 
     48 table, td, th {
     49     border-collapse: collapse;
     50     padding: 5px;
     51     white-space: nowrap;
     52 }
     53 
     54 tr.even {
     55     background: #f6f6f6;
     56 }
     57 
     58 table td {
     59     position: relative;
     60     font-family: monospace;
     61 }
     62 
     63 th, td {
     64     cursor: pointer;
     65     cursor: hand;
     66 }
     67 
     68 th {
     69     background: #e6eeee;
     70     background: -webkit-gradient(linear, left top, left bottom, from(rgb(244, 244, 244)), to(rgb(217, 217, 217)));
     71     border: 1px solid #ccc;
     72 }
     73 
     74 th:after {
     75     content: ' \25B8';
     76 }
     77 
     78 th.headerSortUp:after {
     79     content: ' \25BE';
     80 }
     81 
     82 th.headerSortDown:after {
     83     content: ' \25B4';
     84 }
     85 
     86 td.comparison, td.result {
     87     text-align: right;
     88 }
     89 
     90 td.better {
     91     color: #6c6;
     92 }
     93 
     94 td.worse {
     95     color: #c66;
     96 }
     97 
     98 td.missing {
     99     text-align: center;
    100 }
    101 
    102 .checkbox {
    103     display: inline-block;
    104     background: #eee;
    105     background: -webkit-gradient(linear, left bottom, left top, from(rgb(220, 220, 220)), to(rgb(200, 200, 200)));
    106     border: inset 1px #ddd;
    107     border-radius: 5px;
    108     margin: 10px;
    109     font-size: small;
    110     cursor: pointer;
    111     cursor: hand;
    112     -webkit-user-select: none;
    113     font-weight: bold;
    114 }
    115 
    116 .checkbox span {
    117     display: inline-block;
    118     line-height: 100%;
    119     padding: 5px 8px;
    120     border: outset 1px transparent;
    121 }
    122 
    123 .checkbox .checked {
    124     background: #e6eeee;
    125     background: -webkit-gradient(linear, left top, left bottom, from(rgb(255, 255, 255)), to(rgb(235, 235, 235)));
    126     border: outset 1px #eee;
    127     border-radius: 5px;
    128 }
    129 
    130 </style>
    131 </head>
    132 <body onload="init()">
    133 <div style="padding: 0 10px; white-space: nowrap;">
    134 Result <span id="time-memory" class="checkbox"><span class="checked">Time</span><span>Memory</span></span>
    135 Reference <span id="reference" class="checkbox"></span>
    136 Run Telemetry with --reset-html-results to clear all runs
    137 </div>
    138 <table id="container"></table>
    139 <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.6.4/jquery.min.js"></script>
    140 <script>
    141 %plugins%
    142 </script>
    143 <script>
    144 function TestResult(metric, values, associatedRun) {
    145     if (values[0] instanceof Array) {
    146         var flattenedValues = [];
    147         for (var i = 0; i < values.length; i++)
    148             flattenedValues = flattenedValues.concat(values[i]);
    149         values = flattenedValues;
    150     }
    151 
    152     this.test = function () { return metric; }
    153     this.values = function () { return values.map(function (value) { return metric.scalingFactor() * value; }); }
    154     this.unscaledMean = function () { return Statistics.sum(values) / values.length; }
    155     this.mean = function () { return metric.scalingFactor() * this.unscaledMean(); }
    156     this.min = function () { return metric.scalingFactor() * Statistics.min(values); }
    157     this.max = function () { return metric.scalingFactor() * Statistics.max(values); }
    158     this.confidenceIntervalDelta = function () {
    159         return metric.scalingFactor() * Statistics.confidenceIntervalDelta(0.95, values.length,
    160             Statistics.sum(values), Statistics.squareSum(values));
    161     }
    162     this.confidenceIntervalDeltaRatio = function () { return this.confidenceIntervalDelta() / this.mean(); }
    163     this.percentDifference = function(other) { return (other.unscaledMean() - this.unscaledMean()) / this.unscaledMean(); }
    164     this.isStatisticallySignificant = function (other) {
    165         var diff = Math.abs(other.mean() - this.mean());
    166         return diff > this.confidenceIntervalDelta() && diff > other.confidenceIntervalDelta();
    167     }
    168     this.run = function () { return associatedRun; }
    169 }
    170 
    171 function TestRun(entry) {
    172     this.description = function () { return entry['description']; }
    173     this.revision = function () { return entry['revision']; }
    174     this.label = function () {
    175         var label = 'r' + this.revision();
    176         if (this.description())
    177             label += ' &dash; ' + this.description();
    178         return label;
    179     }
    180 }
    181 
    182 function PerfTestMetric(name, metric, unit, isImportant) {
    183     var testResults = [];
    184     var cachedUnit = null;
    185     var cachedScalingFactor = null;
    186 
    187     // We can't do this in TestResult because all results for each test need to share the same unit and the same scaling factor.
    188     function computeScalingFactorIfNeeded() {
    189         // FIXME: We shouldn't be adjusting units on every test result.
    190         // We can only do this on the first test.
    191         if (!testResults.length || cachedUnit)
    192             return;
    193 
    194         var mean = testResults[0].unscaledMean(); // FIXME: We should look at all values.
    195         var kilo = unit == 'bytes' ? 1024 : 1000;
    196         if (mean > 10 * kilo * kilo && unit != 'ms') {
    197             cachedScalingFactor = 1 / kilo / kilo;
    198             cachedUnit = 'M ' + unit;
    199         } else if (mean > 10 * kilo) {
    200             cachedScalingFactor = 1 / kilo;
    201             cachedUnit = unit == 'ms' ? 's' : ('K ' + unit);
    202         } else {
    203             cachedScalingFactor = 1;
    204             cachedUnit = unit;
    205         }
    206     }
    207 
    208     this.name = function () { return name + ':' + metric; }
    209     this.isImportant = isImportant;
    210     this.isMemoryTest = function () {
    211         return (unit == 'kb' ||
    212                 unit == 'KB' ||
    213                 unit == 'MB' ||
    214                 unit == 'bytes' ||
    215                 !metric.indexOf('V8.') ||
    216                 metric == 'processes');
    217     }
    218     this.addResult = function (newResult) {
    219         testResults.push(newResult);
    220         cachedUnit = null;
    221         cachedScalingFactor = null;
    222     }
    223     this.results = function () { return testResults; }
    224     this.scalingFactor = function() {
    225         computeScalingFactorIfNeeded();
    226         return cachedScalingFactor;
    227     }
    228     this.unit = function () {
    229         computeScalingFactorIfNeeded();
    230         return cachedUnit;
    231     }
    232     this.biggerIsBetter = function () {
    233         if (window.unitToBiggerIsBetter == undefined) {
    234             window.unitToBiggerIsBetter = {};
    235             var units = JSON.parse(document.getElementById('units-json').textContent);
    236             for (var unit in units) {
    237                 if (units[unit].improvement_direction == 'up') {
    238                     window.unitToBiggerIsBetter[unit] = true;
    239                 }
    240             }
    241         }
    242         return window.unitToBiggerIsBetter[unit];
    243     }
    244 }
    245 
    246 var plotColor = 'rgb(230,50,50)';
    247 var subpointsPlotOptions = {
    248     lines: {show:true, lineWidth: 0},
    249     color: plotColor,
    250     points: {show: true, radius: 1},
    251     bars: {show: false}};
    252 
    253 var mainPlotOptions = {
    254     xaxis: {
    255         min: -0.5,
    256         tickSize: 1,
    257     },
    258     crosshair: { mode: 'y' },
    259     series: { shadowSize: 0 },
    260     bars: {show: true, align: 'center', barWidth: 0.5},
    261     lines: { show: false },
    262     points: { show: true },
    263     grid: {
    264         borderWidth: 1,
    265         borderColor: '#ccc',
    266         backgroundColor: '#fff',
    267         hoverable: true,
    268         autoHighlight: false,
    269     }
    270 };
    271 
    272 var timePlotOptions = {
    273     yaxis: { show: false },
    274     xaxis: { show: false },
    275     lines: { show: true },
    276     grid: { borderWidth: 1, borderColor: '#ccc' },
    277     colors: [ plotColor ]
    278 };
    279 
    280 function createPlot(container, test) {
    281     var section = $('<section><div class="plot"></div><div class="time-plots"></div>'
    282         + '<span class="tooltip"></span></section>');
    283     section.children('.plot').css({'width': (100 * test.results().length + 25) + 'px', 'height': '300px'});
    284     $(container).append(section);
    285 
    286     var plotContainer = section.children('.plot');
    287     var minIsZero = true;
    288     attachPlot(test, plotContainer, minIsZero);
    289 
    290     attachTimePlots(test, section.children('.time-plots'));
    291 
    292     var tooltip = section.children('.tooltip');
    293     plotContainer.bind('plothover', function (event, position, item) {
    294         if (item) {
    295             var postfix = item.series.id ? ' (' + item.series.id + ')' : '';
    296             tooltip.html(item.datapoint[1].toPrecision(4) + postfix);
    297             var sectionOffset = $(section).offset();
    298             tooltip.css({left: item.pageX - sectionOffset.left - tooltip.outerWidth() / 2, top: item.pageY - sectionOffset.top + 10});
    299             tooltip.fadeIn(200);
    300         } else
    301             tooltip.hide();
    302     });
    303     plotContainer.mouseout(function () {
    304         tooltip.hide();
    305     });
    306     plotContainer.click(function (event) {
    307         event.preventDefault();
    308         minIsZero = !minIsZero;
    309         attachPlot(test, plotContainer, minIsZero);
    310     });
    311 
    312     return section;
    313 }
    314 
    315 function attachTimePlots(test, container) {
    316     var results = test.results();
    317     var attachedPlot = false;
    318     for (var i = 0; i < results.length; i++) {
    319         container.append('
');
    320         var values = results[i].values();
    321         if (!values)
    322             continue;
    323         attachedPlot = true;
    324 
    325         $.plot(container.children().last(), [values.map(function (value, index) { return [index, value]; })],
    326             $.extend(true, {}, timePlotOptions, {yaxis: {min: Math.min.apply(Math, values) * 0.9, max: Math.max.apply(Math, values) * 1.1},
    327                 xaxis: {min: -0.5, max: values.length - 0.5}}));
    328     }
    329     if (!attachedPlot)
    330         container.children().remove();
    331 }
    332 
    333 function attachPlot(test, plotContainer, minIsZero) {
    334     var results = test.results();
    335 
    336     var values = results.reduce(function (values, result, index) {
    337         var newValues = result.values();
    338         return newValues ? values.concat(newValues.map(function (value) { return [index, value]; })) : values;
    339     }, []);
    340 
    341     var plotData = [$.extend(true, {}, subpointsPlotOptions, {data: values})];
    342     plotData.push({id: '&mu;', data: results.map(function (result, index) { return [index, result.mean()]; }), color: plotColor});
    343 
    344     var overallMax = Statistics.max(results.map(function (result, index) { return result.max(); }));
    345     var overallMin = Statistics.min(results.map(function (result, index) { return result.min(); }));
    346     var margin = (overallMax - overallMin) * 0.1;
    347     var currentPlotOptions = $.extend(true, {}, mainPlotOptions, {yaxis: {
    348         min: minIsZero ? 0 : overallMin - margin,
    349         max: minIsZero ? overallMax * 1.1 : overallMax + margin}});
    350 
    351     currentPlotOptions.xaxis.max = results.length - 0.5;
    352     currentPlotOptions.xaxis.ticks = results.map(function (result, index) { return [index, result.run().label()]; });
    353 
    354     $.plot(plotContainer, plotData, currentPlotOptions);
    355 }
    356 
    357 function toFixedWidthPrecision(value) {
    358     var decimal = value.toFixed(2);
    359     return decimal;
    360 }
    361 
    362 function formatPercentage(fraction) {
    363     var percentage = fraction * 100;
    364     return (fraction * 100).toFixed(2) + '%';
    365 }
    366 
    367 function createTable(tests, runs, shouldIgnoreMemory, referenceIndex) {
    368     $('#container').html('TestUnit' + runs.map(function (run, index) {
    369         return 'referenceIndex ? 2 : 3) + '" class="{sorter: \'comparison\'}">' + run.label() + '';
    370     }).reduce(function (markup, cell) { return markup + cell; }, '') + '');
    371 
    372     var testNames = [];
    373     for (testName in tests)
    374         testNames.push(testName);
    375 
    376     testNames.sort().map(function (testName) {
    377         var test = tests[testName];
    378         if (test.isMemoryTest() != shouldIgnoreMemory)
    379             createTableRow(runs, test, referenceIndex);
    380     });
    381 
    382     $('#container').tablesorter({widgets: ['zebra']});
    383 }
    384 
    385 function linearRegression(points) {
    386     // Implement http://www.easycalculation.com/statistics/learn-correlation.php.
    387     // x = magnitude
    388     // y = iterations
    389     var sumX = 0;
    390     var sumY = 0;
    391     var sumXSquared = 0;
    392     var sumYSquared = 0;
    393     var sumXTimesY = 0;
    394 
    395     for (var i = 0; i < points.length; i++) {
    396         var x = i;
    397         var y = points[i];
    398         sumX += x;
    399         sumY += y;
    400         sumXSquared += x * x;
    401         sumYSquared += y * y;
    402         sumXTimesY += x * y;
    403     }
    404 
    405     var r = (points.length * sumXTimesY - sumX * sumY) /
    406         Math.sqrt((points.length * sumXSquared - sumX * sumX) *
    407                   (points.length * sumYSquared - sumY * sumY));
    408 
    409     if (isNaN(r) || r == Math.Infinity)
    410         r = 0;
    411 
    412     var slope = (points.length * sumXTimesY - sumX * sumY) / (points.length * sumXSquared - sumX * sumX);
    413     var intercept = sumY / points.length - slope * sumX / points.length;
    414     return {slope: slope, intercept: intercept, rSquared: r * r};
    415 }
    416 
    417 var warningSign = '<svg viewBox="0 0 100 100" style="width: 18px; height: 18px; vertical-align: bottom;" version="1.1">'
    418     + '<polygon fill="red" points="50,10 90,80 10,80 50,10" stroke="red" stroke-width="10" stroke-linejoin="round" />'
    419     + '<polygon fill="white" points="47,30 48,29, 50, 28.7, 52,29 53,30 50,60" stroke="white" stroke-width="10" stroke-linejoin="round" />'
    420     + '<circle cx="50" cy="73" r="6" fill="white" />'
    421     + '</svg>';
    422 
    423 function createTableRow(runs, test, referenceIndex) {
    424     var tableRow = $('<tr><td class="test"' + (test.isImportant ? ' style="font-weight:bold"' : '') + '>' + test.name() + '</td><td class="unit">' + test.unit() + '</td></tr>');
    425 
    426     function markupForRun(result, referenceResult) {
    427         var comparisonCell = '';
    428         var hiddenValue = '';
    429         var shouldCompare = result !== referenceResult;
    430         if (shouldCompare && referenceResult) {
    431             var percentDifference = referenceResult.percentDifference(result);
    432             var better = test.biggerIsBetter() ? percentDifference > 0 : percentDifference < 0;
    433             var comparison = '';
    434             var className = 'comparison';
    435             if (referenceResult.isStatisticallySignificant(result)) {
    436                 comparison = formatPercentage(Math.abs(percentDifference)) + (better ? ' Better' : ' Worse&nbsp;');
    437                 className += better ? ' better' : ' worse';
    438             }
    439             hiddenValue = '|' + comparison + '';
    440             comparisonCell = 'className + '">' + comparison + '';
    441         } else if (shouldCompare)
    442             comparisonCell = '';
    443 
    444         var values = result.values();
    445         var warning = '';
    446         var regressionAnalysis = '';
    447         if (values && values.length > 3) {
    448             regressionResult = linearRegression(values);
    449             regressionAnalysis = 'slope=' + toFixedWidthPrecision(regressionResult.slope)
    450                 + ', R^2=' + toFixedWidthPrecision(regressionResult.rSquared);
    451             if (regressionResult.rSquared > 0.6 && Math.abs(regressionResult.slope) > 0.01) {
    452                 warning = ' <span class="regression-warning" title="Detected a time dependency with ' + regressionAnalysis + '">' + warningSign + ' </span>';
    453             }
    454         }
    455 
    456         var statistics = '&sigma;=' + toFixedWidthPrecision(result.confidenceIntervalDelta()) + ', min=' + toFixedWidthPrecision(result.min())
    457             + ', max=' + toFixedWidthPrecision(result.max()) + '\n' + regressionAnalysis;
    458 
    459         // Tablesorter doesn't know about the second cell so put the comparison in the invisible element.
    460         return '<td class="result" title="' + statistics + '">' + toFixedWidthPrecision(result.mean()) + hiddenValue
    461             + '</td><td class="confidenceIntervalDelta" title="' + statistics + '">&plusmn; '
    462             + formatPercentage(result.confidenceIntervalDeltaRatio()) + warning + '</td>' + comparisonCell;
    463     }
    464 
    465     function markupForMissingRun(isReference) {
    466         return '<td colspan="' + (isReference ? 2 : 3) + '" class="missing">Missing</td>';
    467     }
    468 
    469     var runIndex = 0;
    470     var results = test.results();
    471     var referenceResult = undefined;
    472     var resultIndexMap = {};
    473     for (var i = 0; i < results.length; i++) {
    474         while (runs[runIndex] !== results[i].run())
    475             runIndex++;
    476         if (runIndex == referenceIndex)
    477             referenceResult = results[i];
    478         resultIndexMap[runIndex] = i;
    479     }
    480     for (var i = 0; i < runs.length; i++) {
    481         var resultIndex = resultIndexMap[i];
    482         if (resultIndex == undefined)
    483             tableRow.append(markupForMissingRun(i == referenceIndex));
    484         else
    485             tableRow.append(markupForRun(results[resultIndex], referenceResult));
    486     }
    487 
    488     $('#container').children('tbody').last().append(tableRow);
    489 
    490     function toggle() {
    491         var firstCell = tableRow.children('td').first();
    492         if (firstCell.children('section').length) {
    493             firstCell.children('section').remove();
    494             tableRow.children('td').css({'padding-bottom': ''});
    495         } else {
    496             var plot = createPlot(firstCell, test);
    497             plot.css({'position': 'absolute', 'z-index': 2});
    498             var offset = tableRow.offset();
    499             offset.left += 1;
    500             offset.top += tableRow.outerHeight();
    501             plot.offset(offset);
    502             tableRow.children('td').css({'padding-bottom': plot.outerHeight() + 5});
    503         }
    504 
    505         return false;
    506     };
    507 
    508     tableRow.click(function(event) {
    509         if (event.target != tableRow[0] && event.target.parentNode != tableRow[0])
    510             return;
    511 
    512         event.preventDefault();
    513 
    514         toggle();
    515     });
    516 
    517     if (test.isImportant) {
    518         toggle();
    519     }
    520 }
    521 
    522 function init() {
    523     $.tablesorter.addParser({
    524         id: 'comparison',
    525         is: function(s) {
    526             return s.indexOf('|') >= 0;
    527         },
    528         format: function(s) {
    529             var parsed = parseFloat(s.substring(s.indexOf('|') + 1));
    530             return isNaN(parsed) ? 0 : parsed;
    531         },
    532         type: 'numeric',
    533     });
    534 
    535     var runs = [];
    536     var metrics = {};
    537     $.each(JSON.parse(document.getElementById('results-json').textContent), function (index, entry) {
    538         var run = new TestRun(entry);
    539         runs.push(run);
    540 
    541         function addTests(tests, parentFullName) {
    542             for (var testName in tests) {
    543                 var fullTestName = parentFullName + '/' + testName;
    544                 var rawMetrics = tests[testName].metrics;
    545 
    546                 for (var metricName in rawMetrics) {
    547                     var fullMetricName = fullTestName + ':' + metricName;
    548                     var metric = metrics[fullMetricName];
    549                     if (!metric) {
    550                         metric = new PerfTestMetric(fullTestName, metricName, rawMetrics[metricName].units, rawMetrics[metricName].important);
    551                         metrics[fullMetricName] = metric;
    552                     }
    553                     metric.addResult(new TestResult(metric, rawMetrics[metricName].current, run));
    554                 }
    555 
    556                 if (tests[testName].tests)
    557                     addTests(tests[testName].tests, fullTestName);
    558             }
    559         }
    560 
    561         addTests(entry.tests, '');
    562     });
    563 
    564     var shouldIgnoreMemory= true;
    565     var referenceIndex = 0;
    566 
    567     createTable(metrics, runs, shouldIgnoreMemory, referenceIndex);
    568 
    569     $('#time-memory').bind('change', function (event, checkedElement) {
    570         shouldIgnoreMemory = checkedElement.textContent == 'Time';
    571         createTable(metrics, runs, shouldIgnoreMemory, referenceIndex);
    572     });
    573 
    574     runs.map(function (run, index) {
    575         $('#reference').append('<span value="' + index + '"' + (index == referenceIndex ? ' class="checked"' : '') + '>' + run.label() + '</span>');
    576     })
    577 
    578     $('#reference').bind('change', function (event, checkedElement) {
    579         referenceIndex = parseInt(checkedElement.getAttribute('value'));
    580         createTable(metrics, runs, shouldIgnoreMemory, referenceIndex);
    581     });
    582 
    583     $('.checkbox').each(function (index, checkbox) {
    584         $(checkbox).children('span').click(function (event) {
    585             if ($(this).hasClass('checked'))
    586                 return;
    587             $(checkbox).children('span').removeClass('checked');
    588             $(this).addClass('checked');
    589             $(checkbox).trigger('change', $(this));
    590         });
    591     });
    592 }
    593 
    594 </script>
    595 <script id="results-json" type="application/json">%json_results%</script>
    596 <script id="units-json" type="application/json">%json_units%</script>
    597 </body>
    598 </html>
    599