Home | History | Annotate | Download | only in scripts 
      1 /*
      2  * Copyright (C) 2011 Google Inc. All rights reserved.
      3  *
      4  * Redistribution and use in source and binary forms, with or without
      5  * modification, are permitted provided that the following conditions
      6  * are met:
      7  * 1. Redistributions of source code must retain the above copyright
      8  *    notice, this list of conditions and the following disclaimer.
      9  * 2. Redistributions in binary form must reproduce the above copyright
     10  *    notice, this list of conditions and the following disclaimer in the
     11  *    documentation and/or other materials provided with the distribution.
     12  *
     13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
     14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
     15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
     17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
     23  * THE POSSIBILITY OF SUCH DAMAGE.
     24  */
     25 
     26 var results = results || {};
     27 
     28 (function() {
     29 
     30 var kResultsName = 'failing_results.json';
     31 
     32 var kBuildLinkRegexp = /a href="\d+\/"/g;
     33 var kBuildNumberRegexp = /\d+/;
     34 
     35 var PASS = 'PASS';
     36 var TIMEOUT = 'TIMEOUT';
     37 var TEXT = 'TEXT';
     38 var CRASH = 'CRASH';
     39 var IMAGE = 'IMAGE';
     40 var IMAGE_TEXT = 'IMAGE+TEXT';
     41 var AUDIO = 'AUDIO';
     42 var MISSING = 'MISSING';
     43 
     44 var kFailingResults = [TEXT, IMAGE_TEXT, AUDIO];
     45 
     46 var kExpectedImageSuffix = '-expected.png';
     47 var kActualImageSuffix = '-actual.png';
     48 var kImageDiffSuffix = '-diff.png';
     49 var kExpectedAudioSuffix = '-expected.wav';
     50 var kActualAudioSuffix = '-actual.wav';
     51 var kExpectedTextSuffix = '-expected.txt';
     52 var kActualTextSuffix = '-actual.txt';
     53 var kDiffTextSuffix = '-diff.txt';
     54 var kCrashLogSuffix = '-crash-log.txt';
     55 
     56 var kPNGExtension = 'png';
     57 var kTXTExtension = 'txt';
     58 var kWAVExtension = 'wav';
     59 
     60 var kPreferredSuffixOrder = [
     61     kExpectedImageSuffix,
     62     kActualImageSuffix,
     63     kImageDiffSuffix,
     64     kExpectedTextSuffix,
     65     kActualTextSuffix,
     66     kDiffTextSuffix,
     67     kCrashLogSuffix,
     68     kExpectedAudioSuffix,
     69     kActualAudioSuffix,
     70     // FIXME: Add support for the rest of the result types.
     71 ];
     72 
     73 // Kinds of results.
     74 results.kActualKind = 'actual';
     75 results.kExpectedKind = 'expected';
     76 results.kDiffKind = 'diff';
     77 results.kUnknownKind = 'unknown';
     78 
     79 // Types of tests.
     80 results.kImageType = 'image'
     81 results.kAudioType = 'audio'
     82 results.kTextType = 'text'
     83 // FIXME: There are more types of tests.
     84 
     85 function layoutTestResultsURL(platform)
     86 {
     87     return config.kPlatforms[platform].layoutTestResultsURL;
     88 }
     89 
     90 function possibleSuffixListFor(failureTypeList)
     91 {
     92     var suffixList = [];
     93 
     94     function pushImageSuffixes()
     95     {
     96         suffixList.push(kExpectedImageSuffix);
     97         suffixList.push(kActualImageSuffix);
     98         suffixList.push(kImageDiffSuffix);
     99     }
    100 
    101     function pushAudioSuffixes()
    102     {
    103         suffixList.push(kExpectedAudioSuffix);
    104         suffixList.push(kActualAudioSuffix);
    105     }
    106 
    107     function pushTextSuffixes()
    108     {
    109         suffixList.push(kActualTextSuffix);
    110         suffixList.push(kExpectedTextSuffix);
    111         suffixList.push(kDiffTextSuffix);
    112         // '-wdiff.html',
    113         // '-pretty-diff.html',
    114     }
    115 
    116     $.each(failureTypeList, function(index, failureType) {
    117         switch(failureType) {
    118         case IMAGE:
    119             pushImageSuffixes();
    120             break;
    121         case TEXT:
    122             pushTextSuffixes();
    123             break;
    124         case AUDIO:
    125             pushAudioSuffixes();
    126             break;
    127         case IMAGE_TEXT:
    128             pushImageSuffixes();
    129             pushTextSuffixes();
    130             break;
    131         case CRASH:
    132             suffixList.push(kCrashLogSuffix);
    133             break;
    134         case MISSING:
    135             pushImageSuffixes();
    136             pushTextSuffixes();
    137             break;
    138         default:
    139             // FIXME: Add support for the rest of the result types.
    140             // '-expected.html',
    141             // '-expected-mismatch.html',
    142             // ... and possibly more.
    143             break;
    144         }
    145     });
    146 
    147     return base.uniquifyArray(suffixList);
    148 }
    149 
    150 results.failureTypeToExtensionList = function(failureType)
    151 {
    152     switch(failureType) {
    153     case IMAGE:
    154         return [kPNGExtension];
    155     case AUDIO:
    156         return [kWAVExtension];
    157     case TEXT:
    158         return [kTXTExtension];
    159     case MISSING:
    160     case IMAGE_TEXT:
    161         return [kTXTExtension, kPNGExtension];
    162     default:
    163         // FIXME: Add support for the rest of the result types.
    164         // '-expected.html',
    165         // '-expected-mismatch.html',
    166         // ... and possibly more.
    167         return [];
    168     }
    169 };
    170 
    171 results.failureTypeList = function(failureBlob)
    172 {
    173     return failureBlob.split(' ');
    174 };
    175 
    176 results.canRebaseline = function(failureTypeList)
    177 {
    178     return failureTypeList.some(function(element) {
    179         return results.failureTypeToExtensionList(element).length > 0;
    180     });
    181 };
    182 
    183 results.directoryForBuilder = function(builderName)
    184 {
    185     return config.kPlatforms[config.currentPlatform].resultsDirectoryNameFromBuilderName(builderName);
    186 }
    187 
    188 function resultsDirectoryURL(platform, builderName)
    189 {
    190     if (config.useLocalResults)
    191         return '/localresult?path=';
    192     return resultsDirectoryListingURL(platform, builderName) + 'results/layout-test-results/';
    193 }
    194 
    195 function resultsDirectoryListingURL(platform, builderName)
    196 {
    197     return layoutTestResultsURL(platform) + '/' + results.directoryForBuilder(builderName) + '/';
    198 }
    199 
    200 function resultsDirectoryURLForBuildNumber(platform, builderName, buildNumber)
    201 {
    202     return resultsDirectoryListingURL(platform, builderName) + buildNumber + '/';
    203 }
    204 
    205 function resultsSummaryURL(platform, builderName)
    206 {
    207     return resultsDirectoryURL(platform, builderName) + kResultsName;
    208 }
    209 
    210 function resultsSummaryURLForBuildNumber(platform, builderName, buildNumber)
    211 {
    212     return resultsDirectoryURLForBuildNumber(platform, builderName, buildNumber) + kResultsName;
    213 }
    214 
    215 var g_resultsCache = new base.AsynchronousCache(function (key, callback) {
    216     net.jsonp(key, callback);
    217 });
    218 
    219 results.ResultAnalyzer = base.extends(Object, {
    220     init: function(resultNode)
    221     {
    222         this._isUnexpected = resultNode.is_unexpected;
    223         this._actual = resultNode ? results.failureTypeList(resultNode.actual) : [];
    224         this._expected = resultNode ? this._addImpliedExpectations(results.failureTypeList(resultNode.expected)) : [];
    225     },
    226     _addImpliedExpectations: function(resultsList)
    227     {
    228         if (resultsList.indexOf('FAIL') == -1)
    229             return resultsList;
    230         return resultsList.concat(kFailingResults);
    231     },
    232     _hasPass: function(results)
    233     {
    234         return results.indexOf(PASS) != -1;
    235     },
    236     unexpectedResults: function()
    237     {
    238         return this._actual.filter(function(result) {
    239             return this._expected.indexOf(result) == -1;
    240         }, this);
    241     },
    242     succeeded: function()
    243     {
    244         return this._hasPass(this._actual);
    245     },
    246     flaky: function()
    247     {
    248         return this._actual.length > 1;
    249     },
    250     wontfix: function()
    251     {
    252         return this._expected.indexOf('WONTFIX') != -1;
    253     },
    254     hasUnexpectedFailures: function()
    255     {
    256         return this._isUnexpected;
    257     }
    258 })
    259 
    260 function isExpectedFailure(resultNode)
    261 {
    262     var analyzer = new results.ResultAnalyzer(resultNode);
    263     return !analyzer.hasUnexpectedFailures() && !analyzer.succeeded() && !analyzer.flaky() && !analyzer.wontfix();
    264 }
    265 
    266 function isUnexpectedFailure(resultNode)
    267 {
    268     var analyzer = new results.ResultAnalyzer(resultNode);
    269     return analyzer.hasUnexpectedFailures() && !analyzer.succeeded() && !analyzer.flaky() && !analyzer.wontfix();
    270 }
    271 
    272 function isResultNode(node)
    273 {
    274     return !!node.actual;
    275 }
    276 
    277 results.expectedFailures = function(resultsTree)
    278 {
    279     return base.filterTree(resultsTree.tests, isResultNode, isExpectedFailure);
    280 };
    281 
    282 results.unexpectedFailures = function(resultsTree)
    283 {
    284     return base.filterTree(resultsTree.tests, isResultNode, isUnexpectedFailure);
    285 };
    286 
    287 function resultsByTest(resultsByBuilder, filter)
    288 {
    289     var resultsByTest = {};
    290 
    291     $.each(resultsByBuilder, function(builderName, resultsTree) {
    292         $.each(filter(resultsTree), function(testName, resultNode) {
    293             resultsByTest[testName] = resultsByTest[testName] || {};
    294             resultsByTest[testName][builderName] = resultNode;
    295         });
    296     });
    297 
    298     return resultsByTest;
    299 }
    300 
    301 results.expectedFailuresByTest = function(resultsByBuilder)
    302 {
    303     return resultsByTest(resultsByBuilder, results.expectedFailures);
    304 };
    305 
    306 results.unexpectedFailuresByTest = function(resultsByBuilder)
    307 {
    308     return resultsByTest(resultsByBuilder, results.unexpectedFailures);
    309 };
    310 
    311 results.failureInfoForTestAndBuilder = function(resultsByTest, testName, builderName)
    312 {
    313     var failureInfoForTest = {
    314         'testName': testName,
    315         'builderName': builderName,
    316         'failureTypeList': results.failureTypeList(resultsByTest[testName][builderName].actual),
    317     };
    318 
    319     return failureInfoForTest;
    320 };
    321 
    322 results.collectUnexpectedResults = function(dictionaryOfResultNodes)
    323 {
    324     var collectedResults = [];
    325     $.each(dictionaryOfResultNodes, function(key, resultNode) {
    326         var analyzer = new results.ResultAnalyzer(resultNode);
    327         collectedResults = collectedResults.concat(analyzer.unexpectedResults());
    328     });
    329     return base.uniquifyArray(collectedResults);
    330 };
    331 
    332 // Callback data is [{ buildNumber:, url: }]
    333 function historicalResultsLocations(platform, builderName, callback)
    334 {
    335     var listingURL = resultsDirectoryListingURL(platform, builderName);
    336     net.get(listingURL, function(directoryListing) {
    337         var historicalResultsData = directoryListing.match(kBuildLinkRegexp).map(function(buildLink) {
    338             var buildNumber = parseInt(buildLink.match(kBuildNumberRegexp)[0]);
    339             var resultsData = {
    340                 'buildNumber': buildNumber,
    341                 'url': resultsSummaryURLForBuildNumber(platform, builderName, buildNumber)
    342             };
    343             return resultsData;
    344         }).reverse();
    345 
    346         callback(historicalResultsData);
    347     });
    348 }
    349 
    350 function walkHistory(platform, builderName, testName, callback)
    351 {
    352     var indexOfNextKeyToFetch = 0;
    353     var keyList = [];
    354 
    355     function continueWalk()
    356     {
    357         if (indexOfNextKeyToFetch >= keyList.length) {
    358             processResultNode(0, null);
    359             return;
    360         }
    361 
    362         var resultsURL = keyList[indexOfNextKeyToFetch].url;
    363         ++indexOfNextKeyToFetch;
    364         g_resultsCache.get(resultsURL, function(resultsTree) {
    365             if ($.isEmptyObject(resultsTree)) {
    366                 continueWalk();
    367                 return;
    368             }
    369             var resultNode = results.resultNodeForTest(resultsTree, testName);
    370             var revision = parseInt(resultsTree['blink_revision'])
    371             if (isNaN(revision))
    372                 revision = 0;
    373             processResultNode(revision, resultNode);
    374         });
    375     }
    376 
    377     function processResultNode(revision, resultNode)
    378     {
    379         var shouldContinue = callback(revision, resultNode);
    380         if (!shouldContinue)
    381             return;
    382         continueWalk();
    383     }
    384 
    385     historicalResultsLocations(platform, builderName, function(resultsLocations) {
    386         keyList = resultsLocations;
    387         continueWalk();
    388     });
    389 }
    390 
    391 results.regressionRangeForFailure = function(builderName, testName, callback)
    392 {
    393     var oldestFailingRevision = 0;
    394     var newestPassingRevision = 0;
    395 
    396     // FIXME: should treat {platform, builderName} as a tuple
    397     walkHistory(config.currentPlatform, builderName, testName, function(revision, resultNode) {
    398         if (!revision) {
    399             callback(oldestFailingRevision, newestPassingRevision);
    400             return false;
    401         }
    402         if (!resultNode) {
    403             newestPassingRevision = revision;
    404             callback(oldestFailingRevision, newestPassingRevision);
    405             return false;
    406         }
    407         if (isUnexpectedFailure(resultNode)) {
    408             oldestFailingRevision = revision;
    409             return true;
    410         }
    411         if (!oldestFailingRevision)
    412             return true;  // We need to keep looking for a failing revision.
    413         newestPassingRevision = revision;
    414         callback(oldestFailingRevision, newestPassingRevision);
    415         return false;
    416     });
    417 };
    418 
    419 function mergeRegressionRanges(regressionRanges)
    420 {
    421     var mergedRange = {};
    422 
    423     mergedRange.oldestFailingRevision = 0;
    424     mergedRange.newestPassingRevision = 0;
    425 
    426     $.each(regressionRanges, function(builderName, range) {
    427         if (!range.oldestFailingRevision && !range.newestPassingRevision)
    428             return
    429 
    430         if (!mergedRange.oldestFailingRevision)
    431             mergedRange.oldestFailingRevision = range.oldestFailingRevision;
    432         if (!mergedRange.newestPassingRevision)
    433             mergedRange.newestPassingRevision = range.newestPassingRevision;
    434 
    435         if (range.oldestFailingRevision && range.oldestFailingRevision < mergedRange.oldestFailingRevision)
    436             mergedRange.oldestFailingRevision = range.oldestFailingRevision;
    437         if (range.newestPassingRevision > mergedRange.newestPassingRevision)
    438             mergedRange.newestPassingRevision = range.newestPassingRevision;
    439     });
    440 
    441     return mergedRange;
    442 }
    443 
    444 results.unifyRegressionRanges = function(builderNameList, testName, callback)
    445 {
    446     var regressionRanges = {};
    447 
    448     var tracker = new base.RequestTracker(builderNameList.length, function() {
    449         var mergedRange = mergeRegressionRanges(regressionRanges);
    450         callback(mergedRange.oldestFailingRevision, mergedRange.newestPassingRevision);
    451     });
    452 
    453     $.each(builderNameList, function(index, builderName) {
    454         results.regressionRangeForFailure(builderName, testName, function(oldestFailingRevision, newestPassingRevision) {
    455             var range = {};
    456             range.oldestFailingRevision = oldestFailingRevision;
    457             range.newestPassingRevision = newestPassingRevision;
    458             regressionRanges[builderName] = range;
    459             tracker.requestComplete();
    460         });
    461     });
    462 };
    463 
    464 results.resultNodeForTest = function(resultsTree, testName)
    465 {
    466     var testNamePath = testName.split('/');
    467     var currentNode = resultsTree['tests'];
    468     $.each(testNamePath, function(index, segmentName) {
    469         if (!currentNode)
    470             return;
    471         currentNode = (segmentName in currentNode) ? currentNode[segmentName] : null;
    472     });
    473     return currentNode;
    474 };
    475 
    476 results.resultKind = function(url)
    477 {
    478     if (/-actual\.[a-z]+$/.test(url))
    479         return results.kActualKind;
    480     else if (/-expected\.[a-z]+$/.test(url))
    481         return results.kExpectedKind;
    482     else if (/diff\.[a-z]+$/.test(url))
    483         return results.kDiffKind;
    484     return results.kUnknownKind;
    485 }
    486 
    487 results.resultType = function(url)
    488 {
    489     if (/\.png$/.test(url))
    490         return results.kImageType;
    491     if (/\.wav$/.test(url))
    492         return results.kAudioType;
    493     return results.kTextType;
    494 }
    495 
    496 function sortResultURLsBySuffix(urls)
    497 {
    498     var sortedURLs = [];
    499     $.each(kPreferredSuffixOrder, function(i, suffix) {
    500         $.each(urls, function(j, url) {
    501             if (!base.endsWith(url, suffix))
    502                 return;
    503             sortedURLs.push(url);
    504         });
    505     });
    506     if (sortedURLs.length != urls.length)
    507         throw "sortResultURLsBySuffix failed to return the same number of URLs."
    508     return sortedURLs;
    509 }
    510 
    511 results.fetchResultsURLs = function(failureInfo, callback)
    512 {
    513     var testNameStem = base.trimExtension(failureInfo.testName);
    514     var urlStem = resultsDirectoryURL(config.currentPlatform, failureInfo.builderName);
    515 
    516     var suffixList = possibleSuffixListFor(failureInfo.failureTypeList);
    517     var resultURLs = [];
    518     var tracker = new base.RequestTracker(suffixList.length, function() {
    519         callback(sortResultURLsBySuffix(resultURLs));
    520     });
    521     $.each(suffixList, function(index, suffix) {
    522         var url = urlStem + testNameStem + suffix;
    523         net.probe(url, {
    524             success: function() {
    525                 resultURLs.push(url);
    526                 tracker.requestComplete();
    527             },
    528             error: function() {
    529                 tracker.requestComplete();
    530             },
    531         });
    532     });
    533 };
    534 
    535 results.fetchResultsByBuilder = function(builderNameList, callback)
    536 {
    537     var resultsByBuilder = {};
    538     var tracker = new base.RequestTracker(builderNameList.length, function() {
    539         callback(resultsByBuilder);
    540     });
    541     $.each(builderNameList, function(index, builderName) {
    542         var resultsURL = resultsSummaryURL(config.currentPlatform, builderName);
    543         net.jsonp(resultsURL, function(resultsTree) {
    544             resultsByBuilder[builderName] = resultsTree;
    545             tracker.requestComplete();
    546         });
    547     });
    548 };
    549 
    550 })();
    551