Home | History | Annotate | Download | only in scripts 
      1 /*
      2  * Copyright (C) 2011 Google Inc. All rights reserved.
      3  *
      4  * Redistribution and use in source and binary forms, with or without
      5  * modification, are permitted provided that the following conditions
      6  * are met:
      7  * 1. Redistributions of source code must retain the above copyright
      8  *    notice, this list of conditions and the following disclaimer.
      9  * 2. Redistributions in binary form must reproduce the above copyright
     10  *    notice, this list of conditions and the following disclaimer in the
     11  *    documentation and/or other materials provided with the distribution.
     12  *
     13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
     14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
     15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
     17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
     23  * THE POSSIBILITY OF SUCH DAMAGE.
     24  */
     25 
     26 var results = results || {};
     27 
     28 (function() {
     29 
     30 var kResultsName = 'failing_results.json';
     31 
     32 var PASS = 'PASS';
     33 var TIMEOUT = 'TIMEOUT';
     34 var TEXT = 'TEXT';
     35 var CRASH = 'CRASH';
     36 var IMAGE = 'IMAGE';
     37 var IMAGE_TEXT = 'IMAGE+TEXT';
     38 var AUDIO = 'AUDIO';
     39 var MISSING = 'MISSING';
     40 
     41 var kFailingResults = [TEXT, IMAGE_TEXT, AUDIO];
     42 
     43 var kExpectedImageSuffix = '-expected.png';
     44 var kActualImageSuffix = '-actual.png';
     45 var kImageDiffSuffix = '-diff.png';
     46 var kExpectedAudioSuffix = '-expected.wav';
     47 var kActualAudioSuffix = '-actual.wav';
     48 var kExpectedTextSuffix = '-expected.txt';
     49 var kActualTextSuffix = '-actual.txt';
     50 var kDiffTextSuffix = '-diff.txt';
     51 var kCrashLogSuffix = '-crash-log.txt';
     52 
     53 var kPNGExtension = 'png';
     54 var kTXTExtension = 'txt';
     55 var kWAVExtension = 'wav';
     56 
     57 var kPreferredSuffixOrder = [
     58     kExpectedImageSuffix,
     59     kActualImageSuffix,
     60     kImageDiffSuffix,
     61     kExpectedTextSuffix,
     62     kActualTextSuffix,
     63     kDiffTextSuffix,
     64     kCrashLogSuffix,
     65     kExpectedAudioSuffix,
     66     kActualAudioSuffix,
     67     // FIXME: Add support for the rest of the result types.
     68 ];
     69 
     70 // Kinds of results.
     71 results.kActualKind = 'actual';
     72 results.kExpectedKind = 'expected';
     73 results.kDiffKind = 'diff';
     74 results.kUnknownKind = 'unknown';
     75 
     76 // Types of tests.
     77 results.kImageType = 'image'
     78 results.kAudioType = 'audio'
     79 results.kTextType = 'text'
     80 // FIXME: There are more types of tests.
     81 
     82 function layoutTestResultsURL(platform)
     83 {
     84     return config.kPlatforms[platform].layoutTestResultsURL;
     85 }
     86 
     87 function possibleSuffixListFor(failureTypeList)
     88 {
     89     var suffixList = [];
     90 
     91     function pushImageSuffixes()
     92     {
     93         suffixList.push(kExpectedImageSuffix);
     94         suffixList.push(kActualImageSuffix);
     95         suffixList.push(kImageDiffSuffix);
     96     }
     97 
     98     function pushAudioSuffixes()
     99     {
    100         suffixList.push(kExpectedAudioSuffix);
    101         suffixList.push(kActualAudioSuffix);
    102     }
    103 
    104     function pushTextSuffixes()
    105     {
    106         suffixList.push(kActualTextSuffix);
    107         suffixList.push(kExpectedTextSuffix);
    108         suffixList.push(kDiffTextSuffix);
    109         // '-wdiff.html',
    110         // '-pretty-diff.html',
    111     }
    112 
    113     $.each(failureTypeList, function(index, failureType) {
    114         switch(failureType) {
    115         case IMAGE:
    116             pushImageSuffixes();
    117             break;
    118         case TEXT:
    119             pushTextSuffixes();
    120             break;
    121         case AUDIO:
    122             pushAudioSuffixes();
    123             break;
    124         case IMAGE_TEXT:
    125             pushImageSuffixes();
    126             pushTextSuffixes();
    127             break;
    128         case CRASH:
    129             suffixList.push(kCrashLogSuffix);
    130             break;
    131         case MISSING:
    132             pushImageSuffixes();
    133             pushTextSuffixes();
    134             break;
    135         default:
    136             // FIXME: Add support for the rest of the result types.
    137             // '-expected.html',
    138             // '-expected-mismatch.html',
    139             // ... and possibly more.
    140             break;
    141         }
    142     });
    143 
    144     return base.uniquifyArray(suffixList);
    145 }
    146 
    147 results.failureTypeToExtensionList = function(failureType)
    148 {
    149     switch(failureType) {
    150     case IMAGE:
    151         return [kPNGExtension];
    152     case AUDIO:
    153         return [kWAVExtension];
    154     case TEXT:
    155         return [kTXTExtension];
    156     case MISSING:
    157     case IMAGE_TEXT:
    158         return [kTXTExtension, kPNGExtension];
    159     default:
    160         // FIXME: Add support for the rest of the result types.
    161         // '-expected.html',
    162         // '-expected-mismatch.html',
    163         // ... and possibly more.
    164         return [];
    165     }
    166 };
    167 
    168 results.failureTypeList = function(failureBlob)
    169 {
    170     return failureBlob.split(' ');
    171 };
    172 
    173 results.directoryForBuilder = function(builderName)
    174 {
    175     return config.kPlatforms[config.currentPlatform].resultsDirectoryNameFromBuilderName(builderName);
    176 }
    177 
    178 function resultsDirectoryURL(platform, builderName)
    179 {
    180     if (config.useLocalResults)
    181         return '/localresult?path=';
    182     return layoutTestResultsURL(platform) + '/' + results.directoryForBuilder(builderName) + '/results/layout-test-results/';
    183 }
    184 
    185 function resultsPrefixListingURL(platform, builderName, marker)
    186 {
    187     var url =  layoutTestResultsURL(platform) + '/?prefix=' + results.directoryForBuilder(builderName) + '/&delimiter=/';
    188     if (marker)
    189         return url + '&marker=' + marker;
    190     return url;
    191 }
    192 
    193 function resultsDirectoryURLForBuildNumber(platform, builderName, buildNumber)
    194 {
    195     return layoutTestResultsURL(platform) + '/' + results.directoryForBuilder(builderName) + '/' + buildNumber + '/' ;
    196 }
    197 
    198 function resultsSummaryURL(platform, builderName)
    199 {
    200     return resultsDirectoryURL(platform, builderName) + kResultsName;
    201 }
    202 
    203 function resultsSummaryURLForBuildNumber(platform, builderName, buildNumber)
    204 {
    205     return resultsDirectoryURLForBuildNumber(platform, builderName, buildNumber) + kResultsName;
    206 }
    207 
    208 var g_resultsCache = new base.AsynchronousCache(function (key, callback) {
    209     net.jsonp(key, callback);
    210 });
    211 
    212 results.ResultAnalyzer = base.extends(Object, {
    213     init: function(resultNode)
    214     {
    215         this._isUnexpected = resultNode.is_unexpected;
    216         this._actual = resultNode ? results.failureTypeList(resultNode.actual) : [];
    217         this._expected = resultNode ? this._addImpliedExpectations(results.failureTypeList(resultNode.expected)) : [];
    218     },
    219     _addImpliedExpectations: function(resultsList)
    220     {
    221         if (resultsList.indexOf('FAIL') == -1)
    222             return resultsList;
    223         return resultsList.concat(kFailingResults);
    224     },
    225     _hasPass: function(results)
    226     {
    227         return results.indexOf(PASS) != -1;
    228     },
    229     unexpectedResults: function()
    230     {
    231         return this._actual.filter(function(result) {
    232             return this._expected.indexOf(result) == -1;
    233         }, this);
    234     },
    235     succeeded: function()
    236     {
    237         return this._hasPass(this._actual);
    238     },
    239     flaky: function()
    240     {
    241         return this._actual.length > 1;
    242     },
    243     wontfix: function()
    244     {
    245         return this._expected.indexOf('WONTFIX') != -1;
    246     },
    247     hasUnexpectedFailures: function()
    248     {
    249         return this._isUnexpected;
    250     }
    251 })
    252 
    253 function isExpectedFailure(resultNode)
    254 {
    255     var analyzer = new results.ResultAnalyzer(resultNode);
    256     return !analyzer.hasUnexpectedFailures() && !analyzer.succeeded() && !analyzer.flaky() && !analyzer.wontfix();
    257 }
    258 
    259 function isUnexpectedFailure(resultNode)
    260 {
    261     var analyzer = new results.ResultAnalyzer(resultNode);
    262     return analyzer.hasUnexpectedFailures() && !analyzer.succeeded() && !analyzer.flaky() && !analyzer.wontfix();
    263 }
    264 
    265 function isResultNode(node)
    266 {
    267     return !!node.actual;
    268 }
    269 
    270 results.expectedFailures = function(resultsTree)
    271 {
    272     return base.filterTree(resultsTree.tests, isResultNode, isExpectedFailure);
    273 };
    274 
    275 results.unexpectedFailures = function(resultsTree)
    276 {
    277     return base.filterTree(resultsTree.tests, isResultNode, isUnexpectedFailure);
    278 };
    279 
    280 function resultsByTest(resultsByBuilder, filter)
    281 {
    282     var resultsByTest = {};
    283 
    284     $.each(resultsByBuilder, function(builderName, resultsTree) {
    285         $.each(filter(resultsTree), function(testName, resultNode) {
    286             resultsByTest[testName] = resultsByTest[testName] || {};
    287             resultsByTest[testName][builderName] = resultNode;
    288         });
    289     });
    290 
    291     return resultsByTest;
    292 }
    293 
    294 results.expectedFailuresByTest = function(resultsByBuilder)
    295 {
    296     return resultsByTest(resultsByBuilder, results.expectedFailures);
    297 };
    298 
    299 results.unexpectedFailuresByTest = function(resultsByBuilder)
    300 {
    301     return resultsByTest(resultsByBuilder, results.unexpectedFailures);
    302 };
    303 
    304 results.failureInfoForTestAndBuilder = function(resultsByTest, testName, builderName)
    305 {
    306     var failureInfoForTest = {
    307         'testName': testName,
    308         'builderName': builderName,
    309         'failureTypeList': results.failureTypeList(resultsByTest[testName][builderName].actual),
    310     };
    311 
    312     return failureInfoForTest;
    313 };
    314 
    315 results.collectUnexpectedResults = function(dictionaryOfResultNodes)
    316 {
    317     var collectedResults = [];
    318     $.each(dictionaryOfResultNodes, function(key, resultNode) {
    319         var analyzer = new results.ResultAnalyzer(resultNode);
    320         collectedResults = collectedResults.concat(analyzer.unexpectedResults());
    321     });
    322     return base.uniquifyArray(collectedResults);
    323 };
    324 
    325 // Callback data is [{ buildNumber:, url: }]
    326 function historicalResultsLocations(platform, builderName, callback)
    327 {
    328     var historicalResultsData = [];
    329 
    330     function parseListingDocument(prefixListingDocument) {
    331         $(prefixListingDocument).find("Prefix").each(function() {
    332             var buildString = this.textContent.replace(results.directoryForBuilder(builderName) + '/', '');
    333             if (buildString.match(/\d+\//)) {
    334                 var buildNumber = parseInt(buildString);
    335                 var resultsData = {
    336                     'buildNumber': buildNumber,
    337                     'url': resultsSummaryURLForBuildNumber(platform, builderName, buildNumber)
    338                 };
    339                 historicalResultsData.unshift(resultsData);
    340            }
    341         });
    342         var nextMarker = $(prefixListingDocument).find('NextMarker').get();
    343         if (nextMarker.length) {
    344             var nextListingURL = resultsPrefixListingURL(platform, builderName, nextMarker[0].textContent);
    345             net.get(nextListingURL, parseListingDocument);
    346         } else {
    347             callback(historicalResultsData);
    348         }
    349     }
    350 
    351     builders.mostRecentBuildForBuilder(platform, builderName, function (mostRecentBuildNumber) {
    352         var marker = results.directoryForBuilder(builderName) + "/" + (mostRecentBuildNumber - 100) + "/";
    353         var listingURL = resultsPrefixListingURL(platform, builderName, marker);
    354         net.get(listingURL, parseListingDocument);
    355     });
    356 }
    357 
    358 function walkHistory(platform, builderName, testName, callback)
    359 {
    360     var indexOfNextKeyToFetch = 0;
    361     var keyList = [];
    362 
    363     function continueWalk()
    364     {
    365         if (indexOfNextKeyToFetch >= keyList.length) {
    366             processResultNode(0, null);
    367             return;
    368         }
    369 
    370         var resultsURL = keyList[indexOfNextKeyToFetch].url;
    371         ++indexOfNextKeyToFetch;
    372         g_resultsCache.get(resultsURL, function(resultsTree) {
    373             if ($.isEmptyObject(resultsTree)) {
    374                 continueWalk();
    375                 return;
    376             }
    377             var resultNode = results.resultNodeForTest(resultsTree, testName);
    378             var revision = parseInt(resultsTree['blink_revision'])
    379             if (isNaN(revision))
    380                 revision = 0;
    381             processResultNode(revision, resultNode);
    382         });
    383     }
    384 
    385     function processResultNode(revision, resultNode)
    386     {
    387         var shouldContinue = callback(revision, resultNode);
    388         if (!shouldContinue)
    389             return;
    390         continueWalk();
    391     }
    392 
    393     historicalResultsLocations(platform, builderName, function(resultsLocations) {
    394         keyList = resultsLocations;
    395         continueWalk();
    396     });
    397 }
    398 
    399 results.regressionRangeForFailure = function(builderName, testName, callback)
    400 {
    401     var oldestFailingRevision = 0;
    402     var newestPassingRevision = 0;
    403 
    404     // FIXME: should treat {platform, builderName} as a tuple
    405     walkHistory(config.currentPlatform, builderName, testName, function(revision, resultNode) {
    406         if (!revision) {
    407             callback(oldestFailingRevision, newestPassingRevision);
    408             return false;
    409         }
    410         if (!resultNode) {
    411             newestPassingRevision = revision;
    412             callback(oldestFailingRevision, newestPassingRevision);
    413             return false;
    414         }
    415         if (isUnexpectedFailure(resultNode)) {
    416             oldestFailingRevision = revision;
    417             return true;
    418         }
    419         if (!oldestFailingRevision)
    420             return true;  // We need to keep looking for a failing revision.
    421         newestPassingRevision = revision;
    422         callback(oldestFailingRevision, newestPassingRevision);
    423         return false;
    424     });
    425 };
    426 
    427 function mergeRegressionRanges(regressionRanges)
    428 {
    429     var mergedRange = {};
    430 
    431     mergedRange.oldestFailingRevision = 0;
    432     mergedRange.newestPassingRevision = 0;
    433 
    434     $.each(regressionRanges, function(builderName, range) {
    435         if (!range.oldestFailingRevision && !range.newestPassingRevision)
    436             return
    437 
    438         if (!mergedRange.oldestFailingRevision)
    439             mergedRange.oldestFailingRevision = range.oldestFailingRevision;
    440         if (!mergedRange.newestPassingRevision)
    441             mergedRange.newestPassingRevision = range.newestPassingRevision;
    442 
    443         if (range.oldestFailingRevision && range.oldestFailingRevision < mergedRange.oldestFailingRevision)
    444             mergedRange.oldestFailingRevision = range.oldestFailingRevision;
    445         if (range.newestPassingRevision > mergedRange.newestPassingRevision)
    446             mergedRange.newestPassingRevision = range.newestPassingRevision;
    447     });
    448 
    449     return mergedRange;
    450 }
    451 
    452 results.unifyRegressionRanges = function(builderNameList, testName, callback)
    453 {
    454     var regressionRanges = {};
    455 
    456     var tracker = new base.RequestTracker(builderNameList.length, function() {
    457         var mergedRange = mergeRegressionRanges(regressionRanges);
    458         callback(mergedRange.oldestFailingRevision, mergedRange.newestPassingRevision);
    459     });
    460 
    461     $.each(builderNameList, function(index, builderName) {
    462         results.regressionRangeForFailure(builderName, testName, function(oldestFailingRevision, newestPassingRevision) {
    463             var range = {};
    464             range.oldestFailingRevision = oldestFailingRevision;
    465             range.newestPassingRevision = newestPassingRevision;
    466             regressionRanges[builderName] = range;
    467             tracker.requestComplete();
    468         });
    469     });
    470 };
    471 
    472 results.resultNodeForTest = function(resultsTree, testName)
    473 {
    474     var testNamePath = testName.split('/');
    475     var currentNode = resultsTree['tests'];
    476     $.each(testNamePath, function(index, segmentName) {
    477         if (!currentNode)
    478             return;
    479         currentNode = (segmentName in currentNode) ? currentNode[segmentName] : null;
    480     });
    481     return currentNode;
    482 };
    483 
    484 results.resultKind = function(url)
    485 {
    486     if (/-actual\.[a-z]+$/.test(url))
    487         return results.kActualKind;
    488     else if (/-expected\.[a-z]+$/.test(url))
    489         return results.kExpectedKind;
    490     else if (/diff\.[a-z]+$/.test(url))
    491         return results.kDiffKind;
    492     return results.kUnknownKind;
    493 }
    494 
    495 results.resultType = function(url)
    496 {
    497     if (/\.png$/.test(url))
    498         return results.kImageType;
    499     if (/\.wav$/.test(url))
    500         return results.kAudioType;
    501     return results.kTextType;
    502 }
    503 
    504 function sortResultURLsBySuffix(urls)
    505 {
    506     var sortedURLs = [];
    507     $.each(kPreferredSuffixOrder, function(i, suffix) {
    508         $.each(urls, function(j, url) {
    509             if (!base.endsWith(url, suffix))
    510                 return;
    511             sortedURLs.push(url);
    512         });
    513     });
    514     if (sortedURLs.length != urls.length)
    515         throw "sortResultURLsBySuffix failed to return the same number of URLs."
    516     return sortedURLs;
    517 }
    518 
    519 results.fetchResultsURLs = function(failureInfo, callback)
    520 {
    521     var testNameStem = base.trimExtension(failureInfo.testName);
    522     var urlStem = resultsDirectoryURL(config.currentPlatform, failureInfo.builderName);
    523 
    524     var suffixList = possibleSuffixListFor(failureInfo.failureTypeList);
    525     var resultURLs = [];
    526     var tracker = new base.RequestTracker(suffixList.length, function() {
    527         callback(sortResultURLsBySuffix(resultURLs));
    528     });
    529     $.each(suffixList, function(index, suffix) {
    530         var url = urlStem + testNameStem + suffix;
    531         net.probe(url, {
    532             success: function() {
    533                 resultURLs.push(url);
    534                 tracker.requestComplete();
    535             },
    536             error: function() {
    537                 tracker.requestComplete();
    538             },
    539         });
    540     });
    541 };
    542 
    543 results.fetchResultsByBuilder = function(builderNameList, callback)
    544 {
    545     var resultsByBuilder = {};
    546     var tracker = new base.RequestTracker(builderNameList.length, function() {
    547         callback(resultsByBuilder);
    548     });
    549     $.each(builderNameList, function(index, builderName) {
    550         var resultsURL = resultsSummaryURL(config.currentPlatform, builderName);
    551         net.jsonp(resultsURL, function(resultsTree) {
    552             resultsByBuilder[builderName] = resultsTree;
    553             tracker.requestComplete();
    554         });
    555     });
    556 };
    557 
    558 })();
    559