1 /* 2 * Copyright (C) 2011 Google Inc. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' 14 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 15 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS 17 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 18 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 19 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 20 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 21 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 22 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF 23 * THE POSSIBILITY OF SUCH DAMAGE. 24 */ 25 26 var results = results || {}; 27 28 (function() { 29 30 var kResultsName = 'failing_results.json'; 31 32 var PASS = 'PASS'; 33 var TIMEOUT = 'TIMEOUT'; 34 var TEXT = 'TEXT'; 35 var CRASH = 'CRASH'; 36 var IMAGE = 'IMAGE'; 37 var IMAGE_TEXT = 'IMAGE+TEXT'; 38 var AUDIO = 'AUDIO'; 39 var MISSING = 'MISSING'; 40 41 var kFailingResults = [TEXT, IMAGE_TEXT, AUDIO]; 42 43 var kExpectedImageSuffix = '-expected.png'; 44 var kActualImageSuffix = '-actual.png'; 45 var kImageDiffSuffix = '-diff.png'; 46 var kExpectedAudioSuffix = '-expected.wav'; 47 var kActualAudioSuffix = '-actual.wav'; 48 var kExpectedTextSuffix = '-expected.txt'; 49 var kActualTextSuffix = '-actual.txt'; 50 var kDiffTextSuffix = '-diff.txt'; 51 var kCrashLogSuffix = '-crash-log.txt'; 52 53 var kPNGExtension = 'png'; 54 var kTXTExtension = 'txt'; 55 var kWAVExtension = 'wav'; 56 57 var kPreferredSuffixOrder = [ 58 kExpectedImageSuffix, 59 kActualImageSuffix, 60 kImageDiffSuffix, 61 kExpectedTextSuffix, 62 kActualTextSuffix, 63 kDiffTextSuffix, 64 kCrashLogSuffix, 65 kExpectedAudioSuffix, 66 kActualAudioSuffix, 67 // FIXME: Add support for the rest of the result types. 68 ]; 69 70 // Kinds of results. 71 results.kActualKind = 'actual'; 72 results.kExpectedKind = 'expected'; 73 results.kDiffKind = 'diff'; 74 results.kUnknownKind = 'unknown'; 75 76 // Types of tests. 77 results.kImageType = 'image'; 78 results.kAudioType = 'audio'; 79 results.kTextType = 'text'; 80 // FIXME: There are more types of tests. 81 82 function possibleSuffixListFor(failureTypeList) 83 { 84 var suffixList = []; 85 86 function pushImageSuffixes() 87 { 88 suffixList.push(kExpectedImageSuffix); 89 suffixList.push(kActualImageSuffix); 90 suffixList.push(kImageDiffSuffix); 91 } 92 93 function pushAudioSuffixes() 94 { 95 suffixList.push(kExpectedAudioSuffix); 96 suffixList.push(kActualAudioSuffix); 97 } 98 99 function pushTextSuffixes() 100 { 101 suffixList.push(kActualTextSuffix); 102 suffixList.push(kExpectedTextSuffix); 103 suffixList.push(kDiffTextSuffix); 104 // '-wdiff.html', 105 // '-pretty-diff.html', 106 } 107 108 $.each(failureTypeList, function(index, failureType) { 109 switch(failureType) { 110 case IMAGE: 111 pushImageSuffixes(); 112 break; 113 case TEXT: 114 pushTextSuffixes(); 115 break; 116 case AUDIO: 117 pushAudioSuffixes(); 118 break; 119 case IMAGE_TEXT: 120 pushImageSuffixes(); 121 pushTextSuffixes(); 122 break; 123 case CRASH: 124 suffixList.push(kCrashLogSuffix); 125 break; 126 case MISSING: 127 pushImageSuffixes(); 128 pushTextSuffixes(); 129 break; 130 default: 131 // FIXME: Add support for the rest of the result types. 132 // '-expected.html', 133 // '-expected-mismatch.html', 134 // ... and possibly more. 135 break; 136 } 137 }); 138 139 return base.uniquifyArray(suffixList); 140 } 141 142 results.failureTypeToExtensionList = function(failureType) 143 { 144 switch(failureType) { 145 case IMAGE: 146 return [kPNGExtension]; 147 case AUDIO: 148 return [kWAVExtension]; 149 case TEXT: 150 return [kTXTExtension]; 151 case MISSING: 152 case IMAGE_TEXT: 153 return [kTXTExtension, kPNGExtension]; 154 default: 155 // FIXME: Add support for the rest of the result types. 156 // '-expected.html', 157 // '-expected-mismatch.html', 158 // ... and possibly more. 159 return []; 160 } 161 }; 162 163 results.failureTypeList = function(failureBlob) 164 { 165 return failureBlob.split(' '); 166 }; 167 168 function resultsDirectoryURL(builderName) 169 { 170 if (config.useLocalResults) 171 return '/localresult?path='; 172 return config.layoutTestResultsURL + '/' + config.resultsDirectoryNameFromBuilderName(builderName) + '/results/layout-test-results/'; 173 } 174 175 function resultsDirectoryURLForBuildNumber(builderName, buildNumber) 176 { 177 return config.layoutTestResultsURL + '/' + config.resultsDirectoryNameFromBuilderName(builderName) + '/' + buildNumber + '/' ; 178 } 179 180 function resultsSummaryURL(builderName) 181 { 182 return resultsDirectoryURL(builderName) + kResultsName; 183 } 184 185 function resultsSummaryURLForBuildNumber(builderName, buildNumber) 186 { 187 return resultsDirectoryURLForBuildNumber(builderName, buildNumber) + kResultsName; 188 } 189 190 var g_resultsCache = new base.AsynchronousCache(function(key) { 191 return net.jsonp(key); 192 }); 193 194 results.ResultAnalyzer = base.extends(Object, { 195 init: function(resultNode) 196 { 197 this._isUnexpected = resultNode.is_unexpected; 198 this._actual = resultNode ? results.failureTypeList(resultNode.actual) : []; 199 this._expected = resultNode ? this._addImpliedExpectations(results.failureTypeList(resultNode.expected)) : []; 200 }, 201 _addImpliedExpectations: function(resultsList) 202 { 203 if (resultsList.indexOf('FAIL') == -1) 204 return resultsList; 205 return resultsList.concat(kFailingResults); 206 }, 207 _hasPass: function(results) 208 { 209 return results.indexOf(PASS) != -1; 210 }, 211 unexpectedResults: function() 212 { 213 return this._actual.filter(function(result) { 214 return this._expected.indexOf(result) == -1; 215 }, this); 216 }, 217 succeeded: function() 218 { 219 return this._hasPass(this._actual); 220 }, 221 flaky: function() 222 { 223 return this._actual.length > 1; 224 }, 225 wontfix: function() 226 { 227 return this._expected.indexOf('WONTFIX') != -1; 228 }, 229 hasUnexpectedFailures: function() 230 { 231 return this._isUnexpected; 232 } 233 }); 234 235 function isExpectedFailure(resultNode) 236 { 237 var analyzer = new results.ResultAnalyzer(resultNode); 238 return !analyzer.hasUnexpectedFailures() && !analyzer.succeeded() && !analyzer.flaky() && !analyzer.wontfix(); 239 } 240 241 function isUnexpectedFailure(resultNode) 242 { 243 var analyzer = new results.ResultAnalyzer(resultNode); 244 return analyzer.hasUnexpectedFailures() && !analyzer.succeeded() && !analyzer.flaky() && !analyzer.wontfix(); 245 } 246 247 function isResultNode(node) 248 { 249 return !!node.actual; 250 } 251 252 results.expectedFailures = function(resultsTree) 253 { 254 return base.filterTree(resultsTree.tests, isResultNode, isExpectedFailure); 255 }; 256 257 results.unexpectedFailures = function(resultsTree) 258 { 259 return base.filterTree(resultsTree.tests, isResultNode, isUnexpectedFailure); 260 }; 261 262 function resultsByTest(resultsByBuilder, filter) 263 { 264 var resultsByTest = {}; 265 266 $.each(resultsByBuilder, function(builderName, resultsTree) { 267 $.each(filter(resultsTree), function(testName, resultNode) { 268 resultsByTest[testName] = resultsByTest[testName] || {}; 269 resultsByTest[testName][builderName] = resultNode; 270 }); 271 }); 272 273 return resultsByTest; 274 } 275 276 results.expectedFailuresByTest = function(resultsByBuilder) 277 { 278 return resultsByTest(resultsByBuilder, results.expectedFailures); 279 }; 280 281 results.unexpectedFailuresByTest = function(resultsByBuilder) 282 { 283 return resultsByTest(resultsByBuilder, results.unexpectedFailures); 284 }; 285 286 results.failureInfoForTestAndBuilder = function(resultsByTest, testName, builderName) 287 { 288 var failureInfoForTest = { 289 'testName': testName, 290 'builderName': builderName, 291 'failureTypeList': results.failureTypeList(resultsByTest[testName][builderName].actual), 292 }; 293 294 return failureInfoForTest; 295 }; 296 297 results.collectUnexpectedResults = function(dictionaryOfResultNodes) 298 { 299 var collectedResults = []; 300 $.each(dictionaryOfResultNodes, function(key, resultNode) { 301 var analyzer = new results.ResultAnalyzer(resultNode); 302 collectedResults = collectedResults.concat(analyzer.unexpectedResults()); 303 }); 304 return base.uniquifyArray(collectedResults); 305 }; 306 307 // Callback data is [{ buildNumber:, url: }] 308 function historicalResultsLocations(builderName) 309 { 310 return builders.mostRecentBuildForBuilder(builderName).then(function (mostRecentBuildNumber) { 311 var resultsLocations = []; 312 // Return the builds in reverse chronological order in order to load the most recent data first. 313 for (var buildNumber = mostRecentBuildNumber; buildNumber > mostRecentBuildNumber - 100; --buildNumber) { 314 resultsLocations.push({ 315 'buildNumber': buildNumber, 316 'url': resultsDirectoryURLForBuildNumber(builderName, buildNumber) + "failing_results.json" 317 }); 318 } 319 return resultsLocations; 320 }); 321 } 322 323 // This will repeatedly call continueCallback(revision, resultNode) until it returns false. 324 function walkHistory(builderName, testName, continueCallback) 325 { 326 var indexOfNextKeyToFetch = 0; 327 var keyList = []; 328 329 function continueWalk() 330 { 331 if (indexOfNextKeyToFetch >= keyList.length) { 332 processResultNode(0, null); 333 return; 334 } 335 336 var resultsURL = keyList[indexOfNextKeyToFetch].url; 337 ++indexOfNextKeyToFetch; 338 g_resultsCache.get(resultsURL).then(function(resultsTree) { 339 if ($.isEmptyObject(resultsTree)) { 340 continueWalk(); 341 return; 342 } 343 var resultNode = results.resultNodeForTest(resultsTree, testName); 344 var revision = parseInt(resultsTree['blink_revision']); 345 if (isNaN(revision)) 346 revision = 0; 347 processResultNode(revision, resultNode); 348 }); 349 } 350 351 function processResultNode(revision, resultNode) 352 { 353 var shouldContinue = continueCallback(revision, resultNode); 354 if (!shouldContinue) 355 return; 356 continueWalk(); 357 } 358 359 historicalResultsLocations(builderName).then(function(resultsLocations) { 360 keyList = resultsLocations; 361 continueWalk(); 362 }); 363 } 364 365 results.regressionRangeForFailure = function(builderName, testName) { 366 return new Promise(function(resolve, reject) { 367 var oldestFailingRevision = 0; 368 var newestPassingRevision = 0; 369 370 walkHistory(builderName, testName, function(revision, resultNode) { 371 if (!revision) { 372 resolve([oldestFailingRevision, newestPassingRevision]); 373 return false; 374 } 375 if (!resultNode) { 376 newestPassingRevision = revision; 377 resolve([oldestFailingRevision, newestPassingRevision]); 378 return false; 379 } 380 if (isUnexpectedFailure(resultNode)) { 381 oldestFailingRevision = revision; 382 return true; 383 } 384 if (!oldestFailingRevision) 385 return true; // We need to keep looking for a failing revision. 386 newestPassingRevision = revision; 387 resolve([oldestFailingRevision, newestPassingRevision]); 388 return false; 389 }); 390 }); 391 }; 392 393 function mergeRegressionRanges(regressionRanges) 394 { 395 var mergedRange = {}; 396 397 mergedRange.oldestFailingRevision = 0; 398 mergedRange.newestPassingRevision = 0; 399 400 $.each(regressionRanges, function(builderName, range) { 401 if (!range.oldestFailingRevision && !range.newestPassingRevision) 402 return 403 404 if (!mergedRange.oldestFailingRevision) 405 mergedRange.oldestFailingRevision = range.oldestFailingRevision; 406 if (!mergedRange.newestPassingRevision) 407 mergedRange.newestPassingRevision = range.newestPassingRevision; 408 409 if (range.oldestFailingRevision && range.oldestFailingRevision < mergedRange.oldestFailingRevision) 410 mergedRange.oldestFailingRevision = range.oldestFailingRevision; 411 if (range.newestPassingRevision > mergedRange.newestPassingRevision) 412 mergedRange.newestPassingRevision = range.newestPassingRevision; 413 }); 414 415 return mergedRange; 416 } 417 418 results.unifyRegressionRanges = function(builderNameList, testName) { 419 var regressionRanges = {}; 420 421 var rangePromises = []; 422 $.each(builderNameList, function(index, builderName) { 423 rangePromises.push(results.regressionRangeForFailure(builderName, testName) 424 .then(function(result) { 425 var oldestFailingRevision = result[0]; 426 var newestPassingRevision = result[1]; 427 var range = {}; 428 range.oldestFailingRevision = oldestFailingRevision; 429 range.newestPassingRevision = newestPassingRevision; 430 regressionRanges[builderName] = range; 431 })); 432 }); 433 return Promise.all(rangePromises).then(function() { 434 var mergedRange = mergeRegressionRanges(regressionRanges); 435 return [mergedRange.oldestFailingRevision, mergedRange.newestPassingRevision]; 436 }); 437 }; 438 439 results.resultNodeForTest = function(resultsTree, testName) 440 { 441 var testNamePath = testName.split('/'); 442 var currentNode = resultsTree['tests']; 443 $.each(testNamePath, function(index, segmentName) { 444 if (!currentNode) 445 return; 446 currentNode = (segmentName in currentNode) ? currentNode[segmentName] : null; 447 }); 448 return currentNode; 449 }; 450 451 results.resultKind = function(url) 452 { 453 if (/-actual\.[a-z]+$/.test(url)) 454 return results.kActualKind; 455 else if (/-expected\.[a-z]+$/.test(url)) 456 return results.kExpectedKind; 457 else if (/diff\.[a-z]+$/.test(url)) 458 return results.kDiffKind; 459 return results.kUnknownKind; 460 } 461 462 results.resultType = function(url) 463 { 464 if (/\.png$/.test(url)) 465 return results.kImageType; 466 if (/\.wav$/.test(url)) 467 return results.kAudioType; 468 return results.kTextType; 469 } 470 471 function sortResultURLsBySuffix(urls) 472 { 473 var sortedURLs = []; 474 $.each(kPreferredSuffixOrder, function(i, suffix) { 475 $.each(urls, function(j, url) { 476 if (!base.endsWith(url, suffix)) 477 return; 478 sortedURLs.push(url); 479 }); 480 }); 481 if (sortedURLs.length != urls.length) 482 throw "sortResultURLsBySuffix failed to return the same number of URLs."; 483 return sortedURLs; 484 } 485 486 results.fetchResultsURLs = function(failureInfo) 487 { 488 var testNameStem = base.trimExtension(failureInfo.testName); 489 var urlStem = resultsDirectoryURL(failureInfo.builderName); 490 491 var suffixList = possibleSuffixListFor(failureInfo.failureTypeList); 492 var resultURLs = []; 493 var probePromises = []; 494 $.each(suffixList, function(index, suffix) { 495 var url = urlStem + testNameStem + suffix; 496 probePromises.push(net.probe(url).then( 497 function() { 498 resultURLs.push(url); 499 }, 500 function() {})); 501 }); 502 return Promise.all(probePromises).then(function() { 503 return sortResultURLsBySuffix(resultURLs); 504 }); 505 }; 506 507 results.fetchResultsByBuilder = function(builderNameList) 508 { 509 var resultsByBuilder = {}; 510 var fetchPromises = []; 511 $.each(builderNameList, function(index, builderName) { 512 var resultsURL = resultsSummaryURL(builderName); 513 fetchPromises.push(net.jsonp(resultsURL).then(function(resultsTree) { 514 resultsByBuilder[builderName] = resultsTree; 515 })); 516 }); 517 return Promise.all(fetchPromises).then(function() { 518 return resultsByBuilder; 519 }); 520 }; 521 522 })(); 523