Home | History | Annotate | Download | only in static-dashboards 
      1 // Copyright (C) 2013 Google Inc. All rights reserved.
      2 //
      3 // Redistribution and use in source and binary forms, with or without
      4 // modification, are permitted provided that the following conditions are
      5 // met:
      6 //
      7 //         * Redistributions of source code must retain the above copyright
      8 // notice, this list of conditions and the following disclaimer.
      9 //         * Redistributions in binary form must reproduce the above
     10 // copyright notice, this list of conditions and the following disclaimer
     11 // in the documentation and/or other materials provided with the
     12 // distribution.
     13 //         * Neither the name of Google Inc. nor the names of its
     14 // contributors may be used to endorse or promote products derived from
     15 // this software without specific prior written permission.
     16 //
     17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
     18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
     19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
     20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
     21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
     22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
     23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
     24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
     25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
     27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     28 
     29 var results = results || {};
     30 
     31 (function() {
     32 
     33 // Keys in the JSON files.
     34 results.NUM_FAILURES_BY_TYPE = 'num_failures_by_type';
     35 results.FAILURE_MAP = 'failure_map';
     36 results.CHROME_REVISIONS = 'chromeRevision';
     37 results.BLINK_REVISIONS = 'blinkRevision';
     38 results.TIMESTAMPS = 'secondsSinceEpoch';
     39 results.BUILD_NUMBERS = 'buildNumbers';
     40 results.TESTS = 'tests';
     41 
     42 // Failure types.
     43 results.PASS = 'PASS';
     44 results.NO_DATA = 'NO DATA';
     45 results.SKIP = 'SKIP';
     46 results.NOTRUN = 'NOTRUN';
     47 results.WONTFIX = 'WONTFIX';
     48 
     49 // FIXME: Create a ResultsJson class or something similar that abstracts out the JSON
     50 // data format. Code outside this class shouldn't know about the guts of the JSON format.
     51 
     52 // Enum for indexing into the run-length encoded results in the JSON files.
     53 // 0 is where the count is length is stored. 1 is the value.
     54 results.RLE = {
     55     LENGTH: 0,
     56     VALUE: 1
     57 }
     58 
     59 var NON_FAILURE_TYPES = [results.PASS, results.NO_DATA, results.SKIP, results.NOTRUN, results.WONTFIX];
     60 
     61 results.isFailingResult = function(failureMap, failureType)
     62 {
     63     return NON_FAILURE_TYPES.indexOf(failureMap[failureType]) == -1;
     64 }
     65 
     66 results.testCounts = function(failuresByType)
     67 {
     68     var countData = {
     69         totalTests: [],
     70         totalFailingTests: []
     71     };
     72 
     73     for (var failureType in failuresByType) {
     74         if (failureType == results.WONTFIX)
     75             continue;
     76 
     77         var failures = failuresByType[failureType];
     78         failures.forEach(function(count, index) {
     79             if (!countData.totalTests[index]) {
     80                 countData.totalTests[index] = 0;
     81                 countData.totalFailingTests[index] = 0;
     82             }
     83 
     84             countData.totalTests[index] += count;
     85             if (failureType != results.PASS)
     86                 countData.totalFailingTests[index] += count;
     87         });
     88     }
     89     return countData;
     90 }
     91 
     92 results.determineFlakiness = function(failureMap, testResults, out)
     93 {
     94     // FIXME: Ideally this heuristic would be a bit smarter and not consider
     95     // all passes, followed by a few consecutive failures, followed by all passes
     96     // to be flakiness since that's more likely the test actually failing for a
     97     // few runs due to a commit.
     98     var FAILURE_TYPES_TO_IGNORE = [results.NOTRUN, results.NO_DATA, results.SKIP];
     99     var flipCount = 0;
    100     var mostRecentNonIgnorableFailureType;
    101 
    102     for (var i = 0; i < testResults.length; i++) {
    103         var result = testResults[i][results.RLE.VALUE];
    104         var failureType = failureMap[result];
    105         if (failureType != mostRecentNonIgnorableFailureType && FAILURE_TYPES_TO_IGNORE.indexOf(failureType) == -1) {
    106             if (mostRecentNonIgnorableFailureType)
    107                 flipCount++;
    108             mostRecentNonIgnorableFailureType = failureType;
    109         }
    110     }
    111 
    112     out.flipCount = flipCount;
    113     out.isFlaky = flipCount > 1;
    114 }
    115 
    116 })();
    117