Home | History | Annotate | Download | only in gm 
      1 #!/usr/bin/env python
      2 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
      3 # Use of this source code is governed by a BSD-style license that can be
      4 # found in the LICENSE file.
      5 
      6 """Utility to display a summary of JSON-format GM results, and exit with
      7 a nonzero errorcode if there were non-ignored failures in the GM results.
      8 
      9 Usage:
     10   python display_json_results.py <filename>
     11 
     12 TODO(epoger): We may want to add flags to set the following:
     13 - which error types cause a nonzero return code
     14 - maximum number of tests to list for any one ResultAccumulator
     15   (to keep the output reasonably short)
     16 """
     17 
     18 __author__ = 'Elliot Poger'
     19 
     20 
     21 # system-level imports
     22 import sys
     23 
     24 # local imports
     25 import gm_json
     26 
     27 
     28 class ResultAccumulator(object):
     29   """Object that accumulates results of a given type, and can generate a
     30      summary upon request."""
     31 
     32   def __init__(self, name, do_list, do_fail):
     33     """name: name of the category this result type falls into
     34        do_list: whether to list all of the tests with this results type
     35        do_fail: whether to return with nonzero exit code if there are any
     36                 results of this type
     37     """
     38     self._name = name
     39     self._do_list = do_list
     40     self._do_fail = do_fail
     41     self._testnames = []
     42 
     43   def AddResult(self, testname):
     44     """Adds a result of this particular type.
     45        testname: (string) name of the test"""
     46     self._testnames.append(testname)
     47 
     48   def ShouldSignalFailure(self):
     49     """Returns true if this result type is serious (self._do_fail is True)
     50        and there were any results of this type."""
     51     if self._do_fail and self._testnames:
     52       return True
     53     else:
     54       return False
     55 
     56   def GetSummaryLine(self):
     57     """Returns a single-line string summary of all results added to this
     58        accumulator so far."""
     59     summary = ''
     60     if self._do_fail:
     61       summary += '[*] '
     62     else:
     63       summary += '[ ] '
     64     summary += str(len(self._testnames))
     65     summary += ' '
     66     summary += self._name
     67     if self._do_list:
     68       summary += ': '
     69       for testname in self._testnames:
     70         summary += testname
     71         summary += ' '
     72     return summary
     73 
     74 
     75 def Display(filepath):
     76   """Displays a summary of the results in a JSON file.
     77      Returns True if the results are free of any significant failures.
     78      filepath: (string) path to JSON file"""
     79 
     80   # Map labels within the JSON file to the ResultAccumulator for each label.
     81   results_map = {
     82     gm_json.JSONKEY_ACTUALRESULTS_FAILED:
     83         ResultAccumulator(name='ExpectationsMismatch',
     84                           do_list=True, do_fail=True),
     85     gm_json.JSONKEY_ACTUALRESULTS_FAILUREIGNORED:
     86         ResultAccumulator(name='IgnoredExpectationsMismatch',
     87                           do_list=True, do_fail=False),
     88     gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON:
     89         ResultAccumulator(name='MissingExpectations',
     90                           do_list=False, do_fail=False),
     91     gm_json.JSONKEY_ACTUALRESULTS_SUCCEEDED:
     92         ResultAccumulator(name='Passed',
     93                           do_list=False, do_fail=False),
     94   }
     95 
     96   success = True
     97   json_dict = gm_json.LoadFromFile(filepath)
     98   actual_results = json_dict[gm_json.JSONKEY_ACTUALRESULTS]
     99   for label, accumulator in results_map.iteritems():
    100     results = actual_results[label]
    101     if results:
    102       for result in results:
    103         accumulator.AddResult(result)
    104     print accumulator.GetSummaryLine()
    105     if accumulator.ShouldSignalFailure():
    106       success = False
    107   print '(results marked with [*] will cause nonzero return value)'
    108   return success
    109 
    110 
    111 if '__main__' == __name__:
    112   if len(sys.argv) != 2:
    113     raise Exception('usage: %s <input-json-filepath>' % sys.argv[0])
    114   sys.exit(0 if Display(sys.argv[1]) else 1)
    115